An open source, non-profit web search engine
Remember which pages we indexed
Daoud Clarkec949cfbd2d57
diff --git a/crawl.py b/crawl.pyindex 75caaf6..39d7a9a 100644--- a/crawl.py+++ b/crawl.py@@ -1,6 +1,7 @@import loggingimport osimport time+from collections import Counterfrom multiprocessing import Processfrom pathlib import Path@@ -32,6 +33,7 @@ logger = logging.getLogger(__name__)FORMAT = "%(process)d:%(levelname)s:%(name)s:%(message)s"logging.basicConfig(level=logging.INFO, format=FORMAT)+API_KEY = os.environ["MWMBL_API_KEY"]BATCH_QUEUE_KEY = "batch-queue"@@ -91,7 +93,8 @@ def run_indexing():returnlogger.info(f"Got {len(batch_jsons)} batches to index")batches = [HashedBatch.parse_raw(b) for b in batch_jsons]- term_new_doc_count = index_batches(batches, index_path)+ term_new_docs = index_batches(batches, index_path)+ term_new_doc_count = Counter({term: len(docs) for term, docs in term_new_docs.items()})logger.info(f"Indexed, top terms to sync: {term_new_doc_count.most_common(10)}")remote_index = RemoteIndex()@@ -107,11 +110,11 @@ def run_indexing():result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,score=doc.score, term=doc.term, state=doc.state) for doc in new_items]- results = Results(api_key="test", results=result_items)+ results = Results(api_key=API_KEY, results=result_items)response = requests.post("https://beta.mwmbl.org/api/v1/crawler/results", json=results.dict())+ print("Response", response.text)response.raise_for_status()- print("Response", response.text)diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 6d51a86..a572ed8 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -182,7 +182,7 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versiodocuments = [Document(url=result.url, title=result.title, extract=result.extract) for result in results.results]index_path = f"{settings.DATA_PATH}/{settings.INDEX_NAME}"- index_documents(documents, index_path)+ new_doc_count = index_documents(documents, index_path)now = datetime.now(timezone.utc)filename = upload_object(results, now, api_key.user.username, "results")diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.pyindex 480856a..838cbed 100644--- a/mwmbl/indexer/index_batches.py+++ b/mwmbl/indexer/index_batches.py@@ -42,24 +42,29 @@ def get_url_score(url):return 1/len(url)-def index_batches(batch_data: Collection[HashedBatch], index_path: str) -> Counter:+def index_batches(batch_data: Collection[HashedBatch], index_path: str) -> dict[str, list[Document]]:start_time = datetime.utcnow()document_tuples = list(get_documents_from_batches(batch_data))documents = [Document(title, url, extract) for title, url, extract in document_tuples]- end_time, new_page_doc_counts = index_documents(documents, index_path)+ new_page_documents = index_documents(documents, index_path)+ end_time = datetime.utcnow()logger.info(f"Indexing took {end_time - start_time}")- return new_page_doc_counts+ return new_page_documents-def index_documents(documents, index_path):+def index_documents(documents: list[Document], index_path: str) -> dict[str, list[Document]]:page_documents = preprocess_documents(documents, index_path)- new_page_doc_counts = index_pages(index_path, page_documents)- end_time = datetime.utcnow()- return end_time, new_page_doc_counts+ new_page_documents = index_pages(index_path, page_documents)+ return new_page_documents-def index_pages(index_path: str, page_documents: dict[int, list[Document]]) -> Counter:- term_new_doc_counts = Counter()+def index_pages(index_path: str, page_documents: dict[int, list[Document]]) -> dict[str, list[Document]]:+ """+ Index the pages with the given documents.++ Returns a dictionary of term to the new documents that were added.+ """+ term_new_docs = defaultdict(list)with TinyIndex(Document, index_path, 'w') as indexer:ranker = HeuristicRanker(indexer, None, score_threshold=float('-inf'))for page, documents in page_documents.items():@@ -75,12 +80,14 @@ def index_pages(index_path: str, page_documents: dict[int, list[Document]]) -> Cnew_documents.append(document)seen_urls.add(document.url)seen_titles.add(document.title)- logger.info(f"Storing {len(new_documents)} documents for page {page}, originally {len(existing_documents)}")- indexer.store_in_page(page, new_documents)+ num_fitting = indexer.store_in_page(page, new_documents)+ logger.info(f"Stored {num_fitting} documents of {len(new_documents)} for page {page}, originally {len(existing_documents)}")++ for document in new_documents[:num_fitting]:+ if document.state != DocumentState.SYNCED_WITH_MAIN_INDEX.value:+ term_new_docs[document.term].append(document)- term_new_doc_counts.update(document.term for document in new_documents- if document.state != DocumentState.SYNCED_WITH_MAIN_INDEX.value)- return term_new_doc_counts+ return term_new_docsdef sort_documents(documents, all_existing_documents, ranker):diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pyindex cd5b4cb..b81b515 100644--- a/mwmbl/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -141,13 +141,19 @@ def _trim_items_to_page(compressor: ZstdCompressor, page_size: int, items:list[Treturn _binary_search_fitting_size(compressor, page_size, items, 0, len(items))-def _get_page_data(page_size: int, items: list[T]):+def _get_page_data(page_size: int, items: list[T]) -> tuple[bytes, int]:+ """+ Compress the data, removing items that don't fit.++ Returns the compressed data and number of items that are included.+ """+compressor = ZstdCompressor()num_fitting, serialised_data = _trim_items_to_page(compressor, page_size, items)compressed_data = compressor.compress(json.dumps(items[:num_fitting]).encode('utf8'))assert len(compressed_data) <= page_size, "The data shouldn't get bigger"- return _pad_to_page_size(compressed_data, page_size)+ return _pad_to_page_size(compressed_data, page_size), num_fittingdef _pad_to_page_size(data: bytes, page_size: int):@@ -220,21 +226,30 @@ class TinyIndex(Generic[T]):return []return json.loads(decompressed_data.decode('utf8'))- def store_in_page(self, page_index: int, values: list[T]):+ def store_in_page(self, page_index: int, values: list[T]) -> int:+ """+ Store as many items in the page as possible in the given order.++ Returns the number of items stored in the page.+ """value_tuples = [astuple(value) for value in values]- self._write_page(value_tuples, page_index)+ num_fitting = self._write_page(value_tuples, page_index)+ return num_fitting- def _write_page(self, data, i: int):+ def _write_page(self, data, i: int) -> int:"""Serialise the data using JSON, compress it and store it at index i.If the data is too big, it will store the first items in the list and discard the rest.++ Returns the number of items stored in the page."""if self.mode != 'w':raise UnsupportedOperation("The file is open in read mode, you cannot write")- page_data = _get_page_data(self.page_size, data)+ page_data, num_fitting = _get_page_data(self.page_size, data)logger.debug(f"Got page data of length {len(page_data)}")self.mmap[i * self.page_size + METADATA_SIZE:(i+1) * self.page_size + METADATA_SIZE] = page_data+ return num_fitting@staticmethoddef create(item_factory: Callable[..., T], index_path: str, num_pages: int, page_size: int):@@ -245,7 +260,7 @@ class TinyIndex(Generic[T]):metadata_bytes = metadata.to_bytes()metadata_padded = _pad_to_page_size(metadata_bytes, METADATA_SIZE)- page_bytes = _get_page_data(page_size, [])+ page_bytes, num_fitting = _get_page_data(page_size, [])with open(index_path, 'wb') as index_file:index_file.write(metadata_padded)diff --git a/mwmbl/views.py b/mwmbl/views.pyindex ef2bf55..fdc8a47 100644--- a/mwmbl/views.py+++ b/mwmbl/views.py@@ -403,8 +403,8 @@ def _save_to_index(query: str, new_results: list[Document]):doc.state = states.get(doc.url, doc.state)all_documents = documents + other_documents- logger.info(f"Storing {len(all_documents)} documents at page {page_index}")- indexer.store_in_page(page_index, all_documents)+ num_fitting = indexer.store_in_page(page_index, all_documents)+ logger.info(f"Stored {num_fitting} of {len(all_documents)} documents at page {page_index}")return {"curation": "ok"}diff --git a/test/test_indexer.py b/test/test_indexer.pyindex fe67876..bd05ca8 100644--- a/test/test_indexer.py+++ b/test/test_indexer.py@@ -63,7 +63,7 @@ def test_get_page_data_single_doc():# Compare the trimmed data to the actual data we're persisting# We need to pad the trimmmed data, then it should be equal to the data we persistpadded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)- serialized_data = _get_page_data(page_size, items)+ serialized_data, num_fitting = _get_page_data(page_size, items)assert serialized_data == padded_trimmed_data@@ -87,7 +87,7 @@ def test_get_page_data_many_docs_all_fit():# Compare the trimmed data to the actual data we're persisting# We need to pad the trimmed data, then it should be equal to the data we persist- serialized_data = _get_page_data(page_size, items)+ serialized_data, num_fitting = _get_page_data(page_size, items)padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)assert serialized_data == padded_trimmed_data@@ -105,7 +105,7 @@ def test_get_page_data_many_docs_subset_fit():# Trim the itemscompressor = ZstdCompressor()- num_fitting,trimmed_data = _trim_items_to_page(compressor,page_size,items)+ num_fitting, trimmed_data = _trim_items_to_page(compressor,page_size,items)# We should be able to fit a subset of the items onto the pageassert num_fitting > 1@@ -113,7 +113,7 @@ def test_get_page_data_many_docs_subset_fit():# Compare the trimmed data to the actual data we're persisting# We need to pad the trimmed data, then it should be equal to the data we persist- serialized_data = _get_page_data(page_size, items)+ serialized_data, num_fitting = _get_page_data(page_size, items)padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)assert serialized_data == padded_trimmed_data