Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Remember which pages we indexed

Daoud Clarkec949cfbd2d57
diff --git a/crawl.py b/crawl.py
index 75caaf6..39d7a9a 100644
--- a/crawl.py
+++ b/crawl.py
@@ -1,6 +1,7 @@
import logging
import os
import time
+from collections import Counter
from multiprocessing import Process
from pathlib import Path
@@ -32,6 +33,7 @@ logger = logging.getLogger(__name__)
FORMAT = "%(process)d:%(levelname)s:%(name)s:%(message)s"
logging.basicConfig(level=logging.INFO, format=FORMAT)
+API_KEY = os.environ["MWMBL_API_KEY"]
BATCH_QUEUE_KEY = "batch-queue"
@@ -91,7 +93,8 @@ def run_indexing():
return
logger.info(f"Got {len(batch_jsons)} batches to index")
batches = [HashedBatch.parse_raw(b) for b in batch_jsons]
- term_new_doc_count = index_batches(batches, index_path)
+ term_new_docs = index_batches(batches, index_path)
+ term_new_doc_count = Counter({term: len(docs) for term, docs in term_new_docs.items()})
logger.info(f"Indexed, top terms to sync: {term_new_doc_count.most_common(10)}")
remote_index = RemoteIndex()
@@ -107,11 +110,11 @@ def run_indexing():
result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,
score=doc.score, term=doc.term, state=doc.state) for doc in new_items]
- results = Results(api_key="test", results=result_items)
+ results = Results(api_key=API_KEY, results=result_items)
response = requests.post("https://beta.mwmbl.org/api/v1/crawler/results", json=results.dict())
+ print("Response", response.text)
response.raise_for_status()
- print("Response", response.text)
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index 6d51a86..a572ed8 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -182,7 +182,7 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
documents = [Document(url=result.url, title=result.title, extract=result.extract) for result in results.results]
index_path = f"{settings.DATA_PATH}/{settings.INDEX_NAME}"
- index_documents(documents, index_path)
+ new_doc_count = index_documents(documents, index_path)
now = datetime.now(timezone.utc)
filename = upload_object(results, now, api_key.user.username, "results")
diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.py
index 480856a..838cbed 100644
--- a/mwmbl/indexer/index_batches.py
+++ b/mwmbl/indexer/index_batches.py
@@ -42,24 +42,29 @@ def get_url_score(url):
return 1/len(url)
-def index_batches(batch_data: Collection[HashedBatch], index_path: str) -> Counter:
+def index_batches(batch_data: Collection[HashedBatch], index_path: str) -> dict[str, list[Document]]:
start_time = datetime.utcnow()
document_tuples = list(get_documents_from_batches(batch_data))
documents = [Document(title, url, extract) for title, url, extract in document_tuples]
- end_time, new_page_doc_counts = index_documents(documents, index_path)
+ new_page_documents = index_documents(documents, index_path)
+ end_time = datetime.utcnow()
logger.info(f"Indexing took {end_time - start_time}")
- return new_page_doc_counts
+ return new_page_documents
-def index_documents(documents, index_path):
+def index_documents(documents: list[Document], index_path: str) -> dict[str, list[Document]]:
page_documents = preprocess_documents(documents, index_path)
- new_page_doc_counts = index_pages(index_path, page_documents)
- end_time = datetime.utcnow()
- return end_time, new_page_doc_counts
+ new_page_documents = index_pages(index_path, page_documents)
+ return new_page_documents
-def index_pages(index_path: str, page_documents: dict[int, list[Document]]) -> Counter:
- term_new_doc_counts = Counter()
+def index_pages(index_path: str, page_documents: dict[int, list[Document]]) -> dict[str, list[Document]]:
+ """
+ Index the pages with the given documents.
+
+ Returns a dictionary of term to the new documents that were added.
+ """
+ term_new_docs = defaultdict(list)
with TinyIndex(Document, index_path, 'w') as indexer:
ranker = HeuristicRanker(indexer, None, score_threshold=float('-inf'))
for page, documents in page_documents.items():
@@ -75,12 +80,14 @@ def index_pages(index_path: str, page_documents: dict[int, list[Document]]) -> C
new_documents.append(document)
seen_urls.add(document.url)
seen_titles.add(document.title)
- logger.info(f"Storing {len(new_documents)} documents for page {page}, originally {len(existing_documents)}")
- indexer.store_in_page(page, new_documents)
+ num_fitting = indexer.store_in_page(page, new_documents)
+ logger.info(f"Stored {num_fitting} documents of {len(new_documents)} for page {page}, originally {len(existing_documents)}")
+
+ for document in new_documents[:num_fitting]:
+ if document.state != DocumentState.SYNCED_WITH_MAIN_INDEX.value:
+ term_new_docs[document.term].append(document)
- term_new_doc_counts.update(document.term for document in new_documents
- if document.state != DocumentState.SYNCED_WITH_MAIN_INDEX.value)
- return term_new_doc_counts
+ return term_new_docs
def sort_documents(documents, all_existing_documents, ranker):
diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.py
index cd5b4cb..b81b515 100644
--- a/mwmbl/tinysearchengine/indexer.py
+++ b/mwmbl/tinysearchengine/indexer.py
@@ -141,13 +141,19 @@ def _trim_items_to_page(compressor: ZstdCompressor, page_size: int, items:list[T
return _binary_search_fitting_size(compressor, page_size, items, 0, len(items))
-def _get_page_data(page_size: int, items: list[T]):
+def _get_page_data(page_size: int, items: list[T]) -> tuple[bytes, int]:
+ """
+ Compress the data, removing items that don't fit.
+
+ Returns the compressed data and number of items that are included.
+ """
+
compressor = ZstdCompressor()
num_fitting, serialised_data = _trim_items_to_page(compressor, page_size, items)
compressed_data = compressor.compress(json.dumps(items[:num_fitting]).encode('utf8'))
assert len(compressed_data) <= page_size, "The data shouldn't get bigger"
- return _pad_to_page_size(compressed_data, page_size)
+ return _pad_to_page_size(compressed_data, page_size), num_fitting
def _pad_to_page_size(data: bytes, page_size: int):
@@ -220,21 +226,30 @@ class TinyIndex(Generic[T]):
return []
return json.loads(decompressed_data.decode('utf8'))
- def store_in_page(self, page_index: int, values: list[T]):
+ def store_in_page(self, page_index: int, values: list[T]) -> int:
+ """
+ Store as many items in the page as possible in the given order.
+
+ Returns the number of items stored in the page.
+ """
value_tuples = [astuple(value) for value in values]
- self._write_page(value_tuples, page_index)
+ num_fitting = self._write_page(value_tuples, page_index)
+ return num_fitting
- def _write_page(self, data, i: int):
+ def _write_page(self, data, i: int) -> int:
"""
Serialise the data using JSON, compress it and store it at index i.
If the data is too big, it will store the first items in the list and discard the rest.
+
+ Returns the number of items stored in the page.
"""
if self.mode != 'w':
raise UnsupportedOperation("The file is open in read mode, you cannot write")
- page_data = _get_page_data(self.page_size, data)
+ page_data, num_fitting = _get_page_data(self.page_size, data)
logger.debug(f"Got page data of length {len(page_data)}")
self.mmap[i * self.page_size + METADATA_SIZE:(i+1) * self.page_size + METADATA_SIZE] = page_data
+ return num_fitting
@staticmethod
def create(item_factory: Callable[..., T], index_path: str, num_pages: int, page_size: int):
@@ -245,7 +260,7 @@ class TinyIndex(Generic[T]):
metadata_bytes = metadata.to_bytes()
metadata_padded = _pad_to_page_size(metadata_bytes, METADATA_SIZE)
- page_bytes = _get_page_data(page_size, [])
+ page_bytes, num_fitting = _get_page_data(page_size, [])
with open(index_path, 'wb') as index_file:
index_file.write(metadata_padded)
diff --git a/mwmbl/views.py b/mwmbl/views.py
index ef2bf55..fdc8a47 100644
--- a/mwmbl/views.py
+++ b/mwmbl/views.py
@@ -403,8 +403,8 @@ def _save_to_index(query: str, new_results: list[Document]):
doc.state = states.get(doc.url, doc.state)
all_documents = documents + other_documents
- logger.info(f"Storing {len(all_documents)} documents at page {page_index}")
- indexer.store_in_page(page_index, all_documents)
+ num_fitting = indexer.store_in_page(page_index, all_documents)
+ logger.info(f"Stored {num_fitting} of {len(all_documents)} documents at page {page_index}")
return {"curation": "ok"}
diff --git a/test/test_indexer.py b/test/test_indexer.py
index fe67876..bd05ca8 100644
--- a/test/test_indexer.py
+++ b/test/test_indexer.py
@@ -63,7 +63,7 @@ def test_get_page_data_single_doc():
# Compare the trimmed data to the actual data we're persisting
# We need to pad the trimmmed data, then it should be equal to the data we persist
padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)
- serialized_data = _get_page_data(page_size, items)
+ serialized_data, num_fitting = _get_page_data(page_size, items)
assert serialized_data == padded_trimmed_data
@@ -87,7 +87,7 @@ def test_get_page_data_many_docs_all_fit():
# Compare the trimmed data to the actual data we're persisting
# We need to pad the trimmed data, then it should be equal to the data we persist
- serialized_data = _get_page_data(page_size, items)
+ serialized_data, num_fitting = _get_page_data(page_size, items)
padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)
assert serialized_data == padded_trimmed_data
@@ -105,7 +105,7 @@ def test_get_page_data_many_docs_subset_fit():
# Trim the items
compressor = ZstdCompressor()
- num_fitting,trimmed_data = _trim_items_to_page(compressor,page_size,items)
+ num_fitting, trimmed_data = _trim_items_to_page(compressor,page_size,items)
# We should be able to fit a subset of the items onto the page
assert num_fitting > 1
@@ -113,7 +113,7 @@ def test_get_page_data_many_docs_subset_fit():
# Compare the trimmed data to the actual data we're persisting
# We need to pad the trimmed data, then it should be equal to the data we persist
- serialized_data = _get_page_data(page_size, items)
+ serialized_data, num_fitting = _get_page_data(page_size, items)
padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)
assert serialized_data == padded_trimmed_data