An open source, non-profit web search engine
Fix new crawler: always use enum for DocumentState
Daoud Clarke58c985dc42e8
diff --git a/analyse/add_term_info.py b/analyse/add_term_info.pyindex 9cf9795..c1030c2 100644--- a/analyse/add_term_info.py+++ b/analyse/add_term_info.py@@ -10,7 +10,7 @@ from random import Randomimport numpy as npfrom scipy.stats import sem-from mwmbl.tinysearchengine.indexer import TinyIndex, Document, _trim_items_to_page, astuple+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, _trim_items_to_pagefrom zstandard import ZstdCompressor@@ -37,7 +37,7 @@ def run():term_document = add_term_info(document, index, i)term_documents.append(term_document)- value_tuples = [astuple(value) for value in term_documents]+ value_tuples = [document.as_tuple() for document in term_documents]num_fitting, compressed = _trim_items_to_page(compressor, index.page_size, value_tuples)new_sizes.append(num_fitting)diff --git a/crawl.py b/crawl.pyindex 31b0c2d..09f5e41 100644--- a/crawl.py+++ b/crawl.py@@ -99,15 +99,13 @@ def run_indexing():remote_index = RemoteIndex()with TinyIndex(Document, index_path, 'w') as local_index:- for term, count in term_new_doc_count.most_common(10):+ for term, count in term_new_doc_count.most_common(100):logger.info(f"Syncing term {term} with {count} new local items")remote_items = remote_index.retrieve(term)remote_item_urls = {item.url for item in remote_items}local_items = local_index.retrieve(term)new_items = [item for item in local_items if item.url not in remote_item_urls]logger.info(f"Found {len(new_items)} new items for term {term}")- if len(new_items) == 0:- continueterms = tokenize(term)remote_item_scores = [score_result(terms, item, True) for item in remote_items]@@ -116,27 +114,30 @@ def run_indexing():max_local_score = max(local_scores, default=0.0)logger.info(f"Max local score: {max_local_score}, min remote score: {min_remote_score}")- if max_local_score < min_remote_score:- continue+ new_high_score = max_local_score < min_remote_score- result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,- score=doc.score, term=doc.term, state=doc.state) for doc in new_items]- results = Results(api_key=API_KEY, results=result_items)- logger.info(f"Posting {len(result_items)} results")- response = requests.post("https://mwmbl.org/api/v1/crawler/results", json=results.dict())- logger.info(f"Response: {response.text}")- response.raise_for_status()+ if new_high_score:+ result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,+ score=doc.score, term=doc.term, state=doc.state) for doc in new_items]+ results = Results(api_key=API_KEY, results=result_items)+ logger.info(f"Posting {len(result_items)} results")+ response = requests.post("https://mwmbl.org/api/v1/crawler/results", json=results.dict())+ logger.info(f"Response: {response.text}")+ response.raise_for_status()new_remote_items = remote_index.retrieve(term, refresh=True)# Check how many of our items were indexednew_remote_item_urls = {item.url for item in new_remote_items}indexed_items = sum(1 for item in new_items if item.url in new_remote_item_urls)- logger.info(f"Indexed items: {indexed_items}/{len(new_items)} for term {term}")+ logger.info(f'Indexed items: {indexed_items}/{len(new_items)} for term "{term}"')page_index = local_index.get_key_page_index(term)index_pages(index_path, {page_index: new_remote_items}, mark_synced=True)logger.info(f"Completed indexing for term {term}")+ new_page_content = local_index.get_page(page_index)+ logger.info(f"Page content: {new_page_content}")+if __name__ == "__main__":run()diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.pyindex 2a60db6..47a5feb 100644--- a/mwmbl/indexer/index_batches.py+++ b/mwmbl/indexer/index_batches.py@@ -64,27 +64,31 @@ def index_pages(index_path: str, page_documents: dict[int, list[Document]], markranker = HeuristicRanker(indexer, None, score_threshold=float('-inf'))for page, documents in page_documents.items():existing_documents = indexer.get_page(page)- sorted_documents = sort_documents(documents, existing_documents, ranker)-- seen_urls = set()- seen_titles = set()- combined_documents = []- for document in sorted_documents:- if document.title in seen_titles or document.url in seen_urls:- continue- if mark_synced:- document.state = DocumentState.SYNCED_WITH_MAIN_INDEX.value- combined_documents.append(document)- seen_urls.add(document.url)- seen_titles.add(document.title)+ combined_documents = combine_documents(existing_documents, documents, mark_synced, ranker)logger.info(f"Storing {len(combined_documents)} documents for page {page}, originally {len(existing_documents)}")indexer.store_in_page(page, combined_documents)term_new_doc_counts.update(document.term for document in combined_documents- if document.state != DocumentState.SYNCED_WITH_MAIN_INDEX.value)+ if document.state != DocumentState.SYNCED_WITH_MAIN_INDEX)return term_new_doc_counts+def combine_documents(existing_documents, documents, mark_synced, ranker):+ sorted_documents = sort_documents(documents, existing_documents, ranker)+ seen_urls = set()+ seen_titles = set()+ combined_documents = []+ for document in sorted_documents:+ if document.title in seen_titles or document.url in seen_urls:+ continue+ if mark_synced:+ document.state = DocumentState.SYNCED_WITH_MAIN_INDEX+ combined_documents.append(document)+ seen_urls.add(document.url)+ seen_titles.add(document.title)+ return combined_documents++def sort_documents(documents, all_existing_documents, ranker):curated_documents = [doc for doc in all_existing_documents if doc.state in CURATED_STATES]existing_documents = [doc for doc in all_existing_documents if doc.state not in CURATED_STATES]diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pyindex 985c684..a84d627 100644--- a/mwmbl/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -20,16 +20,6 @@ PAGE_SIZE = 4096logger = getLogger(__name__)-def astuple(dc):- """- Convert a type to a tuple - values at the end that are None can be truncated.- """- value = tuple(dc.__dict__.values())- while value[-1] is None:- value = value[:-1]- return value--class DocumentState(IntEnum):"""The state of the document in the index. A value of None indicates an organic search result.@@ -45,7 +35,7 @@ class DocumentState(IntEnum):FROM_WIKI_APPROVED = 10-CURATED_STATES = {state.value for state in DocumentState if state.value >= 7}+CURATED_STATES = {state for state in DocumentState if state.value >= 7}@dataclass@@ -64,7 +54,7 @@ class Document:extract: str,score: Optional[float] = None,term: Optional[str] = None,- state: Optional[int] = None+ state: Optional[int | DocumentState] = None):# Sometimes the title or extract may be None, probably because of user generated content# It's not allowed to be None though, or things will break@@ -73,7 +63,19 @@ class Document:self.extract = extract if extract is not None else ''self.score = scoreself.term = term- self.state = state+ self.state = None if state is None else DocumentState(state)++ def as_tuple(self):+ """+ Convert a type to a tuple - values at the end that are None can be truncated.+ """+ values = list(self.__dict__.values())+ if values[-1] is not None:+ values[-1] = values[-1].value++ while values[-1] is None:+ values = values[:-1]+ return tuple(values)@dataclass@@ -224,7 +226,7 @@ class TinyIndex(Generic[T]):return json.loads(decompressed_data.decode('utf8'))def store_in_page(self, page_index: int, values: list[T]):- value_tuples = [astuple(value) for value in values]+ value_tuples = [value.as_tuple() for value in values]self._write_page(value_tuples, page_index)def _write_page(self, data, i: int):diff --git a/test/test_index_batches.py b/test/test_index_batches.pyindex 57fac78..754609f 100644--- a/test/test_index_batches.py+++ b/test/test_index_batches.py@@ -1,4 +1,4 @@-from mwmbl.indexer.index_batches import sort_documents+from mwmbl.indexer.index_batches import sort_documents, combine_documentsfrom mwmbl.tinysearchengine.indexer import Document, DocumentState@@ -62,3 +62,22 @@ def test_sort_documents_curated_items_first():Document(title="title5", url="2", extract="extract5", term="term1"),Document(title="title2", url="4", extract="extract2", term="term2"),]+++def test_sort_documents_duplicates_keep_synced_state():+ existing_documents = [+ Document(title="title1", url="1", extract="extract1", term="term1", state=DocumentState.SYNCED_WITH_MAIN_INDEX),+ ]++ documents = [+ Document(title="title1", url="1", extract="extract1", term="term1"),+ ]++ # Sort the documents+ combined_documents = combine_documents(documents, existing_documents, mark_synced=False, ranker=UrlRanker())+ print("Combined documents", combined_documents)++ # Curated items should be first+ assert combined_documents == [+ Document(title="title1", url="1", extract="extract1", term="term1", state=DocumentState.SYNCED_WITH_MAIN_INDEX),+ ]diff --git a/test/test_indexer.py b/test/test_indexer.pyindex fe67876..41cf2a1 100644--- a/test/test_indexer.py+++ b/test/test_indexer.py@@ -3,7 +3,7 @@ from tempfile import TemporaryDirectoryfrom zstandard import ZstdCompressor-from mwmbl.tinysearchengine.indexer import TinyIndex, Document, _binary_search_fitting_size, astuple, \+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, _binary_search_fitting_size, \_trim_items_to_page, _pad_to_page_size, _get_page_data@@ -45,12 +45,12 @@ def test_binary_search_fitting_size_none_fit():# We should not fit anythingassert count_fit == -1assert data is None-++def test_get_page_data_single_doc():document1 = Document(title='title1',url='url1',extract='extract1',score=1.0)- documents = [document1]- items = [astuple(value) for value in documents]-+ items = [document1.as_tuple()]+compressor = ZstdCompressor()page_size = 4096@@ -76,7 +76,7 @@ def test_get_page_data_many_docs_all_fit():txt = 'text{}'.format(x)document = Document(title=txt,url=txt,extract=txt,score=x)documents.append(document)- items = [astuple(value) for value in documents]+ items = [document.as_tuple() for document in documents]# Trim the itemscompressor = ZstdCompressor()@@ -91,7 +91,8 @@ def test_get_page_data_many_docs_all_fit():padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)assert serialized_data == padded_trimmed_data-++def test_get_page_data_many_docs_subset_fit():# Build giant documents itemdocuments = []@@ -101,7 +102,7 @@ def test_get_page_data_many_docs_subset_fit():txt = 'text{}'.format(x)document = Document(title=txt,url=txt,extract=txt,score=x)documents.append(document)- items = [astuple(value) for value in documents]+ items = [document.as_tuple() for document in documents]# Trim the itemscompressor = ZstdCompressor()