Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Fix new crawler: always use enum for DocumentState

Daoud Clarke58c985dc42e8
diff --git a/analyse/add_term_info.py b/analyse/add_term_info.py
index 9cf9795..c1030c2 100644
--- a/analyse/add_term_info.py
+++ b/analyse/add_term_info.py
@@ -10,7 +10,7 @@ from random import Random
import numpy as np
from scipy.stats import sem
-from mwmbl.tinysearchengine.indexer import TinyIndex, Document, _trim_items_to_page, astuple
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, _trim_items_to_page
from zstandard import ZstdCompressor
@@ -37,7 +37,7 @@ def run():
term_document = add_term_info(document, index, i)
term_documents.append(term_document)
- value_tuples = [astuple(value) for value in term_documents]
+ value_tuples = [document.as_tuple() for document in term_documents]
num_fitting, compressed = _trim_items_to_page(compressor, index.page_size, value_tuples)
new_sizes.append(num_fitting)
diff --git a/crawl.py b/crawl.py
index 31b0c2d..09f5e41 100644
--- a/crawl.py
+++ b/crawl.py
@@ -99,15 +99,13 @@ def run_indexing():
remote_index = RemoteIndex()
with TinyIndex(Document, index_path, 'w') as local_index:
- for term, count in term_new_doc_count.most_common(10):
+ for term, count in term_new_doc_count.most_common(100):
logger.info(f"Syncing term {term} with {count} new local items")
remote_items = remote_index.retrieve(term)
remote_item_urls = {item.url for item in remote_items}
local_items = local_index.retrieve(term)
new_items = [item for item in local_items if item.url not in remote_item_urls]
logger.info(f"Found {len(new_items)} new items for term {term}")
- if len(new_items) == 0:
- continue
terms = tokenize(term)
remote_item_scores = [score_result(terms, item, True) for item in remote_items]
@@ -116,27 +114,30 @@ def run_indexing():
max_local_score = max(local_scores, default=0.0)
logger.info(f"Max local score: {max_local_score}, min remote score: {min_remote_score}")
- if max_local_score < min_remote_score:
- continue
+ new_high_score = max_local_score < min_remote_score
- result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,
- score=doc.score, term=doc.term, state=doc.state) for doc in new_items]
- results = Results(api_key=API_KEY, results=result_items)
- logger.info(f"Posting {len(result_items)} results")
- response = requests.post("https://mwmbl.org/api/v1/crawler/results", json=results.dict())
- logger.info(f"Response: {response.text}")
- response.raise_for_status()
+ if new_high_score:
+ result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,
+ score=doc.score, term=doc.term, state=doc.state) for doc in new_items]
+ results = Results(api_key=API_KEY, results=result_items)
+ logger.info(f"Posting {len(result_items)} results")
+ response = requests.post("https://mwmbl.org/api/v1/crawler/results", json=results.dict())
+ logger.info(f"Response: {response.text}")
+ response.raise_for_status()
new_remote_items = remote_index.retrieve(term, refresh=True)
# Check how many of our items were indexed
new_remote_item_urls = {item.url for item in new_remote_items}
indexed_items = sum(1 for item in new_items if item.url in new_remote_item_urls)
- logger.info(f"Indexed items: {indexed_items}/{len(new_items)} for term {term}")
+ logger.info(f'Indexed items: {indexed_items}/{len(new_items)} for term "{term}"')
page_index = local_index.get_key_page_index(term)
index_pages(index_path, {page_index: new_remote_items}, mark_synced=True)
logger.info(f"Completed indexing for term {term}")
+ new_page_content = local_index.get_page(page_index)
+ logger.info(f"Page content: {new_page_content}")
+
if __name__ == "__main__":
run()
diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.py
index 2a60db6..47a5feb 100644
--- a/mwmbl/indexer/index_batches.py
+++ b/mwmbl/indexer/index_batches.py
@@ -64,27 +64,31 @@ def index_pages(index_path: str, page_documents: dict[int, list[Document]], mark
ranker = HeuristicRanker(indexer, None, score_threshold=float('-inf'))
for page, documents in page_documents.items():
existing_documents = indexer.get_page(page)
- sorted_documents = sort_documents(documents, existing_documents, ranker)
-
- seen_urls = set()
- seen_titles = set()
- combined_documents = []
- for document in sorted_documents:
- if document.title in seen_titles or document.url in seen_urls:
- continue
- if mark_synced:
- document.state = DocumentState.SYNCED_WITH_MAIN_INDEX.value
- combined_documents.append(document)
- seen_urls.add(document.url)
- seen_titles.add(document.title)
+ combined_documents = combine_documents(existing_documents, documents, mark_synced, ranker)
logger.info(f"Storing {len(combined_documents)} documents for page {page}, originally {len(existing_documents)}")
indexer.store_in_page(page, combined_documents)
term_new_doc_counts.update(document.term for document in combined_documents
- if document.state != DocumentState.SYNCED_WITH_MAIN_INDEX.value)
+ if document.state != DocumentState.SYNCED_WITH_MAIN_INDEX)
return term_new_doc_counts
+def combine_documents(existing_documents, documents, mark_synced, ranker):
+ sorted_documents = sort_documents(documents, existing_documents, ranker)
+ seen_urls = set()
+ seen_titles = set()
+ combined_documents = []
+ for document in sorted_documents:
+ if document.title in seen_titles or document.url in seen_urls:
+ continue
+ if mark_synced:
+ document.state = DocumentState.SYNCED_WITH_MAIN_INDEX
+ combined_documents.append(document)
+ seen_urls.add(document.url)
+ seen_titles.add(document.title)
+ return combined_documents
+
+
def sort_documents(documents, all_existing_documents, ranker):
curated_documents = [doc for doc in all_existing_documents if doc.state in CURATED_STATES]
existing_documents = [doc for doc in all_existing_documents if doc.state not in CURATED_STATES]
diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.py
index 985c684..a84d627 100644
--- a/mwmbl/tinysearchengine/indexer.py
+++ b/mwmbl/tinysearchengine/indexer.py
@@ -20,16 +20,6 @@ PAGE_SIZE = 4096
logger = getLogger(__name__)
-def astuple(dc):
- """
- Convert a type to a tuple - values at the end that are None can be truncated.
- """
- value = tuple(dc.__dict__.values())
- while value[-1] is None:
- value = value[:-1]
- return value
-
-
class DocumentState(IntEnum):
"""
The state of the document in the index. A value of None indicates an organic search result.
@@ -45,7 +35,7 @@ class DocumentState(IntEnum):
FROM_WIKI_APPROVED = 10
-CURATED_STATES = {state.value for state in DocumentState if state.value >= 7}
+CURATED_STATES = {state for state in DocumentState if state.value >= 7}
@dataclass
@@ -64,7 +54,7 @@ class Document:
extract: str,
score: Optional[float] = None,
term: Optional[str] = None,
- state: Optional[int] = None
+ state: Optional[int | DocumentState] = None
):
# Sometimes the title or extract may be None, probably because of user generated content
# It's not allowed to be None though, or things will break
@@ -73,7 +63,19 @@ class Document:
self.extract = extract if extract is not None else ''
self.score = score
self.term = term
- self.state = state
+ self.state = None if state is None else DocumentState(state)
+
+ def as_tuple(self):
+ """
+ Convert a type to a tuple - values at the end that are None can be truncated.
+ """
+ values = list(self.__dict__.values())
+ if values[-1] is not None:
+ values[-1] = values[-1].value
+
+ while values[-1] is None:
+ values = values[:-1]
+ return tuple(values)
@dataclass
@@ -224,7 +226,7 @@ class TinyIndex(Generic[T]):
return json.loads(decompressed_data.decode('utf8'))
def store_in_page(self, page_index: int, values: list[T]):
- value_tuples = [astuple(value) for value in values]
+ value_tuples = [value.as_tuple() for value in values]
self._write_page(value_tuples, page_index)
def _write_page(self, data, i: int):
diff --git a/test/test_index_batches.py b/test/test_index_batches.py
index 57fac78..754609f 100644
--- a/test/test_index_batches.py
+++ b/test/test_index_batches.py
@@ -1,4 +1,4 @@
-from mwmbl.indexer.index_batches import sort_documents
+from mwmbl.indexer.index_batches import sort_documents, combine_documents
from mwmbl.tinysearchengine.indexer import Document, DocumentState
@@ -62,3 +62,22 @@ def test_sort_documents_curated_items_first():
Document(title="title5", url="2", extract="extract5", term="term1"),
Document(title="title2", url="4", extract="extract2", term="term2"),
]
+
+
+def test_sort_documents_duplicates_keep_synced_state():
+ existing_documents = [
+ Document(title="title1", url="1", extract="extract1", term="term1", state=DocumentState.SYNCED_WITH_MAIN_INDEX),
+ ]
+
+ documents = [
+ Document(title="title1", url="1", extract="extract1", term="term1"),
+ ]
+
+ # Sort the documents
+ combined_documents = combine_documents(documents, existing_documents, mark_synced=False, ranker=UrlRanker())
+ print("Combined documents", combined_documents)
+
+ # Curated items should be first
+ assert combined_documents == [
+ Document(title="title1", url="1", extract="extract1", term="term1", state=DocumentState.SYNCED_WITH_MAIN_INDEX),
+ ]
diff --git a/test/test_indexer.py b/test/test_indexer.py
index fe67876..41cf2a1 100644
--- a/test/test_indexer.py
+++ b/test/test_indexer.py
@@ -3,7 +3,7 @@ from tempfile import TemporaryDirectory
from zstandard import ZstdCompressor
-from mwmbl.tinysearchengine.indexer import TinyIndex, Document, _binary_search_fitting_size, astuple, \
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, _binary_search_fitting_size, \
_trim_items_to_page, _pad_to_page_size, _get_page_data
@@ -45,12 +45,12 @@ def test_binary_search_fitting_size_none_fit():
# We should not fit anything
assert count_fit == -1
assert data is None
-
+
+
def test_get_page_data_single_doc():
document1 = Document(title='title1',url='url1',extract='extract1',score=1.0)
- documents = [document1]
- items = [astuple(value) for value in documents]
-
+ items = [document1.as_tuple()]
+
compressor = ZstdCompressor()
page_size = 4096
@@ -76,7 +76,7 @@ def test_get_page_data_many_docs_all_fit():
txt = 'text{}'.format(x)
document = Document(title=txt,url=txt,extract=txt,score=x)
documents.append(document)
- items = [astuple(value) for value in documents]
+ items = [document.as_tuple() for document in documents]
# Trim the items
compressor = ZstdCompressor()
@@ -91,7 +91,8 @@ def test_get_page_data_many_docs_all_fit():
padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)
assert serialized_data == padded_trimmed_data
-
+
+
def test_get_page_data_many_docs_subset_fit():
# Build giant documents item
documents = []
@@ -101,7 +102,7 @@ def test_get_page_data_many_docs_subset_fit():
txt = 'text{}'.format(x)
document = Document(title=txt,url=txt,extract=txt,score=x)
documents.append(document)
- items = [astuple(value) for value in documents]
+ items = [document.as_tuple() for document in documents]
# Trim the items
compressor = ZstdCompressor()