An open source, non-profit web search engine
Gate submissions on how much the main index would keep
Daoud Clarkef07b46372268
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.pyindex 98e0a43..6104f8a 100644--- a/mwmbl/crawl.py+++ b/mwmbl/crawl.py@@ -55,6 +55,10 @@ _curated_domains_cache: set[str] = set()_curated_domains_fetched_at: float = 0.0CURATED_DOMAINS_CACHE_SECONDS = 300.0+# How many of a term's new items must be expected to survive in the main index before we+# submit the term at all. See count_new_index_entries for where the expectation comes from.+MIN_NEW_INDEX_ENTRIES = 1+def _fetch_curated_domains() -> set[str]:global _curated_domains_cache, _curated_domains_fetched_at@@ -77,37 +81,37 @@ def _fetch_curated_domains() -> set[str]:return _curated_domains_cache-def is_new_high_score(term: str, new_items: list[Document], remote_items: list[Document]) -> bool:+def count_new_index_entries(term: str, new_items: list[Document], remote_items: list[Document]) -> int:"""- Decide whether newly crawled items are worth contributing to the main index.-- score_result returns higher values for better matches, so we promote only when the best- new local item beats the best item the remote index already holds for the term. Comparing- against the best rather than the worst remote item keeps the bar where the docstring of- run_indexing says it is: a single weak remote result should not open the gate.-- An empty remote result set scores 0.0, so any local item that matches the term at all is- promoted. That is deliberate: the main index has nothing for this term, and a result is- better than no result.-- The answer is per term, not per item: when it is yes the caller submits every new local- item for the term, including ones that lose to the remote index. That is not the leak it- looks like. The server re-tokenizes each submitted document and files it under every term- it matches, ranking it against that term's existing documents on the way in- (index_batches.sort_documents), so an item that is weak for this term cannot jump the- queue - it loses here and may win on a term we never asked about. Submitting a URL the- index already holds is useful in its own right: combine_documents keeps the largest- last_crawled per URL, so the re-submission refreshes the stored crawl date (the one the- API reports; recrawl scheduling reads FoundURL.last_crawled from the URL database, which- this path does not touch).+ Estimate how many of our new local items the main index will actually keep for a term.++ The main index holds a fixed number of bytes per page and drops the tail that does not+ fit, so contributing to a term the index already covers means displacing something. The+ server ranks each term's documents by score_result on the way in and keeps the best of+ them (index_batches.sort_documents, TinyIndex.store), so a new item survives roughly when+ it outranks one of the documents currently occupying the term's slots. Merging our scores+ into the remote ones and counting how many of ours land in the top len(remote_items) is+ that question asked directly. Ties go to the incumbent: matching what is already indexed+ gains the index nothing.++ A term the index has never seen has nothing to displace, so every item that matches the+ term at all is a new entry.++ This is an estimate, not a simulation. The remote page also holds documents for other+ terms that hash to it, which we cannot see, so the real capacity is lower than+ len(remote_items) suggests and blacklisting can drop items server-side. run_indexing logs+ the estimate next to the number the index actually kept, which is how to calibrate it."""terms = tokenize(term)- remote_item_scores = [score_result(terms, item, True) for item in remote_items]- max_remote_score = max(remote_item_scores, default=0.0)- local_scores = [score_result(terms, item, True) for item in new_items]- max_local_score = max(local_scores, default=0.0)- logger.info(f"Max local score: {max_local_score}, max remote score: {max_remote_score}")- return max_local_score > max_remote_score+ new_scores = [score_result(terms, item, True) for item in new_items]+ matching_new_scores = [score for score in new_scores if score > 0.0]+ if not remote_items:+ return len(matching_new_scores)++ remote_scores = [score_result(terms, item, True) for item in remote_items]+ candidates = [(score, True) for score in matching_new_scores] + [(score, False) for score in remote_scores]+ candidates.sort(key=lambda candidate: (-candidate[0], candidate[1]))+ return sum(1 for _, is_new in candidates[: len(remote_scores)] if is_new)# Validate environment variables when actually needed@@ -229,10 +233,17 @@ class Crawler:to the main Mwmbl search index, while also keeping the local index updatedwith the latest remote results for better search quality.- A term's new local items are submitted only when the best of them beats the best- result the main index already holds for that term, so a term the main index already- covers well is left alone. See is_new_high_score for where that bar sits and why the- whole term goes when it is cleared.+ A term is submitted when enough of its new local items are expected to survive in+ the main index - see count_new_index_entries and MIN_NEW_INDEX_ENTRIES. Once a term+ clears that bar all of its new items go, not just the ones that cleared it: the+ server re-tokenizes each submitted document and files it under every term it matches,+ ranking it against that term's existing documents on the way in+ (index_batches.sort_documents), so an item that is weak here cannot jump the queue -+ it loses here and may win on a term we never asked about. Submitting a URL the index+ already holds is useful in its own right: combine_documents keeps the largest+ last_crawled per URL, so the re-submission refreshes the stored crawl date (the one+ the API reports; recrawl scheduling reads FoundURL.last_crawled from the URL+ database, which this path does not touch)."""index_path = data_path / settings.INDEX_NAMEbatch_jsons = self.redis.lpop(BATCH_QUEUE_KEY, 10)@@ -256,7 +267,13 @@ class Crawler:new_items = [item for item in local_items if item.url not in remote_item_urls]logger.info(f"Found {len(new_items)} new items for term {term}")- if is_new_high_score(term, new_items, remote_items):+ estimated_new_entries = count_new_index_entries(term, new_items, remote_items)+ logger.info(+ f"Estimated {estimated_new_entries} of {len(new_items)} new items would be "+ f'kept by the main index for term "{term}"'+ )++ if estimated_new_entries >= MIN_NEW_INDEX_ENTRIES:result_items = [Result(url=doc.url,@@ -285,7 +302,10 @@ class Crawler:# Check how many of our items were indexednew_remote_item_urls = {item.url for item in new_remote_items}indexed_items = sum(1 for item in new_items if item.url in new_remote_item_urls)- logger.info(f'Indexed items: {indexed_items}/{len(new_items)} for term "{term}"')+ logger.info(+ f'Indexed items: {indexed_items}/{len(new_items)} for term "{term}" '+ f"(estimated {estimated_new_entries})"+ )page_index = local_index.get_key_page_index(term)index_pages(index_path, {page_index: new_remote_items}, mark_synced=True)diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 734ae40..e954695 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -56,7 +56,14 @@ def score_result(terms: list[str], result: Document, is_complete: bool):+ features["match_score_path"])- if features["match_terms"] <= len(terms) / 2 and result.state is None:+ # Documents a person or an external source deliberately attached to a query are exempt+ # from the majority-terms rule: they are here because someone said they belong here, not+ # because they matched. That exemption was written as "state is not None" when every+ # state meant exactly that (FROM_USER, FROM_GOOGLE, FROM_WIKI). The negative states are+ # later bookkeeping - SYNCED_WITH_MAIN_INDEX marks a crawler's local mirror of the main+ # index - and nobody curated those, so they get the rule like anything else.+ is_curated = result.state is not None and result.state > 0+ if features["match_terms"] <= len(terms) / 2 and not is_curated:return 0.0if match_score > MATCH_SCORE_THRESHOLD:diff --git a/test/test_crawl_promotion.py b/test/test_crawl_promotion.pyindex 5eac95e..2931c9e 100644--- a/test/test_crawl_promotion.py+++ b/test/test_crawl_promotion.py@@ -1,11 +1,14 @@"""Which crawled results the crawler contributes to the main index.-The gate compares the best new local result against the best result the remote index-already holds for a term, and higher scores are better. It used to compare the other way-round - `max_local_score < min_remote_score` - which promoted results precisely when they-were the weaker ones, and, because scores are never negative, promoted nothing at all-whenever any remote item scored 0.0 or the term was missing from the remote index-entirely. These cover both directions so it stays the right way round.+Contributing to a term the main index already covers means displacing something: pages hold+a fixed number of bytes and the server keeps the best-ranked documents that fit. So the gate+asks how many of our new items would survive that, and submits the term only when enough of+them would - see count_new_index_entries.++The comparison used to run the other way round entirely (`max_local_score < min_remote_score`),+which promoted results precisely when they were the weaker ones and, because scores are never+negative, promoted nothing at all whenever any remote item scored 0.0 or the term was missing+from the remote index. These cover both directions so it stays the right way round."""import time@@ -15,9 +18,11 @@ from unittest.mock import MagicMock, patchimport fakeredisimport pytest-from mwmbl.crawl import Crawler, is_new_high_score+from mwmbl.crawl import Crawler, count_new_index_entriesfrom mwmbl.crawler.batch import HashedBatch, Item, ItemContent-from mwmbl.tinysearchengine.indexer import Document+from mwmbl.tinysearchengine.indexer import Document, DocumentState+from mwmbl.tinysearchengine.rank import score_result+from mwmbl.tokenizer import tokenizeTERM = "python"@@ -33,6 +38,11 @@ MEDIUM = Document(url="https://example.com/python/tutorial",extract="Learn python here",)+WEAKER_MEDIUM = Document(+ title="Some notes on python",+ url="https://example.net/blog/archive/notes/python",+ extract="A few things I learned about python",+)WEAK = Document(title="Cooking recipes",url="https://example.com/recipes/some/long/path",@@ -45,44 +55,78 @@ def fake_redis():return fakeredis.FakeRedis(decode_responses=True, health_check_interval=30)-def test_better_local_results_are_promoted():- assert is_new_high_score(TERM, [STRONG], [MEDIUM])+def test_the_fixtures_are_ordered_as_the_tests_assume():+ """Every expectation below rests on this ordering, so state it once and check it."""+ terms = tokenize(TERM)+ scores = [score_result(terms, doc, True) for doc in [STRONG, MEDIUM, WEAKER_MEDIUM, WEAK]]+ assert scores[0] > scores[1] > scores[2] > 0.0+ assert scores[3] == 0.0+++def test_an_item_that_outranks_the_incumbent_is_a_new_entry():+ assert count_new_index_entries(TERM, [STRONG], [MEDIUM]) == 1-def test_worse_local_results_are_not_promoted():- assert not is_new_high_score(TERM, [MEDIUM], [STRONG])+def test_an_item_the_incumbent_outranks_is_not():+ assert count_new_index_entries(TERM, [MEDIUM], [STRONG]) == 0-def test_equally_good_local_results_are_not_promoted():+def test_ties_go_to_the_incumbent():"""Nothing to gain from resubmitting what the main index already has."""- assert not is_new_high_score(TERM, [MEDIUM], [MEDIUM])+ assert count_new_index_entries(TERM, [MEDIUM], [MEDIUM]) == 0-def test_a_term_the_remote_index_has_never_seen_is_promoted():+def test_every_matching_item_is_a_new_entry_for_a_term_the_index_has_never_seen():"""The perverse case under the old comparison: fresh results for an unknown term.- An empty remote result set scores 0.0, so any local item that matches the term at all- wins. That is the intent - a result beats no result.+ There is nothing to displace, so anything that matches the term at all is a new entry.+ """+ assert count_new_index_entries(TERM, [MEDIUM, WEAKER_MEDIUM], []) == 2+++def test_items_that_match_nothing_are_never_new_entries():+ """A 0.0 score is not a contribution, whether or not the index has the term."""+ assert count_new_index_entries(TERM, [WEAK], []) == 0+ assert count_new_index_entries(TERM, [WEAK], [MEDIUM]) == 0+++def test_a_synced_mirror_document_that_matches_nothing_is_not_a_new_entry():+ """SYNCED_WITH_MAIN_INDEX is bookkeeping, not curation.++ score_result exempts curated documents from its majority-terms rule. This local index+ is full of documents marked SYNCED_WITH_MAIN_INDEX by run_indexing, and they used to+ ride that exemption: a mirrored document scored above 0.0 for a term it did not match,+ which is exactly the score that clears the bar for an unseen term."""- assert is_new_high_score(TERM, [MEDIUM], [])+ mirrored = Document(+ title=WEAK.title, url=WEAK.url, extract=WEAK.extract, state=DocumentState.SYNCED_WITH_MAIN_INDEX+ )+ assert count_new_index_entries(TERM, [mirrored], []) == 0-def test_local_results_that_match_nothing_are_not_promoted():- """A 0.0 local score must not clear the empty-remote bar."""- assert not is_new_high_score(TERM, [WEAK], [])+def test_a_curated_document_keeps_its_exemption():+ """The exemption is still there for the documents it was written for."""+ curated = Document(title=WEAK.title, url=WEAK.url, extract=WEAK.extract, state=DocumentState.FROM_USER)+ assert score_result(tokenize(TERM), curated, True) > 0.0-def test_no_new_local_items_is_not_a_high_score():- assert not is_new_high_score(TERM, [], [MEDIUM])+def test_nothing_new_is_nothing_to_contribute():+ assert count_new_index_entries(TERM, [], [MEDIUM]) == 0-def test_one_weak_remote_result_does_not_open_the_gate():- """The bar is the best remote result, not the worst.+def test_one_weak_remote_result_leaves_room_for_a_middling_one():+ """The bar is displacement, not beating the best result there is.- Weak matches score 0.0 and are common, so comparing against the minimum would promote- anything at all as soon as one poor result was indexed for the term.+ Comparing against the best remote result made contribution all-or-nothing: any term the+ main index already covered well was closed to us, however much we had to add. A result+ that outranks the weakest of two incumbents takes its slot."""- assert not is_new_high_score(TERM, [MEDIUM], [WEAK, STRONG])+ assert count_new_index_entries(TERM, [MEDIUM], [WEAK, STRONG]) == 1+++def test_the_estimate_is_capped_by_the_slots_there_are():+ """Three good items cannot all land in a term the index holds two documents for."""+ assert count_new_index_entries(TERM, [STRONG, MEDIUM, WEAKER_MEDIUM], [WEAK, WEAK]) == 2def run_indexing_with(redis, local_items: list[Document], remote_items: list[Document]) -> MagicMock:@@ -136,3 +180,17 @@ def test_run_indexing_keeps_quiet_about_results_the_remote_index_beats(fake_redipost = run_indexing_with(fake_redis, [MEDIUM], [STRONG])post.assert_not_called()+++def test_run_indexing_submits_the_whole_term_once_the_gate_opens(fake_redis):+ """Deliberate: the gate is per term, the payload is every new item for it.++ A weak item cannot jump the queue - the server ranks it against the term's existing+ documents on the way in - and it may be the right answer for a term we never asked+ about. Submitting one already in the index also refreshes its stored crawl date.+ """+ post = run_indexing_with(fake_redis, [STRONG, WEAK], [MEDIUM])++ post.assert_called_once()+ submitted_urls = {result["url"] for result in post.call_args.kwargs["json"]["results"]}+ assert submitted_urls == {STRONG.url, WEAK.url}