Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Gate submissions on how much the main index would keep

Daoud Clarkef07b46372268
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.py
index 98e0a43..6104f8a 100644
--- a/mwmbl/crawl.py
+++ b/mwmbl/crawl.py
@@ -55,6 +55,10 @@ _curated_domains_cache: set[str] = set()
_curated_domains_fetched_at: float = 0.0
CURATED_DOMAINS_CACHE_SECONDS = 300.0
+# How many of a term's new items must be expected to survive in the main index before we
+# submit the term at all. See count_new_index_entries for where the expectation comes from.
+MIN_NEW_INDEX_ENTRIES = 1
+
def _fetch_curated_domains() -> set[str]:
global _curated_domains_cache, _curated_domains_fetched_at
@@ -77,37 +81,37 @@ def _fetch_curated_domains() -> set[str]:
return _curated_domains_cache
-def is_new_high_score(term: str, new_items: list[Document], remote_items: list[Document]) -> bool:
+def count_new_index_entries(term: str, new_items: list[Document], remote_items: list[Document]) -> int:
"""
- Decide whether newly crawled items are worth contributing to the main index.
-
- score_result returns higher values for better matches, so we promote only when the best
- new local item beats the best item the remote index already holds for the term. Comparing
- against the best rather than the worst remote item keeps the bar where the docstring of
- run_indexing says it is: a single weak remote result should not open the gate.
-
- An empty remote result set scores 0.0, so any local item that matches the term at all is
- promoted. That is deliberate: the main index has nothing for this term, and a result is
- better than no result.
-
- The answer is per term, not per item: when it is yes the caller submits every new local
- item for the term, including ones that lose to the remote index. That is not the leak it
- looks like. The server re-tokenizes each submitted document and files it under every term
- it matches, ranking it against that term's existing documents on the way in
- (index_batches.sort_documents), so an item that is weak for this term cannot jump the
- queue - it loses here and may win on a term we never asked about. Submitting a URL the
- index already holds is useful in its own right: combine_documents keeps the largest
- last_crawled per URL, so the re-submission refreshes the stored crawl date (the one the
- API reports; recrawl scheduling reads FoundURL.last_crawled from the URL database, which
- this path does not touch).
+ Estimate how many of our new local items the main index will actually keep for a term.
+
+ The main index holds a fixed number of bytes per page and drops the tail that does not
+ fit, so contributing to a term the index already covers means displacing something. The
+ server ranks each term's documents by score_result on the way in and keeps the best of
+ them (index_batches.sort_documents, TinyIndex.store), so a new item survives roughly when
+ it outranks one of the documents currently occupying the term's slots. Merging our scores
+ into the remote ones and counting how many of ours land in the top len(remote_items) is
+ that question asked directly. Ties go to the incumbent: matching what is already indexed
+ gains the index nothing.
+
+ A term the index has never seen has nothing to displace, so every item that matches the
+ term at all is a new entry.
+
+ This is an estimate, not a simulation. The remote page also holds documents for other
+ terms that hash to it, which we cannot see, so the real capacity is lower than
+ len(remote_items) suggests and blacklisting can drop items server-side. run_indexing logs
+ the estimate next to the number the index actually kept, which is how to calibrate it.
"""
terms = tokenize(term)
- remote_item_scores = [score_result(terms, item, True) for item in remote_items]
- max_remote_score = max(remote_item_scores, default=0.0)
- local_scores = [score_result(terms, item, True) for item in new_items]
- max_local_score = max(local_scores, default=0.0)
- logger.info(f"Max local score: {max_local_score}, max remote score: {max_remote_score}")
- return max_local_score > max_remote_score
+ new_scores = [score_result(terms, item, True) for item in new_items]
+ matching_new_scores = [score for score in new_scores if score > 0.0]
+ if not remote_items:
+ return len(matching_new_scores)
+
+ remote_scores = [score_result(terms, item, True) for item in remote_items]
+ candidates = [(score, True) for score in matching_new_scores] + [(score, False) for score in remote_scores]
+ candidates.sort(key=lambda candidate: (-candidate[0], candidate[1]))
+ return sum(1 for _, is_new in candidates[: len(remote_scores)] if is_new)
# Validate environment variables when actually needed
@@ -229,10 +233,17 @@ class Crawler:
to the main Mwmbl search index, while also keeping the local index updated
with the latest remote results for better search quality.
- A term's new local items are submitted only when the best of them beats the best
- result the main index already holds for that term, so a term the main index already
- covers well is left alone. See is_new_high_score for where that bar sits and why the
- whole term goes when it is cleared.
+ A term is submitted when enough of its new local items are expected to survive in
+ the main index - see count_new_index_entries and MIN_NEW_INDEX_ENTRIES. Once a term
+ clears that bar all of its new items go, not just the ones that cleared it: the
+ server re-tokenizes each submitted document and files it under every term it matches,
+ ranking it against that term's existing documents on the way in
+ (index_batches.sort_documents), so an item that is weak here cannot jump the queue -
+ it loses here and may win on a term we never asked about. Submitting a URL the index
+ already holds is useful in its own right: combine_documents keeps the largest
+ last_crawled per URL, so the re-submission refreshes the stored crawl date (the one
+ the API reports; recrawl scheduling reads FoundURL.last_crawled from the URL
+ database, which this path does not touch).
"""
index_path = data_path / settings.INDEX_NAME
batch_jsons = self.redis.lpop(BATCH_QUEUE_KEY, 10)
@@ -256,7 +267,13 @@ class Crawler:
new_items = [item for item in local_items if item.url not in remote_item_urls]
logger.info(f"Found {len(new_items)} new items for term {term}")
- if is_new_high_score(term, new_items, remote_items):
+ estimated_new_entries = count_new_index_entries(term, new_items, remote_items)
+ logger.info(
+ f"Estimated {estimated_new_entries} of {len(new_items)} new items would be "
+ f'kept by the main index for term "{term}"'
+ )
+
+ if estimated_new_entries >= MIN_NEW_INDEX_ENTRIES:
result_items = [
Result(
url=doc.url,
@@ -285,7 +302,10 @@ class Crawler:
# Check how many of our items were indexed
new_remote_item_urls = {item.url for item in new_remote_items}
indexed_items = sum(1 for item in new_items if item.url in new_remote_item_urls)
- logger.info(f'Indexed items: {indexed_items}/{len(new_items)} for term "{term}"')
+ logger.info(
+ f'Indexed items: {indexed_items}/{len(new_items)} for term "{term}" '
+ f"(estimated {estimated_new_entries})"
+ )
page_index = local_index.get_key_page_index(term)
index_pages(index_path, {page_index: new_remote_items}, mark_synced=True)
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 734ae40..e954695 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -56,7 +56,14 @@ def score_result(terms: list[str], result: Document, is_complete: bool):
+ features["match_score_path"]
)
- if features["match_terms"] <= len(terms) / 2 and result.state is None:
+ # Documents a person or an external source deliberately attached to a query are exempt
+ # from the majority-terms rule: they are here because someone said they belong here, not
+ # because they matched. That exemption was written as "state is not None" when every
+ # state meant exactly that (FROM_USER, FROM_GOOGLE, FROM_WIKI). The negative states are
+ # later bookkeeping - SYNCED_WITH_MAIN_INDEX marks a crawler's local mirror of the main
+ # index - and nobody curated those, so they get the rule like anything else.
+ is_curated = result.state is not None and result.state > 0
+ if features["match_terms"] <= len(terms) / 2 and not is_curated:
return 0.0
if match_score > MATCH_SCORE_THRESHOLD:
diff --git a/test/test_crawl_promotion.py b/test/test_crawl_promotion.py
index 5eac95e..2931c9e 100644
--- a/test/test_crawl_promotion.py
+++ b/test/test_crawl_promotion.py
@@ -1,11 +1,14 @@
"""Which crawled results the crawler contributes to the main index.
-The gate compares the best new local result against the best result the remote index
-already holds for a term, and higher scores are better. It used to compare the other way
-round - `max_local_score < min_remote_score` - which promoted results precisely when they
-were the weaker ones, and, because scores are never negative, promoted nothing at all
-whenever any remote item scored 0.0 or the term was missing from the remote index
-entirely. These cover both directions so it stays the right way round.
+Contributing to a term the main index already covers means displacing something: pages hold
+a fixed number of bytes and the server keeps the best-ranked documents that fit. So the gate
+asks how many of our new items would survive that, and submits the term only when enough of
+them would - see count_new_index_entries.
+
+The comparison used to run the other way round entirely (`max_local_score < min_remote_score`),
+which promoted results precisely when they were the weaker ones and, because scores are never
+negative, promoted nothing at all whenever any remote item scored 0.0 or the term was missing
+from the remote index. These cover both directions so it stays the right way round.
"""
import time
@@ -15,9 +18,11 @@ from unittest.mock import MagicMock, patch
import fakeredis
import pytest
-from mwmbl.crawl import Crawler, is_new_high_score
+from mwmbl.crawl import Crawler, count_new_index_entries
from mwmbl.crawler.batch import HashedBatch, Item, ItemContent
-from mwmbl.tinysearchengine.indexer import Document
+from mwmbl.tinysearchengine.indexer import Document, DocumentState
+from mwmbl.tinysearchengine.rank import score_result
+from mwmbl.tokenizer import tokenize
TERM = "python"
@@ -33,6 +38,11 @@ MEDIUM = Document(
url="https://example.com/python/tutorial",
extract="Learn python here",
)
+WEAKER_MEDIUM = Document(
+ title="Some notes on python",
+ url="https://example.net/blog/archive/notes/python",
+ extract="A few things I learned about python",
+)
WEAK = Document(
title="Cooking recipes",
url="https://example.com/recipes/some/long/path",
@@ -45,44 +55,78 @@ def fake_redis():
return fakeredis.FakeRedis(decode_responses=True, health_check_interval=30)
-def test_better_local_results_are_promoted():
- assert is_new_high_score(TERM, [STRONG], [MEDIUM])
+def test_the_fixtures_are_ordered_as_the_tests_assume():
+ """Every expectation below rests on this ordering, so state it once and check it."""
+ terms = tokenize(TERM)
+ scores = [score_result(terms, doc, True) for doc in [STRONG, MEDIUM, WEAKER_MEDIUM, WEAK]]
+ assert scores[0] > scores[1] > scores[2] > 0.0
+ assert scores[3] == 0.0
+
+
+def test_an_item_that_outranks_the_incumbent_is_a_new_entry():
+ assert count_new_index_entries(TERM, [STRONG], [MEDIUM]) == 1
-def test_worse_local_results_are_not_promoted():
- assert not is_new_high_score(TERM, [MEDIUM], [STRONG])
+def test_an_item_the_incumbent_outranks_is_not():
+ assert count_new_index_entries(TERM, [MEDIUM], [STRONG]) == 0
-def test_equally_good_local_results_are_not_promoted():
+def test_ties_go_to_the_incumbent():
"""Nothing to gain from resubmitting what the main index already has."""
- assert not is_new_high_score(TERM, [MEDIUM], [MEDIUM])
+ assert count_new_index_entries(TERM, [MEDIUM], [MEDIUM]) == 0
-def test_a_term_the_remote_index_has_never_seen_is_promoted():
+def test_every_matching_item_is_a_new_entry_for_a_term_the_index_has_never_seen():
"""The perverse case under the old comparison: fresh results for an unknown term.
- An empty remote result set scores 0.0, so any local item that matches the term at all
- wins. That is the intent - a result beats no result.
+ There is nothing to displace, so anything that matches the term at all is a new entry.
+ """
+ assert count_new_index_entries(TERM, [MEDIUM, WEAKER_MEDIUM], []) == 2
+
+
+def test_items_that_match_nothing_are_never_new_entries():
+ """A 0.0 score is not a contribution, whether or not the index has the term."""
+ assert count_new_index_entries(TERM, [WEAK], []) == 0
+ assert count_new_index_entries(TERM, [WEAK], [MEDIUM]) == 0
+
+
+def test_a_synced_mirror_document_that_matches_nothing_is_not_a_new_entry():
+ """SYNCED_WITH_MAIN_INDEX is bookkeeping, not curation.
+
+ score_result exempts curated documents from its majority-terms rule. This local index
+ is full of documents marked SYNCED_WITH_MAIN_INDEX by run_indexing, and they used to
+ ride that exemption: a mirrored document scored above 0.0 for a term it did not match,
+ which is exactly the score that clears the bar for an unseen term.
"""
- assert is_new_high_score(TERM, [MEDIUM], [])
+ mirrored = Document(
+ title=WEAK.title, url=WEAK.url, extract=WEAK.extract, state=DocumentState.SYNCED_WITH_MAIN_INDEX
+ )
+ assert count_new_index_entries(TERM, [mirrored], []) == 0
-def test_local_results_that_match_nothing_are_not_promoted():
- """A 0.0 local score must not clear the empty-remote bar."""
- assert not is_new_high_score(TERM, [WEAK], [])
+def test_a_curated_document_keeps_its_exemption():
+ """The exemption is still there for the documents it was written for."""
+ curated = Document(title=WEAK.title, url=WEAK.url, extract=WEAK.extract, state=DocumentState.FROM_USER)
+ assert score_result(tokenize(TERM), curated, True) > 0.0
-def test_no_new_local_items_is_not_a_high_score():
- assert not is_new_high_score(TERM, [], [MEDIUM])
+def test_nothing_new_is_nothing_to_contribute():
+ assert count_new_index_entries(TERM, [], [MEDIUM]) == 0
-def test_one_weak_remote_result_does_not_open_the_gate():
- """The bar is the best remote result, not the worst.
+def test_one_weak_remote_result_leaves_room_for_a_middling_one():
+ """The bar is displacement, not beating the best result there is.
- Weak matches score 0.0 and are common, so comparing against the minimum would promote
- anything at all as soon as one poor result was indexed for the term.
+ Comparing against the best remote result made contribution all-or-nothing: any term the
+ main index already covered well was closed to us, however much we had to add. A result
+ that outranks the weakest of two incumbents takes its slot.
"""
- assert not is_new_high_score(TERM, [MEDIUM], [WEAK, STRONG])
+ assert count_new_index_entries(TERM, [MEDIUM], [WEAK, STRONG]) == 1
+
+
+def test_the_estimate_is_capped_by_the_slots_there_are():
+ """Three good items cannot all land in a term the index holds two documents for."""
+ assert count_new_index_entries(TERM, [STRONG, MEDIUM, WEAKER_MEDIUM], [WEAK, WEAK]) == 2
def run_indexing_with(redis, local_items: list[Document], remote_items: list[Document]) -> MagicMock:
@@ -136,3 +180,17 @@ def test_run_indexing_keeps_quiet_about_results_the_remote_index_beats(fake_redi
post = run_indexing_with(fake_redis, [MEDIUM], [STRONG])
post.assert_not_called()
+
+
+def test_run_indexing_submits_the_whole_term_once_the_gate_opens(fake_redis):
+ """Deliberate: the gate is per term, the payload is every new item for it.
+
+ A weak item cannot jump the queue - the server ranks it against the term's existing
+ documents on the way in - and it may be the right answer for a term we never asked
+ about. Submitting one already in the index also refreshes its stored crawl date.
+ """
+ post = run_indexing_with(fake_redis, [STRONG, WEAK], [MEDIUM])
+
+ post.assert_called_once()
+ submitted_urls = {result["url"] for result in post.call_args.kwargs["json"]["results"]}
+ assert submitted_urls == {STRONG.url, WEAK.url}