Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add tests for moderation evidence crawl, dataset stats and domain links; update coverage notes

blueheron786fa159b62b7aa
diff --git a/docs/coverage.md b/docs/coverage.md
index 9c84a27..c28d547 100644
--- a/docs/coverage.md
+++ b/docs/coverage.md
@@ -64,12 +64,17 @@ ranking, crawling, moderation, and external-source adapters over tests that only
call trivial accessors.
The baseline when this workflow was introduced was 70.83% Python, 66.36% Rust,
-and 70.42% combined; including the extension-boundary tests lifts Rust to 91.6% and
-the combined figure to 72.7%. The largest actionable Python gaps were `platform/api.py`,
-`views.py`, moderation training, crawler retrieval, and crawler application
-behavior. Standalone ranking-evaluation reports also account for many misses;
-test maintained reports through their parsing and scoring functions, and delete
-obsolete reports rather than hiding them from coverage.
+and 70.42% combined. Including the extension-boundary tests lifted Rust to 91.6%,
+and new Python tests for views, the crawler, moderation training, Super Search
+sources and the platform API lifted Python to 80.1% and the combined figure to 81.2%.
+
+What remains uncovered is mostly not worth a test:
+
+- standalone `rankeval/` report and experiment scripts (about 1,100 lines at 0%);
+ test the ones that are still run through their parsing and scoring functions, and
+ delete the rest rather than hiding them from coverage;
+- modules nothing imports - `database.py`, `retry.py`, `indexer/links.py`,
+ `indexer/dedupe.py` and `indexer/domains/*` - which should be deleted, not tested.
## CI and badges
diff --git a/test/test_moderation_evidence_crawl.py b/test/test_moderation_evidence_crawl.py
new file mode 100644
index 0000000..2bce50c
--- /dev/null
+++ b/test/test_moderation_evidence_crawl.py
@@ -0,0 +1,159 @@
+"""
+Crawling a submitted domain for moderation evidence: which follow-up pages are picked,
+how redirects and failures are reported, and when the blacklist lookup is unavailable.
+"""
+
+import pytest
+
+from mwmbl.crawler.batch import HashedBatch, Item, ItemContent
+from mwmbl.indexer.index_batches import get_documents_from_batches
+from mwmbl.models import DomainEvidence
+from mwmbl.moderation import evidence
+from mwmbl.moderation.evidence import _final_domain, _follow_up_urls, crawl_domain, page_texts
+
+
+def _homepage(links=(), extra_links=()):
+ return {"content": {"links": list(links), "extra_links": list(extra_links)}}
+
+
+def test_follow_ups_stay_on_the_site_skip_boilerplate_and_prefer_shallow_paths():
+ homepage = _homepage(
+ links=[
+ "https://example.com/blog/2024/deep/article",
+ "https://example.com/about",
+ "https://example.com/privacy-policy",
+ "https://example.com/",
+ "https://other.test/about",
+ ],
+ extra_links=["https://www.example.com/products"],
+ )
+
+ urls = _follow_up_urls("example.com", homepage, limit=5)
+
+ assert urls == [
+ "https://example.com/about",
+ "https://www.example.com/products",
+ "https://example.com/blog/2024/deep/article",
+ ]
+
+
+def test_follow_ups_are_capped_and_nothing_is_picked_for_a_zero_limit():
+ homepage = _homepage(links=["https://example.com/a", "https://example.com/b"])
+
+ assert len(_follow_up_urls("example.com", homepage, limit=1)) == 1
+ assert _follow_up_urls("example.com", homepage, limit=0) == []
+ assert _follow_up_urls("example.com", {"content": None}, limit=3) == []
+
+
+ "resolved_url, expected",
+ [
+ ("https://www.example.com/home", "www.example.com"),
+ ("https://example.com/home", ""),
+ ("", ""),
+ (None, ""),
+ ],
+)
+def test_final_domain_is_only_reported_after_a_redirect(resolved_url, expected):
+ assert _final_domain({"resolved_url": resolved_url}, "example.com") == expected
+
+
+def _crawled(url: str, links=()):
+ return {
+ "url": url,
+ "resolved_url": url,
+ "status": 200,
+ "content": {"title": "T", "extract": "E", "links": list(links), "extra_links": []},
+ "error": None,
+ }
+
+
+def test_crawling_a_domain_fetches_the_homepage_then_its_follow_up_pages(monkeypatch):
+ fetched = []
+
+ def fake_crawl(url, redis):
+ fetched.append(url)
+ links = ["https://example.com/about", "https://example.com/shop"] if url == "https://example.com/" else []
+ return _crawled(url, links)
+
+ monkeypatch.setattr(evidence, "crawl_url", fake_crawl)
+ monkeypatch.setattr(evidence, "CRAWL_DELAY_SECONDS", 0)
+
+ result = crawl_domain("example.com", redis=None, max_pages=3)
+
+ assert fetched == ["https://example.com/", "https://example.com/shop", "https://example.com/about"]
+ assert result["http_status"] == 200
+ assert result["error"] == ""
+ assert len(result["pages"]) == 3
+ assert result["signals"]["https"] is True
+
+
+def test_the_delay_between_pages_is_honoured(monkeypatch):
+ sleeps = []
+
+ def fake_crawl(url, redis):
+ return _crawled(url, ["https://example.com/about"])
+
+ monkeypatch.setattr(evidence, "crawl_url", fake_crawl)
+ monkeypatch.setattr(evidence, "CRAWL_DELAY_SECONDS", 2)
+ monkeypatch.setattr(evidence.time, "sleep", sleeps.append)
+
+ crawl_domain("example.com", redis=None, max_pages=2)
+
+ assert sleeps == [2]
+
+
+def test_an_unreachable_domain_reports_the_error_rather_than_raising(monkeypatch):
+ failure = {"url": "x", "status": None, "content": None, "error": {"name": "AbortError", "message": "down"}}
+ monkeypatch.setattr(evidence, "crawl_url", lambda url, redis: failure)
+
+ result = crawl_domain("down.example", redis=None, max_pages=3)
+
+ assert result["error"] == "AbortError"
+ assert result["signals"]["https"] is None
+ assert result["signals"]["num_pages_fetched"] == 0
+
+
+def test_a_missing_blacklist_snapshot_is_one_missing_signal_not_a_failure(monkeypatch):
+ def unavailable():
+ raise ConnectionError("redis not ready")
+
+ monkeypatch.setattr(evidence, "crawl_url", lambda url, redis: _crawled(url))
+ monkeypatch.setattr(evidence, "get_snapshot_blacklist", unavailable)
+
+ result = crawl_domain("example.com", redis=None, max_pages=1)
+
+ assert result["signals"]["blacklisted"] is False
+
+
+def test_page_texts_join_title_and_extract():
+ assert page_texts(None) == []
+ row = DomainEvidence(domain="a.example", pages=[{"title": "Home", "extract": "Welcome"}, {"extract": "Only text"}])
+ assert page_texts(row) == ["Home Welcome", "Only text"]
+ assert page_texts(DomainEvidence(domain="b.example", pages=None)) == []
+
+
+def test_only_crawled_pages_with_content_become_documents():
+ batch = HashedBatch(
+ user_id_hash="h",
+ timestamp=1.0,
+ items=[
+ Item(
+ url="https://a.example/",
+ status=200,
+ timestamp=1704067200000.0,
+ content=ItemContent(title="A", extract="Extract A"),
+ ),
+ Item(
+ url="https://b.example/",
+ status=200,
+ timestamp=1704067200000.0,
+ content=ItemContent(title="B", extract="x", links_only=True),
+ ),
+ Item(url="https://c.example/", status=404, timestamp=1704067200000.0),
+ ],
+ )
+
+ documents = list(get_documents_from_batches([batch]))
+
+ assert [(d.title, d.url, d.last_crawled) for d in documents] == [("A", "https://a.example/", 1704067200)]
diff --git a/test/test_small_modules.py b/test/test_small_modules.py
new file mode 100644
index 0000000..00dd109
--- /dev/null
+++ b/test/test_small_modules.py
@@ -0,0 +1,125 @@
+"""
+Small helpers that decide what is stored or counted: tagging documents with the term whose
+page they live on, the dataset counters, and the domain-link filters.
+"""
+
+from datetime import date
+from types import SimpleNamespace
+
+import fakeredis
+import pytest
+
+from mwmbl.crawler import domains
+from mwmbl.crawler.domains import DomainLinkDatabase
+from mwmbl.crawler.stats import DATASET_QUERIES_COUNT_KEY, DATASET_RESULTS_COUNT_KEY, StatsManager
+from mwmbl.tinysearchengine.indexer import Document
+from mwmbl.utils import add_term_info, add_term_infos, batch, get_domain
+
+
+def test_batch_splits_into_chunks_with_a_short_last_one():
+ assert list(batch([1, 2, 3, 4, 5], 2)) == [[1, 2], [3, 4], [5]]
+ assert list(batch([], 3)) == []
+
+
+def test_get_domain_requires_a_scheme():
+ assert get_domain("https://www.example.com/page") == "www.example.com"
+ with pytest.raises(ValueError):
+ get_domain("example.com")
+
+
+class FakeIndex:
+ """An index where every token lives on a page decided by the token's first letter."""
+
+ def get_key_page_index(self, token: str) -> int:
+ return ord(token[0]) % 7
+
+
+def _document(**fields) -> Document:
+ return Document(
+ title="Pasta recipes", url="https://example.com/pasta", extract="Cooking pasta", score=1.0, **fields
+ )
+
+
+def test_a_document_is_tagged_with_a_term_stored_on_its_page():
+ index = FakeIndex()
+ document = _document()
+ tokens = ["pasta", "recipes", "cooking", "example"]
+ page_index = index.get_key_page_index("recipes")
+
+ tagged = add_term_info(document, index, page_index)
+
+ assert tagged.term in tokens
+ assert index.get_key_page_index(tagged.term) == page_index
+ assert tagged.url == document.url
+
+
+def test_a_document_with_no_term_on_the_page_is_an_error():
+ with pytest.raises(ValueError, match="Could not find token"):
+ add_term_info(_document(), FakeIndex(), page_index=-1)
+
+
+def test_documents_without_a_term_are_tagged_and_unmatched_ones_dropped():
+ index = FakeIndex()
+ already = _document(term="existing")
+ page_index = index.get_key_page_index("recipes")
+ untaggable = Document(title="", url="", extract="", score=1.0)
+
+ result = list(add_term_infos([already, _document(), untaggable], index, page_index))
+
+ assert result[0] is already
+ assert len(result) == 2
+ assert result[1].term is not None
+
+
+def _dataset(day: str, successes: list[int]):
+ results = [SimpleNamespace(success=True, results=[None] * n) for n in successes]
+ results.append(SimpleNamespace(success=False, results=[None] * 50))
+ return SimpleNamespace(date=day, queryDataset=[None] * 3, searchResults=results)
+
+
+def test_dataset_submissions_count_queries_and_only_successful_results():
+ redis = fakeredis.FakeStrictRedis()
+
+ StatsManager(redis).record_dataset(_dataset("2024-03-05", [2, 5]))
+
+ day = date(2024, 3, 5)
+ assert int(redis.get(DATASET_QUERIES_COUNT_KEY.format(date=day))) == 3
+ assert int(redis.get(DATASET_RESULTS_COUNT_KEY.format(date=day))) == 7
+
+
+def test_a_dataset_with_an_unreadable_date_is_counted_today(monkeypatch):
+ redis = fakeredis.FakeStrictRedis()
+ monkeypatch.setattr("mwmbl.crawler.stats.utc_today", lambda: date(2030, 1, 1))
+
+ StatsManager(redis).record_dataset(_dataset("not-a-date", [1]))
+
+ assert int(redis.get(DATASET_QUERIES_COUNT_KEY.format(date=date(2030, 1, 1)))) == 3
+
+
+def bloom_files(settings, tmp_path):
+ settings.DOMAIN_LINKS_BLOOM_FILTER_PATH = str(tmp_path / "links_{domain_group}.bloom")
+ settings.NUM_DOMAINS_IN_BLOOM_FILTER = 1000
+
+
+def test_links_from_a_top_domain_count_towards_the_top_group(bloom_files, monkeypatch):
+ monkeypatch.setattr(domains, "TOP_DOMAINS", {"top.example"})
+ monkeypatch.setattr(domains, "OTHER_DOMAINS", {"other.example"})
+
+ with DomainLinkDatabase() as database:
+ database.update_domain_links("top.example", {"a.example"})
+ database.update_domain_links("other.example", {"b.example"})
+ database.update_domain_links("unknown.example", {"c.example"})
+
+ assert database.get_domain_score("a.example") == 5
+ assert database.get_domain_score("b.example") == 1
+ assert database.get_domain_score("c.example") == 0
+
+
+def test_link_filters_persist_between_openings(bloom_files, monkeypatch):
+ monkeypatch.setattr(domains, "TOP_DOMAINS", {"top.example"})
+ with DomainLinkDatabase() as database:
+ database.update_domain_links("top.example", {"a.example"})
+
+ with DomainLinkDatabase() as reopened:
+ assert reopened.get_domain_score("a.example") == 5