An open source, non-profit web search engine
Add tests for moderation evidence crawl, dataset stats and domain links; update coverage notes
blueheron786fa159b62b7aa
diff --git a/docs/coverage.md b/docs/coverage.mdindex 9c84a27..c28d547 100644--- a/docs/coverage.md+++ b/docs/coverage.md@@ -64,12 +64,17 @@ ranking, crawling, moderation, and external-source adapters over tests that onlycall trivial accessors.The baseline when this workflow was introduced was 70.83% Python, 66.36% Rust,-and 70.42% combined; including the extension-boundary tests lifts Rust to 91.6% and-the combined figure to 72.7%. The largest actionable Python gaps were `platform/api.py`,-`views.py`, moderation training, crawler retrieval, and crawler application-behavior. Standalone ranking-evaluation reports also account for many misses;-test maintained reports through their parsing and scoring functions, and delete-obsolete reports rather than hiding them from coverage.+and 70.42% combined. Including the extension-boundary tests lifted Rust to 91.6%,+and new Python tests for views, the crawler, moderation training, Super Search+sources and the platform API lifted Python to 80.1% and the combined figure to 81.2%.++What remains uncovered is mostly not worth a test:++- standalone `rankeval/` report and experiment scripts (about 1,100 lines at 0%);+ test the ones that are still run through their parsing and scoring functions, and+ delete the rest rather than hiding them from coverage;+- modules nothing imports - `database.py`, `retry.py`, `indexer/links.py`,+ `indexer/dedupe.py` and `indexer/domains/*` - which should be deleted, not tested.## CI and badgesdiff --git a/test/test_moderation_evidence_crawl.py b/test/test_moderation_evidence_crawl.pynew file mode 100644index 0000000..2bce50c--- /dev/null+++ b/test/test_moderation_evidence_crawl.py@@ -0,0 +1,159 @@+"""+Crawling a submitted domain for moderation evidence: which follow-up pages are picked,+how redirects and failures are reported, and when the blacklist lookup is unavailable.+"""++import pytest++from mwmbl.crawler.batch import HashedBatch, Item, ItemContent+from mwmbl.indexer.index_batches import get_documents_from_batches+from mwmbl.models import DomainEvidence+from mwmbl.moderation import evidence+from mwmbl.moderation.evidence import _final_domain, _follow_up_urls, crawl_domain, page_texts+++def _homepage(links=(), extra_links=()):+ return {"content": {"links": list(links), "extra_links": list(extra_links)}}+++def test_follow_ups_stay_on_the_site_skip_boilerplate_and_prefer_shallow_paths():+ homepage = _homepage(+ links=[+ "https://example.com/blog/2024/deep/article",+ "https://example.com/about",+ "https://example.com/privacy-policy",+ "https://example.com/",+ "https://other.test/about",+ ],+ extra_links=["https://www.example.com/products"],+ )++ urls = _follow_up_urls("example.com", homepage, limit=5)++ assert urls == [+ "https://example.com/about",+ "https://www.example.com/products",+ "https://example.com/blog/2024/deep/article",+ ]+++def test_follow_ups_are_capped_and_nothing_is_picked_for_a_zero_limit():+ homepage = _homepage(links=["https://example.com/a", "https://example.com/b"])++ assert len(_follow_up_urls("example.com", homepage, limit=1)) == 1+ assert _follow_up_urls("example.com", homepage, limit=0) == []+ assert _follow_up_urls("example.com", {"content": None}, limit=3) == []+++ "resolved_url, expected",+ [+ ("https://www.example.com/home", "www.example.com"),+ ("https://example.com/home", ""),+ ("", ""),+ (None, ""),+ ],+)+def test_final_domain_is_only_reported_after_a_redirect(resolved_url, expected):+ assert _final_domain({"resolved_url": resolved_url}, "example.com") == expected+++def _crawled(url: str, links=()):+ return {+ "url": url,+ "resolved_url": url,+ "status": 200,+ "content": {"title": "T", "extract": "E", "links": list(links), "extra_links": []},+ "error": None,+ }+++def test_crawling_a_domain_fetches_the_homepage_then_its_follow_up_pages(monkeypatch):+ fetched = []++ def fake_crawl(url, redis):+ fetched.append(url)+ links = ["https://example.com/about", "https://example.com/shop"] if url == "https://example.com/" else []+ return _crawled(url, links)++ monkeypatch.setattr(evidence, "crawl_url", fake_crawl)+ monkeypatch.setattr(evidence, "CRAWL_DELAY_SECONDS", 0)++ result = crawl_domain("example.com", redis=None, max_pages=3)++ assert fetched == ["https://example.com/", "https://example.com/shop", "https://example.com/about"]+ assert result["http_status"] == 200+ assert result["error"] == ""+ assert len(result["pages"]) == 3+ assert result["signals"]["https"] is True+++def test_the_delay_between_pages_is_honoured(monkeypatch):+ sleeps = []++ def fake_crawl(url, redis):+ return _crawled(url, ["https://example.com/about"])++ monkeypatch.setattr(evidence, "crawl_url", fake_crawl)+ monkeypatch.setattr(evidence, "CRAWL_DELAY_SECONDS", 2)+ monkeypatch.setattr(evidence.time, "sleep", sleeps.append)++ crawl_domain("example.com", redis=None, max_pages=2)++ assert sleeps == [2]+++def test_an_unreachable_domain_reports_the_error_rather_than_raising(monkeypatch):+ failure = {"url": "x", "status": None, "content": None, "error": {"name": "AbortError", "message": "down"}}+ monkeypatch.setattr(evidence, "crawl_url", lambda url, redis: failure)++ result = crawl_domain("down.example", redis=None, max_pages=3)++ assert result["error"] == "AbortError"+ assert result["signals"]["https"] is None+ assert result["signals"]["num_pages_fetched"] == 0+++def test_a_missing_blacklist_snapshot_is_one_missing_signal_not_a_failure(monkeypatch):+ def unavailable():+ raise ConnectionError("redis not ready")++ monkeypatch.setattr(evidence, "crawl_url", lambda url, redis: _crawled(url))+ monkeypatch.setattr(evidence, "get_snapshot_blacklist", unavailable)++ result = crawl_domain("example.com", redis=None, max_pages=1)++ assert result["signals"]["blacklisted"] is False+++def test_page_texts_join_title_and_extract():+ assert page_texts(None) == []+ row = DomainEvidence(domain="a.example", pages=[{"title": "Home", "extract": "Welcome"}, {"extract": "Only text"}])+ assert page_texts(row) == ["Home Welcome", "Only text"]+ assert page_texts(DomainEvidence(domain="b.example", pages=None)) == []+++def test_only_crawled_pages_with_content_become_documents():+ batch = HashedBatch(+ user_id_hash="h",+ timestamp=1.0,+ items=[+ Item(+ url="https://a.example/",+ status=200,+ timestamp=1704067200000.0,+ content=ItemContent(title="A", extract="Extract A"),+ ),+ Item(+ url="https://b.example/",+ status=200,+ timestamp=1704067200000.0,+ content=ItemContent(title="B", extract="x", links_only=True),+ ),+ Item(url="https://c.example/", status=404, timestamp=1704067200000.0),+ ],+ )++ documents = list(get_documents_from_batches([batch]))++ assert [(d.title, d.url, d.last_crawled) for d in documents] == [("A", "https://a.example/", 1704067200)]diff --git a/test/test_small_modules.py b/test/test_small_modules.pynew file mode 100644index 0000000..00dd109--- /dev/null+++ b/test/test_small_modules.py@@ -0,0 +1,125 @@+"""+Small helpers that decide what is stored or counted: tagging documents with the term whose+page they live on, the dataset counters, and the domain-link filters.+"""++from datetime import date+from types import SimpleNamespace++import fakeredis+import pytest++from mwmbl.crawler import domains+from mwmbl.crawler.domains import DomainLinkDatabase+from mwmbl.crawler.stats import DATASET_QUERIES_COUNT_KEY, DATASET_RESULTS_COUNT_KEY, StatsManager+from mwmbl.tinysearchengine.indexer import Document+from mwmbl.utils import add_term_info, add_term_infos, batch, get_domain+++def test_batch_splits_into_chunks_with_a_short_last_one():+ assert list(batch([1, 2, 3, 4, 5], 2)) == [[1, 2], [3, 4], [5]]+ assert list(batch([], 3)) == []+++def test_get_domain_requires_a_scheme():+ assert get_domain("https://www.example.com/page") == "www.example.com"+ with pytest.raises(ValueError):+ get_domain("example.com")+++class FakeIndex:+ """An index where every token lives on a page decided by the token's first letter."""++ def get_key_page_index(self, token: str) -> int:+ return ord(token[0]) % 7+++def _document(**fields) -> Document:+ return Document(+ title="Pasta recipes", url="https://example.com/pasta", extract="Cooking pasta", score=1.0, **fields+ )+++def test_a_document_is_tagged_with_a_term_stored_on_its_page():+ index = FakeIndex()+ document = _document()+ tokens = ["pasta", "recipes", "cooking", "example"]+ page_index = index.get_key_page_index("recipes")++ tagged = add_term_info(document, index, page_index)++ assert tagged.term in tokens+ assert index.get_key_page_index(tagged.term) == page_index+ assert tagged.url == document.url+++def test_a_document_with_no_term_on_the_page_is_an_error():+ with pytest.raises(ValueError, match="Could not find token"):+ add_term_info(_document(), FakeIndex(), page_index=-1)+++def test_documents_without_a_term_are_tagged_and_unmatched_ones_dropped():+ index = FakeIndex()+ already = _document(term="existing")+ page_index = index.get_key_page_index("recipes")+ untaggable = Document(title="", url="", extract="", score=1.0)++ result = list(add_term_infos([already, _document(), untaggable], index, page_index))++ assert result[0] is already+ assert len(result) == 2+ assert result[1].term is not None+++def _dataset(day: str, successes: list[int]):+ results = [SimpleNamespace(success=True, results=[None] * n) for n in successes]+ results.append(SimpleNamespace(success=False, results=[None] * 50))+ return SimpleNamespace(date=day, queryDataset=[None] * 3, searchResults=results)+++def test_dataset_submissions_count_queries_and_only_successful_results():+ redis = fakeredis.FakeStrictRedis()++ StatsManager(redis).record_dataset(_dataset("2024-03-05", [2, 5]))++ day = date(2024, 3, 5)+ assert int(redis.get(DATASET_QUERIES_COUNT_KEY.format(date=day))) == 3+ assert int(redis.get(DATASET_RESULTS_COUNT_KEY.format(date=day))) == 7+++def test_a_dataset_with_an_unreadable_date_is_counted_today(monkeypatch):+ redis = fakeredis.FakeStrictRedis()+ monkeypatch.setattr("mwmbl.crawler.stats.utc_today", lambda: date(2030, 1, 1))++ StatsManager(redis).record_dataset(_dataset("not-a-date", [1]))++ assert int(redis.get(DATASET_QUERIES_COUNT_KEY.format(date=date(2030, 1, 1)))) == 3+++def bloom_files(settings, tmp_path):+ settings.DOMAIN_LINKS_BLOOM_FILTER_PATH = str(tmp_path / "links_{domain_group}.bloom")+ settings.NUM_DOMAINS_IN_BLOOM_FILTER = 1000+++def test_links_from_a_top_domain_count_towards_the_top_group(bloom_files, monkeypatch):+ monkeypatch.setattr(domains, "TOP_DOMAINS", {"top.example"})+ monkeypatch.setattr(domains, "OTHER_DOMAINS", {"other.example"})++ with DomainLinkDatabase() as database:+ database.update_domain_links("top.example", {"a.example"})+ database.update_domain_links("other.example", {"b.example"})+ database.update_domain_links("unknown.example", {"c.example"})++ assert database.get_domain_score("a.example") == 5+ assert database.get_domain_score("b.example") == 1+ assert database.get_domain_score("c.example") == 0+++def test_link_filters_persist_between_openings(bloom_files, monkeypatch):+ monkeypatch.setattr(domains, "TOP_DOMAINS", {"top.example"})+ with DomainLinkDatabase() as database:+ database.update_domain_links("top.example", {"a.example"})++ with DomainLinkDatabase() as reopened:+ assert reopened.get_domain_score("a.example") == 5