An open source, non-profit web search engine
Add tests for Super Search sources, justext, platform user API, backfill, background loops
blueheron7861754b208ceca
diff --git a/test/test_backfill_domain_evidence.py b/test/test_backfill_domain_evidence.pynew file mode 100644index 0000000..680519b--- /dev/null+++ b/test/test_backfill_domain_evidence.py@@ -0,0 +1,107 @@+"""+The evidence backfill: which submitted domains it picks, and that one failing domain does+not end the run.+"""++from datetime import date, datetime, timezone+from types import SimpleNamespace++import pytest+from django.core.management import call_command++from mwmbl.management.commands import backfill_domain_evidence as backfill+from mwmbl.models import DomainEvidence, DomainSubmission, MwmblUser+++def submitter(db):+ return MwmblUser.objects.create_user(username="submitter", email="[email protected]", password="pw")+++def _submit(submitter, name, status="PENDING", year=2025):+ return DomainSubmission.objects.create(+ name=name,+ status=status,+ submitted_by=submitter,+ submitted_on=datetime(year, 6, 1, tzinfo=timezone.utc),+ )+++def _options(**overrides):+ return {"status": None, "since": None, "limit": None, **overrides}+++def test_domains_already_crawled_duplicates_and_malformed_names_are_skipped(submitter):+ _submit(submitter, "new.example")+ _submit(submitter, "new.example")+ _submit(submitter, "done.example")+ _submit(submitter, "Bad-Case.example")+ DomainEvidence.objects.create(domain="done.example", state=DomainEvidence.State.READY)++ domains = backfill.Command._domains_to_crawl(_options())++ assert domains == ["new.example"]+++def test_domains_can_be_filtered_by_status_date_and_limit(submitter):+ _submit(submitter, "old-approved.example", status="APPROVED", year=2023)+ _submit(submitter, "new-approved.example", status="APPROVED", year=2025)+ _submit(submitter, "new-pending.example", status="PENDING", year=2025)++ by_status = backfill.Command._domains_to_crawl(_options(status=["APPROVED"]))+ by_date = backfill.Command._domains_to_crawl(_options(since=date(2024, 1, 1)))+ limited = backfill.Command._domains_to_crawl(_options(limit=1))++ assert by_status == ["old-approved.example", "new-approved.example"]+ assert by_date == ["new-approved.example", "new-pending.example"]+ assert limited == ["old-approved.example"]+++def test_one_failing_domain_is_recorded_and_the_run_continues(submitter, monkeypatch, capsys):+ _submit(submitter, "broken.example")+ _submit(submitter, "fine.example")+ stored, failures = [], []++ def crawl(domain, redis):+ if domain == "broken.example":+ raise TimeoutError("timed out")+ return domain++ def store(domain, crawled):+ stored.append(domain)+ return SimpleNamespace(suggested_action="APPROVE", state="READY")++ monkeypatch.setattr(backfill, "get_redis", lambda: None)+ monkeypatch.setattr(backfill, "crawl_domain", crawl)+ monkeypatch.setattr(backfill, "store_evidence", store)+ monkeypatch.setattr(backfill, "refresh_suggestion", lambda evidence: None)+ monkeypatch.setattr(backfill, "store_failure", lambda domain, reason: failures.append((domain, reason)))++ call_command("backfill_domain_evidence", "--delay", "0")++ captured = capsys.readouterr()+ assert stored == ["fine.example"]+ assert failures == [("broken.example", "TimeoutError")]+ assert "2 domains to crawl" in captured.out+ assert "fine.example: APPROVE (READY)" in captured.out+ assert "broken.example: FAILED (timed out)" in captured.err+++def test_the_delay_between_domains_is_honoured(submitter, monkeypatch):+ _submit(submitter, "one.example")+ sleeps = []+ monkeypatch.setattr(backfill, "get_redis", lambda: None)+ monkeypatch.setattr(backfill, "crawl_domain", lambda domain, redis: domain)+ monkeypatch.setattr(+ backfill, "store_evidence", lambda domain, crawled: SimpleNamespace(suggested_action="", state="X")+ )+ monkeypatch.setattr(backfill, "refresh_suggestion", lambda evidence: None)+ monkeypatch.setattr(backfill.time, "sleep", sleeps.append)++ call_command("backfill_domain_evidence", "--delay", "2.5")++ assert sleeps == [2.5]diff --git a/test/test_justext.py b/test/test_justext.pynew file mode 100644index 0000000..dc742e8--- /dev/null+++ b/test/test_justext.py@@ -0,0 +1,119 @@+"""+The vendored jusText: decoding HTML of unknown encoding, and the paragraph classes that+decide what becomes a page's extract.+"""++import pytest++from mwmbl.justext import core+from mwmbl.justext.core import JustextError, decode_html, html_to_dom, justext_from_dom+from mwmbl.justext.utils import get_stoplist++PROSE = (+ "This is a long paragraph of ordinary English prose that talks about the history of the town and "+ "all of the people who have lived in it for the last several hundred years, which is what a reader "+ "of the page would come to it to find out about."+)+++def _classes(body: str) -> dict[str, str]:+ dom = html_to_dom(f"<html><body>{body}</body></html>".encode())+ return {p.text: p.class_type for p in justext_from_dom(dom, get_stoplist("English"))}+++def test_text_is_returned_unchanged():+ assert decode_html("déjà vu") == "déjà vu"+++def test_an_explicit_encoding_wins_over_the_page_declaration():+ html = '<meta charset="utf-8">caf\xe9'.encode("latin-1")++ assert decode_html(html, encoding="latin-1").endswith("café")+++def test_the_meta_charset_is_used_when_no_encoding_is_given():+ html = '<meta charset="latin-1">caf\xe9'.encode("latin-1")++ assert decode_html(html).endswith("café")+++def test_an_unknown_declared_charset_is_treated_as_undeclared():+ assert decode_html('<meta charset="not-a-codec">café'.encode()).endswith("café")+++def test_bytes_that_are_not_utf8_fall_back_to_the_default_encoding():+ assert decode_html(b"caf\xe9", default_encoding="latin-1") == "café"+++def test_undecodable_bytes_raise_when_errors_are_strict():+ with pytest.raises(JustextError, match="Unable to decode"):+ decode_html(b"caf\xe9", default_encoding="ascii", errors="strict")+++def test_text_carrying_an_xml_encoding_declaration_still_parses():+ dom = html_to_dom('<?xml version="1.0" encoding="utf-8"?><html><body><p>Hello</p></body></html>')++ assert dom.xpath("//p")[0].text == "Hello"+++def test_text_input_is_parsed_directly():+ dom = html_to_dom("<html><body><p>Héllo</p></body></html>", encoding="utf-8")++ assert dom.xpath("//p")[0].text == "Héllo"+++def test_long_prose_is_good_and_a_copyright_line_is_bad():+ classes = _classes(f"<p>{PROSE}</p><p>\xa9 2024 Example Ltd. All rights reserved.</p>")++ assert classes[PROSE] == "good"+ assert classes["\xa9 2024 Example Ltd. All rights reserved."] == "bad"+++def test_a_paragraph_that_is_mostly_links_is_bad():+ link_text = "Read more about the history of the town and the people who lived in it, " * 4+ classes = _classes(f"<p>{PROSE}</p><p><a href='/a'>{link_text}</a></p>")++ assert classes[PROSE] == "good"+ assert classes[link_text.strip()] == "bad"+++def test_a_short_linked_label_is_bad_but_a_short_plain_line_between_good_paragraphs_is_good():+ classes = _classes(f"<p>{PROSE}</p><p>Short line.</p><p>{PROSE} Again.</p><p><a href='/x'>Menu</a></p>")++ assert classes["Short line."] == "good"+ assert classes["Menu"] == "bad"+++def test_a_short_line_between_boilerplate_is_bad():+ classes = _classes("<p><a href='/a'>Home</a></p><p>Short line.</p><p><a href='/b'>About</a></p>")++ assert classes["Short line."] == "bad"+++def test_a_heading_just_before_good_text_is_kept():+ classes = _classes(f"<h2>History</h2><p>{PROSE}</p>")++ assert classes["History"] == "good"+++def test_a_heading_far_from_good_text_is_dropped():+ classes = _classes("<h2>History</h2><p><a href='/a'>Home</a></p>" + "<p><a href='/b'>About us now</a></p>" * 20)++ assert classes["History"] == "bad"+++def test_br_br_separates_paragraphs():+ dom = html_to_dom("<html><body><p>First line.<br><br>Second line.</p></body></html>".encode())++ paragraphs = justext_from_dom(dom, get_stoplist("English"))++ assert [p.text for p in paragraphs] == ["First line.", "Second line."]+++def test_ignored_swallows_only_the_listed_exceptions():+ with core.ignored(LookupError):+ raise LookupError++ with pytest.raises(ValueError):+ with core.ignored(LookupError):+ raise ValueErrordiff --git a/test/test_platform_user_api.py b/test/test_platform_user_api.pynew file mode 100644index 0000000..6c5411f--- /dev/null+++ b/test/test_platform_user_api.py@@ -0,0 +1,135 @@+"""+Per-user platform endpoints: agreements, devices, the profile, and the moderator's+re-crawl of a submission.+"""++import pytest+from allauth.account.models import EmailAddress+from django.contrib.auth import get_user_model+from django.contrib.auth.models import Permission+from ninja_jwt.tokens import RefreshToken++from mwmbl.models import Device, DomainEvidence, DomainSubmission, UserAgreement+from mwmbl.platform import api++User = get_user_model()++BASE = "/api/v1/platform"+++def _user(username: str):+ user = User.objects.create_user(username=username, email=f"{username}@example.com", password="correctpassword")+ EmailAddress.objects.create(user=user, email=user.email, verified=True, primary=True)+ return user+++def _auth(user) -> dict:+ return {"Authorization": f"Bearer {RefreshToken.for_user(user).access_token}"}+++def member(db):+ return _user("member")+++def moderator(db):+ user = _user("moderator")+ user.user_permissions.add(Permission.objects.get(codename="change_domain_submission_status"))+ return user+++def test_accepting_an_agreement_records_the_current_version(client, member, settings):+ response = client.post(+ f"{BASE}/agreements/",+ data={"agreement_type": "TERMS_OF_SERVICE_API"},+ content_type="application/json",+ headers=_auth(member),+ )++ assert response.status_code == 200+ assert response.json()["version_id"] == settings.CURRENT_AGREEMENT_VERSIONS["TERMS_OF_SERVICE_API"]+ assert UserAgreement.objects.filter(user=member, agreement_type="TERMS_OF_SERVICE_API").count() == 1+++def test_current_agreements_list_the_latest_acceptance_of_each_type(client, member):+ for version in ("v1", "v2"):+ UserAgreement.objects.create(user=member, agreement_type="TERMS_OF_SERVICE_GUI", version_id=version)++ current = client.get(f"{BASE}/agreements/", headers=_auth(member))+ history = client.get(f"{BASE}/agreements/history/", headers=_auth(member))++ assert [(a["agreement_type"], a["version_id"]) for a in current.json()] == [("TERMS_OF_SERVICE_GUI", "v2")]+ assert [a["version_id"] for a in history.json()["items"]] == ["v2", "v1"]+++def test_devices_are_listed_newest_first_and_only_for_their_owner(client, member):+ stranger = _user("stranger")+ Device.objects.create(user=member, hostname="old-laptop")+ Device.objects.create(user=member, hostname="new-laptop")+ Device.objects.create(user=stranger, hostname="not-mine")++ response = client.get(f"{BASE}/devices/", headers=_auth(member))++ assert [d["hostname"] for d in response.json()] == ["new-laptop", "old-laptop"]+++def test_a_device_can_be_renamed_by_its_owner_only(client, member):+ stranger = _user("stranger")+ device = Device.objects.create(user=member, hostname="laptop")+ url = f"{BASE}/devices/{device.id}"++ refused = client.patch(+ url, data={"friendly_name": "Hijacked"}, content_type="application/json", headers=_auth(stranger)+ )+ renamed = client.patch(+ url, data={"friendly_name": "Office"}, content_type="application/json", headers=_auth(member)+ )++ assert refused.status_code == 404+ assert renamed.status_code == 200+ assert renamed.json()["friendly_name"] == "Office"+ device.refresh_from_db()+ assert device.friendly_name == "Office"+++def test_profile_reports_the_account_and_plan(client, member):+ response = client.get(f"{BASE}/user", headers=_auth(member))++ body = response.json()+ assert response.status_code == 200+ assert body["username"] == "member"+ assert body["email"] == "[email protected]"+ assert body["email_confirmed"] is True+ assert body["plan"] == "free"+++def test_refetching_clears_the_old_evidence_and_queues_a_crawl(client, member, moderator, monkeypatch):+ submission = DomainSubmission.objects.create(name="example.org", submitted_by=member)+ DomainEvidence.objects.create(domain="example.org", state=DomainEvidence.State.READY)+ queued = []+ monkeypatch.setattr(api, "enrich_domain_submission", queued.append)+ url = f"{BASE}/domain-submissions/ids/{submission.id}/refetch"++ refused = client.post(url, headers=_auth(member))+ accepted = client.post(url, headers=_auth(moderator))++ assert refused.status_code == 403+ assert accepted.status_code == 200+ assert queued == ["example.org"]+ assert not DomainEvidence.objects.filter(domain="example.org").exists()+++def test_refetching_a_missing_submission_is_a_404(client, moderator):+ response = client.post(f"{BASE}/domain-submissions/ids/999999/refetch", headers=_auth(moderator))++ assert response.status_code == 404+++def test_index_stats_are_empty_when_redis_cannot_be_read(monkeypatch):+ def unreachable(domain):+ raise ConnectionError("redis down")++ monkeypatch.setattr(api.stats_manager, "get_stats_for_domain", unreachable)++ assert api._index_stats("example.org") == {}diff --git a/test/test_super_search_sources_http.py b/test/test_super_search_sources_http.pynew file mode 100644index 0000000..35f1700--- /dev/null+++ b/test/test_super_search_sources_http.py@@ -0,0 +1,303 @@+"""+The Guardian, NHS, Wikidata and homepage-guesser sources, and the recipe smoke check, against+canned HTTP responses: what each turns a response into, and that a failing upstream yields no+results rather than an exception.+"""++from types import SimpleNamespace++import httpx+import pytest++from mwmbl.tinysearchengine.indexer import Document+from mwmbl.tinysearchengine.super_search_sources import guardian, homepage, nhs, smoke, wikidata++TRAIL = (+ "<p>"+ + "The prime minister said on Tuesday that the <b>government</b> would publish its plan for the economy. " * 4+ + "</p>"+)+++def _client(handler) -> httpx.AsyncClient:+ return httpx.AsyncClient(transport=httpx.MockTransport(handler))+++def _json(payload, status: int = 200):+ return lambda request: httpx.Response(status, json=payload)+++def _status(status: int):+ return lambda request: httpx.Response(status)+++# ------------------------------------------------------------------- guardian+++async def test_guardian_without_a_key_makes_no_request(settings):+ settings.GUARDIAN_API_KEY = ""++ def forbidden(request):+ raise AssertionError("no request expected without an API key")++ async with _client(forbidden) as client:+ assert await guardian.search(client, "news", 5) == []+++async def test_guardian_results_become_documents_with_cleaned_trail_text(settings):+ settings.GUARDIAN_API_KEY = "key"+ seen = {}++ def handler(request):+ seen.update(dict(request.url.params))+ return httpx.Response(+ 200,+ json={+ "response": {+ "results": [+ {+ "webTitle": "Headline",+ "webUrl": "https://www.theguardian.com/a",+ "fields": {"trailText": TRAIL},+ },+ {"webTitle": "No trail", "webUrl": "https://www.theguardian.com/b"},+ {"webTitle": "", "webUrl": "https://www.theguardian.com/c"},+ ]+ }+ },+ )++ async with _client(handler) as client:+ docs = await guardian.search(client, "election", 50)++ assert [d.url for d in docs] == ["https://www.theguardian.com/a", "https://www.theguardian.com/b"]+ assert "government" in docs[0].extract and "<" not in docs[0].extract+ assert docs[1].extract == ""+ assert seen["page-size"] == "10"+ assert seen["api-key"] == "key"++[email protected]("handler", [_status(500), lambda request: httpx.Response(200, text="not json")])+async def test_guardian_failures_give_no_results(settings, handler):+ settings.GUARDIAN_API_KEY = "key"++ async with _client(handler) as client:+ assert await guardian.search(client, "election", 5) == []+++# ------------------------------------------------------------------------ nhs++NHS_PAGE = """+<html><body>+<a class="app-search-results-item" href="/click?url=%2Fconditions%2Fasthma%2F&q=asthma">Asthma - NHS</a>+<a class="app-search-results-item" href="/conditions/cough/">Cough</a>+<a class="app-search-results-item" href="/conditions/cold/">Cold</a>+<a class="other" href="/ignored/">Ignored</a>+</body></html>+"""+++async def test_nhs_results_use_the_tracked_target_and_respect_the_limit():+ async with _client(lambda request: httpx.Response(200, text=NHS_PAGE)) as client:+ docs = await nhs.search(client, "asthma", 2)++ assert [(d.title, d.url) for d in docs] == [+ ("Asthma - NHS", "https://www.nhs.uk/conditions/asthma/"),+ ("Cough", "https://www.nhs.uk/conditions/cough/"),+ ]+++async def test_nhs_failure_gives_no_results():+ async with _client(_status(503)) as client:+ assert await nhs.search(client, "asthma", 5) == []+++# ------------------------------------------------------------------- wikidata+++def _wikidata_handler(search_hits, entities, entity_status: int = 200):+ def handler(request):+ if request.url.params["action"] == "wbsearchentities":+ return httpx.Response(200, json={"search": search_hits})+ return httpx.Response(entity_status, json={"entities": entities})++ return handler+++def _entity(label, description, *urls):+ claims = [{"mainsnak": {"datavalue": {"value": url}}} for url in urls]+ return {+ "labels": {"en": {"value": label}},+ "descriptions": {"en": {"value": description}},+ "claims": {"P856": claims},+ }+++async def test_wikidata_returns_one_official_site_per_entity_in_search_order():+ handler = _wikidata_handler(+ [{"id": "Q2"}, {"id": "Q1"}, {"id": "Q3"}],+ {+ "Q1": _entity("One", "first", "https://one.example/", "https://one-alt.example/"),+ "Q2": _entity("Two", "second", "https://two.example/"),+ "Q3": {"claims": {"P856": [{"mainsnak": {}}]}},+ },+ )++ async with _client(handler) as client:+ docs = await wikidata.search(client, "numbers", 5)++ assert [(d.title, d.url, d.extract) for d in docs] == [+ ("Two", "https://two.example/", "second"),+ ("One", "https://one.example/", "first"),+ ]+++async def test_wikidata_with_no_matching_entities_makes_no_second_request():+ async with _client(_wikidata_handler([], {})) as client:+ assert await wikidata.search(client, "nothing", 5) == []+++async def test_wikidata_failure_in_either_step_gives_no_results():+ async with _client(_status(500)) as client:+ assert await wikidata.search(client, "x", 5) == []++ handler = _wikidata_handler([{"id": "Q1"}], {}, entity_status=500)+ async with _client(handler) as client:+ assert await wikidata.search(client, "x", 5) == []+++# ------------------------------------------------------------------- homepage+++def _page(title: str, content_type: str = "text/html", status: int = 200):+ return httpx.Response(status, text=f"<html><head><title>{title}</title></head></html>", headers={+ "content-type": content_type+ }) # fmt: skip+++async def test_homepage_guesser_keeps_pages_whose_title_mentions_the_query():+ def handler(request):+ host = request.url.host+ if host == "solvera.com":+ return _page("Solvera Jewellery")+ if host == "solvera.co.uk":+ return _page("Totally unrelated parking page")+ if host == "solvera.org":+ return _page("Solvera", content_type="application/pdf")+ return _page("Solvera", status=404)++ async with _client(handler) as client:+ docs = await homepage.search(client, "Solvera", 5)++ assert [(d.title, d.url) for d in docs] == [("Solvera Jewellery", "https://solvera.com/")]+++async def test_homepage_guesser_stops_at_the_limit_and_skips_unreachable_hosts():+ def handler(request):+ if request.url.host == "brand.com":+ raise httpx.ConnectError("refused")+ return _page("Brand home")++ async with _client(handler) as client:+ docs = await homepage.search(client, "brand", 1)++ assert [d.url for d in docs] == ["https://brand.co.uk/"]++[email protected]("query", ["ab", "a", " ", "!!"])+async def test_homepage_guesser_ignores_queries_too_short_to_guess_from(query):+ def forbidden(request):+ raise AssertionError("no request expected")++ async with _client(forbidden) as client:+ assert await homepage.search(client, query, 5) == []+++# ---------------------------------------------------------------------- smoke+++def _docs(*urls_and_titles):+ return [Document(title=title, url=url, extract="") for url, title in urls_and_titles]+++def _recipe(**smoke_config):+ return SimpleNamespace(smoke=smoke_config)+++def _searching(monkeypatch, real, control):+ async def fake_search(client, recipe, query, limit):+ return real if query == "real query" else control++ monkeypatch.setattr(smoke, "search_with_recipe", fake_search)+++def test_control_overlap_is_a_jaccard_of_result_urls():+ one = _docs(("a", "A"), ("b", "B"))+ two = _docs(("b", "B"), ("c", "C"))++ assert smoke.control_overlap(one, two) == pytest.approx(1 / 3)+ assert smoke.control_overlap(one, []) is None+++async def test_a_recipe_without_smoke_settings_fails():+ ok, reason = await smoke.check_recipe(None, _recipe(query="real query"))++ assert not ok+ assert "missing" in reason+++async def test_a_recipe_that_returns_nothing_fails(monkeypatch):+ _searching(monkeypatch, [], [])++ ok, reason = await smoke.check_recipe(None, _recipe(query="real query", expect_title_contains="x"))++ assert (ok, reason) == (False, "no results for 'real query'")+++async def test_a_recipe_whose_top_result_lacks_a_url_fails(monkeypatch):+ _searching(monkeypatch, _docs(("", "Title")), [])++ ok, reason = await smoke.check_recipe(None, _recipe(query="real query", expect_title_contains="title"))++ assert not ok+ assert "missing url/title" in reason+++async def test_a_recipe_with_no_matching_title_fails(monkeypatch):+ _searching(monkeypatch, _docs(("a", "Cats"), ("b", "Dogs")), [])++ ok, reason = await smoke.check_recipe(None, _recipe(query="real query", expect_title_contains="pasta"))++ assert not ok+ assert "no title contains 'pasta'" in reason+++async def test_a_recipe_returning_the_same_results_for_gibberish_is_scraping_boilerplate(monkeypatch):+ same = _docs(("a", "Pasta"), ("b", "Pasta bake"))+ _searching(monkeypatch, same, same)++ ok, reason = await smoke.check_recipe(None, _recipe(query="real query", expect_title_contains="pasta"))++ assert not ok+ assert "barely change" in reason+++async def test_a_query_sensitive_recipe_passes(monkeypatch):+ _searching(monkeypatch, _docs(("a", "Pasta")), _docs(("z", "Something else")))++ ok, reason = await smoke.check_recipe(None, _recipe(query="real query", expect_title_contains="pasta"))++ assert (ok, reason) == (True, "")+++async def test_an_erroring_control_search_does_not_fail_the_recipe(monkeypatch):+ async def fake_search(client, recipe, query, limit):+ if query == "real query":+ return _docs(("a", "Pasta"))+ raise RuntimeError("boom")++ monkeypatch.setattr(smoke, "search_with_recipe", fake_search)++ ok, _ = await smoke.check_recipe(None, _recipe(query="real query", expect_title_contains="pasta"))++ assert ok