An open source, non-profit web search engine
Publish seed domains in daily batches, to crawl-scoped keys only
Claudef14e1074e857
diff --git a/mwmbl/indexer/seed_domains.py b/mwmbl/indexer/seed_domains.pyindex bb6cd75..f3d9066 100644--- a/mwmbl/indexer/seed_domains.py+++ b/mwmbl/indexer/seed_domains.py@@ -6,13 +6,18 @@ that turned out to be new pages, averaged over its last SEED_DOMAIN_RECENT_CRAWLit falls as crawls saturate the domain. And it should be likely to answer searches: Staanreturning it often, across everyone's seed queries, is the evidence for that. A domain'sscore is the product of the two, and GET /api/v2/combined-search/seed-domains lists the-domains by it.+domains by it, to crawl-scoped API keys only.++That list is published in batches, rebuilt at most every SEED_DOMAINS_PUBLISH_SECONDS. Were+it live, comparing it before and after one crawl would show which domains Staan returned+for that crawl's query, and queries are private.The user whose crawl first met a domain is kept against it, so that discovering a domainthat turns out well can be rewarded later. Nothing shows it yet."""from django.conf import settings+from django.core.cache import cachefrom django.db import transactionfrom django.db.models import F@@ -67,5 +72,22 @@ def record_seed_domain_crawls(pages_indexed: dict[str, int]) -> None:seed_domain.save(update_fields=["new_page_score", "score"])-def top_seed_domains(limit: int) -> list[SeedDomain]:- return list(SeedDomain.objects.order_by("-score", "domain")[:limit])+PUBLISHED_KEY = "seed-domains:published"+++def published_seed_domains() -> list[dict]:+ """The best SEED_DOMAINS_PUBLISHED seed domains, as of the last time the list was published."""+ published = cache.get(PUBLISHED_KEY)+ if published is None:+ top = SeedDomain.objects.order_by("-score", "domain")[: settings.SEED_DOMAINS_PUBLISHED]+ published = [+ {+ "domain": seed_domain.domain,+ "new_page_score": seed_domain.new_page_score,+ "staan_results": seed_domain.staan_results,+ "score": seed_domain.score,+ }+ for seed_domain in top+ ]+ cache.set(PUBLISHED_KEY, published, settings.SEED_DOMAINS_PUBLISH_SECONDS)+ return publisheddiff --git a/mwmbl/search_auth.py b/mwmbl/search_auth.pyindex 74f70a8..060873c 100644--- a/mwmbl/search_auth.py+++ b/mwmbl/search_auth.py@@ -62,6 +62,21 @@ class SearchApiKeyAuth(APIKeyHeader):return api_key+class CrawlApiKeyAuth(APIKeyHeader):+ """A crawl-scoped key in X-API-Key, for endpoints only trusted crawlers may use."""++ param_name = "X-API-Key"++ def authenticate(self, request, key: str | None):+ if not key:+ return None+ key_hash = hashlib.sha256(key.encode()).hexdigest()+ api_key = ApiKey.objects.filter(key=key_hash, scopes__contains=[ApiKey.Scope.CRAWL]).first()+ if api_key is not None:+ ApiKey.objects.filter(pk=api_key.pk).update(last_used=datetime.now(timezone.utc))+ return api_key++async def authenticate_user(request) -> MwmblUser:"""Resolve the requesting user from either an X-API-Key header or a JWT.diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex 35c3caf..ee8fb86 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -320,6 +320,10 @@ SEED_CRAWL_RECORD_TTL_SECONDS = 7 * 24 * 60 * 60# A seed domain's new-page score averages only its most recent crawls, so a domain the# crawls have saturated drops down the list rather than living off its first crawl.SEED_DOMAIN_RECENT_CRAWLS = 5+# The seed domain list crawlers read is republished at most this often, so it cannot be+# diffed to learn what one private query returned - see mwmbl.indexer.seed_domains.+SEED_DOMAINS_PUBLISH_SECONDS = 24 * 60 * 60+SEED_DOMAINS_PUBLISHED = 1000# Jev (TypeSafe), which orders Combined Search's results - see mwmbl.tinysearchengine.jev_rank.# Without a key, or when a request fails or times out, Combined Search serves the LTR'sdiff --git a/mwmbl/tinysearchengine/combined_search.py b/mwmbl/tinysearchengine/combined_search.pyindex 62b983e..2fdd395 100644--- a/mwmbl/tinysearchengine/combined_search.py+++ b/mwmbl/tinysearchengine/combined_search.py@@ -50,7 +50,7 @@ from mwmbl.indexer.seed_crawl import (get_seed_crawl_summary,start_seed_crawl,)-from mwmbl.indexer.seed_domains import top_seed_domains+from mwmbl.indexer.seed_domains import published_seed_domainsfrom mwmbl.membership import combined_search_monthly_limitfrom mwmbl.models import Membership, UserBillingfrom mwmbl.quota import (@@ -60,7 +60,7 @@ from mwmbl.quota import (increment_monthly_combined_search,increment_monthly_combined_search_api_if_below,)-from mwmbl.search_auth import authenticate_user+from mwmbl.search_auth import CrawlApiKeyAuth, authenticate_userfrom mwmbl.search_setup import index_pathfrom mwmbl.tinysearchengine.indexer import Documentfrom mwmbl.tinysearchengine.rank import find_blacklisted_urls@@ -362,11 +362,14 @@ def init_router(ranker) -> None:@router.get("seed-domains",response=list[SeedDomainResponse],- auth=None,+ auth=CrawlApiKeyAuth(),summary="Seed domains, best first",description="The domains EUSP has returned for seed crawls, ordered by `score`: how likely ""a crawl of the domain is to add new pages to the index, times how often EUSP returns it. "- "For crawlers deciding where to go next.",+ "For crawlers deciding where to go next; requires a crawl-scoped API key in `X-API-Key`.\n\n"+ "The list is republished at most once every "+ f"{settings.SEED_DOMAINS_PUBLISH_SECONDS // 3600} hours, so that it cannot reveal what any "+ "one search returned.",)- def seed_domains(request, limit: int = Query(100, ge=1, le=1000)):- return top_seed_domains(limit)+ def seed_domains(request, limit: int = Query(100, ge=1, le=settings.SEED_DOMAINS_PUBLISHED)):+ return published_seed_domains()[:limit]diff --git a/test/test_seed_crawl.py b/test/test_seed_crawl.pyindex fcb886e..6ceb86c 100644--- a/test/test_seed_crawl.py+++ b/test/test_seed_crawl.py@@ -8,12 +8,13 @@ from pathlib import Pathimport pytestfrom django.contrib.auth import get_user_model+from django.core.cache import cachefrom django.test import Clientfrom ninja_jwt.tokens import RefreshToken-from mwmbl.indexer import index_batches, seed_crawl+from mwmbl.indexer import index_batches, seed_crawl, seed_domainsfrom mwmbl.indexer.index_batches import index_new_documents-from mwmbl.models import SeedDomain, SeedDomainCrawl+from mwmbl.models import ApiKey, SeedDomain, SeedDomainCrawl, generate_api_keyfrom mwmbl.tinysearchengine.indexer import Document, DocumentSource, TinyIndexUser = get_user_model()@@ -518,8 +519,21 @@ def test_new_pages_reports_progress_and_domains(fake_web, redis_cache, user, indassert "discovered_by" not in response.json()["domains"][0]+SEED_DOMAINS_URL = "/api/v2/combined-search/seed-domains"+++def _api_key(user, scope):+ raw, hashed = generate_api_key()+ ApiKey.objects.create(user=user, key=hashed, name="seed-domains", scopes=[scope])+ return raw+++def _seed_domains(user, query=""):+ return Client().get(f"{SEED_DOMAINS_URL}{query}", HTTP_X_API_KEY=_api_key(user, ApiKey.Scope.CRAWL))++@pytest.mark.django_db-def test_the_seed_domains_endpoint_lists_the_best_first_without_their_discoverers(user):+def test_the_seed_domains_endpoint_lists_the_best_first_without_their_discoverers(redis_cache, user):SeedDomain.objects.create(domain="low.example.com", discovered_by=user, new_page_score=0.1, staan_results=1, score=0.1)@@ -527,12 +541,36 @@ def test_the_seed_domains_endpoint_lists_the_best_first_without_their_discovererdomain="high.example.com", discovered_by=user, new_page_score=0.5, staan_results=4, score=2)- response = Client().get("/api/v2/combined-search/seed-domains?limit=1")+ response = _seed_domains(user, "?limit=1")assert response.status_code == 200assert response.json() == [{"domain": "high.example.com", "new_page_score": 0.5, "staan_results": 4, "score": 2.0}]+def test_the_seed_domains_endpoint_requires_a_crawl_scoped_key(redis_cache, user):+ search_key = _api_key(user, ApiKey.Scope.SEARCH)++ assert Client().get(SEED_DOMAINS_URL).status_code == 401+ assert Client().get(SEED_DOMAINS_URL, HTTP_X_API_KEY=search_key).status_code == 401+++def test_seed_domains_are_published_in_batches(redis_cache, user):+ SeedDomain.objects.create(domain="old.example.com", new_page_score=0.1, staan_results=1, score=0.1)+ assert [domain["domain"] for domain in _seed_domains(user).json()] == ["old.example.com"]++ # What a crawl changes is not shown until the list is next published.+ SeedDomain.objects.create(domain="new.example.com", new_page_score=0.5, staan_results=1, score=0.5)+ SeedDomain.objects.filter(domain="old.example.com").update(score=0.0)+ assert _seed_domains(user).json() == [+ {"domain": "old.example.com", "new_page_score": 0.1, "staan_results": 1, "score": 0.1}+ ]++ cache.delete(seed_domains.PUBLISHED_KEY)+ assert [domain["domain"] for domain in _seed_domains(user).json()] == ["new.example.com", "old.example.com"]++def test_index_new_documents_returns_only_urls_the_index_lacked(index_path, monkeypatch):monkeypatch.setattr(index_batches, "filter_blacklisted_documents", lambda documents: documents)first = Document("Tokio tutorial", "https://tokio.rs/tutorial", "Learn tokio.")