Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Publish seed domains in daily batches, to crawl-scoped keys only

Claudef14e1074e857
diff --git a/mwmbl/indexer/seed_domains.py b/mwmbl/indexer/seed_domains.py
index bb6cd75..f3d9066 100644
--- a/mwmbl/indexer/seed_domains.py
+++ b/mwmbl/indexer/seed_domains.py
@@ -6,13 +6,18 @@ that turned out to be new pages, averaged over its last SEED_DOMAIN_RECENT_CRAWL
it falls as crawls saturate the domain. And it should be likely to answer searches: Staan
returning it often, across everyone's seed queries, is the evidence for that. A domain's
score is the product of the two, and GET /api/v2/combined-search/seed-domains lists the
-domains by it.
+domains by it, to crawl-scoped API keys only.
+
+That list is published in batches, rebuilt at most every SEED_DOMAINS_PUBLISH_SECONDS. Were
+it live, comparing it before and after one crawl would show which domains Staan returned
+for that crawl's query, and queries are private.
The user whose crawl first met a domain is kept against it, so that discovering a domain
that turns out well can be rewarded later. Nothing shows it yet.
"""
from django.conf import settings
+from django.core.cache import cache
from django.db import transaction
from django.db.models import F
@@ -67,5 +72,22 @@ def record_seed_domain_crawls(pages_indexed: dict[str, int]) -> None:
seed_domain.save(update_fields=["new_page_score", "score"])
-def top_seed_domains(limit: int) -> list[SeedDomain]:
- return list(SeedDomain.objects.order_by("-score", "domain")[:limit])
+PUBLISHED_KEY = "seed-domains:published"
+
+
+def published_seed_domains() -> list[dict]:
+ """The best SEED_DOMAINS_PUBLISHED seed domains, as of the last time the list was published."""
+ published = cache.get(PUBLISHED_KEY)
+ if published is None:
+ top = SeedDomain.objects.order_by("-score", "domain")[: settings.SEED_DOMAINS_PUBLISHED]
+ published = [
+ {
+ "domain": seed_domain.domain,
+ "new_page_score": seed_domain.new_page_score,
+ "staan_results": seed_domain.staan_results,
+ "score": seed_domain.score,
+ }
+ for seed_domain in top
+ ]
+ cache.set(PUBLISHED_KEY, published, settings.SEED_DOMAINS_PUBLISH_SECONDS)
+ return published
diff --git a/mwmbl/search_auth.py b/mwmbl/search_auth.py
index 74f70a8..060873c 100644
--- a/mwmbl/search_auth.py
+++ b/mwmbl/search_auth.py
@@ -62,6 +62,21 @@ class SearchApiKeyAuth(APIKeyHeader):
return api_key
+class CrawlApiKeyAuth(APIKeyHeader):
+ """A crawl-scoped key in X-API-Key, for endpoints only trusted crawlers may use."""
+
+ param_name = "X-API-Key"
+
+ def authenticate(self, request, key: str | None):
+ if not key:
+ return None
+ key_hash = hashlib.sha256(key.encode()).hexdigest()
+ api_key = ApiKey.objects.filter(key=key_hash, scopes__contains=[ApiKey.Scope.CRAWL]).first()
+ if api_key is not None:
+ ApiKey.objects.filter(pk=api_key.pk).update(last_used=datetime.now(timezone.utc))
+ return api_key
+
+
async def authenticate_user(request) -> MwmblUser:
"""Resolve the requesting user from either an X-API-Key header or a JWT.
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index 35c3caf..ee8fb86 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -320,6 +320,10 @@ SEED_CRAWL_RECORD_TTL_SECONDS = 7 * 24 * 60 * 60
# A seed domain's new-page score averages only its most recent crawls, so a domain the
# crawls have saturated drops down the list rather than living off its first crawl.
SEED_DOMAIN_RECENT_CRAWLS = 5
+# The seed domain list crawlers read is republished at most this often, so it cannot be
+# diffed to learn what one private query returned - see mwmbl.indexer.seed_domains.
+SEED_DOMAINS_PUBLISH_SECONDS = 24 * 60 * 60
+SEED_DOMAINS_PUBLISHED = 1000
# Jev (TypeSafe), which orders Combined Search's results - see mwmbl.tinysearchengine.jev_rank.
# Without a key, or when a request fails or times out, Combined Search serves the LTR's
diff --git a/mwmbl/tinysearchengine/combined_search.py b/mwmbl/tinysearchengine/combined_search.py
index 62b983e..2fdd395 100644
--- a/mwmbl/tinysearchengine/combined_search.py
+++ b/mwmbl/tinysearchengine/combined_search.py
@@ -50,7 +50,7 @@ from mwmbl.indexer.seed_crawl import (
get_seed_crawl_summary,
start_seed_crawl,
)
-from mwmbl.indexer.seed_domains import top_seed_domains
+from mwmbl.indexer.seed_domains import published_seed_domains
from mwmbl.membership import combined_search_monthly_limit
from mwmbl.models import Membership, UserBilling
from mwmbl.quota import (
@@ -60,7 +60,7 @@ from mwmbl.quota import (
increment_monthly_combined_search,
increment_monthly_combined_search_api_if_below,
)
-from mwmbl.search_auth import authenticate_user
+from mwmbl.search_auth import CrawlApiKeyAuth, authenticate_user
from mwmbl.search_setup import index_path
from mwmbl.tinysearchengine.indexer import Document
from mwmbl.tinysearchengine.rank import find_blacklisted_urls
@@ -362,11 +362,14 @@ def init_router(ranker) -> None:
@router.get(
"seed-domains",
response=list[SeedDomainResponse],
- auth=None,
+ auth=CrawlApiKeyAuth(),
summary="Seed domains, best first",
description="The domains EUSP has returned for seed crawls, ordered by `score`: how likely "
"a crawl of the domain is to add new pages to the index, times how often EUSP returns it. "
- "For crawlers deciding where to go next.",
+ "For crawlers deciding where to go next; requires a crawl-scoped API key in `X-API-Key`.\n\n"
+ "The list is republished at most once every "
+ f"{settings.SEED_DOMAINS_PUBLISH_SECONDS // 3600} hours, so that it cannot reveal what any "
+ "one search returned.",
)
- def seed_domains(request, limit: int = Query(100, ge=1, le=1000)):
- return top_seed_domains(limit)
+ def seed_domains(request, limit: int = Query(100, ge=1, le=settings.SEED_DOMAINS_PUBLISHED)):
+ return published_seed_domains()[:limit]
diff --git a/test/test_seed_crawl.py b/test/test_seed_crawl.py
index fcb886e..6ceb86c 100644
--- a/test/test_seed_crawl.py
+++ b/test/test_seed_crawl.py
@@ -8,12 +8,13 @@ from pathlib import Path
import pytest
from django.contrib.auth import get_user_model
+from django.core.cache import cache
from django.test import Client
from ninja_jwt.tokens import RefreshToken
-from mwmbl.indexer import index_batches, seed_crawl
+from mwmbl.indexer import index_batches, seed_crawl, seed_domains
from mwmbl.indexer.index_batches import index_new_documents
-from mwmbl.models import SeedDomain, SeedDomainCrawl
+from mwmbl.models import ApiKey, SeedDomain, SeedDomainCrawl, generate_api_key
from mwmbl.tinysearchengine.indexer import Document, DocumentSource, TinyIndex
User = get_user_model()
@@ -518,8 +519,21 @@ def test_new_pages_reports_progress_and_domains(fake_web, redis_cache, user, ind
assert "discovered_by" not in response.json()["domains"][0]
+SEED_DOMAINS_URL = "/api/v2/combined-search/seed-domains"
+
+
+def _api_key(user, scope):
+ raw, hashed = generate_api_key()
+ ApiKey.objects.create(user=user, key=hashed, name="seed-domains", scopes=[scope])
+ return raw
+
+
+def _seed_domains(user, query=""):
+ return Client().get(f"{SEED_DOMAINS_URL}{query}", HTTP_X_API_KEY=_api_key(user, ApiKey.Scope.CRAWL))
+
+
@pytest.mark.django_db
-def test_the_seed_domains_endpoint_lists_the_best_first_without_their_discoverers(user):
+def test_the_seed_domains_endpoint_lists_the_best_first_without_their_discoverers(redis_cache, user):
SeedDomain.objects.create(
domain="low.example.com", discovered_by=user, new_page_score=0.1, staan_results=1, score=0.1
)
@@ -527,12 +541,36 @@ def test_the_seed_domains_endpoint_lists_the_best_first_without_their_discoverer
domain="high.example.com", discovered_by=user, new_page_score=0.5, staan_results=4, score=2
)
- response = Client().get("/api/v2/combined-search/seed-domains?limit=1")
+ response = _seed_domains(user, "?limit=1")
assert response.status_code == 200
assert response.json() == [{"domain": "high.example.com", "new_page_score": 0.5, "staan_results": 4, "score": 2.0}]
+def test_the_seed_domains_endpoint_requires_a_crawl_scoped_key(redis_cache, user):
+ search_key = _api_key(user, ApiKey.Scope.SEARCH)
+
+ assert Client().get(SEED_DOMAINS_URL).status_code == 401
+ assert Client().get(SEED_DOMAINS_URL, HTTP_X_API_KEY=search_key).status_code == 401
+
+
+def test_seed_domains_are_published_in_batches(redis_cache, user):
+ SeedDomain.objects.create(domain="old.example.com", new_page_score=0.1, staan_results=1, score=0.1)
+ assert [domain["domain"] for domain in _seed_domains(user).json()] == ["old.example.com"]
+
+ # What a crawl changes is not shown until the list is next published.
+ SeedDomain.objects.create(domain="new.example.com", new_page_score=0.5, staan_results=1, score=0.5)
+ SeedDomain.objects.filter(domain="old.example.com").update(score=0.0)
+ assert _seed_domains(user).json() == [
+ {"domain": "old.example.com", "new_page_score": 0.1, "staan_results": 1, "score": 0.1}
+ ]
+
+ cache.delete(seed_domains.PUBLISHED_KEY)
+ assert [domain["domain"] for domain in _seed_domains(user).json()] == ["new.example.com", "old.example.com"]
+
+
def test_index_new_documents_returns_only_urls_the_index_lacked(index_path, monkeypatch):
monkeypatch.setattr(index_batches, "filter_blacklisted_documents", lambda documents: documents)
first = Document("Tokio tutorial", "https://tokio.rs/tutorial", "Learn tokio.")