Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Name the crawler to our API by version and username, not email

Daoud Clarke085c1e64d82b
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.py
index 82a1421..512704c 100644
--- a/mwmbl/crawl.py
+++ b/mwmbl/crawl.py
@@ -6,6 +6,8 @@ import time
from datetime import datetime, timedelta, timezone
from multiprocessing import Process
from pathlib import Path
+from typing import Optional
+from urllib.parse import quote
import django
import requests
@@ -40,7 +42,7 @@ from mwmbl.crawler.env_vars import (
SUBMIT_MODE_DRY_RUN,
SUBMIT_MODE_OFF,
)
-from mwmbl.crawler.retrieve import CRAWLER_VERSION, USER_AGENT, crawl_url
+from mwmbl.crawler.retrieve import CRAWLER_VERSION, crawl_url
from mwmbl.indexer.index_batches import index_batches, index_pages
from mwmbl.indexer.update_urls import record_urls_in_database
from mwmbl.rankeval.evaluation.remote_index import RemoteIndex
@@ -55,6 +57,19 @@ BATCH_QUEUE_KEY = "batch-queue"
# its traffic and to production for the other half.
REMOTE_SERVER = MWMBL_REMOTE_SERVER
+# How the crawler names itself to our own API, most of which is the index sync: one raw search
+# per term, for the top hundred terms of every indexing pass. The version is there so that
+# crawlers running old code can be told apart and asked to upgrade; the username is the
+# account's public name, looked up from the API key, so that a crawler causing trouble can be
+# traced to an account without its operator's email address ending up in the API's logs and
+# user agent counters. The contact address stays in USER_AGENT, which goes only to the
+# third-party sites being crawled.
+API_PRODUCT = f"mwmbl-crawler/{CRAWLER_VERSION}"
+PROJECT_URL = "https://github.com/mwmbl/mwmbl"
+USER_URL = f"{REMOTE_SERVER}/api/v1/platform/user"
+
+_username: Optional[str] = None
+
_curated_domains_cache: set[str] = set()
_curated_domains_fetched_at: float = 0.0
CURATED_DOMAINS_CACHE_SECONDS = 300.0
@@ -73,7 +88,7 @@ def _fetch_curated_domains() -> set[str]:
response = requests.get(
f"{REMOTE_SERVER}/api/v1/crawler/curated-domains",
timeout=10,
- headers={"User-Agent": USER_AGENT},
+ headers={"User-Agent": api_user_agent(_fetch_username())},
)
response.raise_for_status()
data = response.json()
@@ -85,6 +100,36 @@ def _fetch_curated_domains() -> set[str]:
return _curated_domains_cache
+def _fetch_username() -> Optional[str]:
+ """The public username MWMBL_API_KEY belongs to, or None if it cannot be found out.
+
+ Remembered once found, per process. Not remembered when it fails, so a crawler that started
+ while the API was unreachable, or before the API accepted keys here, names itself on a
+ later pass - that is one request per indexing pass beside the hundred it already makes.
+ """
+ global _username
+ if _username is not None or not MWMBL_API_KEY:
+ return _username
+ try:
+ response = requests.get(
+ USER_URL,
+ timeout=10,
+ headers={"User-Agent": api_user_agent(None), "X-API-Key": MWMBL_API_KEY},
+ )
+ response.raise_for_status()
+ _username = response.json()["username"]
+ logger.info(f"Identifying to the index as user {_username}")
+ except requests.RequestException:
+ logger.warning("Could not look up the username for MWMBL_API_KEY; requests to the API will not include it")
+ return _username
+
+
+def api_user_agent(username: Optional[str]) -> str:
+ # Django allows non-ASCII letters in a username, and a header value has to be latin-1.
+ identity = f"+{PROJECT_URL}; user {quote(username, safe='@.+-_')}" if username else f"+{PROJECT_URL}"
+ return f"{API_PRODUCT} ({identity})"
+
+
def count_new_index_entries(term: str, new_items: list[Document], remote_items: list[Document]) -> int:
"""
Estimate how many of our new local items the main index will actually keep for a term.
@@ -261,7 +306,7 @@ class Crawler:
term_new_doc_count = index_batches(batches, index_path)
logger.info(f"Indexed, top terms to sync: {term_new_doc_count.most_common(10)}")
- remote_index = RemoteIndex()
+ remote_index = RemoteIndex(user_agent=api_user_agent(_fetch_username()))
with TinyIndex(Document, index_path, "w") as local_index:
for term, count in term_new_doc_count.most_common(100):
logger.info(f"Syncing term {term} with {count} new local items")
diff --git a/mwmbl/platform/api.py b/mwmbl/platform/api.py
index e252d38..76fd6c8 100644
--- a/mwmbl/platform/api.py
+++ b/mwmbl/platform/api.py
@@ -75,7 +75,7 @@ from mwmbl.platform.schemas import (
VoteStats,
VoteStatsRequest,
)
-from mwmbl.search_auth import invalidate_api_key_cache, invalidate_user_api_key_cache
+from mwmbl.search_auth import AccountApiKeyAuth, invalidate_api_key_cache, invalidate_user_api_key_cache
from mwmbl.signals import schedule_blacklist_rebuild
from mwmbl.usernames import generate_username
from mwmbl.utils import normalize_domain, validate_domain
@@ -1112,10 +1112,14 @@ def delete_api_key(request, key_id: int):
@router.get(
"/user",
- auth=JWTAuth(),
+ auth=[JWTAuth(), AccountApiKeyAuth()],
response=UserProfileResponse,
summary="Get current user profile",
- description="Returns the authenticated user's username, email, plan, and email confirmation status.",
+ description=(
+ "Returns the authenticated user's username, email, plan, and email confirmation status. "
+ "Accepts a Bearer token or an API key of any scope in the `X-API-Key` header, so a "
+ "crawler can look up the username its key belongs to."
+ ),
tags=["Users"],
)
def get_current_user(request):
diff --git a/mwmbl/rankeval/evaluation/remote_index.py b/mwmbl/rankeval/evaluation/remote_index.py
index 626eee6..8fad349 100644
--- a/mwmbl/rankeval/evaluation/remote_index.py
+++ b/mwmbl/rankeval/evaluation/remote_index.py
@@ -25,15 +25,18 @@ USER_AGENT = f"mwmbl-remote-index/{REMOTE_INDEX_VERSION} (+https://github.com/mw
class RemoteIndex:
- def __init__(self, remote_server: str = MWMBL_REMOTE_SERVER):
+ def __init__(self, remote_server: str = MWMBL_REMOTE_SERVER, user_agent: str = USER_AGENT):
self.remote_server = remote_server
+ # A caller that knows more about itself than this module does - the crawler, with its
+ # version and its user's name - says so here rather than being lumped in with scripts.
+ self.user_agent = user_agent
self.url = f"{remote_server}/api/v1/search/raw?s="
def retrieve(self, query: str, refresh: bool = False):
url = self.url + query
response = None
with request_cache() as session:
- session.headers["User-Agent"] = USER_AGENT
+ session.headers["User-Agent"] = self.user_agent
for i in range(3):
try:
response = session.get(url, timeout=15, force_refresh=refresh)
diff --git a/mwmbl/search_auth.py b/mwmbl/search_auth.py
index a1e06a6..5fb9101 100644
--- a/mwmbl/search_auth.py
+++ b/mwmbl/search_auth.py
@@ -55,6 +55,28 @@ class SearchApiKeyAuth(APIKeyHeader):
return api_key
+class AccountApiKeyAuth(APIKeyHeader):
+ """Identify the account an API key belongs to, whatever the key's scopes.
+
+ Scopes gate what a key may do, and saying whose key it is is not one of those things: a
+ crawler holds a crawl-scoped key and still needs to know its own username. Not cached,
+ unlike SearchApiKeyAuth, because nothing calls this per search. Sets request.user so that
+ a view written for JWTAuth works unchanged under either.
+ """
+
+ param_name = "X-API-Key"
+
+ def authenticate(self, request, key: str | None):
+ if not key:
+ return None
+ key_hash = hashlib.sha256(key.encode()).hexdigest()
+ api_key = ApiKey.objects.select_related("user").filter(key=key_hash).first()
+ if api_key is None:
+ return None
+ request.user = api_key.user
+ return api_key.user
+
+
async def authenticate_user(request) -> MwmblUser:
"""Resolve the requesting user from either an X-API-Key header or a JWT.
diff --git a/test/test_remote_index.py b/test/test_remote_index.py
index c5155b3..9892654 100644
--- a/test/test_remote_index.py
+++ b/test/test_remote_index.py
@@ -2,14 +2,20 @@
Two things are pinned here, both of which were true of api.mwmbl.org's traffic before they
were fixed. It sends a User-Agent that names it, so that its share of the API's traffic is
-attributable rather than arriving as an anonymous python-requests. And the server it talks
-to comes from the environment, so that an evaluation sweep is not aimed at production by
-the mere fact of calling RemoteIndex() with no argument.
+attributable rather than arriving as an anonymous python-requests - and the crawler's index
+sync names itself more precisely, with its version and its user's public name. And the server
+it talks to comes from the environment, so that an evaluation sweep is not aimed at production
+by the mere fact of calling RemoteIndex() with no argument.
"""
import importlib
from unittest.mock import MagicMock, patch
+import pytest
+import requests
+
+import mwmbl.crawl as crawl
+from mwmbl.crawler.retrieve import CRAWLER_VERSION
from mwmbl.rankeval.evaluation.remote_index import USER_AGENT, RemoteIndex
@@ -60,3 +66,74 @@ def test_default_server_is_production_when_unset():
assert env_vars.MWMBL_REMOTE_SERVER == "https://api.mwmbl.org"
assert RemoteIndex().remote_server == "https://api.mwmbl.org"
+
+
+def test_retrieve_sends_the_user_agent_it_was_given():
+ cache, session = _session_returning({"results": []})
+ with patch("mwmbl.rankeval.evaluation.remote_index.request_cache", return_value=cache):
+ RemoteIndex(user_agent="mwmbl-crawler/9.9.9 (+x; user swift_falcon_379)").retrieve("cat")
+
+ assert session.headers["User-Agent"] == "mwmbl-crawler/9.9.9 (+x; user swift_falcon_379)"
+
+
+# ---------------------------------------------------------------------------
+# The crawler's index sync
+# ---------------------------------------------------------------------------
+
+
+def forget_username():
+ crawl._username = None
+ yield
+ crawl._username = None
+
+
+def test_api_user_agent_carries_the_crawler_version_and_the_username():
+ user_agent = crawl.api_user_agent("swift_falcon_379")
+
+ assert user_agent == (f"mwmbl-crawler/{CRAWLER_VERSION} (+https://github.com/mwmbl/mwmbl; user swift_falcon_379)")
+
+
+def test_api_user_agent_still_carries_the_version_without_a_username():
+ assert crawl.api_user_agent(None) == f"mwmbl-crawler/{CRAWLER_VERSION} (+https://github.com/mwmbl/mwmbl)"
+
+
+def test_api_user_agent_is_a_valid_header_for_a_non_ascii_username():
+ user_agent = crawl.api_user_agent("zoë")
+
+ assert "user zo%C3%AB" in user_agent
+ user_agent.encode("latin-1")
+
+
+def test_the_username_comes_from_the_api_key_and_is_remembered(forget_username):
+ response = MagicMock(json=MagicMock(return_value={"username": "swift_falcon_379"}))
+ with (
+ patch("mwmbl.crawl.MWMBL_API_KEY", "a-key"),
+ patch("mwmbl.crawl.requests.get", return_value=response) as get,
+ ):
+ assert crawl._fetch_username() == "swift_falcon_379"
+ assert crawl._fetch_username() == "swift_falcon_379"
+
+ get.assert_called_once()
+ assert get.call_args.args[0] == crawl.USER_URL
+ assert get.call_args.kwargs["headers"]["X-API-Key"] == "a-key"
+
+
+def test_a_failed_username_lookup_is_tried_again_next_time(forget_username):
+ response = MagicMock(json=MagicMock(return_value={"username": "swift_falcon_379"}))
+ with (
+ patch("mwmbl.crawl.MWMBL_API_KEY", "a-key"),
+ patch("mwmbl.crawl.requests.get", side_effect=[requests.ConnectionError(), response]),
+ ):
+ assert crawl._fetch_username() is None
+ assert crawl._fetch_username() == "swift_falcon_379"
+
+
+def test_no_username_lookup_without_an_api_key(forget_username):
+ with (
+ patch("mwmbl.crawl.MWMBL_API_KEY", ""),
+ patch("mwmbl.crawl.requests.get") as get,
+ ):
+ assert crawl._fetch_username() is None
+
+ get.assert_not_called()
diff --git a/test/test_search_api_key.py b/test/test_search_api_key.py
index 908cebd..d5ffd2b 100644
--- a/test/test_search_api_key.py
+++ b/test/test_search_api_key.py
@@ -635,6 +635,45 @@ def test_post_results_sets_user_id(api_client, crawl_api_key, verified_user):
assert captured[0].user_ids == [verified_user.id]
+# ---------------------------------------------------------------------------
+# Current user — GET /api/v1/platform/user with a token or an API key
+# ---------------------------------------------------------------------------
+
+
+def test_current_user_with_token(api_client, verified_user, access_token):
+ response = api_client.get("/api/v1/platform/user", HTTP_AUTHORIZATION=f"Bearer {access_token}")
+ assert response.status_code == 200
+ assert response.json()["username"] == verified_user.username
+
+
+def test_current_user_with_crawl_key(api_client, verified_user, crawl_api_key):
+ """What the crawler calls to find the username it names itself with."""
+ response = api_client.get("/api/v1/platform/user", **api_key_header(crawl_api_key.raw_key))
+ assert response.status_code == 200
+ assert response.json()["username"] == verified_user.username
+
+
+def test_current_user_with_search_key(api_client, verified_user, search_api_key):
+ response = api_client.get("/api/v1/platform/user", **api_key_header(search_api_key.raw_key))
+ assert response.status_code == 200
+ assert response.json()["username"] == verified_user.username
+
+
+def test_current_user_with_unknown_key_returns_401(api_client, verified_user):
+ response = api_client.get("/api/v1/platform/user", **api_key_header("not-a-real-key"))
+ assert response.status_code == 401
+
+
+def test_current_user_without_credentials_returns_401(api_client):
+ response = api_client.get("/api/v1/platform/user")
+ assert response.status_code == 401
+
+
# ---------------------------------------------------------------------------
# Background tasks
# ---------------------------------------------------------------------------