An open source, non-profit web search engine
Count search requests and the user agents that send them
Daoud Clarkec047c36ce3b0
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex bb868a3..352e21f 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -52,6 +52,7 @@ INSTALLED_APPS = [MIDDLEWARE = ["debug_toolbar.middleware.DebugToolbarMiddleware","django.middleware.security.SecurityMiddleware",+ "mwmbl.traffic_middleware.SearchTrafficMiddleware","django.contrib.sessions.middleware.SessionMiddleware","django.middleware.common.CommonMiddleware","django.middleware.csrf.CsrfViewMiddleware",@@ -341,6 +342,10 @@ BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS = 600# evicts them - #359 measured that as a net NDCG loss - and because a separate file never# enters the candidate pool for anyone else's query.EXTERNAL_CACHE_ENABLED = os.environ.get("EXTERNAL_CACHE_ENABLED", "true").lower() != "false"++# Counting search traffic (mwmbl.traffic). Off in the test settings: it is the only thing in+# the request path that reaches for Redis, and the CI test job runs none.+SEARCH_TRAFFIC_COUNTING = os.environ.get("SEARCH_TRAFFIC_COUNTING", "true").lower() != "false"# One TTL for every provider. Wikipedia articles move rarely, which is what justifies six# months; a provider whose results turn over faster wants its own, and the place to add that# is here, keyed by source.diff --git a/mwmbl/settings_test.py b/mwmbl/settings_test.pyindex 8cd5738..f60e6a4 100644--- a/mwmbl/settings_test.py+++ b/mwmbl/settings_test.py@@ -14,6 +14,11 @@ NUM_PAGES = 10EXTERNAL_CACHE_INDEX_NAME = "external-cache.tinysearch"EXTERNAL_CACHE_NUM_PAGES = 10+# The traffic counters are the only Redis touch on the request path, and every test that+# drives the Django test client would hit them. Tests that exercise them turn this on with+# override_settings and a fakeredis client of their own.+SEARCH_TRAFFIC_COUNTING = False+# Use fakeredis for cache so tests don't need a real Redis serverCACHES = {"default": {diff --git a/mwmbl/traffic.py b/mwmbl/traffic.pynew file mode 100644index 0000000..f03d926--- /dev/null+++ b/mwmbl/traffic.py@@ -0,0 +1,214 @@+"""Counting search traffic, and assuming nothing about it.++Phase 0 of #410: before we can decide what an anonymous visitor is allowed to cost us, we+need to know how much of the traffic is people. This counts; nothing acts on the answer and+no request's behaviour changes.++Deliberately raw. Classifying a request as a bot or a browser here would fix a taxonomy in+the Redis keys before anyone has looked at the traffic, and a key dimension is not something+you can change your mind about later. What is counted is what arrived: requests per+endpoint, split by which of the two headers were present, and requests per verbatim user+agent. Any rule about what a bot looks like then applies when the numbers are read, over+strings we already have, and can be applied differently next week. #412 is where a+classification earns its place, once there is data to design it against.+"""++import random+from collections import Counter+from datetime import date+from logging import getLogger+from typing import Optional++import redis+from django.conf import settings+from django.http import HttpRequest+from redis.exceptions import RedisError++from mwmbl.utils import utc_today++logger = getLogger(__name__)++# nginx accepts an 8 KB header; this bounds what one request can make us store.+MAX_USER_AGENT_LENGTH = 512++# Endpoint labels. Keyed on resolver_match.view_name rather than request.path. The names are unique across+# the two ninja APIs and the legacy one because each mounts under its own namespace, and+# they are bounded by construction - a path-derived label would grow keys from a string the+# caller chooses. Anything not in here is not search traffic and is not counted.+ENDPOINT_LABELS = {+ "search-0.1:search": "legacy_search",+ "search-0.1:complete": "legacy_complete",+ "search-0.1:raw": "legacy_raw",+ "api-v1:search": "v1_search",+ "api-v1:complete": "v1_complete",+ "api-v1:raw": "v1_raw",+ "api-v2:search": "v2_search",+ "api-v2:complete": "v2_complete",+ "api-v2:raw": "v2_raw",+ "api-v2:super_search": "super_search",+}++# The Django UI is two views and three kinds of request, so it cannot be a straight lookup.+UI_INDEX_VIEW_NAME = "index"+UI_FRAGMENT_VIEW_NAME = "home"+UI_SEARCH_LABEL = "ui_search"+UI_KEYSTROKE_LABEL = "ui_keystroke"++ALL_ENDPOINT_LABELS = sorted(set(ENDPOINT_LABELS.values()) | {UI_SEARCH_LABEL, UI_KEYSTROKE_LABEL})+++def _ui_endpoint_label(request: HttpRequest) -> Optional[str]:+ """Which of the Django UI's three kinds of request this is.++ Without the keystroke split the UI's search count is inflated roughly tenfold:+ home_fragment is hit on every keystroke and only the debounced call carries external=1.+ """+ if not request.GET.get("q"):+ return None+ if request.resolver_match.view_name == UI_INDEX_VIEW_NAME:+ return UI_SEARCH_LABEL+ return UI_SEARCH_LABEL if request.GET.get("external") == "1" else UI_KEYSTROKE_LABEL+++def endpoint_label(request: HttpRequest) -> Optional[str]:+ """The counter label for this request, or None if it is not search traffic.++ resolver_match is populated by the time a response-phase middleware runs, but is None+ for a 404 and for CommonMiddleware's APPEND_SLASH redirect, both of which this sits+ outside of.+ """+ if request.resolver_match is None:+ return None+ view_name = request.resolver_match.view_name+ if view_name in (UI_INDEX_VIEW_NAME, UI_FRAGMENT_VIEW_NAME):+ return _ui_endpoint_label(request)+ return ENDPOINT_LABELS.get(view_name)+++# The one split on the request counter, and a description rather than a judgement: which of+# the two headers arrived, not what that means. A request with neither is what our own+# server-side render sends today, and a missing Accept-Language is the cheapest hint that a+# client is not a browser, since browsers send one and HTTP clients do not.+HEADER_LABELS = ["user_agent+language", "user_agent", "language", "neither"]+++def header_label(request: HttpRequest) -> str:+ has_user_agent = bool(request.headers.get("User-Agent"))+ has_language = bool(request.headers.get("Accept-Language"))+ if has_user_agent:+ return "user_agent+language" if has_language else "user_agent"+ return "language" if has_language else "neither"+++REQUEST_COUNT_KEY = "traffic:req:{date}:{endpoint}:{headers}"+USER_AGENT_COUNT_KEY = "traffic:user-agents:{date}"++# Thirty days, matching the public daily crawler counters in mwmbl/crawler/stats.py.+# Production Redis runs allkeys-lru (see mwmbl/indexer/blacklist_snapshot.py), so a longer+# retention would be a promise the eviction policy does not keep.+TRAFFIC_EXPIRE_SECONDS = 60 * 60 * 24 * 30++# Seven days for the user agents: they are here to be read and argued about, not to be a+# permanent series, and a short window limits what a verbatim string can later be used for.+USER_AGENT_EXPIRE_SECONDS = 60 * 60 * 24 * 7++# How many user agents a day survives, which is both the memory bound on a sorted set whose+# members the caller chooses and the privacy control. What it drops is the tail, and a user+# agent seen once is both most of a browser fingerprint and a lone person; what survives is+# by construction a client somebody runs at volume.+TRACKED_USER_AGENTS = 2000++# Trimming on one request in five hundred holds the set within a few hundred rows of the+# limit. There is no counter shared between workers to do it on a schedule, and a ZCARD to+# decide would cost a round trip on every request.+USER_AGENT_TRIM_PROBABILITY = 1 / 500++# A missing user agent is a real answer, so it gets a row rather than being dropped.+MISSING_USER_AGENT = "(none)"++# Aggressive by design. django-redis is configured with 5 s timeouts, which is fine for a+# cache lookup a view is waiting on and far too slow for a counter: this runs on every+# request, on the one thread asgiref gives the whole sync middleware chain.+REDIS_CONNECT_TIMEOUT_SECONDS = 0.1+REDIS_TIMEOUT_SECONDS = 0.25++_redis: Optional[redis.Redis] = None+++def get_redis() -> redis.Redis:+ global _redis+ if _redis is None:+ _redis = redis.from_url(+ settings.REDIS_URL,+ socket_connect_timeout=REDIS_CONNECT_TIMEOUT_SECONDS,+ socket_timeout=REDIS_TIMEOUT_SECONDS,+ decode_responses=True,+ )+ return _redis+++def record_request(request: HttpRequest) -> None:+ """Count one search request. Never raises: a counter that costs a search its results is+ worse than no counter, which is the rule external_cache already follows."""+ if not settings.SEARCH_TRAFFIC_COUNTING:+ return++ endpoint = endpoint_label(request)+ if endpoint is None:+ return++ today = utc_today()+ try:+ pipeline = get_redis().pipeline()++ request_key = REQUEST_COUNT_KEY.format(date=today, endpoint=endpoint, headers=header_label(request))+ pipeline.incr(request_key)+ pipeline.expire(request_key, TRAFFIC_EXPIRE_SECONDS)++ # Its own key, not a dimension on the counter above: nothing joins a user agent to+ # an address or a query, so no row says a particular person searched for something.+ user_agent_key = USER_AGENT_COUNT_KEY.format(date=today)+ user_agent = request.headers.get("User-Agent", "")[:MAX_USER_AGENT_LENGTH] or MISSING_USER_AGENT+ pipeline.zincrby(user_agent_key, 1, user_agent)+ pipeline.expire(user_agent_key, USER_AGENT_EXPIRE_SECONDS)+ if random.random() < USER_AGENT_TRIM_PROBABILITY:+ pipeline.zremrangebyrank(user_agent_key, 0, -TRACKED_USER_AGENTS - 1)++ pipeline.execute()+ except RedisError:+ logger.warning("Could not record search traffic for %s", endpoint, exc_info=True)+++def read_request_counts(redis_client, days: list[date]) -> dict[tuple[date, str, str], int]:+ """Request counts for the given days, keyed by (day, endpoint, header label).++ One mget over the known labels rather than a scan: nothing indexes which traffic keys+ exist, and SCAN on a shared Redis is not something a readout should do. Combinations+ with no traffic are absent rather than zero.+ """+ coordinates = [+ (day, endpoint, headers) for day in days for endpoint in ALL_ENDPOINT_LABELS for headers in HEADER_LABELS+ ]+ counts = redis_client.mget(+ [+ REQUEST_COUNT_KEY.format(date=day, endpoint=endpoint, headers=headers)+ for day, endpoint, headers in coordinates+ ]+ )+ return {coordinate: int(count) for coordinate, count in zip(coordinates, counts) if count}+++def read_user_agent_counts(redis_client, days: list[date], limit: int) -> list[tuple[str, int]]:+ """The most-seen user agents over the window, most requests first.++ Summed across days, so a client that appears every day outranks one that appeared once.+ """+ pipeline = redis_client.pipeline()+ for day in days:+ pipeline.zrevrange(USER_AGENT_COUNT_KEY.format(date=day), 0, TRACKED_USER_AGENTS, withscores=True)++ totals: Counter = Counter()+ for day_counts in pipeline.execute():+ for user_agent, count in day_counts:+ totals[user_agent] += int(count)+ return totals.most_common(limit)diff --git a/mwmbl/traffic_middleware.py b/mwmbl/traffic_middleware.pynew file mode 100644index 0000000..63a6608--- /dev/null+++ b/mwmbl/traffic_middleware.py@@ -0,0 +1,27 @@+"""Count every search request as it leaves.++Sync-only, and measured rather than assumed: the app runs under uvicorn workers, but+allauth's AccountMiddleware is sync-only and Django adapts everything above a sync+middleware to sync too, so an async branch here would never execute.+test_the_chain_runs_sync_because_of_allauth pins that.++Two things follow. The sync chain already runs off the event loop, on the thread asgiref+hands it, so a blocking Redis call here cannot block the loop. And it is one thread for the+whole chain, so the work stays a single pipeline with short socket timeouts.+"""++from mwmbl.traffic import record_request+++class SearchTrafficMiddleware:+ def __init__(self, get_response):+ self.get_response = get_response++ def __call__(self, request):+ response = self.get_response(request)+ # After the view, so resolver_match is populated. For the Super Search SSE view this+ # is the start of the stream rather than the end of it, which is the right moment to+ # count a search - but it means the response has no body to look at yet, and reading+ # response.content would raise. Nothing here touches it.+ record_request(request)+ return responsediff --git a/test/test_traffic.py b/test/test_traffic.pynew file mode 100644index 0000000..b60e958--- /dev/null+++ b/test/test_traffic.py@@ -0,0 +1,257 @@+"""Tests for the search traffic counters and the middleware that writes them.++Counting is off in mwmbl.settings_test, because it is the only thing in the request path+that reaches for Redis and the CI job runs none. Everything here turns it back on and wires+in a fakeredis, the way test_admin_blacklist_status.py does.+"""++from datetime import timedelta++import fakeredis+import pytest+from asgiref.sync import iscoroutinefunction, markcoroutinefunction+from django.conf import settings+from django.core.handlers.base import BaseHandler+from django.http import StreamingHttpResponse+from django.test import override_settings+from redis.exceptions import ConnectionError as RedisConnectionError++from mwmbl import traffic+from mwmbl.traffic_middleware import SearchTrafficMiddleware+from mwmbl.utils import utc_today++BROWSER_HEADERS = {+ "HTTP_USER_AGENT": "Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0",+ "HTTP_ACCEPT_LANGUAGE": "en-GB,en;q=0.9",+}+FIREFOX = BROWSER_HEADERS["HTTP_USER_AGENT"]++_PROBE_HANDLER_WAS_ASYNC = []+++class ModeProbeMiddleware:+ """Reports whether Django gave it an async handler, from where our middleware sits."""++ sync_capable = True+ async_capable = True++ def __init__(self, get_response):+ self.get_response = get_response+ handler_is_async = iscoroutinefunction(get_response)+ _PROBE_HANDLER_WAS_ASYNC.append(handler_is_async)+ if handler_is_async:+ markcoroutinefunction(self)++ def __call__(self, request):+ return self.get_response(request)+++def counting(monkeypatch, settings):+ settings.SEARCH_TRAFFIC_COUNTING = True+ client = fakeredis.FakeStrictRedis(decode_responses=True)+ monkeypatch.setattr(traffic, "_redis", client)+ return client+++def count_for(endpoint, headers="user_agent+language"):+ key = traffic.REQUEST_COUNT_KEY.format(date=utc_today(), endpoint=endpoint, headers=headers)+ return traffic.get_redis().get(key)+++def user_agent_counts():+ key = traffic.USER_AGENT_COUNT_KEY.format(date=utc_today())+ return dict(traffic.get_redis().zrange(key, 0, -1, withscores=True))+++def test_the_middleware_is_installed_after_the_security_middleware():+ assert settings.MIDDLEWARE.index("mwmbl.traffic_middleware.SearchTrafficMiddleware") == (+ settings.MIDDLEWARE.index("django.middleware.security.SecurityMiddleware") + 1+ )+++def test_the_chain_runs_sync_because_of_allauth():+ """Why SearchTrafficMiddleware has no async path.++ allauth's AccountMiddleware is sync-only, so Django adapts every middleware above it to+ sync even under ASGI. If this fails, allauth has gained async support, this middleware+ is now the thing forcing the chain sync, and it should grow an async branch.+ """+ probe_position = settings.MIDDLEWARE.index("mwmbl.traffic_middleware.SearchTrafficMiddleware")+ middleware = list(settings.MIDDLEWARE)+ # By __name__: test/ has no __init__.py, so a dotted path would import a second copy of+ # this module with its own module-level state, and the probe would record nothing.+ middleware.insert(probe_position, f"{__name__}.ModeProbeMiddleware")++ _PROBE_HANDLER_WAS_ASYNC.clear()+ with override_settings(MIDDLEWARE=middleware):+ BaseHandler().load_middleware(is_async=True)++ assert _PROBE_HANDLER_WAS_ASYNC == [False]+++ "path,query,expected_endpoint",+ [+ ("/", {"q": "python"}, "ui_search"),+ ("/app/home/", {"q": "pyth"}, "ui_keystroke"),+ ("/app/home/", {"q": "python", "external": "1"}, "ui_search"),+ ("/api/v1/search/", {"s": "python"}, "v1_search"),+ ("/api/v2/search/", {"q": "python"}, "v2_search"),+ ("/api/v2/search/complete", {"q": "pyth"}, "v2_complete"),+ ("/search/", {"s": "python"}, "legacy_search"),+ ],+)+def test_each_kind_of_search_lands_in_its_own_bucket(counting, client, path, query, expected_endpoint):+ client.get(path, query, **BROWSER_HEADERS)++ assert count_for(expected_endpoint) == "1"+ others = [label for label in traffic.ALL_ENDPOINT_LABELS if label != expected_endpoint]+ assert all(count_for(label) is None for label in others)++[email protected]("path,query", [("/", {}), ("/app/home/", {}), ("/api/v1/docs", {})])+def test_requests_that_are_not_searches_are_not_counted(counting, client, path, query):+ client.get(path, query, **BROWSER_HEADERS)++ assert counting.keys("traffic:*") == []+++def test_every_search_route_has_a_label():+ """A renamed route must fail here rather than going silently uncounted."""+ from django.urls import get_resolver++ def view_names(resolver, namespace=None):+ for pattern in resolver.url_patterns:+ if hasattr(pattern, "url_patterns"):+ yield from view_names(pattern, pattern.namespace or namespace)+ elif pattern.name:+ yield f"{namespace}:{pattern.name}" if namespace else pattern.name++ searchy = {+ name+ for name in view_names(get_resolver())+ if name.split(":")[-1] in {"search", "complete", "raw", "super_search"}+ or name in {traffic.UI_INDEX_VIEW_NAME, traffic.UI_FRAGMENT_VIEW_NAME}+ }+ labelled = set(traffic.ENDPOINT_LABELS) | {traffic.UI_INDEX_VIEW_NAME, traffic.UI_FRAGMENT_VIEW_NAME}++ assert searchy - labelled == set()+++ "headers,expected",+ [+ ({"HTTP_USER_AGENT": FIREFOX, "HTTP_ACCEPT_LANGUAGE": "en-GB"}, "user_agent+language"),+ ({"HTTP_USER_AGENT": FIREFOX}, "user_agent"),+ ({"HTTP_ACCEPT_LANGUAGE": "en-GB"}, "language"),+ ({}, "neither"),+ ],+)+def test_the_headers_that_arrived_are_counted_and_nothing_is_inferred_from_them(counting, client, headers, expected):+ client.get("/api/v1/search/", {"s": "python"}, **headers)++ assert count_for("v1_search", expected) == "1"+++def test_the_counters_expire(counting, client):+ client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)++ requests_key = traffic.REQUEST_COUNT_KEY.format(+ date=utc_today(), endpoint="v1_search", headers="user_agent+language"+ )+ user_agents_key = traffic.USER_AGENT_COUNT_KEY.format(date=utc_today())+ assert 0 < counting.ttl(requests_key) <= traffic.TRAFFIC_EXPIRE_SECONDS+ assert 0 < counting.ttl(user_agents_key) <= traffic.USER_AGENT_EXPIRE_SECONDS+++def test_a_user_agent_is_counted_verbatim(counting, client):+ client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)+ client.get("/api/v1/search/", {"s": "python"}, HTTP_USER_AGENT="Googlebot/2.1")++ assert user_agent_counts() == {FIREFOX: 1, "Googlebot/2.1": 1}+++ "sent,counted",+ [+ # A missing user agent is what the front end's own server-side render sends, so it+ # is a number we want rather than a row to drop.+ (None, traffic.MISSING_USER_AGENT),+ ("A" * 2000, "A" * traffic.MAX_USER_AGENT_LENGTH),+ ],+)+def test_an_awkward_user_agent_still_gets_a_row(counting, client, sent, counted):+ headers = {"HTTP_USER_AGENT": sent} if sent else {}+ client.get("/api/v1/search/", {"s": "python"}, **headers)++ assert user_agent_counts() == {counted: 1}+++def test_the_busiest_user_agents_survive_the_trim(counting, client, monkeypatch):+ """The bound on the key, and the reason the tail is not kept."""+ monkeypatch.setattr(traffic, "TRACKED_USER_AGENTS", 2)+ monkeypatch.setattr(traffic, "USER_AGENT_TRIM_PROBABILITY", 0)+ for user_agent, requests in [("busiest", 3), ("second", 2), ("seen once", 1)]:+ for _ in range(requests):+ client.get("/api/v1/search/", {"s": "python"}, HTTP_USER_AGENT=user_agent)++ monkeypatch.setattr(traffic, "USER_AGENT_TRIM_PROBABILITY", 1)+ client.get("/api/v1/search/", {"s": "python"}, HTTP_USER_AGENT="busiest")++ assert user_agent_counts() == {"busiest": 4, "second": 2}+++def test_the_readout_sums_user_agents_across_the_window(counting, client):+ yesterday = utc_today() - timedelta(days=1)+ counting.zincrby(traffic.USER_AGENT_COUNT_KEY.format(date=yesterday), 5, FIREFOX)+ client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)++ assert traffic.read_user_agent_counts(counting, [yesterday, utc_today()], 10) == [(FIREFOX, 6)]+++def test_the_readout_returns_a_count_per_day_endpoint_and_header_combination(counting, client):+ client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)+ client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)+ client.get("/api/v2/search/", {"q": "python"})++ counts = traffic.read_request_counts(counting, [utc_today()])++ assert counts == {+ (utc_today(), "v1_search", "user_agent+language"): 2,+ (utc_today(), "v2_search", "neither"): 1,+ }+++def test_the_readout_survives_a_day_with_no_traffic(counting):+ days = [utc_today() - timedelta(days=offset) for offset in range(3)]++ assert traffic.read_request_counts(counting, days) == {}+ assert traffic.read_user_agent_counts(counting, days, 10) == []+++def test_a_dead_redis_does_not_change_the_response(client, monkeypatch, settings):+ class DeadRedis:+ def pipeline(self):+ raise RedisConnectionError("Connection refused")++ with_counting_off = client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)++ settings.SEARCH_TRAFFIC_COUNTING = True+ monkeypatch.setattr(traffic, "_redis", DeadRedis())+ with_dead_redis = client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)++ assert with_dead_redis.status_code == with_counting_off.status_code == 200+ assert with_dead_redis.content == with_counting_off.content+++def test_a_streaming_response_is_counted_without_being_consumed(counting, rf):+ """The Super Search view streams; touching response.content would raise on it."""+ request = rf.get("/api/v2/super-search/", {"q": "python"}, **BROWSER_HEADERS)+ request.resolver_match = type("Match", (), {"view_name": "api-v2:super_search"})()++ response = SearchTrafficMiddleware(lambda _: StreamingHttpResponse(iter([b"a", b"b"])))(request)++ assert count_for("super_search") == "1"+ assert b"".join(response.streaming_content) == b"ab"