An open source, non-profit web search engine
Count search callers per week under an expiring salt, and keep only totals
Claudeb853de3d556a
diff --git a/mwmbl/admin_views.py b/mwmbl/admin_views.pyindex e8b9f20..cbea20c 100644--- a/mwmbl/admin_views.py+++ b/mwmbl/admin_views.py@@ -34,7 +34,7 @@ from django.http import HttpRequestfrom django.shortcuts import renderfrom redis import RedisError-from mwmbl import count_urls, pricing, quota, search_setup, traffic+from mwmbl import count_urls, pricing, quota, search_callers, search_setup, trafficfrom mwmbl.crawler.stats import BLACKLISTED_REMOVED_COUNT_KEYfrom mwmbl.curated_domains import get_curated_domainsfrom mwmbl.indexer import blacklist_snapshot, purge_queue@@ -49,14 +49,10 @@ from mwmbl.membership import TIERS, VOLUNTEER_TIERS, combined_search_monthly_limfrom mwmbl.models import ApiKey, Membership, MwmblUserfrom mwmbl.traffic import (HEADER_LABELS,- IDENTITY_ENDPOINTS,- IDENTITY_EXPIRE_SECONDS,STATUS_LABELS,- TRACKED_IDENTITIES,TRACKED_USER_AGENTS,TRAFFIC_EXPIRE_SECONDS,USER_AGENT_EXPIRE_SECONDS,- read_identity_counts,read_request_counts,read_user_agent_counts,)@@ -231,10 +227,9 @@ TRAFFIC_MAX_DAYS = TRAFFIC_EXPIRE_SECONDS // SECONDS_PER_DAYTRAFFIC_DEFAULT_DAYS = 7TRAFFIC_WINDOW_CHOICES = [1, 7, TRAFFIC_MAX_DAYS]USER_AGENT_MAX_DAYS = USER_AGENT_EXPIRE_SECONDS // SECONDS_PER_DAY-IDENTITY_MAX_DAYS = IDENTITY_EXPIRE_SECONDS // SECONDS_PER_DAY-# The share of a day's searches carried by its busiest N callers, at each of these N.-IDENTITY_SHARE_RANKS = [1, 10, 100, 1000]+# This week, still filling, and as many finished weeks as their summaries are kept for.+CALLER_WINDOWS = search_callers.SUMMARY_EXPIRE_SECONDS // (SECONDS_PER_DAY * search_callers.WINDOW_DAYS) + 1# Enough to see the shape of the tail without putting two thousand caller-supplied strings# through the template.@@ -317,34 +312,42 @@ def _user_agents(redis_client, days: list[date], counts: dict) -> dict:}-def _identities(redis_client, days: list[date], counts: dict) -> dict:- """How many distinct callers searched each day, and how concentrated their searches are.+def _callers(redis_client) -> dict:+ """Distinct callers per week, how concentrated their searches are, and how many search often.- The shares are of the day's served searches as the request counter has them, so the- callers the trim dropped still count towards the total they are a share of.+ Weeks with no callers are left out, which before this was deployed is all of them: a row+ of zeros would read as nobody searching rather than as nothing counted."""- identity_days = days[:IDENTITY_MAX_DAYS]- searches_by_day: Counter = Counter()- for coordinate, count in counts.items():- if coordinate[ENDPOINT] in IDENTITY_ENDPOINTS and coordinate[STATUS] == "2xx":- searches_by_day[coordinate[DAY]] += count-+ this_window = search_callers.window_start(utc_today())+ windows = [this_window - timedelta(days=search_callers.WINDOW_DAYS * offset) for offset in range(CALLER_WINDOWS)]rows = []- for day, (distinct, tracked) in read_identity_counts(redis_client, identity_days).items():- searches = searches_by_day[day]+ for window in windows:+ if window == this_window:+ totals = search_callers.read_live(+ redis_client, window, traffic.caller_window_searches(redis_client, window)+ )+ else:+ totals = search_callers.read_summary(redis_client, window)+ # Last week in the hour before it is summarised is absent too, briefly.+ if totals is None or totals["distinct"] == 0:+ continue+ searches = totals["searches"]rows.append({- "day": day,+ "window": window,+ "summarised": window != this_window,"searches": searches,- "distinct": distinct,- "shares": [100 * sum(tracked[:rank]) / searches if searches else 0 for rank in IDENTITY_SHARE_RANKS],+ "distinct": totals["distinct"],+ "shares": [100 * busiest / searches if searches else 0 for busiest in totals["busiest_searches"]],+ "over": totals["over"],})return {"rows": rows,- "ranks": IDENTITY_SHARE_RANKS,- "endpoints": sorted(IDENTITY_ENDPOINTS),- "tracked_limit": TRACKED_IDENTITIES,+ "ranks": search_callers.SHARE_RANKS,+ "thresholds": search_callers.THRESHOLDS,+ "endpoints": sorted(search_callers.IDENTITY_ENDPOINTS),+ "tracked_limit": search_callers.TRACKED_CALLERS,}@@ -516,7 +519,7 @@ def search_traffic_view(request):context["by_headers"] = _requests_by_label(counts, HEADERS, HEADER_LABELS)context["by_status"] = _requests_by_label(counts, STATUS, STATUS_LABELS)context["user_agents"] = _user_agents(redis_client, days, counts)- context["identities"] = _identities(redis_client, days, counts)+ context["callers"] = _callers(redis_client)except RedisError as e:logger.exception("Could not read search traffic from Redis")context["redis_error"] = str(e)diff --git a/mwmbl/apps.py b/mwmbl/apps.pyindex 10b6890..c3f0df9 100644--- a/mwmbl/apps.py+++ b/mwmbl/apps.py@@ -147,6 +147,7 @@ class MwmblConfig(AppConfig):refresh_blacklist_snapshot,report_usage_to_polar,retrain_domain_moderation_model,+ summarise_search_callers,sync_search_counts,)@@ -157,6 +158,7 @@ class MwmblConfig(AppConfig):MODERATION_RETRAIN_TASK = "mwmbl.background.retrain_domain_moderation_model"INDEX_COUNT_TASK = "mwmbl.background.count_index_urls"FLUSH_TOKENS_TASK = "mwmbl.background.flush_expired_tokens"+ SUMMARISE_CALLERS_TASK = "mwmbl.background.summarise_search_callers"# Sync search counts once per hour (3600 seconds)if not Task.objects.filter(task_name=SYNC_TASK).exists():@@ -201,6 +203,11 @@ class MwmblConfig(AppConfig):if not Task.objects.filter(task_name=FLUSH_TOKENS_TASK).exists():flush_expired_tokens(repeat=Task.DAILY, repeat_until=None)+ # Hourly, so a finished week's per-caller counts are replaced by its totals well+ # inside the grace day before they expire - see mwmbl.search_callers.+ if not Task.objects.filter(task_name=SUMMARISE_CALLERS_TASK).exists():+ summarise_search_callers(repeat=Task.HOURLY, repeat_until=None)+except Exception:# Don't prevent startup if background task scheduling failslog.exception("Failed to schedule background tasks")diff --git a/mwmbl/background.py b/mwmbl/background.pyindex bb806c7..c159cf5 100644--- a/mwmbl/background.py+++ b/mwmbl/background.py@@ -7,6 +7,7 @@ Django Background Tasks for periodic maintenance:- refresh_blacklist_snapshot: rebuilds the blacklist the search path filters against- purge_blacklisted_from_queue: removes retrieval-filtered documents from the index- count_index_urls: counts the unique URLs and domains in the index once a week+ - summarise_search_callers: replaces last week's per-caller counts with totals"""import logging@@ -23,7 +24,7 @@ from django.core.management import call_commandfrom django.db import transactionfrom redis import Redis-from mwmbl import pricing+from mwmbl import pricing, trafficfrom mwmbl.count_urls import count_urls_step, get_redisfrom mwmbl.crawler.stats import StatsManagerfrom mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist, refresh_snapshot@@ -356,3 +357,10 @@ def retrain_domain_moderation_model():def flush_expired_tokens():"""Delete expired JWT refresh tokens from the blacklist tables, which gain a row per refresh."""call_command("flushexpiredtokens")+++@background(schedule=0)+def summarise_search_callers():+ """Roll last week's search callers into totals and delete the per-caller keys."""+ if traffic.summarise_last_caller_window(traffic.get_redis()):+ logger.info("Summarised last week's search callers")diff --git a/mwmbl/search_callers.py b/mwmbl/search_callers.pynew file mode 100644index 0000000..0f3bf91--- /dev/null+++ b/mwmbl/search_callers.py@@ -0,0 +1,208 @@+"""Counting distinct search callers per week, without keeping who they were.++Sizing for a k-anonymous count of query n-grams: how many distinct callers search in a+week, how much of the traffic the busiest carry, and how many search often enough to+matter. A SearxNG instance forwards no client address, so it is one caller however many+people use it. Only searches somebody typed and got an answer to, since those are what+n-grams would be counted from: /complete is keystrokes and /raw is mostly the crawler's+index sync.++A caller is an IPv4 address or an IPv6 /64, stored only as an HMAC under a random salt+made for the week. The salt lives in Redis and dies a day after the week ends, along with+everything keyed on it, so from then on nobody - us included, with SECRET_KEY and the+whole Redis - can tell which address a token was. Before that, summarise_window rolls the+week into a handful of totals and deletes the per-caller keys, so a finished week is+never more than those totals. The week rather than the day because that is the window+n-grams would be published over: tokens cannot be joined across windows, so callers+counted per day could not be added up to size it.+"""++import hashlib+import hmac+import ipaddress+import json+import random+import secrets+from dataclasses import dataclass+from datetime import date, datetime, time, timedelta, timezone++import redis+from django.http import HttpRequest++IDENTITY_ENDPOINTS = {"legacy_search", "v1_search", "v2_search", "ui_search"}++WINDOW_DAYS = 7++SALT_KEY = "traffic:caller-salt:{window}"+DISTINCT_KEY = "traffic:callers:{window}"+COUNT_KEY = "traffic:caller-counts:{window}"+OVER_KEY = "traffic:callers-over:{window}:{threshold}"+SUMMARY_KEY = "traffic:caller-summary:{window}"++# The per-caller keys outlive the week by a day so the hourly summary has time to run. If it+# never does, they expire anyway: the week is lost, which is the safe way to fail.+RAW_GRACE = timedelta(days=1)++# Thirteen weeks of summaries. They are a few numbers each, so the length is about how far+# back is worth comparing, not memory.+SUMMARY_EXPIRE_SECONDS = 60 * 60 * 24 * 7 * 13++# The memory bound on the per-caller counts, trimmed the way the user agents are. A caller+# the trim drops restarts from zero, which understates the busiest shares slightly; the+# shares are read no further down than the thousandth, well inside what is kept.+TRACKED_CALLERS = 2000+TRIM_PROBABILITY = 1 / 500++# The share of the week's searches carried by its busiest N callers, at each of these N.+SHARE_RANKS = [1, 10, 100, 1000]++# How many callers searched at least this often. Counted as each caller crosses the line,+# since the trimmed counts lose exactly the tail these are about.+THRESHOLDS = [10, 100, 1000]++# A household or a small network gets a /64, so one IPv6 caller is the whole prefix.+IPV6_PREFIX = 64++# 64 bits: collisions are negligible at a few million callers a week.+TOKEN_LENGTH = 16++# This process's copy of each window's salt, so a request costs no extra round trip.+_salts: dict[date, str] = {}+++def window_start(day: date) -> date:+ """The Monday starting the week that day is in."""+ return day - timedelta(days=day.weekday())+++def _raw_expiry(window: date) -> datetime:+ window_end = datetime.combine(window + timedelta(days=WINDOW_DAYS), time(), tzinfo=timezone.utc)+ return window_end + RAW_GRACE+++def client_network(request: HttpRequest) -> str:+ """The caller's address, or its /64 for IPv6.++ The last X-Forwarded-For entry is the one our nginx saw: dokku sets the header to+ $remote_addr, and anything earlier in the list would be the client's own say-so.+ REMOTE_ADDR is absent when uvicorn has no peer address, as on a unix socket, and this+ runs inside a counter that must never fail a search.+ """+ forwarded_for = request.headers.get("X-Forwarded-For")+ address = forwarded_for.split(",")[-1].strip() if forwarded_for else request.META.get("REMOTE_ADDR", "")+ try:+ ip = ipaddress.ip_address(address)+ except ValueError:+ # Still a caller, just not one with a parseable address.+ return address+ # Without this every IPv4-mapped address would share ::/64.+ if ip.version == 6 and ip.ipv4_mapped:+ return str(ip.ipv4_mapped)+ if ip.version == 6:+ return str(ipaddress.ip_network(f"{ip}/{IPV6_PREFIX}", strict=False))+ return str(ip)+++def _salt(redis_client: redis.Redis, window: date) -> str:+ """The window's salt, made by whichever process asks first."""+ if window not in _salts:+ salt_key = SALT_KEY.format(window=window)+ pipeline = redis_client.pipeline()+ pipeline.set(salt_key, secrets.token_hex(32), nx=True, exat=_raw_expiry(window))+ pipeline.get(salt_key)+ _salts.clear()+ _salts[window] = pipeline.execute()[1]+ return _salts[window]+++def caller_token(salt: str, network: str) -> str:+ return hmac.new(salt.encode(), network.encode(), hashlib.sha256).hexdigest()[:TOKEN_LENGTH]+++@dataclass+class QueuedCaller:+ window: date+ token: str+ count_index: int+++def queue_caller(redis_client: redis.Redis, pipeline, request: HttpRequest, day: date) -> QueuedCaller:+ """Add one search by this request's caller to the end of the pipeline, which must have+ nothing queued after it."""+ window = window_start(day)+ expiry = _raw_expiry(window)+ salt = _salt(redis_client, window)+ token = caller_token(salt, client_network(request))++ # Every request rewrites the salt if it is missing. That keeps it warm under allkeys-lru,+ # where a key read once a week per process is the first to go, and if it was evicted+ # anyway, puts back the one this process is still hashing with rather than a new one.+ pipeline.set(SALT_KEY.format(window=window), salt, nx=True, exat=expiry)+ distinct_key = DISTINCT_KEY.format(window=window)+ pipeline.pfadd(distinct_key, token)+ pipeline.expireat(distinct_key, expiry)+ count_key = COUNT_KEY.format(window=window)+ pipeline.zincrby(count_key, 1, token)+ pipeline.expireat(count_key, expiry)+ # Counted back from the end, so this needs nothing from whatever else is in the pipeline.+ count_index = -2+ if random.random() < TRIM_PROBABILITY:+ pipeline.zremrangebyrank(count_key, 0, -TRACKED_CALLERS - 1)+ count_index = -3+ return QueuedCaller(window=window, token=token, count_index=count_index)+++def record_thresholds(redis_client: redis.Redis, queued: QueuedCaller, results: list) -> None:+ """Note the caller crossing a threshold, which is a second round trip once per caller per+ threshold. A caller the trim dropped crosses again, and the HyperLogLog counts them once."""+ count = int(results[queued.count_index])+ if count not in THRESHOLDS:+ return+ over_key = OVER_KEY.format(window=queued.window, threshold=count)+ pipeline = redis_client.pipeline()+ pipeline.pfadd(over_key, queued.token)+ pipeline.expireat(over_key, _raw_expiry(queued.window))+ pipeline.execute()+++def _raw_keys(window: date) -> list[str]:+ over_keys = [OVER_KEY.format(window=window, threshold=threshold) for threshold in THRESHOLDS]+ return [SALT_KEY.format(window=window), DISTINCT_KEY.format(window=window), COUNT_KEY.format(window=window)] + (+ over_keys+ )+++def read_summary(redis_client: redis.Redis, window: date) -> dict | None:+ summary = redis_client.get(SUMMARY_KEY.format(window=window))+ return None if summary is None else json.loads(summary)+++def read_live(redis_client: redis.Redis, window: date, searches: int) -> dict:+ """A window's totals, read from its per-caller keys.++ searches is the window's served searches from the request counter, which the shares are+ of, so callers the trim dropped still count towards the total they are a share of.+ """+ pipeline = redis_client.pipeline()+ pipeline.pfcount(DISTINCT_KEY.format(window=window))+ pipeline.zrevrange(COUNT_KEY.format(window=window), 0, max(SHARE_RANKS) - 1, withscores=True)+ for threshold in THRESHOLDS:+ pipeline.pfcount(OVER_KEY.format(window=window, threshold=threshold))+ distinct, busiest, *over = pipeline.execute()++ counts = [int(count) for _, count in busiest]+ return {+ "searches": searches,+ "distinct": distinct,+ "busiest_searches": [sum(counts[:rank]) for rank in SHARE_RANKS],+ "over": over,+ }+++def summarise_window(redis_client: redis.Redis, window: date, searches: int) -> None:+ """Replace a finished window's per-caller keys with its totals."""+ totals = read_live(redis_client, window, searches)+ pipeline = redis_client.pipeline()+ pipeline.set(SUMMARY_KEY.format(window=window), json.dumps(totals), ex=SUMMARY_EXPIRE_SECONDS)+ pipeline.delete(*_raw_keys(window))+ pipeline.execute()diff --git a/mwmbl/templates/admin/search_traffic.html b/mwmbl/templates/admin/search_traffic.htmlindex 219aa01..c3ceec4 100644--- a/mwmbl/templates/admin/search_traffic.html+++ b/mwmbl/templates/admin/search_traffic.html@@ -141,33 +141,38 @@</div><div class="module">- <h2>Distinct callers</h2>+ <h2>Distinct callers per week</h2><table><thead><tr>- <th>Day</th><th class="num">Searches</th><th class="num">Distinct callers</th>- {% for rank in identities.ranks %}<th class="num">Top {{ rank|intcomma }}</th>{% endfor %}+ <th>Week from (UTC)</th><th class="num">Searches</th><th class="num">Distinct callers</th>+ {% for rank in callers.ranks %}<th class="num">Top {{ rank|intcomma }}</th>{% endfor %}+ {% for threshold in callers.thresholds %}<th class="num">≥ {{ threshold|intcomma }} searches</th>{% endfor %}</tr></thead><tbody>- {% for row in identities.rows %}+ {% for row in callers.rows %}<tr>- <td>{{ row.day }}</td>+ <td>{{ row.window }}{% if not row.summarised %} <span class="note">(filling)</span>{% endif %}</td><td class="num">{{ row.searches|intcomma }}</td><td class="num">{{ row.distinct|intcomma }}</td>{% for share in row.shares %}<td class="num">{{ share|floatformat:1 }}%</td>{% endfor %}+ {% for count in row.over %}<td class="num">{{ count|intcomma }}</td>{% endfor %}</tr>{% endfor %}</tbody></table><p class="note">- Served searches on {% for endpoint in identities.endpoints %}<code>{{ endpoint }}</code>{% if not forloop.last %}, {% endif %}{% endfor %},- the ones query n-grams would be counted from. A caller is an IPv4 address or an IPv6- /64, so a SearxNG instance is one caller however many people use it. The distinct- count is a HyperLogLog, accurate to about 1%. The Top columns are the share of the- day's searches made by its busiest callers, from the {{ identities.tracked_limit|intcomma }}- kept per day. Callers are stored only as a token keyed on the day and the secret key,- so no address is kept and a caller cannot be followed from one day to the next.+ Served searches on {% for endpoint in callers.endpoints %}<code>{{ endpoint }}</code>{% if not forloop.last %}, {% endif %}{% endfor %},+ the ones query n-grams would be counted from, in Monday-to-Sunday weeks. A caller is+ an IPv4 address or an IPv6 /64, so a SearxNG instance is one caller however many+ people use it, and so is our own front end's server-side render. Distinct counts are+ HyperLogLogs, accurate to about 1%. The Top columns are the share of the week's+ searches made by its busiest callers, from the {{ callers.tracked_limit|intcomma }}+ kept; a caller the trim dropped restarts from zero, so they slightly understate it.+ Callers are hashed under a random salt made for the week, which expires a day after+ it ends. Within the hour after a week ends its per-caller counts are replaced by+ these totals, so a finished week is only the numbers in its row.</p></div>diff --git a/mwmbl/traffic.py b/mwmbl/traffic.pyindex c96e370..8a5f072 100644--- a/mwmbl/traffic.py+++ b/mwmbl/traffic.py@@ -14,19 +14,18 @@ differently next week. #412 is where aclassification earns its place, once there is data to design it against."""-import ipaddressimport randomfrom collections import Counter-from datetime import date+from datetime import date, timedeltafrom logging import getLoggerfrom typing import Optionalimport redisfrom django.conf import settingsfrom django.http import HttpRequest, HttpResponse-from django.utils.crypto import salted_hmacfrom redis.exceptions import RedisError+from mwmbl import search_callersfrom mwmbl.utils import utc_todaylogger = getLogger(__name__)@@ -126,8 +125,6 @@ def status_label(response: HttpResponse) -> str:REQUEST_COUNT_KEY = "traffic:req:{date}:{endpoint}:{headers}:{status}"USER_AGENT_COUNT_KEY = "traffic:user-agents:{date}"-IDENTITY_DISTINCT_KEY = "traffic:identities:{date}"-IDENTITY_COUNT_KEY = "traffic:identity-counts:{date}"# Thirty days, matching the public daily crawler counters in mwmbl/crawler/stats.py.# Production Redis runs allkeys-lru (see mwmbl/indexer/blacklist_snapshot.py), so a longer@@ -151,24 +148,6 @@ USER_AGENT_TRIM_PROBABILITY = 1 / 500# A missing user agent is a real answer, so it gets a row rather than being dropped.MISSING_USER_AGENT = "(none)"-# Who is searching, for sizing a k-anonymous count of query n-grams: how many distinct-# callers there are in a day, and how much of the traffic the busiest carry. A SearxNG-# instance forwards no client address, so it is one caller however many people use it.-# Only searches somebody typed and got an answer to, since those are what n-grams would be-# counted from: /complete is keystrokes and /raw is mostly the crawler's index sync.-IDENTITY_ENDPOINTS = {"legacy_search", "v1_search", "v2_search", "ui_search"}--# Kept as long as the user agents and trimmed the same way, for the same reasons.-IDENTITY_EXPIRE_SECONDS = USER_AGENT_EXPIRE_SECONDS-TRACKED_IDENTITIES = 2000-IDENTITY_TRIM_PROBABILITY = 1 / 500--# A household or a small network gets a /64, so one IPv6 caller is the whole prefix.-IPV6_IDENTITY_PREFIX = 64--# 64 bits: collisions are negligible at a few million callers a day.-IDENTITY_HASH_LENGTH = 16-# Aggressive by design. django-redis is configured with 5 s timeouts, which is fine for a# cache lookup a view is waiting on and far too slow for a counter: this runs on every# request, on the one thread asgiref gives the whole sync middleware chain.@@ -190,32 +169,6 @@ def get_redis() -> redis.Redis:return _redis-def client_network(request: HttpRequest) -> str:- """The caller's address, or its /64 for IPv6.-- The last X-Forwarded-For entry is the one our nginx saw: dokku sets the header to- $remote_addr, and anything earlier in the list would be the client's own say-so.- """- forwarded_for = request.headers.get("X-Forwarded-For")- address = forwarded_for.split(",")[-1].strip() if forwarded_for else request.META["REMOTE_ADDR"]- try:- ip = ipaddress.ip_address(address)- except ValueError:- # Still a caller, just not one with a parseable address, and record_request must not raise.- return address- if ip.version == 6:- return str(ipaddress.ip_network(f"{ip}/{IPV6_IDENTITY_PREFIX}", strict=False))- return str(ip)---def client_identity(request: HttpRequest, day: date) -> str:- """An opaque per-day token for the caller. Keyed on the day so tokens cannot be joined- across days, and on SECRET_KEY so the address space cannot be hashed to reverse them- without it. No address is ever stored."""- digest = salted_hmac(f"mwmbl.traffic.identity:{day}", client_network(request), algorithm="sha256")- return digest.hexdigest()[:IDENTITY_HASH_LENGTH]--def record_request(request: HttpRequest, response: HttpResponse) -> None:"""Count one search request. Never raises: a counter that costs a search its results isworse than no counter, which is the rule external_cache already follows."""@@ -246,18 +199,13 @@ def record_request(request: HttpRequest, response: HttpResponse) -> None:if random.random() < USER_AGENT_TRIM_PROBABILITY:pipeline.zremrangebyrank(user_agent_key, 0, -TRACKED_USER_AGENTS - 1)- if endpoint in IDENTITY_ENDPOINTS and response.status_code // 100 == 2:- identity = client_identity(request, today)- distinct_key = IDENTITY_DISTINCT_KEY.format(date=today)- pipeline.pfadd(distinct_key, identity)- pipeline.expire(distinct_key, IDENTITY_EXPIRE_SECONDS)- identity_count_key = IDENTITY_COUNT_KEY.format(date=today)- pipeline.zincrby(identity_count_key, 1, identity)- pipeline.expire(identity_count_key, IDENTITY_EXPIRE_SECONDS)- if random.random() < IDENTITY_TRIM_PROBABILITY:- pipeline.zremrangebyrank(identity_count_key, 0, -TRACKED_IDENTITIES - 1)-- pipeline.execute()+ queued_caller = None+ if endpoint in search_callers.IDENTITY_ENDPOINTS and response.status_code // 100 == 2:+ queued_caller = search_callers.queue_caller(get_redis(), pipeline, request, today)++ results = pipeline.execute()+ if queued_caller is not None:+ search_callers.record_thresholds(get_redis(), queued_caller, results)except RedisError:logger.warning("Could not record search traffic for %s", endpoint, exc_info=True)@@ -301,19 +249,28 @@ def read_user_agent_counts(redis_client, days: list[date], limit: int) -> list[treturn totals.most_common(limit)-def read_identity_counts(redis_client, days: list[date]) -> dict[date, tuple[int, list[int]]]:- """Per day, the distinct callers and the tracked callers' search counts, busiest first.+def caller_window_searches(redis_client, window: date) -> int:+ """A week's served searches on the endpoints callers are counted for: what their shares are of."""+ days = [window + timedelta(days=offset) for offset in range(search_callers.WINDOW_DAYS)]+ counts = read_request_counts(redis_client, days)+ return sum(+ count+ for (_, endpoint, _, status), count in counts.items()+ if endpoint in search_callers.IDENTITY_ENDPOINTS and status == "2xx"+ )+- Per day rather than summed, because the tokens change at midnight and the same caller- on two days is two rows that cannot be matched up.- """- pipeline = redis_client.pipeline()- for day in days:- pipeline.pfcount(IDENTITY_DISTINCT_KEY.format(date=day))- pipeline.zrevrange(IDENTITY_COUNT_KEY.format(date=day), 0, TRACKED_IDENTITIES - 1, withscores=True)- results = pipeline.execute()-- readout = {}- for day, distinct, tracked in zip(days, results[0::2], results[1::2]):- readout[day] = (distinct, [int(count) for _, count in tracked])- return readout+def summarise_last_caller_window(redis_client) -> bool:+ """Roll last week's callers into totals, if that has not happened yet. Hourly, so a week+ is summarised within an hour of ending and well inside its per-caller keys' grace day."""+ last_window = search_callers.window_start(utc_today()) - timedelta(days=search_callers.WINDOW_DAYS)+ # A request that read the clock just before midnight can recreate last week's keys after+ # its summary. They expire at the end of the grace day; summarising them again would+ # replace the week's totals with that one search.+ if search_callers.read_summary(redis_client, last_window) is not None:+ return False+ if not redis_client.exists(search_callers.DISTINCT_KEY.format(window=last_window)):+ return False+ searches = caller_window_searches(redis_client, last_window)+ search_callers.summarise_window(redis_client, last_window, searches)+ return Truediff --git a/test/test_admin_search_traffic.py b/test/test_admin_search_traffic.pyindex 20e0fb0..d5669ad 100644--- a/test/test_admin_search_traffic.py+++ b/test/test_admin_search_traffic.py@@ -232,43 +232,3 @@ def test_view_renders_when_redis_is_down(staff_client, monkeypatch):def test_admin_index_links_to_the_traffic_page(staff_client):assert TRAFFIC_URL in staff_client.get("/admin/").content.decode()---def record_callers(redis_client, searches_by_caller, day=None):- """Seed the caller keys the way record_request writes them, and the matching requests."""- day = day or utc_today()- for caller, searches in searches_by_caller.items():- redis_client.pfadd(traffic.IDENTITY_DISTINCT_KEY.format(date=day), caller)- redis_client.zincrby(traffic.IDENTITY_COUNT_KEY.format(date=day), searches, caller)- record(redis_client, "v1_search", day=day, count=sum(searches_by_caller.values()))---def test_callers_are_counted_and_their_share_is_of_all_searches(redis_client):- record_callers(redis_client, {"instance": 90, "browser": 6, "phone": 4})- # Served searches by callers the trim dropped, and traffic that is not a typed search.- record(redis_client, "ui_search", count=100)- record(redis_client, "v1_raw", count=1000)- record(redis_client, "v1_search", status="4xx", count=1000)-- identities = admin_views._identities(traffic.get_redis(), *window(1))-- [today] = identities["rows"]- assert today["searches"] == 200- assert today["distinct"] == 3- assert today["shares"] == [45, 50, 50, 50]---def test_the_caller_window_stops_at_the_caller_expiry(redis_client):- identities = admin_views._identities(traffic.get_redis(), *window(traffic.TRAFFIC_EXPIRE_SECONDS // 86400))-- assert len(identities["rows"]) == admin_views.IDENTITY_MAX_DAYS---def test_view_renders_the_callers(staff_client, redis_client):- record_callers(redis_client, {"instance": 1234})-- response = staff_client.get(TRAFFIC_URL)-- assert response.status_code == 200- assert "Distinct callers" in response.content.decode()- assert response.context["identities"]["rows"][0]["shares"][0] == 100diff --git a/test/test_search_callers.py b/test/test_search_callers.pynew file mode 100644index 0000000..2d7649b--- /dev/null+++ b/test/test_search_callers.py@@ -0,0 +1,261 @@+"""Tests for the weekly distinct-caller counts (mwmbl.search_callers) and their admin readout.++Counting is off in mwmbl.settings_test, so these turn it back on and wire in a fakeredis,+as test_traffic.py does.+"""++from datetime import date, timedelta++import fakeredis+import pytest+from django.contrib.auth import get_user_model+from django.http import HttpResponseServerError++from mwmbl import admin_views, background, search_callers, traffic+from mwmbl.utils import utc_today++User = get_user_model()++SEARCH = "/api/v1/search/"+++def counting(monkeypatch, settings):+ settings.SEARCH_TRAFFIC_COUNTING = True+ client = fakeredis.FakeStrictRedis(decode_responses=True)+ monkeypatch.setattr(traffic, "_redis", client)+ monkeypatch.setattr(search_callers, "_salts", {})+ return client+++def staff_client(client, db):+ user = User.objects.create_user(username="staff_member_1", password="correctpassword", is_staff=True)+ client.force_login(user)+ return client+++def this_window():+ return search_callers.window_start(utc_today())+++def search(client, address, times=1):+ for _ in range(times):+ client.get(SEARCH, {"s": "python"}, REMOTE_ADDR=address)+++def caller_counts(redis_client):+ key = search_callers.COUNT_KEY.format(window=this_window())+ return sorted(int(count) for _, count in redis_client.zrange(key, 0, -1, withscores=True))+++def distinct_callers(redis_client):+ return redis_client.pfcount(search_callers.DISTINCT_KEY.format(window=this_window()))+++def test_a_window_is_a_monday_to_sunday_week():+ monday = date(2026, 10, 12)++ assert [search_callers.window_start(monday + timedelta(days=offset)) for offset in range(8)] == [monday] * 7 + [+ monday + timedelta(days=7)+ ]+++def test_each_address_is_one_caller(counting, client):+ search(client, "203.0.113.1", times=3)+ search(client, "203.0.113.2")++ assert distinct_callers(counting) == 2+ assert caller_counts(counting) == [1, 3]+++def test_an_ipv6_caller_is_its_whole_64(counting, client):+ search(client, "2001:db8:1:2::1")+ search(client, "2001:db8:1:2:ffff::9")+ search(client, "2001:db8:1:3::1")++ assert caller_counts(counting) == [1, 2]+++ "headers,expected",+ [+ # Earlier X-Forwarded-For entries are whatever the client claimed.+ ({"HTTP_X_FORWARDED_FOR": "10.0.0.1, 198.51.100.7", "REMOTE_ADDR": "172.17.0.1"}, "198.51.100.7"),+ ({"REMOTE_ADDR": "::ffff:198.51.100.7"}, "198.51.100.7"),+ ({"HTTP_X_FORWARDED_FOR": "unknown"}, "unknown"),+ ],+)+def test_the_caller_is_the_address_our_nginx_saw(rf, headers, expected):+ assert search_callers.client_network(rf.get("/", **headers)) == expected+++def test_a_request_with_no_address_is_still_counted(counting, rf):+ """uvicorn on a unix socket gives Django no REMOTE_ADDR, and counting must not fail the search."""+ request = rf.get(SEARCH, {"s": "python"})+ del request.META["REMOTE_ADDR"]+ request.resolver_match = type("Match", (), {"view_name": "api-v1:search"})()++ traffic.record_request(request, type("Response", (), {"status_code": 200})())++ assert distinct_callers(counting) == 1+++def test_no_address_is_stored_and_tokens_depend_on_the_window_salt(counting, client):+ search(client, "203.0.113.1")++ stored = counting.zrange(search_callers.COUNT_KEY.format(window=this_window()), 0, -1)+ assert stored and all("203.0.113" not in token for token in stored)+ salt = counting.get(search_callers.SALT_KEY.format(window=this_window()))+ assert stored == [search_callers.caller_token(salt, "203.0.113.1")]+ assert search_callers.caller_token("another week's salt", "203.0.113.1") != stored[0]+++def test_an_evicted_salt_is_put_back_rather_than_replaced(counting, client):+ search(client, "203.0.113.1")+ salt_key = search_callers.SALT_KEY.format(window=this_window())+ salt = counting.get(salt_key)+ counting.delete(salt_key)++ search(client, "203.0.113.1")++ assert counting.get(salt_key) == salt+ assert caller_counts(counting) == [2]+++ "path,query",+ [("/api/v1/search/complete", {"q": "pyth"}), ("/api/v1/search/raw", {"s": "python"}), ("/app/home/", {"q": "py"})],+)+def test_only_typed_searches_are_attributed_to_a_caller(counting, client, path, query):+ client.get(path, query)++ assert distinct_callers(counting) == 0+++def test_an_unserved_search_is_not_attributed_to_a_caller(counting, rf):+ request = rf.get(SEARCH, {"s": "python"})+ request.resolver_match = type("Match", (), {"view_name": "api-v1:search"})()++ traffic.record_request(request, HttpResponseServerError())++ assert distinct_callers(counting) == 0+++def test_the_per_caller_keys_die_a_day_after_the_week(counting, client):+ search(client, "203.0.113.1", times=10)++ expiry = search_callers._raw_expiry(this_window())+ assert expiry.date() == this_window() + timedelta(days=8)+ # Ten searches cross the first threshold only, so the higher thresholds' keys do not exist.+ written = [key for key in search_callers._raw_keys(this_window()) if counting.exists(key)]+ assert len(written) == 4+ assert all(counting.expiretime(key) == int(expiry.timestamp()) for key in written)+++def test_callers_are_counted_as_they_cross_each_threshold(counting, client, monkeypatch):+ monkeypatch.setattr(search_callers, "THRESHOLDS", [2, 3])+ search(client, "203.0.113.1", times=3)+ search(client, "203.0.113.2", times=2)+ search(client, "203.0.113.3")++ totals = search_callers.read_live(counting, this_window(), searches=6)++ assert totals["over"] == [2, 1]+++def test_the_busiest_callers_survive_the_trim(counting, client, monkeypatch):+ monkeypatch.setattr(search_callers, "TRACKED_CALLERS", 1)+ monkeypatch.setattr(search_callers, "TRIM_PROBABILITY", 0)+ search(client, "203.0.113.1", times=2)+ search(client, "203.0.113.2")+ monkeypatch.setattr(search_callers, "TRIM_PROBABILITY", 1)+ search(client, "203.0.113.1")++ assert caller_counts(counting) == [3]+++def test_the_shares_are_of_every_served_search(counting, client):+ search(client, "203.0.113.1", times=3)+ search(client, "203.0.113.2")+ # A served search the trim would have dropped, and traffic that is not a typed search.+ counting.incrby(+ traffic.REQUEST_COUNT_KEY.format(date=utc_today(), endpoint="ui_search", headers="neither", status="2xx"), 4+ )+ client.get("/api/v1/search/raw", {"s": "python"})++ searches = traffic.caller_window_searches(counting, this_window())+ totals = search_callers.read_live(counting, this_window(), searches)++ assert searches == 8+ assert totals["busiest_searches"] == [3, 4, 4, 4]+++def last_window_with(redis_client, searches_by_caller):+ """Seed last week's keys the way record_request would have written them."""+ last_window = this_window() - timedelta(days=7)+ for caller, searches in searches_by_caller.items():+ redis_client.pfadd(search_callers.DISTINCT_KEY.format(window=last_window), caller)+ redis_client.zincrby(search_callers.COUNT_KEY.format(window=last_window), searches, caller)+ redis_client.set(search_callers.SALT_KEY.format(window=last_window), "salt")+ redis_client.incrby(+ traffic.REQUEST_COUNT_KEY.format(date=last_window, endpoint="v1_search", headers="neither", status="2xx"),+ sum(searches_by_caller.values()),+ )+ return last_window+++def test_a_finished_week_is_replaced_by_its_totals(counting):+ last_window = last_window_with(counting, {"instance": 90, "browser": 10})++ background.summarise_search_callers.now()++ assert all(not counting.exists(key) for key in search_callers._raw_keys(last_window))+ assert search_callers.read_summary(counting, last_window) == {+ "searches": 100,+ "distinct": 2,+ "busiest_searches": [90, 100, 100, 100],+ "over": [0, 0, 0],+ }+ assert 0 < counting.ttl(search_callers.SUMMARY_KEY.format(window=last_window))+++def test_a_late_search_does_not_replace_the_summary(counting):+ last_window = last_window_with(counting, {"instance": 90})+ traffic.summarise_last_caller_window(counting)+ counting.pfadd(search_callers.DISTINCT_KEY.format(window=last_window), "straggler")++ assert traffic.summarise_last_caller_window(counting) is False+ assert search_callers.read_summary(counting, last_window)["distinct"] == 1+++def test_this_week_is_not_summarised(counting, client):+ search(client, "203.0.113.1")++ assert traffic.summarise_last_caller_window(counting) is False+ assert distinct_callers(counting) == 1+++def test_the_page_shows_this_week_live_and_finished_weeks_from_their_summaries(staff_client, counting, client):+ last_window = last_window_with(counting, {"instance": 90, "browser": 10})+ traffic.summarise_last_caller_window(counting)+ search(client, "203.0.113.1", times=2)++ response = staff_client.get("/admin/search-traffic/")++ assert response.status_code == 200+ assert "Distinct callers per week" in response.content.decode()+ rows = response.context["callers"]["rows"]+ assert [(row["window"], row["summarised"], row["distinct"]) for row in rows] == [+ (this_window(), False, 1),+ (last_window, True, 2),+ ]+ assert rows[0]["shares"][0] == 100+ assert rows[1]["shares"][0] == 90+++def test_weeks_with_no_callers_are_left_out(staff_client, counting):+ response = staff_client.get("/admin/search-traffic/")++ assert response.context["callers"]["rows"] == []+ assert admin_views.CALLER_WINDOWS == 14diff --git a/test/test_traffic.py b/test/test_traffic.pyindex 72e76b0..a448918 100644--- a/test/test_traffic.py+++ b/test/test_traffic.py@@ -332,105 +332,3 @@ def test_a_streaming_response_is_counted_without_being_consumed(counting, rf):assert count_for("super_search") == "1"assert b"".join(response.streaming_content) == b"ab"---def identity_counts():- key = traffic.IDENTITY_COUNT_KEY.format(date=utc_today())- return sorted(int(count) for _, count in traffic.get_redis().zrange(key, 0, -1, withscores=True))---def distinct_identities():- return traffic.get_redis().pfcount(traffic.IDENTITY_DISTINCT_KEY.format(date=utc_today()))---def test_each_address_is_one_caller(counting, client):- for address, searches in [("203.0.113.1", 3), ("203.0.113.2", 1)]:- for _ in range(searches):- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR=address)-- assert distinct_identities() == 2- assert identity_counts() == [1, 3]---def test_an_ipv6_caller_is_its_whole_64(counting, client):- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="2001:db8:1:2::1")- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="2001:db8:1:2:ffff::9")- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="2001:db8:1:3::1")-- assert identity_counts() == [1, 2]---def test_the_caller_is_the_address_our_nginx_saw(rf):- """Earlier X-Forwarded-For entries are whatever the client claimed."""- request = rf.get("/", HTTP_X_FORWARDED_FOR="10.0.0.1, 198.51.100.7", REMOTE_ADDR="172.17.0.1")-- assert traffic.client_network(request) == "198.51.100.7"---def test_an_unparseable_address_is_still_a_caller(rf):- request = rf.get("/", HTTP_X_FORWARDED_FOR="unknown")-- assert traffic.client_network(request) == "unknown"---def test_no_address_is_stored_and_tokens_change_daily(counting, client, rf):- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.1")-- stored = [- str(value)- for key in counting.keys("traffic:identity*")- if counting.type(key) == "zset"- for value in counting.zrange(key, 0, -1)- ]- assert stored and all("203.0.113" not in value for value in stored)- request = rf.get("/", REMOTE_ADDR="203.0.113.1")- yesterday = utc_today() - timedelta(days=1)- assert traffic.client_identity(request, utc_today()) != traffic.client_identity(request, yesterday)--- "path,query,endpoint",- [- ("/api/v1/search/complete", {"q": "pyth"}, "v1_complete"),- ("/api/v1/search/raw", {"s": "python"}, "v1_raw"),- ("/app/home/", {"q": "py"}, "ui_keystroke"),- ],-)-def test_only_typed_searches_are_attributed_to_a_caller(counting, client, path, query, endpoint):- client.get(path, query, **BROWSER_HEADERS)-- assert count_for(endpoint) == "1"- assert distinct_identities() == 0---def test_an_unserved_search_is_not_attributed_to_a_caller(counting, rf):- request = rf.get("/api/v1/search/", {"s": "python"})- request.resolver_match = type("Match", (), {"view_name": "api-v1:search"})()-- traffic.record_request(request, HttpResponseServerError())-- assert distinct_identities() == 0---def test_the_caller_keys_expire_and_are_trimmed(counting, client, monkeypatch):- monkeypatch.setattr(traffic, "TRACKED_IDENTITIES", 1)- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.1")- monkeypatch.setattr(traffic, "IDENTITY_TRIM_PROBABILITY", 1)- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.1")- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.2")-- assert identity_counts() == [2]- for key in [traffic.IDENTITY_DISTINCT_KEY, traffic.IDENTITY_COUNT_KEY]:- assert 0 < counting.ttl(key.format(date=utc_today())) <= traffic.IDENTITY_EXPIRE_SECONDS---def test_the_readout_gives_distinct_callers_and_counts_per_day(counting, client):- yesterday = utc_today() - timedelta(days=1)- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.1")- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.1")- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.2")-- assert traffic.read_identity_counts(counting, [utc_today(), yesterday]) == {- utc_today(): (2, [2, 1]),- yesterday: (0, []),- }