Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Count search callers per week under an expiring salt, and keep only totals

Claudeb853de3d556a
diff --git a/mwmbl/admin_views.py b/mwmbl/admin_views.py
index e8b9f20..cbea20c 100644
--- a/mwmbl/admin_views.py
+++ b/mwmbl/admin_views.py
@@ -34,7 +34,7 @@ from django.http import HttpRequest
from django.shortcuts import render
from redis import RedisError
-from mwmbl import count_urls, pricing, quota, search_setup, traffic
+from mwmbl import count_urls, pricing, quota, search_callers, search_setup, traffic
from mwmbl.crawler.stats import BLACKLISTED_REMOVED_COUNT_KEY
from mwmbl.curated_domains import get_curated_domains
from mwmbl.indexer import blacklist_snapshot, purge_queue
@@ -49,14 +49,10 @@ from mwmbl.membership import TIERS, VOLUNTEER_TIERS, combined_search_monthly_lim
from mwmbl.models import ApiKey, Membership, MwmblUser
from mwmbl.traffic import (
HEADER_LABELS,
- IDENTITY_ENDPOINTS,
- IDENTITY_EXPIRE_SECONDS,
STATUS_LABELS,
- TRACKED_IDENTITIES,
TRACKED_USER_AGENTS,
TRAFFIC_EXPIRE_SECONDS,
USER_AGENT_EXPIRE_SECONDS,
- read_identity_counts,
read_request_counts,
read_user_agent_counts,
)
@@ -231,10 +227,9 @@ TRAFFIC_MAX_DAYS = TRAFFIC_EXPIRE_SECONDS // SECONDS_PER_DAY
TRAFFIC_DEFAULT_DAYS = 7
TRAFFIC_WINDOW_CHOICES = [1, 7, TRAFFIC_MAX_DAYS]
USER_AGENT_MAX_DAYS = USER_AGENT_EXPIRE_SECONDS // SECONDS_PER_DAY
-IDENTITY_MAX_DAYS = IDENTITY_EXPIRE_SECONDS // SECONDS_PER_DAY
-# The share of a day's searches carried by its busiest N callers, at each of these N.
-IDENTITY_SHARE_RANKS = [1, 10, 100, 1000]
+# This week, still filling, and as many finished weeks as their summaries are kept for.
+CALLER_WINDOWS = search_callers.SUMMARY_EXPIRE_SECONDS // (SECONDS_PER_DAY * search_callers.WINDOW_DAYS) + 1
# Enough to see the shape of the tail without putting two thousand caller-supplied strings
# through the template.
@@ -317,34 +312,42 @@ def _user_agents(redis_client, days: list[date], counts: dict) -> dict:
}
-def _identities(redis_client, days: list[date], counts: dict) -> dict:
- """How many distinct callers searched each day, and how concentrated their searches are.
+def _callers(redis_client) -> dict:
+ """Distinct callers per week, how concentrated their searches are, and how many search often.
- The shares are of the day's served searches as the request counter has them, so the
- callers the trim dropped still count towards the total they are a share of.
+ Weeks with no callers are left out, which before this was deployed is all of them: a row
+ of zeros would read as nobody searching rather than as nothing counted.
"""
- identity_days = days[:IDENTITY_MAX_DAYS]
- searches_by_day: Counter = Counter()
- for coordinate, count in counts.items():
- if coordinate[ENDPOINT] in IDENTITY_ENDPOINTS and coordinate[STATUS] == "2xx":
- searches_by_day[coordinate[DAY]] += count
-
+ this_window = search_callers.window_start(utc_today())
+ windows = [this_window - timedelta(days=search_callers.WINDOW_DAYS * offset) for offset in range(CALLER_WINDOWS)]
rows = []
- for day, (distinct, tracked) in read_identity_counts(redis_client, identity_days).items():
- searches = searches_by_day[day]
+ for window in windows:
+ if window == this_window:
+ totals = search_callers.read_live(
+ redis_client, window, traffic.caller_window_searches(redis_client, window)
+ )
+ else:
+ totals = search_callers.read_summary(redis_client, window)
+ # Last week in the hour before it is summarised is absent too, briefly.
+ if totals is None or totals["distinct"] == 0:
+ continue
+ searches = totals["searches"]
rows.append(
{
- "day": day,
+ "window": window,
+ "summarised": window != this_window,
"searches": searches,
- "distinct": distinct,
- "shares": [100 * sum(tracked[:rank]) / searches if searches else 0 for rank in IDENTITY_SHARE_RANKS],
+ "distinct": totals["distinct"],
+ "shares": [100 * busiest / searches if searches else 0 for busiest in totals["busiest_searches"]],
+ "over": totals["over"],
}
)
return {
"rows": rows,
- "ranks": IDENTITY_SHARE_RANKS,
- "endpoints": sorted(IDENTITY_ENDPOINTS),
- "tracked_limit": TRACKED_IDENTITIES,
+ "ranks": search_callers.SHARE_RANKS,
+ "thresholds": search_callers.THRESHOLDS,
+ "endpoints": sorted(search_callers.IDENTITY_ENDPOINTS),
+ "tracked_limit": search_callers.TRACKED_CALLERS,
}
@@ -516,7 +519,7 @@ def search_traffic_view(request):
context["by_headers"] = _requests_by_label(counts, HEADERS, HEADER_LABELS)
context["by_status"] = _requests_by_label(counts, STATUS, STATUS_LABELS)
context["user_agents"] = _user_agents(redis_client, days, counts)
- context["identities"] = _identities(redis_client, days, counts)
+ context["callers"] = _callers(redis_client)
except RedisError as e:
logger.exception("Could not read search traffic from Redis")
context["redis_error"] = str(e)
diff --git a/mwmbl/apps.py b/mwmbl/apps.py
index 10b6890..c3f0df9 100644
--- a/mwmbl/apps.py
+++ b/mwmbl/apps.py
@@ -147,6 +147,7 @@ class MwmblConfig(AppConfig):
refresh_blacklist_snapshot,
report_usage_to_polar,
retrain_domain_moderation_model,
+ summarise_search_callers,
sync_search_counts,
)
@@ -157,6 +158,7 @@ class MwmblConfig(AppConfig):
MODERATION_RETRAIN_TASK = "mwmbl.background.retrain_domain_moderation_model"
INDEX_COUNT_TASK = "mwmbl.background.count_index_urls"
FLUSH_TOKENS_TASK = "mwmbl.background.flush_expired_tokens"
+ SUMMARISE_CALLERS_TASK = "mwmbl.background.summarise_search_callers"
# Sync search counts once per hour (3600 seconds)
if not Task.objects.filter(task_name=SYNC_TASK).exists():
@@ -201,6 +203,11 @@ class MwmblConfig(AppConfig):
if not Task.objects.filter(task_name=FLUSH_TOKENS_TASK).exists():
flush_expired_tokens(repeat=Task.DAILY, repeat_until=None)
+ # Hourly, so a finished week's per-caller counts are replaced by its totals well
+ # inside the grace day before they expire - see mwmbl.search_callers.
+ if not Task.objects.filter(task_name=SUMMARISE_CALLERS_TASK).exists():
+ summarise_search_callers(repeat=Task.HOURLY, repeat_until=None)
+
except Exception:
# Don't prevent startup if background task scheduling fails
log.exception("Failed to schedule background tasks")
diff --git a/mwmbl/background.py b/mwmbl/background.py
index bb806c7..c159cf5 100644
--- a/mwmbl/background.py
+++ b/mwmbl/background.py
@@ -7,6 +7,7 @@ Django Background Tasks for periodic maintenance:
- refresh_blacklist_snapshot: rebuilds the blacklist the search path filters against
- purge_blacklisted_from_queue: removes retrieval-filtered documents from the index
- count_index_urls: counts the unique URLs and domains in the index once a week
+ - summarise_search_callers: replaces last week's per-caller counts with totals
"""
import logging
@@ -23,7 +24,7 @@ from django.core.management import call_command
from django.db import transaction
from redis import Redis
-from mwmbl import pricing
+from mwmbl import pricing, traffic
from mwmbl.count_urls import count_urls_step, get_redis
from mwmbl.crawler.stats import StatsManager
from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist, refresh_snapshot
@@ -356,3 +357,10 @@ def retrain_domain_moderation_model():
def flush_expired_tokens():
"""Delete expired JWT refresh tokens from the blacklist tables, which gain a row per refresh."""
call_command("flushexpiredtokens")
+
+
+@background(schedule=0)
+def summarise_search_callers():
+ """Roll last week's search callers into totals and delete the per-caller keys."""
+ if traffic.summarise_last_caller_window(traffic.get_redis()):
+ logger.info("Summarised last week's search callers")
diff --git a/mwmbl/search_callers.py b/mwmbl/search_callers.py
new file mode 100644
index 0000000..0f3bf91
--- /dev/null
+++ b/mwmbl/search_callers.py
@@ -0,0 +1,208 @@
+"""Counting distinct search callers per week, without keeping who they were.
+
+Sizing for a k-anonymous count of query n-grams: how many distinct callers search in a
+week, how much of the traffic the busiest carry, and how many search often enough to
+matter. A SearxNG instance forwards no client address, so it is one caller however many
+people use it. Only searches somebody typed and got an answer to, since those are what
+n-grams would be counted from: /complete is keystrokes and /raw is mostly the crawler's
+index sync.
+
+A caller is an IPv4 address or an IPv6 /64, stored only as an HMAC under a random salt
+made for the week. The salt lives in Redis and dies a day after the week ends, along with
+everything keyed on it, so from then on nobody - us included, with SECRET_KEY and the
+whole Redis - can tell which address a token was. Before that, summarise_window rolls the
+week into a handful of totals and deletes the per-caller keys, so a finished week is
+never more than those totals. The week rather than the day because that is the window
+n-grams would be published over: tokens cannot be joined across windows, so callers
+counted per day could not be added up to size it.
+"""
+
+import hashlib
+import hmac
+import ipaddress
+import json
+import random
+import secrets
+from dataclasses import dataclass
+from datetime import date, datetime, time, timedelta, timezone
+
+import redis
+from django.http import HttpRequest
+
+IDENTITY_ENDPOINTS = {"legacy_search", "v1_search", "v2_search", "ui_search"}
+
+WINDOW_DAYS = 7
+
+SALT_KEY = "traffic:caller-salt:{window}"
+DISTINCT_KEY = "traffic:callers:{window}"
+COUNT_KEY = "traffic:caller-counts:{window}"
+OVER_KEY = "traffic:callers-over:{window}:{threshold}"
+SUMMARY_KEY = "traffic:caller-summary:{window}"
+
+# The per-caller keys outlive the week by a day so the hourly summary has time to run. If it
+# never does, they expire anyway: the week is lost, which is the safe way to fail.
+RAW_GRACE = timedelta(days=1)
+
+# Thirteen weeks of summaries. They are a few numbers each, so the length is about how far
+# back is worth comparing, not memory.
+SUMMARY_EXPIRE_SECONDS = 60 * 60 * 24 * 7 * 13
+
+# The memory bound on the per-caller counts, trimmed the way the user agents are. A caller
+# the trim drops restarts from zero, which understates the busiest shares slightly; the
+# shares are read no further down than the thousandth, well inside what is kept.
+TRACKED_CALLERS = 2000
+TRIM_PROBABILITY = 1 / 500
+
+# The share of the week's searches carried by its busiest N callers, at each of these N.
+SHARE_RANKS = [1, 10, 100, 1000]
+
+# How many callers searched at least this often. Counted as each caller crosses the line,
+# since the trimmed counts lose exactly the tail these are about.
+THRESHOLDS = [10, 100, 1000]
+
+# A household or a small network gets a /64, so one IPv6 caller is the whole prefix.
+IPV6_PREFIX = 64
+
+# 64 bits: collisions are negligible at a few million callers a week.
+TOKEN_LENGTH = 16
+
+# This process's copy of each window's salt, so a request costs no extra round trip.
+_salts: dict[date, str] = {}
+
+
+def window_start(day: date) -> date:
+ """The Monday starting the week that day is in."""
+ return day - timedelta(days=day.weekday())
+
+
+def _raw_expiry(window: date) -> datetime:
+ window_end = datetime.combine(window + timedelta(days=WINDOW_DAYS), time(), tzinfo=timezone.utc)
+ return window_end + RAW_GRACE
+
+
+def client_network(request: HttpRequest) -> str:
+ """The caller's address, or its /64 for IPv6.
+
+ The last X-Forwarded-For entry is the one our nginx saw: dokku sets the header to
+ $remote_addr, and anything earlier in the list would be the client's own say-so.
+ REMOTE_ADDR is absent when uvicorn has no peer address, as on a unix socket, and this
+ runs inside a counter that must never fail a search.
+ """
+ forwarded_for = request.headers.get("X-Forwarded-For")
+ address = forwarded_for.split(",")[-1].strip() if forwarded_for else request.META.get("REMOTE_ADDR", "")
+ try:
+ ip = ipaddress.ip_address(address)
+ except ValueError:
+ # Still a caller, just not one with a parseable address.
+ return address
+ # Without this every IPv4-mapped address would share ::/64.
+ if ip.version == 6 and ip.ipv4_mapped:
+ return str(ip.ipv4_mapped)
+ if ip.version == 6:
+ return str(ipaddress.ip_network(f"{ip}/{IPV6_PREFIX}", strict=False))
+ return str(ip)
+
+
+def _salt(redis_client: redis.Redis, window: date) -> str:
+ """The window's salt, made by whichever process asks first."""
+ if window not in _salts:
+ salt_key = SALT_KEY.format(window=window)
+ pipeline = redis_client.pipeline()
+ pipeline.set(salt_key, secrets.token_hex(32), nx=True, exat=_raw_expiry(window))
+ pipeline.get(salt_key)
+ _salts.clear()
+ _salts[window] = pipeline.execute()[1]
+ return _salts[window]
+
+
+def caller_token(salt: str, network: str) -> str:
+ return hmac.new(salt.encode(), network.encode(), hashlib.sha256).hexdigest()[:TOKEN_LENGTH]
+
+
+@dataclass
+class QueuedCaller:
+ window: date
+ token: str
+ count_index: int
+
+
+def queue_caller(redis_client: redis.Redis, pipeline, request: HttpRequest, day: date) -> QueuedCaller:
+ """Add one search by this request's caller to the end of the pipeline, which must have
+ nothing queued after it."""
+ window = window_start(day)
+ expiry = _raw_expiry(window)
+ salt = _salt(redis_client, window)
+ token = caller_token(salt, client_network(request))
+
+ # Every request rewrites the salt if it is missing. That keeps it warm under allkeys-lru,
+ # where a key read once a week per process is the first to go, and if it was evicted
+ # anyway, puts back the one this process is still hashing with rather than a new one.
+ pipeline.set(SALT_KEY.format(window=window), salt, nx=True, exat=expiry)
+ distinct_key = DISTINCT_KEY.format(window=window)
+ pipeline.pfadd(distinct_key, token)
+ pipeline.expireat(distinct_key, expiry)
+ count_key = COUNT_KEY.format(window=window)
+ pipeline.zincrby(count_key, 1, token)
+ pipeline.expireat(count_key, expiry)
+ # Counted back from the end, so this needs nothing from whatever else is in the pipeline.
+ count_index = -2
+ if random.random() < TRIM_PROBABILITY:
+ pipeline.zremrangebyrank(count_key, 0, -TRACKED_CALLERS - 1)
+ count_index = -3
+ return QueuedCaller(window=window, token=token, count_index=count_index)
+
+
+def record_thresholds(redis_client: redis.Redis, queued: QueuedCaller, results: list) -> None:
+ """Note the caller crossing a threshold, which is a second round trip once per caller per
+ threshold. A caller the trim dropped crosses again, and the HyperLogLog counts them once."""
+ count = int(results[queued.count_index])
+ if count not in THRESHOLDS:
+ return
+ over_key = OVER_KEY.format(window=queued.window, threshold=count)
+ pipeline = redis_client.pipeline()
+ pipeline.pfadd(over_key, queued.token)
+ pipeline.expireat(over_key, _raw_expiry(queued.window))
+ pipeline.execute()
+
+
+def _raw_keys(window: date) -> list[str]:
+ over_keys = [OVER_KEY.format(window=window, threshold=threshold) for threshold in THRESHOLDS]
+ return [SALT_KEY.format(window=window), DISTINCT_KEY.format(window=window), COUNT_KEY.format(window=window)] + (
+ over_keys
+ )
+
+
+def read_summary(redis_client: redis.Redis, window: date) -> dict | None:
+ summary = redis_client.get(SUMMARY_KEY.format(window=window))
+ return None if summary is None else json.loads(summary)
+
+
+def read_live(redis_client: redis.Redis, window: date, searches: int) -> dict:
+ """A window's totals, read from its per-caller keys.
+
+ searches is the window's served searches from the request counter, which the shares are
+ of, so callers the trim dropped still count towards the total they are a share of.
+ """
+ pipeline = redis_client.pipeline()
+ pipeline.pfcount(DISTINCT_KEY.format(window=window))
+ pipeline.zrevrange(COUNT_KEY.format(window=window), 0, max(SHARE_RANKS) - 1, withscores=True)
+ for threshold in THRESHOLDS:
+ pipeline.pfcount(OVER_KEY.format(window=window, threshold=threshold))
+ distinct, busiest, *over = pipeline.execute()
+
+ counts = [int(count) for _, count in busiest]
+ return {
+ "searches": searches,
+ "distinct": distinct,
+ "busiest_searches": [sum(counts[:rank]) for rank in SHARE_RANKS],
+ "over": over,
+ }
+
+
+def summarise_window(redis_client: redis.Redis, window: date, searches: int) -> None:
+ """Replace a finished window's per-caller keys with its totals."""
+ totals = read_live(redis_client, window, searches)
+ pipeline = redis_client.pipeline()
+ pipeline.set(SUMMARY_KEY.format(window=window), json.dumps(totals), ex=SUMMARY_EXPIRE_SECONDS)
+ pipeline.delete(*_raw_keys(window))
+ pipeline.execute()
diff --git a/mwmbl/templates/admin/search_traffic.html b/mwmbl/templates/admin/search_traffic.html
index 219aa01..c3ceec4 100644
--- a/mwmbl/templates/admin/search_traffic.html
+++ b/mwmbl/templates/admin/search_traffic.html
@@ -141,33 +141,38 @@
</div>
<div class="module">
- <h2>Distinct callers</h2>
+ <h2>Distinct callers per week</h2>
<table>
<thead>
<tr>
- <th>Day</th><th class="num">Searches</th><th class="num">Distinct callers</th>
- {% for rank in identities.ranks %}<th class="num">Top {{ rank|intcomma }}</th>{% endfor %}
+ <th>Week from (UTC)</th><th class="num">Searches</th><th class="num">Distinct callers</th>
+ {% for rank in callers.ranks %}<th class="num">Top {{ rank|intcomma }}</th>{% endfor %}
+ {% for threshold in callers.thresholds %}<th class="num">&ge; {{ threshold|intcomma }} searches</th>{% endfor %}
</tr>
</thead>
<tbody>
- {% for row in identities.rows %}
+ {% for row in callers.rows %}
<tr>
- <td>{{ row.day }}</td>
+ <td>{{ row.window }}{% if not row.summarised %} <span class="note">(filling)</span>{% endif %}</td>
<td class="num">{{ row.searches|intcomma }}</td>
<td class="num">{{ row.distinct|intcomma }}</td>
{% for share in row.shares %}<td class="num">{{ share|floatformat:1 }}%</td>{% endfor %}
+ {% for count in row.over %}<td class="num">{{ count|intcomma }}</td>{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
<p class="note">
- Served searches on {% for endpoint in identities.endpoints %}<code>{{ endpoint }}</code>{% if not forloop.last %}, {% endif %}{% endfor %},
- the ones query n-grams would be counted from. A caller is an IPv4 address or an IPv6
- /64, so a SearxNG instance is one caller however many people use it. The distinct
- count is a HyperLogLog, accurate to about 1%. The Top columns are the share of the
- day's searches made by its busiest callers, from the {{ identities.tracked_limit|intcomma }}
- kept per day. Callers are stored only as a token keyed on the day and the secret key,
- so no address is kept and a caller cannot be followed from one day to the next.
+ Served searches on {% for endpoint in callers.endpoints %}<code>{{ endpoint }}</code>{% if not forloop.last %}, {% endif %}{% endfor %},
+ the ones query n-grams would be counted from, in Monday-to-Sunday weeks. A caller is
+ an IPv4 address or an IPv6 /64, so a SearxNG instance is one caller however many
+ people use it, and so is our own front end's server-side render. Distinct counts are
+ HyperLogLogs, accurate to about 1%. The Top columns are the share of the week's
+ searches made by its busiest callers, from the {{ callers.tracked_limit|intcomma }}
+ kept; a caller the trim dropped restarts from zero, so they slightly understate it.
+ Callers are hashed under a random salt made for the week, which expires a day after
+ it ends. Within the hour after a week ends its per-caller counts are replaced by
+ these totals, so a finished week is only the numbers in its row.
</p>
</div>
diff --git a/mwmbl/traffic.py b/mwmbl/traffic.py
index c96e370..8a5f072 100644
--- a/mwmbl/traffic.py
+++ b/mwmbl/traffic.py
@@ -14,19 +14,18 @@ differently next week. #412 is where a
classification earns its place, once there is data to design it against.
"""
-import ipaddress
import random
from collections import Counter
-from datetime import date
+from datetime import date, timedelta
from logging import getLogger
from typing import Optional
import redis
from django.conf import settings
from django.http import HttpRequest, HttpResponse
-from django.utils.crypto import salted_hmac
from redis.exceptions import RedisError
+from mwmbl import search_callers
from mwmbl.utils import utc_today
logger = getLogger(__name__)
@@ -126,8 +125,6 @@ def status_label(response: HttpResponse) -> str:
REQUEST_COUNT_KEY = "traffic:req:{date}:{endpoint}:{headers}:{status}"
USER_AGENT_COUNT_KEY = "traffic:user-agents:{date}"
-IDENTITY_DISTINCT_KEY = "traffic:identities:{date}"
-IDENTITY_COUNT_KEY = "traffic:identity-counts:{date}"
# Thirty days, matching the public daily crawler counters in mwmbl/crawler/stats.py.
# Production Redis runs allkeys-lru (see mwmbl/indexer/blacklist_snapshot.py), so a longer
@@ -151,24 +148,6 @@ USER_AGENT_TRIM_PROBABILITY = 1 / 500
# A missing user agent is a real answer, so it gets a row rather than being dropped.
MISSING_USER_AGENT = "(none)"
-# Who is searching, for sizing a k-anonymous count of query n-grams: how many distinct
-# callers there are in a day, and how much of the traffic the busiest carry. A SearxNG
-# instance forwards no client address, so it is one caller however many people use it.
-# Only searches somebody typed and got an answer to, since those are what n-grams would be
-# counted from: /complete is keystrokes and /raw is mostly the crawler's index sync.
-IDENTITY_ENDPOINTS = {"legacy_search", "v1_search", "v2_search", "ui_search"}
-
-# Kept as long as the user agents and trimmed the same way, for the same reasons.
-IDENTITY_EXPIRE_SECONDS = USER_AGENT_EXPIRE_SECONDS
-TRACKED_IDENTITIES = 2000
-IDENTITY_TRIM_PROBABILITY = 1 / 500
-
-# A household or a small network gets a /64, so one IPv6 caller is the whole prefix.
-IPV6_IDENTITY_PREFIX = 64
-
-# 64 bits: collisions are negligible at a few million callers a day.
-IDENTITY_HASH_LENGTH = 16
-
# Aggressive by design. django-redis is configured with 5 s timeouts, which is fine for a
# cache lookup a view is waiting on and far too slow for a counter: this runs on every
# request, on the one thread asgiref gives the whole sync middleware chain.
@@ -190,32 +169,6 @@ def get_redis() -> redis.Redis:
return _redis
-def client_network(request: HttpRequest) -> str:
- """The caller's address, or its /64 for IPv6.
-
- The last X-Forwarded-For entry is the one our nginx saw: dokku sets the header to
- $remote_addr, and anything earlier in the list would be the client's own say-so.
- """
- forwarded_for = request.headers.get("X-Forwarded-For")
- address = forwarded_for.split(",")[-1].strip() if forwarded_for else request.META["REMOTE_ADDR"]
- try:
- ip = ipaddress.ip_address(address)
- except ValueError:
- # Still a caller, just not one with a parseable address, and record_request must not raise.
- return address
- if ip.version == 6:
- return str(ipaddress.ip_network(f"{ip}/{IPV6_IDENTITY_PREFIX}", strict=False))
- return str(ip)
-
-
-def client_identity(request: HttpRequest, day: date) -> str:
- """An opaque per-day token for the caller. Keyed on the day so tokens cannot be joined
- across days, and on SECRET_KEY so the address space cannot be hashed to reverse them
- without it. No address is ever stored."""
- digest = salted_hmac(f"mwmbl.traffic.identity:{day}", client_network(request), algorithm="sha256")
- return digest.hexdigest()[:IDENTITY_HASH_LENGTH]
-
-
def record_request(request: HttpRequest, response: HttpResponse) -> None:
"""Count one search request. Never raises: a counter that costs a search its results is
worse than no counter, which is the rule external_cache already follows."""
@@ -246,18 +199,13 @@ def record_request(request: HttpRequest, response: HttpResponse) -> None:
if random.random() < USER_AGENT_TRIM_PROBABILITY:
pipeline.zremrangebyrank(user_agent_key, 0, -TRACKED_USER_AGENTS - 1)
- if endpoint in IDENTITY_ENDPOINTS and response.status_code // 100 == 2:
- identity = client_identity(request, today)
- distinct_key = IDENTITY_DISTINCT_KEY.format(date=today)
- pipeline.pfadd(distinct_key, identity)
- pipeline.expire(distinct_key, IDENTITY_EXPIRE_SECONDS)
- identity_count_key = IDENTITY_COUNT_KEY.format(date=today)
- pipeline.zincrby(identity_count_key, 1, identity)
- pipeline.expire(identity_count_key, IDENTITY_EXPIRE_SECONDS)
- if random.random() < IDENTITY_TRIM_PROBABILITY:
- pipeline.zremrangebyrank(identity_count_key, 0, -TRACKED_IDENTITIES - 1)
-
- pipeline.execute()
+ queued_caller = None
+ if endpoint in search_callers.IDENTITY_ENDPOINTS and response.status_code // 100 == 2:
+ queued_caller = search_callers.queue_caller(get_redis(), pipeline, request, today)
+
+ results = pipeline.execute()
+ if queued_caller is not None:
+ search_callers.record_thresholds(get_redis(), queued_caller, results)
except RedisError:
logger.warning("Could not record search traffic for %s", endpoint, exc_info=True)
@@ -301,19 +249,28 @@ def read_user_agent_counts(redis_client, days: list[date], limit: int) -> list[t
return totals.most_common(limit)
-def read_identity_counts(redis_client, days: list[date]) -> dict[date, tuple[int, list[int]]]:
- """Per day, the distinct callers and the tracked callers' search counts, busiest first.
+def caller_window_searches(redis_client, window: date) -> int:
+ """A week's served searches on the endpoints callers are counted for: what their shares are of."""
+ days = [window + timedelta(days=offset) for offset in range(search_callers.WINDOW_DAYS)]
+ counts = read_request_counts(redis_client, days)
+ return sum(
+ count
+ for (_, endpoint, _, status), count in counts.items()
+ if endpoint in search_callers.IDENTITY_ENDPOINTS and status == "2xx"
+ )
+
- Per day rather than summed, because the tokens change at midnight and the same caller
- on two days is two rows that cannot be matched up.
- """
- pipeline = redis_client.pipeline()
- for day in days:
- pipeline.pfcount(IDENTITY_DISTINCT_KEY.format(date=day))
- pipeline.zrevrange(IDENTITY_COUNT_KEY.format(date=day), 0, TRACKED_IDENTITIES - 1, withscores=True)
- results = pipeline.execute()
-
- readout = {}
- for day, distinct, tracked in zip(days, results[0::2], results[1::2]):
- readout[day] = (distinct, [int(count) for _, count in tracked])
- return readout
+def summarise_last_caller_window(redis_client) -> bool:
+ """Roll last week's callers into totals, if that has not happened yet. Hourly, so a week
+ is summarised within an hour of ending and well inside its per-caller keys' grace day."""
+ last_window = search_callers.window_start(utc_today()) - timedelta(days=search_callers.WINDOW_DAYS)
+ # A request that read the clock just before midnight can recreate last week's keys after
+ # its summary. They expire at the end of the grace day; summarising them again would
+ # replace the week's totals with that one search.
+ if search_callers.read_summary(redis_client, last_window) is not None:
+ return False
+ if not redis_client.exists(search_callers.DISTINCT_KEY.format(window=last_window)):
+ return False
+ searches = caller_window_searches(redis_client, last_window)
+ search_callers.summarise_window(redis_client, last_window, searches)
+ return True
diff --git a/test/test_admin_search_traffic.py b/test/test_admin_search_traffic.py
index 20e0fb0..d5669ad 100644
--- a/test/test_admin_search_traffic.py
+++ b/test/test_admin_search_traffic.py
@@ -232,43 +232,3 @@ def test_view_renders_when_redis_is_down(staff_client, monkeypatch):
def test_admin_index_links_to_the_traffic_page(staff_client):
assert TRAFFIC_URL in staff_client.get("/admin/").content.decode()
-
-
-def record_callers(redis_client, searches_by_caller, day=None):
- """Seed the caller keys the way record_request writes them, and the matching requests."""
- day = day or utc_today()
- for caller, searches in searches_by_caller.items():
- redis_client.pfadd(traffic.IDENTITY_DISTINCT_KEY.format(date=day), caller)
- redis_client.zincrby(traffic.IDENTITY_COUNT_KEY.format(date=day), searches, caller)
- record(redis_client, "v1_search", day=day, count=sum(searches_by_caller.values()))
-
-
-def test_callers_are_counted_and_their_share_is_of_all_searches(redis_client):
- record_callers(redis_client, {"instance": 90, "browser": 6, "phone": 4})
- # Served searches by callers the trim dropped, and traffic that is not a typed search.
- record(redis_client, "ui_search", count=100)
- record(redis_client, "v1_raw", count=1000)
- record(redis_client, "v1_search", status="4xx", count=1000)
-
- identities = admin_views._identities(traffic.get_redis(), *window(1))
-
- [today] = identities["rows"]
- assert today["searches"] == 200
- assert today["distinct"] == 3
- assert today["shares"] == [45, 50, 50, 50]
-
-
-def test_the_caller_window_stops_at_the_caller_expiry(redis_client):
- identities = admin_views._identities(traffic.get_redis(), *window(traffic.TRAFFIC_EXPIRE_SECONDS // 86400))
-
- assert len(identities["rows"]) == admin_views.IDENTITY_MAX_DAYS
-
-
-def test_view_renders_the_callers(staff_client, redis_client):
- record_callers(redis_client, {"instance": 1234})
-
- response = staff_client.get(TRAFFIC_URL)
-
- assert response.status_code == 200
- assert "Distinct callers" in response.content.decode()
- assert response.context["identities"]["rows"][0]["shares"][0] == 100
diff --git a/test/test_search_callers.py b/test/test_search_callers.py
new file mode 100644
index 0000000..2d7649b
--- /dev/null
+++ b/test/test_search_callers.py
@@ -0,0 +1,261 @@
+"""Tests for the weekly distinct-caller counts (mwmbl.search_callers) and their admin readout.
+
+Counting is off in mwmbl.settings_test, so these turn it back on and wire in a fakeredis,
+as test_traffic.py does.
+"""
+
+from datetime import date, timedelta
+
+import fakeredis
+import pytest
+from django.contrib.auth import get_user_model
+from django.http import HttpResponseServerError
+
+from mwmbl import admin_views, background, search_callers, traffic
+from mwmbl.utils import utc_today
+
+User = get_user_model()
+
+SEARCH = "/api/v1/search/"
+
+
+def counting(monkeypatch, settings):
+ settings.SEARCH_TRAFFIC_COUNTING = True
+ client = fakeredis.FakeStrictRedis(decode_responses=True)
+ monkeypatch.setattr(traffic, "_redis", client)
+ monkeypatch.setattr(search_callers, "_salts", {})
+ return client
+
+
+def staff_client(client, db):
+ user = User.objects.create_user(username="staff_member_1", password="correctpassword", is_staff=True)
+ client.force_login(user)
+ return client
+
+
+def this_window():
+ return search_callers.window_start(utc_today())
+
+
+def search(client, address, times=1):
+ for _ in range(times):
+ client.get(SEARCH, {"s": "python"}, REMOTE_ADDR=address)
+
+
+def caller_counts(redis_client):
+ key = search_callers.COUNT_KEY.format(window=this_window())
+ return sorted(int(count) for _, count in redis_client.zrange(key, 0, -1, withscores=True))
+
+
+def distinct_callers(redis_client):
+ return redis_client.pfcount(search_callers.DISTINCT_KEY.format(window=this_window()))
+
+
+def test_a_window_is_a_monday_to_sunday_week():
+ monday = date(2026, 10, 12)
+
+ assert [search_callers.window_start(monday + timedelta(days=offset)) for offset in range(8)] == [monday] * 7 + [
+ monday + timedelta(days=7)
+ ]
+
+
+def test_each_address_is_one_caller(counting, client):
+ search(client, "203.0.113.1", times=3)
+ search(client, "203.0.113.2")
+
+ assert distinct_callers(counting) == 2
+ assert caller_counts(counting) == [1, 3]
+
+
+def test_an_ipv6_caller_is_its_whole_64(counting, client):
+ search(client, "2001:db8:1:2::1")
+ search(client, "2001:db8:1:2:ffff::9")
+ search(client, "2001:db8:1:3::1")
+
+ assert caller_counts(counting) == [1, 2]
+
+
+ "headers,expected",
+ [
+ # Earlier X-Forwarded-For entries are whatever the client claimed.
+ ({"HTTP_X_FORWARDED_FOR": "10.0.0.1, 198.51.100.7", "REMOTE_ADDR": "172.17.0.1"}, "198.51.100.7"),
+ ({"REMOTE_ADDR": "::ffff:198.51.100.7"}, "198.51.100.7"),
+ ({"HTTP_X_FORWARDED_FOR": "unknown"}, "unknown"),
+ ],
+)
+def test_the_caller_is_the_address_our_nginx_saw(rf, headers, expected):
+ assert search_callers.client_network(rf.get("/", **headers)) == expected
+
+
+def test_a_request_with_no_address_is_still_counted(counting, rf):
+ """uvicorn on a unix socket gives Django no REMOTE_ADDR, and counting must not fail the search."""
+ request = rf.get(SEARCH, {"s": "python"})
+ del request.META["REMOTE_ADDR"]
+ request.resolver_match = type("Match", (), {"view_name": "api-v1:search"})()
+
+ traffic.record_request(request, type("Response", (), {"status_code": 200})())
+
+ assert distinct_callers(counting) == 1
+
+
+def test_no_address_is_stored_and_tokens_depend_on_the_window_salt(counting, client):
+ search(client, "203.0.113.1")
+
+ stored = counting.zrange(search_callers.COUNT_KEY.format(window=this_window()), 0, -1)
+ assert stored and all("203.0.113" not in token for token in stored)
+ salt = counting.get(search_callers.SALT_KEY.format(window=this_window()))
+ assert stored == [search_callers.caller_token(salt, "203.0.113.1")]
+ assert search_callers.caller_token("another week's salt", "203.0.113.1") != stored[0]
+
+
+def test_an_evicted_salt_is_put_back_rather_than_replaced(counting, client):
+ search(client, "203.0.113.1")
+ salt_key = search_callers.SALT_KEY.format(window=this_window())
+ salt = counting.get(salt_key)
+ counting.delete(salt_key)
+
+ search(client, "203.0.113.1")
+
+ assert counting.get(salt_key) == salt
+ assert caller_counts(counting) == [2]
+
+
+ "path,query",
+ [("/api/v1/search/complete", {"q": "pyth"}), ("/api/v1/search/raw", {"s": "python"}), ("/app/home/", {"q": "py"})],
+)
+def test_only_typed_searches_are_attributed_to_a_caller(counting, client, path, query):
+ client.get(path, query)
+
+ assert distinct_callers(counting) == 0
+
+
+def test_an_unserved_search_is_not_attributed_to_a_caller(counting, rf):
+ request = rf.get(SEARCH, {"s": "python"})
+ request.resolver_match = type("Match", (), {"view_name": "api-v1:search"})()
+
+ traffic.record_request(request, HttpResponseServerError())
+
+ assert distinct_callers(counting) == 0
+
+
+def test_the_per_caller_keys_die_a_day_after_the_week(counting, client):
+ search(client, "203.0.113.1", times=10)
+
+ expiry = search_callers._raw_expiry(this_window())
+ assert expiry.date() == this_window() + timedelta(days=8)
+ # Ten searches cross the first threshold only, so the higher thresholds' keys do not exist.
+ written = [key for key in search_callers._raw_keys(this_window()) if counting.exists(key)]
+ assert len(written) == 4
+ assert all(counting.expiretime(key) == int(expiry.timestamp()) for key in written)
+
+
+def test_callers_are_counted_as_they_cross_each_threshold(counting, client, monkeypatch):
+ monkeypatch.setattr(search_callers, "THRESHOLDS", [2, 3])
+ search(client, "203.0.113.1", times=3)
+ search(client, "203.0.113.2", times=2)
+ search(client, "203.0.113.3")
+
+ totals = search_callers.read_live(counting, this_window(), searches=6)
+
+ assert totals["over"] == [2, 1]
+
+
+def test_the_busiest_callers_survive_the_trim(counting, client, monkeypatch):
+ monkeypatch.setattr(search_callers, "TRACKED_CALLERS", 1)
+ monkeypatch.setattr(search_callers, "TRIM_PROBABILITY", 0)
+ search(client, "203.0.113.1", times=2)
+ search(client, "203.0.113.2")
+ monkeypatch.setattr(search_callers, "TRIM_PROBABILITY", 1)
+ search(client, "203.0.113.1")
+
+ assert caller_counts(counting) == [3]
+
+
+def test_the_shares_are_of_every_served_search(counting, client):
+ search(client, "203.0.113.1", times=3)
+ search(client, "203.0.113.2")
+ # A served search the trim would have dropped, and traffic that is not a typed search.
+ counting.incrby(
+ traffic.REQUEST_COUNT_KEY.format(date=utc_today(), endpoint="ui_search", headers="neither", status="2xx"), 4
+ )
+ client.get("/api/v1/search/raw", {"s": "python"})
+
+ searches = traffic.caller_window_searches(counting, this_window())
+ totals = search_callers.read_live(counting, this_window(), searches)
+
+ assert searches == 8
+ assert totals["busiest_searches"] == [3, 4, 4, 4]
+
+
+def last_window_with(redis_client, searches_by_caller):
+ """Seed last week's keys the way record_request would have written them."""
+ last_window = this_window() - timedelta(days=7)
+ for caller, searches in searches_by_caller.items():
+ redis_client.pfadd(search_callers.DISTINCT_KEY.format(window=last_window), caller)
+ redis_client.zincrby(search_callers.COUNT_KEY.format(window=last_window), searches, caller)
+ redis_client.set(search_callers.SALT_KEY.format(window=last_window), "salt")
+ redis_client.incrby(
+ traffic.REQUEST_COUNT_KEY.format(date=last_window, endpoint="v1_search", headers="neither", status="2xx"),
+ sum(searches_by_caller.values()),
+ )
+ return last_window
+
+
+def test_a_finished_week_is_replaced_by_its_totals(counting):
+ last_window = last_window_with(counting, {"instance": 90, "browser": 10})
+
+ background.summarise_search_callers.now()
+
+ assert all(not counting.exists(key) for key in search_callers._raw_keys(last_window))
+ assert search_callers.read_summary(counting, last_window) == {
+ "searches": 100,
+ "distinct": 2,
+ "busiest_searches": [90, 100, 100, 100],
+ "over": [0, 0, 0],
+ }
+ assert 0 < counting.ttl(search_callers.SUMMARY_KEY.format(window=last_window))
+
+
+def test_a_late_search_does_not_replace_the_summary(counting):
+ last_window = last_window_with(counting, {"instance": 90})
+ traffic.summarise_last_caller_window(counting)
+ counting.pfadd(search_callers.DISTINCT_KEY.format(window=last_window), "straggler")
+
+ assert traffic.summarise_last_caller_window(counting) is False
+ assert search_callers.read_summary(counting, last_window)["distinct"] == 1
+
+
+def test_this_week_is_not_summarised(counting, client):
+ search(client, "203.0.113.1")
+
+ assert traffic.summarise_last_caller_window(counting) is False
+ assert distinct_callers(counting) == 1
+
+
+def test_the_page_shows_this_week_live_and_finished_weeks_from_their_summaries(staff_client, counting, client):
+ last_window = last_window_with(counting, {"instance": 90, "browser": 10})
+ traffic.summarise_last_caller_window(counting)
+ search(client, "203.0.113.1", times=2)
+
+ response = staff_client.get("/admin/search-traffic/")
+
+ assert response.status_code == 200
+ assert "Distinct callers per week" in response.content.decode()
+ rows = response.context["callers"]["rows"]
+ assert [(row["window"], row["summarised"], row["distinct"]) for row in rows] == [
+ (this_window(), False, 1),
+ (last_window, True, 2),
+ ]
+ assert rows[0]["shares"][0] == 100
+ assert rows[1]["shares"][0] == 90
+
+
+def test_weeks_with_no_callers_are_left_out(staff_client, counting):
+ response = staff_client.get("/admin/search-traffic/")
+
+ assert response.context["callers"]["rows"] == []
+ assert admin_views.CALLER_WINDOWS == 14
diff --git a/test/test_traffic.py b/test/test_traffic.py
index 72e76b0..a448918 100644
--- a/test/test_traffic.py
+++ b/test/test_traffic.py
@@ -332,105 +332,3 @@ def test_a_streaming_response_is_counted_without_being_consumed(counting, rf):
assert count_for("super_search") == "1"
assert b"".join(response.streaming_content) == b"ab"
-
-
-def identity_counts():
- key = traffic.IDENTITY_COUNT_KEY.format(date=utc_today())
- return sorted(int(count) for _, count in traffic.get_redis().zrange(key, 0, -1, withscores=True))
-
-
-def distinct_identities():
- return traffic.get_redis().pfcount(traffic.IDENTITY_DISTINCT_KEY.format(date=utc_today()))
-
-
-def test_each_address_is_one_caller(counting, client):
- for address, searches in [("203.0.113.1", 3), ("203.0.113.2", 1)]:
- for _ in range(searches):
- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR=address)
-
- assert distinct_identities() == 2
- assert identity_counts() == [1, 3]
-
-
-def test_an_ipv6_caller_is_its_whole_64(counting, client):
- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="2001:db8:1:2::1")
- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="2001:db8:1:2:ffff::9")
- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="2001:db8:1:3::1")
-
- assert identity_counts() == [1, 2]
-
-
-def test_the_caller_is_the_address_our_nginx_saw(rf):
- """Earlier X-Forwarded-For entries are whatever the client claimed."""
- request = rf.get("/", HTTP_X_FORWARDED_FOR="10.0.0.1, 198.51.100.7", REMOTE_ADDR="172.17.0.1")
-
- assert traffic.client_network(request) == "198.51.100.7"
-
-
-def test_an_unparseable_address_is_still_a_caller(rf):
- request = rf.get("/", HTTP_X_FORWARDED_FOR="unknown")
-
- assert traffic.client_network(request) == "unknown"
-
-
-def test_no_address_is_stored_and_tokens_change_daily(counting, client, rf):
- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.1")
-
- stored = [
- str(value)
- for key in counting.keys("traffic:identity*")
- if counting.type(key) == "zset"
- for value in counting.zrange(key, 0, -1)
- ]
- assert stored and all("203.0.113" not in value for value in stored)
- request = rf.get("/", REMOTE_ADDR="203.0.113.1")
- yesterday = utc_today() - timedelta(days=1)
- assert traffic.client_identity(request, utc_today()) != traffic.client_identity(request, yesterday)
-
-
- "path,query,endpoint",
- [
- ("/api/v1/search/complete", {"q": "pyth"}, "v1_complete"),
- ("/api/v1/search/raw", {"s": "python"}, "v1_raw"),
- ("/app/home/", {"q": "py"}, "ui_keystroke"),
- ],
-)
-def test_only_typed_searches_are_attributed_to_a_caller(counting, client, path, query, endpoint):
- client.get(path, query, **BROWSER_HEADERS)
-
- assert count_for(endpoint) == "1"
- assert distinct_identities() == 0
-
-
-def test_an_unserved_search_is_not_attributed_to_a_caller(counting, rf):
- request = rf.get("/api/v1/search/", {"s": "python"})
- request.resolver_match = type("Match", (), {"view_name": "api-v1:search"})()
-
- traffic.record_request(request, HttpResponseServerError())
-
- assert distinct_identities() == 0
-
-
-def test_the_caller_keys_expire_and_are_trimmed(counting, client, monkeypatch):
- monkeypatch.setattr(traffic, "TRACKED_IDENTITIES", 1)
- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.1")
- monkeypatch.setattr(traffic, "IDENTITY_TRIM_PROBABILITY", 1)
- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.1")
- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.2")
-
- assert identity_counts() == [2]
- for key in [traffic.IDENTITY_DISTINCT_KEY, traffic.IDENTITY_COUNT_KEY]:
- assert 0 < counting.ttl(key.format(date=utc_today())) <= traffic.IDENTITY_EXPIRE_SECONDS
-
-
-def test_the_readout_gives_distinct_callers_and_counts_per_day(counting, client):
- yesterday = utc_today() - timedelta(days=1)
- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.1")
- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.1")
- client.get("/api/v1/search/", {"s": "python"}, REMOTE_ADDR="203.0.113.2")
-
- assert traffic.read_identity_counts(counting, [utc_today(), yesterday]) == {
- utc_today(): (2, [2, 1]),
- yesterday: (0, []),
- }