Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Count search requests and the user agents that send them

Daoud Clarkec047c36ce3b0
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index bb868a3..352e21f 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -52,6 +52,7 @@ INSTALLED_APPS = [
MIDDLEWARE = [
"debug_toolbar.middleware.DebugToolbarMiddleware",
"django.middleware.security.SecurityMiddleware",
+ "mwmbl.traffic_middleware.SearchTrafficMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
@@ -341,6 +342,10 @@ BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS = 600
# evicts them - #359 measured that as a net NDCG loss - and because a separate file never
# enters the candidate pool for anyone else's query.
EXTERNAL_CACHE_ENABLED = os.environ.get("EXTERNAL_CACHE_ENABLED", "true").lower() != "false"
+
+# Counting search traffic (mwmbl.traffic). Off in the test settings: it is the only thing in
+# the request path that reaches for Redis, and the CI test job runs none.
+SEARCH_TRAFFIC_COUNTING = os.environ.get("SEARCH_TRAFFIC_COUNTING", "true").lower() != "false"
# One TTL for every provider. Wikipedia articles move rarely, which is what justifies six
# months; a provider whose results turn over faster wants its own, and the place to add that
# is here, keyed by source.
diff --git a/mwmbl/settings_test.py b/mwmbl/settings_test.py
index 8cd5738..f60e6a4 100644
--- a/mwmbl/settings_test.py
+++ b/mwmbl/settings_test.py
@@ -14,6 +14,11 @@ NUM_PAGES = 10
EXTERNAL_CACHE_INDEX_NAME = "external-cache.tinysearch"
EXTERNAL_CACHE_NUM_PAGES = 10
+# The traffic counters are the only Redis touch on the request path, and every test that
+# drives the Django test client would hit them. Tests that exercise them turn this on with
+# override_settings and a fakeredis client of their own.
+SEARCH_TRAFFIC_COUNTING = False
+
# Use fakeredis for cache so tests don't need a real Redis server
CACHES = {
"default": {
diff --git a/mwmbl/traffic.py b/mwmbl/traffic.py
new file mode 100644
index 0000000..f03d926
--- /dev/null
+++ b/mwmbl/traffic.py
@@ -0,0 +1,214 @@
+"""Counting search traffic, and assuming nothing about it.
+
+Phase 0 of #410: before we can decide what an anonymous visitor is allowed to cost us, we
+need to know how much of the traffic is people. This counts; nothing acts on the answer and
+no request's behaviour changes.
+
+Deliberately raw. Classifying a request as a bot or a browser here would fix a taxonomy in
+the Redis keys before anyone has looked at the traffic, and a key dimension is not something
+you can change your mind about later. What is counted is what arrived: requests per
+endpoint, split by which of the two headers were present, and requests per verbatim user
+agent. Any rule about what a bot looks like then applies when the numbers are read, over
+strings we already have, and can be applied differently next week. #412 is where a
+classification earns its place, once there is data to design it against.
+"""
+
+import random
+from collections import Counter
+from datetime import date
+from logging import getLogger
+from typing import Optional
+
+import redis
+from django.conf import settings
+from django.http import HttpRequest
+from redis.exceptions import RedisError
+
+from mwmbl.utils import utc_today
+
+logger = getLogger(__name__)
+
+# nginx accepts an 8 KB header; this bounds what one request can make us store.
+MAX_USER_AGENT_LENGTH = 512
+
+# Endpoint labels. Keyed on resolver_match.view_name rather than request.path. The names are unique across
+# the two ninja APIs and the legacy one because each mounts under its own namespace, and
+# they are bounded by construction - a path-derived label would grow keys from a string the
+# caller chooses. Anything not in here is not search traffic and is not counted.
+ENDPOINT_LABELS = {
+ "search-0.1:search": "legacy_search",
+ "search-0.1:complete": "legacy_complete",
+ "search-0.1:raw": "legacy_raw",
+ "api-v1:search": "v1_search",
+ "api-v1:complete": "v1_complete",
+ "api-v1:raw": "v1_raw",
+ "api-v2:search": "v2_search",
+ "api-v2:complete": "v2_complete",
+ "api-v2:raw": "v2_raw",
+ "api-v2:super_search": "super_search",
+}
+
+# The Django UI is two views and three kinds of request, so it cannot be a straight lookup.
+UI_INDEX_VIEW_NAME = "index"
+UI_FRAGMENT_VIEW_NAME = "home"
+UI_SEARCH_LABEL = "ui_search"
+UI_KEYSTROKE_LABEL = "ui_keystroke"
+
+ALL_ENDPOINT_LABELS = sorted(set(ENDPOINT_LABELS.values()) | {UI_SEARCH_LABEL, UI_KEYSTROKE_LABEL})
+
+
+def _ui_endpoint_label(request: HttpRequest) -> Optional[str]:
+ """Which of the Django UI's three kinds of request this is.
+
+ Without the keystroke split the UI's search count is inflated roughly tenfold:
+ home_fragment is hit on every keystroke and only the debounced call carries external=1.
+ """
+ if not request.GET.get("q"):
+ return None
+ if request.resolver_match.view_name == UI_INDEX_VIEW_NAME:
+ return UI_SEARCH_LABEL
+ return UI_SEARCH_LABEL if request.GET.get("external") == "1" else UI_KEYSTROKE_LABEL
+
+
+def endpoint_label(request: HttpRequest) -> Optional[str]:
+ """The counter label for this request, or None if it is not search traffic.
+
+ resolver_match is populated by the time a response-phase middleware runs, but is None
+ for a 404 and for CommonMiddleware's APPEND_SLASH redirect, both of which this sits
+ outside of.
+ """
+ if request.resolver_match is None:
+ return None
+ view_name = request.resolver_match.view_name
+ if view_name in (UI_INDEX_VIEW_NAME, UI_FRAGMENT_VIEW_NAME):
+ return _ui_endpoint_label(request)
+ return ENDPOINT_LABELS.get(view_name)
+
+
+# The one split on the request counter, and a description rather than a judgement: which of
+# the two headers arrived, not what that means. A request with neither is what our own
+# server-side render sends today, and a missing Accept-Language is the cheapest hint that a
+# client is not a browser, since browsers send one and HTTP clients do not.
+HEADER_LABELS = ["user_agent+language", "user_agent", "language", "neither"]
+
+
+def header_label(request: HttpRequest) -> str:
+ has_user_agent = bool(request.headers.get("User-Agent"))
+ has_language = bool(request.headers.get("Accept-Language"))
+ if has_user_agent:
+ return "user_agent+language" if has_language else "user_agent"
+ return "language" if has_language else "neither"
+
+
+REQUEST_COUNT_KEY = "traffic:req:{date}:{endpoint}:{headers}"
+USER_AGENT_COUNT_KEY = "traffic:user-agents:{date}"
+
+# Thirty days, matching the public daily crawler counters in mwmbl/crawler/stats.py.
+# Production Redis runs allkeys-lru (see mwmbl/indexer/blacklist_snapshot.py), so a longer
+# retention would be a promise the eviction policy does not keep.
+TRAFFIC_EXPIRE_SECONDS = 60 * 60 * 24 * 30
+
+# Seven days for the user agents: they are here to be read and argued about, not to be a
+# permanent series, and a short window limits what a verbatim string can later be used for.
+USER_AGENT_EXPIRE_SECONDS = 60 * 60 * 24 * 7
+
+# How many user agents a day survives, which is both the memory bound on a sorted set whose
+# members the caller chooses and the privacy control. What it drops is the tail, and a user
+# agent seen once is both most of a browser fingerprint and a lone person; what survives is
+# by construction a client somebody runs at volume.
+TRACKED_USER_AGENTS = 2000
+
+# Trimming on one request in five hundred holds the set within a few hundred rows of the
+# limit. There is no counter shared between workers to do it on a schedule, and a ZCARD to
+# decide would cost a round trip on every request.
+USER_AGENT_TRIM_PROBABILITY = 1 / 500
+
+# A missing user agent is a real answer, so it gets a row rather than being dropped.
+MISSING_USER_AGENT = "(none)"
+
+# Aggressive by design. django-redis is configured with 5 s timeouts, which is fine for a
+# cache lookup a view is waiting on and far too slow for a counter: this runs on every
+# request, on the one thread asgiref gives the whole sync middleware chain.
+REDIS_CONNECT_TIMEOUT_SECONDS = 0.1
+REDIS_TIMEOUT_SECONDS = 0.25
+
+_redis: Optional[redis.Redis] = None
+
+
+def get_redis() -> redis.Redis:
+ global _redis
+ if _redis is None:
+ _redis = redis.from_url(
+ settings.REDIS_URL,
+ socket_connect_timeout=REDIS_CONNECT_TIMEOUT_SECONDS,
+ socket_timeout=REDIS_TIMEOUT_SECONDS,
+ decode_responses=True,
+ )
+ return _redis
+
+
+def record_request(request: HttpRequest) -> None:
+ """Count one search request. Never raises: a counter that costs a search its results is
+ worse than no counter, which is the rule external_cache already follows."""
+ if not settings.SEARCH_TRAFFIC_COUNTING:
+ return
+
+ endpoint = endpoint_label(request)
+ if endpoint is None:
+ return
+
+ today = utc_today()
+ try:
+ pipeline = get_redis().pipeline()
+
+ request_key = REQUEST_COUNT_KEY.format(date=today, endpoint=endpoint, headers=header_label(request))
+ pipeline.incr(request_key)
+ pipeline.expire(request_key, TRAFFIC_EXPIRE_SECONDS)
+
+ # Its own key, not a dimension on the counter above: nothing joins a user agent to
+ # an address or a query, so no row says a particular person searched for something.
+ user_agent_key = USER_AGENT_COUNT_KEY.format(date=today)
+ user_agent = request.headers.get("User-Agent", "")[:MAX_USER_AGENT_LENGTH] or MISSING_USER_AGENT
+ pipeline.zincrby(user_agent_key, 1, user_agent)
+ pipeline.expire(user_agent_key, USER_AGENT_EXPIRE_SECONDS)
+ if random.random() < USER_AGENT_TRIM_PROBABILITY:
+ pipeline.zremrangebyrank(user_agent_key, 0, -TRACKED_USER_AGENTS - 1)
+
+ pipeline.execute()
+ except RedisError:
+ logger.warning("Could not record search traffic for %s", endpoint, exc_info=True)
+
+
+def read_request_counts(redis_client, days: list[date]) -> dict[tuple[date, str, str], int]:
+ """Request counts for the given days, keyed by (day, endpoint, header label).
+
+ One mget over the known labels rather than a scan: nothing indexes which traffic keys
+ exist, and SCAN on a shared Redis is not something a readout should do. Combinations
+ with no traffic are absent rather than zero.
+ """
+ coordinates = [
+ (day, endpoint, headers) for day in days for endpoint in ALL_ENDPOINT_LABELS for headers in HEADER_LABELS
+ ]
+ counts = redis_client.mget(
+ [
+ REQUEST_COUNT_KEY.format(date=day, endpoint=endpoint, headers=headers)
+ for day, endpoint, headers in coordinates
+ ]
+ )
+ return {coordinate: int(count) for coordinate, count in zip(coordinates, counts) if count}
+
+
+def read_user_agent_counts(redis_client, days: list[date], limit: int) -> list[tuple[str, int]]:
+ """The most-seen user agents over the window, most requests first.
+
+ Summed across days, so a client that appears every day outranks one that appeared once.
+ """
+ pipeline = redis_client.pipeline()
+ for day in days:
+ pipeline.zrevrange(USER_AGENT_COUNT_KEY.format(date=day), 0, TRACKED_USER_AGENTS, withscores=True)
+
+ totals: Counter = Counter()
+ for day_counts in pipeline.execute():
+ for user_agent, count in day_counts:
+ totals[user_agent] += int(count)
+ return totals.most_common(limit)
diff --git a/mwmbl/traffic_middleware.py b/mwmbl/traffic_middleware.py
new file mode 100644
index 0000000..63a6608
--- /dev/null
+++ b/mwmbl/traffic_middleware.py
@@ -0,0 +1,27 @@
+"""Count every search request as it leaves.
+
+Sync-only, and measured rather than assumed: the app runs under uvicorn workers, but
+allauth's AccountMiddleware is sync-only and Django adapts everything above a sync
+middleware to sync too, so an async branch here would never execute.
+test_the_chain_runs_sync_because_of_allauth pins that.
+
+Two things follow. The sync chain already runs off the event loop, on the thread asgiref
+hands it, so a blocking Redis call here cannot block the loop. And it is one thread for the
+whole chain, so the work stays a single pipeline with short socket timeouts.
+"""
+
+from mwmbl.traffic import record_request
+
+
+class SearchTrafficMiddleware:
+ def __init__(self, get_response):
+ self.get_response = get_response
+
+ def __call__(self, request):
+ response = self.get_response(request)
+ # After the view, so resolver_match is populated. For the Super Search SSE view this
+ # is the start of the stream rather than the end of it, which is the right moment to
+ # count a search - but it means the response has no body to look at yet, and reading
+ # response.content would raise. Nothing here touches it.
+ record_request(request)
+ return response
diff --git a/test/test_traffic.py b/test/test_traffic.py
new file mode 100644
index 0000000..b60e958
--- /dev/null
+++ b/test/test_traffic.py
@@ -0,0 +1,257 @@
+"""Tests for the search traffic counters and the middleware that writes them.
+
+Counting is off in mwmbl.settings_test, because it is the only thing in the request path
+that reaches for Redis and the CI job runs none. Everything here turns it back on and wires
+in a fakeredis, the way test_admin_blacklist_status.py does.
+"""
+
+from datetime import timedelta
+
+import fakeredis
+import pytest
+from asgiref.sync import iscoroutinefunction, markcoroutinefunction
+from django.conf import settings
+from django.core.handlers.base import BaseHandler
+from django.http import StreamingHttpResponse
+from django.test import override_settings
+from redis.exceptions import ConnectionError as RedisConnectionError
+
+from mwmbl import traffic
+from mwmbl.traffic_middleware import SearchTrafficMiddleware
+from mwmbl.utils import utc_today
+
+BROWSER_HEADERS = {
+ "HTTP_USER_AGENT": "Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0",
+ "HTTP_ACCEPT_LANGUAGE": "en-GB,en;q=0.9",
+}
+FIREFOX = BROWSER_HEADERS["HTTP_USER_AGENT"]
+
+_PROBE_HANDLER_WAS_ASYNC = []
+
+
+class ModeProbeMiddleware:
+ """Reports whether Django gave it an async handler, from where our middleware sits."""
+
+ sync_capable = True
+ async_capable = True
+
+ def __init__(self, get_response):
+ self.get_response = get_response
+ handler_is_async = iscoroutinefunction(get_response)
+ _PROBE_HANDLER_WAS_ASYNC.append(handler_is_async)
+ if handler_is_async:
+ markcoroutinefunction(self)
+
+ def __call__(self, request):
+ return self.get_response(request)
+
+
+def counting(monkeypatch, settings):
+ settings.SEARCH_TRAFFIC_COUNTING = True
+ client = fakeredis.FakeStrictRedis(decode_responses=True)
+ monkeypatch.setattr(traffic, "_redis", client)
+ return client
+
+
+def count_for(endpoint, headers="user_agent+language"):
+ key = traffic.REQUEST_COUNT_KEY.format(date=utc_today(), endpoint=endpoint, headers=headers)
+ return traffic.get_redis().get(key)
+
+
+def user_agent_counts():
+ key = traffic.USER_AGENT_COUNT_KEY.format(date=utc_today())
+ return dict(traffic.get_redis().zrange(key, 0, -1, withscores=True))
+
+
+def test_the_middleware_is_installed_after_the_security_middleware():
+ assert settings.MIDDLEWARE.index("mwmbl.traffic_middleware.SearchTrafficMiddleware") == (
+ settings.MIDDLEWARE.index("django.middleware.security.SecurityMiddleware") + 1
+ )
+
+
+def test_the_chain_runs_sync_because_of_allauth():
+ """Why SearchTrafficMiddleware has no async path.
+
+ allauth's AccountMiddleware is sync-only, so Django adapts every middleware above it to
+ sync even under ASGI. If this fails, allauth has gained async support, this middleware
+ is now the thing forcing the chain sync, and it should grow an async branch.
+ """
+ probe_position = settings.MIDDLEWARE.index("mwmbl.traffic_middleware.SearchTrafficMiddleware")
+ middleware = list(settings.MIDDLEWARE)
+ # By __name__: test/ has no __init__.py, so a dotted path would import a second copy of
+ # this module with its own module-level state, and the probe would record nothing.
+ middleware.insert(probe_position, f"{__name__}.ModeProbeMiddleware")
+
+ _PROBE_HANDLER_WAS_ASYNC.clear()
+ with override_settings(MIDDLEWARE=middleware):
+ BaseHandler().load_middleware(is_async=True)
+
+ assert _PROBE_HANDLER_WAS_ASYNC == [False]
+
+
+ "path,query,expected_endpoint",
+ [
+ ("/", {"q": "python"}, "ui_search"),
+ ("/app/home/", {"q": "pyth"}, "ui_keystroke"),
+ ("/app/home/", {"q": "python", "external": "1"}, "ui_search"),
+ ("/api/v1/search/", {"s": "python"}, "v1_search"),
+ ("/api/v2/search/", {"q": "python"}, "v2_search"),
+ ("/api/v2/search/complete", {"q": "pyth"}, "v2_complete"),
+ ("/search/", {"s": "python"}, "legacy_search"),
+ ],
+)
+def test_each_kind_of_search_lands_in_its_own_bucket(counting, client, path, query, expected_endpoint):
+ client.get(path, query, **BROWSER_HEADERS)
+
+ assert count_for(expected_endpoint) == "1"
+ others = [label for label in traffic.ALL_ENDPOINT_LABELS if label != expected_endpoint]
+ assert all(count_for(label) is None for label in others)
+
+
[email protected]("path,query", [("/", {}), ("/app/home/", {}), ("/api/v1/docs", {})])
+def test_requests_that_are_not_searches_are_not_counted(counting, client, path, query):
+ client.get(path, query, **BROWSER_HEADERS)
+
+ assert counting.keys("traffic:*") == []
+
+
+def test_every_search_route_has_a_label():
+ """A renamed route must fail here rather than going silently uncounted."""
+ from django.urls import get_resolver
+
+ def view_names(resolver, namespace=None):
+ for pattern in resolver.url_patterns:
+ if hasattr(pattern, "url_patterns"):
+ yield from view_names(pattern, pattern.namespace or namespace)
+ elif pattern.name:
+ yield f"{namespace}:{pattern.name}" if namespace else pattern.name
+
+ searchy = {
+ name
+ for name in view_names(get_resolver())
+ if name.split(":")[-1] in {"search", "complete", "raw", "super_search"}
+ or name in {traffic.UI_INDEX_VIEW_NAME, traffic.UI_FRAGMENT_VIEW_NAME}
+ }
+ labelled = set(traffic.ENDPOINT_LABELS) | {traffic.UI_INDEX_VIEW_NAME, traffic.UI_FRAGMENT_VIEW_NAME}
+
+ assert searchy - labelled == set()
+
+
+ "headers,expected",
+ [
+ ({"HTTP_USER_AGENT": FIREFOX, "HTTP_ACCEPT_LANGUAGE": "en-GB"}, "user_agent+language"),
+ ({"HTTP_USER_AGENT": FIREFOX}, "user_agent"),
+ ({"HTTP_ACCEPT_LANGUAGE": "en-GB"}, "language"),
+ ({}, "neither"),
+ ],
+)
+def test_the_headers_that_arrived_are_counted_and_nothing_is_inferred_from_them(counting, client, headers, expected):
+ client.get("/api/v1/search/", {"s": "python"}, **headers)
+
+ assert count_for("v1_search", expected) == "1"
+
+
+def test_the_counters_expire(counting, client):
+ client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)
+
+ requests_key = traffic.REQUEST_COUNT_KEY.format(
+ date=utc_today(), endpoint="v1_search", headers="user_agent+language"
+ )
+ user_agents_key = traffic.USER_AGENT_COUNT_KEY.format(date=utc_today())
+ assert 0 < counting.ttl(requests_key) <= traffic.TRAFFIC_EXPIRE_SECONDS
+ assert 0 < counting.ttl(user_agents_key) <= traffic.USER_AGENT_EXPIRE_SECONDS
+
+
+def test_a_user_agent_is_counted_verbatim(counting, client):
+ client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)
+ client.get("/api/v1/search/", {"s": "python"}, HTTP_USER_AGENT="Googlebot/2.1")
+
+ assert user_agent_counts() == {FIREFOX: 1, "Googlebot/2.1": 1}
+
+
+ "sent,counted",
+ [
+ # A missing user agent is what the front end's own server-side render sends, so it
+ # is a number we want rather than a row to drop.
+ (None, traffic.MISSING_USER_AGENT),
+ ("A" * 2000, "A" * traffic.MAX_USER_AGENT_LENGTH),
+ ],
+)
+def test_an_awkward_user_agent_still_gets_a_row(counting, client, sent, counted):
+ headers = {"HTTP_USER_AGENT": sent} if sent else {}
+ client.get("/api/v1/search/", {"s": "python"}, **headers)
+
+ assert user_agent_counts() == {counted: 1}
+
+
+def test_the_busiest_user_agents_survive_the_trim(counting, client, monkeypatch):
+ """The bound on the key, and the reason the tail is not kept."""
+ monkeypatch.setattr(traffic, "TRACKED_USER_AGENTS", 2)
+ monkeypatch.setattr(traffic, "USER_AGENT_TRIM_PROBABILITY", 0)
+ for user_agent, requests in [("busiest", 3), ("second", 2), ("seen once", 1)]:
+ for _ in range(requests):
+ client.get("/api/v1/search/", {"s": "python"}, HTTP_USER_AGENT=user_agent)
+
+ monkeypatch.setattr(traffic, "USER_AGENT_TRIM_PROBABILITY", 1)
+ client.get("/api/v1/search/", {"s": "python"}, HTTP_USER_AGENT="busiest")
+
+ assert user_agent_counts() == {"busiest": 4, "second": 2}
+
+
+def test_the_readout_sums_user_agents_across_the_window(counting, client):
+ yesterday = utc_today() - timedelta(days=1)
+ counting.zincrby(traffic.USER_AGENT_COUNT_KEY.format(date=yesterday), 5, FIREFOX)
+ client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)
+
+ assert traffic.read_user_agent_counts(counting, [yesterday, utc_today()], 10) == [(FIREFOX, 6)]
+
+
+def test_the_readout_returns_a_count_per_day_endpoint_and_header_combination(counting, client):
+ client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)
+ client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)
+ client.get("/api/v2/search/", {"q": "python"})
+
+ counts = traffic.read_request_counts(counting, [utc_today()])
+
+ assert counts == {
+ (utc_today(), "v1_search", "user_agent+language"): 2,
+ (utc_today(), "v2_search", "neither"): 1,
+ }
+
+
+def test_the_readout_survives_a_day_with_no_traffic(counting):
+ days = [utc_today() - timedelta(days=offset) for offset in range(3)]
+
+ assert traffic.read_request_counts(counting, days) == {}
+ assert traffic.read_user_agent_counts(counting, days, 10) == []
+
+
+def test_a_dead_redis_does_not_change_the_response(client, monkeypatch, settings):
+ class DeadRedis:
+ def pipeline(self):
+ raise RedisConnectionError("Connection refused")
+
+ with_counting_off = client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)
+
+ settings.SEARCH_TRAFFIC_COUNTING = True
+ monkeypatch.setattr(traffic, "_redis", DeadRedis())
+ with_dead_redis = client.get("/api/v1/search/", {"s": "python"}, **BROWSER_HEADERS)
+
+ assert with_dead_redis.status_code == with_counting_off.status_code == 200
+ assert with_dead_redis.content == with_counting_off.content
+
+
+def test_a_streaming_response_is_counted_without_being_consumed(counting, rf):
+ """The Super Search view streams; touching response.content would raise on it."""
+ request = rf.get("/api/v2/super-search/", {"q": "python"}, **BROWSER_HEADERS)
+ request.resolver_match = type("Match", (), {"view_name": "api-v2:super_search"})()
+
+ response = SearchTrafficMiddleware(lambda _: StreamingHttpResponse(iter([b"a", b"b"])))(request)
+
+ assert count_for("super_search") == "1"
+ assert b"".join(response.streaming_content) == b"ab"