An open source, non-profit web search engine
Report errors the crawler loops catch to Sentry via the server
Claude5e0060e50ecf
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.pyindex e68b08b..1136b9b 100644--- a/mwmbl/crawl.py+++ b/mwmbl/crawl.py@@ -2,6 +2,7 @@ import argparseimport loggingimport osimport time+import tracebackfrom datetime import datetime, timedelta, timezonefrom multiprocessing import Processfrom pathlib import Path@@ -29,10 +30,11 @@ print("Mwmbl crawling statistics: https://mwmbl.org/stats")django.setup()-from mwmbl.crawler.batch import HashedBatch, Result, Results+from mwmbl.crawler.batch import CrawlerErrorReport, ErrorFrame, HashedBatch, Result, Resultsfrom mwmbl.crawler.device import get_device_namefrom mwmbl.crawler.env_vars import (CRAWL_DELAY_SECONDS,+ CRAWL_REPORT_ERRORS,CRAWL_SUBMIT_MODE,CRAWL_THREADS,CRAWLER_WORKERS,@@ -68,6 +70,10 @@ MIN_NEW_INDEX_ENTRIES = 1# again often enough that the cache grew by around 800MB a day until the disk filled.REMOTE_INDEX_CACHE_EXPIRY = timedelta(hours=1)+# The server indexes submitted results before it answers, so allow it well over a typical+# request's time - but not forever.+RESULTS_POST_TIMEOUT_SECONDS = 120+def _fetch_curated_domains() -> set[str]:global _curated_domains_cache, _curated_domains_fetched_at@@ -90,6 +96,58 @@ def _fetch_curated_domains() -> set[str]:return _curated_domains_cache+# How long to wait before reporting the same error from the same process again. The loops+# retry every ten seconds, so a persistent failure would otherwise send a report each time.+ERROR_REPORT_INTERVAL_SECONDS = 3600.0+_error_last_reported_at: dict[tuple[str, str, str, int | None], float] = {}+++def report_error(context: str, error: Exception):+ """+ Send an error the crawl loops caught to the Mwmbl server, which passes it on to Sentry.++ The loops catch everything and retry, which keeps a volunteer's crawler running but means+ a crawler failing on every pass looks, from our side, exactly like one that has stopped.+ Reporting goes through our own endpoint rather than straight to Sentry so that volunteers'+ machines only ever talk to Mwmbl, and so each report arrives tied to an API key.+ """+ if not CRAWL_REPORT_ERRORS or CRAWL_SUBMIT_MODE == SUBMIT_MODE_OFF:+ return++ frames = traceback.extract_tb(error.__traceback__)+ exception_type = f"{type(error).__module__}.{type(error).__qualname__}"+ innermost = frames[-1]+ error_key = (context, exception_type, innermost.filename, innermost.lineno)+ now = time.monotonic()+ last_reported_at = _error_last_reported_at.get(error_key)+ if last_reported_at is not None and now - last_reported_at < ERROR_REPORT_INTERVAL_SECONDS:+ return+ _error_last_reported_at[error_key] = now++ report = CrawlerErrorReport(+ context=context,+ exception_type=exception_type,+ message=str(error),+ frames=[+ ErrorFrame(filename=frame.filename, lineno=frame.lineno, function=frame.name, line=frame.line)+ for frame in frames+ ],+ crawler_version=CRAWLER_VERSION,+ device_name=get_device_name(data_path),+ )+ # Failing to report must not become a second error in the loop that caught the first.+ try:+ response = requests.post(+ f"{REMOTE_SERVER}/api/v1/crawler/errors",+ json=report.dict(),+ headers={"X-API-Key": MWMBL_API_KEY, "User-Agent": USER_AGENT},+ timeout=10,+ )+ response.raise_for_status()+ except requests.RequestException:+ logger.exception("Failed to report error to the Mwmbl server")++def count_new_index_entries(term: str, new_items: list[Document], remote_items: list[Document]) -> int:"""Estimate how many of our new local items the main index will actually keep for a term.@@ -312,7 +370,14 @@ class Crawler:if CRAWL_SUBMIT_MODE == SUBMIT_MODE_DRY_RUN:results_url += "?dry_run=true"logger.info(f"Posting {len(result_items)} results to {results_url}")- response = requests.post(results_url, json=results.dict(), headers={"X-API-Key": MWMBL_API_KEY})+ # A post with no timeout that never gets an answer hangs this loop for+ # good: no exception, so nothing logged or reported, and no more syncing.+ response = requests.post(+ results_url,+ json=results.dict(),+ headers={"X-API-Key": MWMBL_API_KEY},+ timeout=RESULTS_POST_TIMEOUT_SECONDS,+ )logger.info(f"Response: {response.text}")response.raise_for_status()@@ -349,6 +414,7 @@ class Crawler:self.process_batch()except Exception as err:logger.exception(f"Error processing batch: '{err}'")+ report_error("batch", err)time.sleep(10)def run_indexing_continuously(self):@@ -365,6 +431,7 @@ class Crawler:self.run_indexing()except Exception as err:logger.exception(f"Error running indexing: '{err}'")+ report_error("indexing", err)time.sleep(10)def run_once(self):diff --git a/mwmbl/crawler/README.md b/mwmbl/crawler/README.mdindex 6391018..5cdf420 100644--- a/mwmbl/crawler/README.md+++ b/mwmbl/crawler/README.md@@ -21,6 +21,9 @@ Batches crawled by different workers at the same time can share domains, since e### Rate Limiting Configuration- `CRAWL_DELAY_SECONDS` (default: 0.0) - Delay in seconds each crawl thread waits between the URLs it crawls. Includes 10% random fuzz (±10%) to avoid synchronized requests across workers. Set to 0 to disable delays.+### Error Reporting+- `CRAWL_REPORT_ERRORS` (default: true) - Send the errors the crawler catches and recovers from to the Mwmbl server (`POST /api/v1/crawler/errors`), so the maintainers can see when crawlers are failing. A report holds the exception, its traceback, the crawler version and the device name. Each distinct error is reported at most once an hour per process. Set to `false` to opt out. Nothing is reported when `CRAWL_SUBMIT_MODE` is `off`.+### Redis Configuration- `REDIS_URL` (default: 'redis://127.0.0.1:6379') - Redis connection URL for URL queues and statsdiff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex dbfc58a..fef988f 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -8,17 +8,20 @@ from uuid import uuid4import boto3import requests+import sentry_sdkfrom django.conf import settingsfrom ninja import NinjaAPI, Router, Schemafrom ninja.errors import HttpErrorfrom redis import Redisfrom mwmbl.crawler.batch import (+ CrawlerErrorReport,DatasetRequest,Error,HashedDataset,PostResultsResponse,Results,+ StatusResponse,)from mwmbl.crawler.stats import MwmblStats, StatsManagerfrom mwmbl.devices import record_device@@ -44,6 +47,8 @@ stats_manager = StatsManager(logger = getLogger(__name__)+INVALID_CRAWL_KEY_MESSAGE = "Invalid API key or insufficient scope (crawl scope required)."+# Module-level router used by the unified v1 APIrouter = Router(tags=["Crawler"])@@ -226,17 +231,9 @@ def _register_routes(r: Router | NinjaAPI):if not raw_key:return 401, {"message": "API key required. Pass it in the X-API-Key header."}- key_hash = hashlib.sha256(raw_key.encode()).hexdigest()- api_key = (- ApiKey.objects.filter(- key=key_hash,- scopes__contains=[ApiKey.Scope.CRAWL],- )- .select_related("user")- .first()- )+ api_key = _get_crawl_api_key(raw_key)if api_key is None:- return 401, {"message": "Invalid API key or insufficient scope (crawl scope required)."}+ return 401, {"message": INVALID_CRAWL_KEY_MESSAGE}now = datetime.now(timezone.utc)ApiKey.objects.filter(pk=api_key.pk).update(last_used=now)@@ -280,6 +277,28 @@ def _register_routes(r: Router | NinjaAPI):"url": f"{PUBLIC_URL_PREFIX}{filename}",}+ @r.post(+ "/errors",+ response={200: StatusResponse, 401: Error},+ summary="Report a crawler error",+ description=(+ "Report an exception a crawler caught and recovered from. The crawler loops retry "+ "after every error, so without this a crawler can fail on every pass - and stop "+ "submitting results - with nothing to show for it but its own logs. Requires a "+ "crawl-scoped API key in the `X-API-Key` request header."+ ),+ )+ def post_error(request, report: CrawlerErrorReport):+ api_key = _get_crawl_api_key(request.headers.get("X-API-Key", ""))+ if api_key is None:+ return 401, {"message": INVALID_CRAWL_KEY_MESSAGE}++ logger.error(+ f"Crawler error from user {api_key.user.id} ({report.context}): {report.exception_type}: {report.message}"+ )+ sentry_sdk.capture_event(_crawler_error_event(report, api_key.user.id))+ return {"status": "ok"}+@r.post("/dataset",summary="Submit Firefox extension dataset",@@ -331,6 +350,60 @@ def create_router(version: str) -> NinjaAPI:return api+def _get_crawl_api_key(raw_key: str) -> ApiKey | None:+ if not raw_key:+ return None+ key_hash = hashlib.sha256(raw_key.encode()).hexdigest()+ return (+ ApiKey.objects.filter(+ key=key_hash,+ scopes__contains=[ApiKey.Scope.CRAWL],+ )+ .select_related("user")+ .first()+ )+++def _crawler_error_event(report: CrawlerErrorReport, user_id: int) -> dict:+ """+ Build the Sentry event for a crawler's error as though Sentry had caught it itself.++ Sending the traceback as a structured exception rather than a message is what lets Sentry+ group reports by where they were raised, so a hundred crawlers hitting the same bug make+ one issue, not a hundred.+ """+ frames = [+ {+ "filename": frame.filename,+ "lineno": frame.lineno,+ "function": frame.function,+ "context_line": frame.line,+ }+ for frame in report.frames+ ]+ tags = {+ "source": "crawler",+ "crawler_context": report.context,+ "crawler_version": report.crawler_version,+ "crawler_device": report.device_name,+ }+ return {+ "level": "error",+ "logger": "mwmbl.crawler.remote",+ "exception": {+ "values": [+ {+ "type": report.exception_type,+ "value": report.message,+ "stacktrace": {"frames": frames},+ }+ ]+ },+ "tags": {tag: value for tag, value in tags.items() if value is not None},+ "user": {"id": str(user_id)},+ }++def _get_user_id_hash(batch: DatasetRequest):return hashlib.sha3_256(batch.user_id.encode("utf8")).hexdigest()diff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.pyindex acb7aed..473f4eb 100644--- a/mwmbl/crawler/batch.py+++ b/mwmbl/crawler/batch.py@@ -222,6 +222,36 @@ class PostResultsResponse(Schema):)+class ErrorFrame(Schema):+ """One frame of the traceback of an exception a crawler caught."""++ filename: str = Field(example="/venv/lib/python3.11/site-packages/mwmbl/crawl.py")+ lineno: Optional[int] = Field(example=312)+ function: str = Field(example="run_indexing")+ line: Optional[str] = Field(default=None, example="response.raise_for_status()")+++class CrawlerErrorReport(Schema):+ """An exception a crawler caught and recovered from, reported so it is not lost in local logs."""++ context: str = Field(+ description="Which crawler loop caught the exception.",+ example="indexing",+ )+ exception_type: str = Field(example="requests.exceptions.HTTPError")+ message: str = Field(example="502 Server Error: Bad Gateway")+ frames: list[ErrorFrame] = Field(+ description="The traceback, outermost frame first.",+ max_length=200,+ )+ crawler_version: Optional[str] = Field(default=None, example="1.2.0")+ device_name: Optional[str] = Field(default=None, example="crawler-3f7a1c9e")+++class StatusResponse(Schema):+ status: str = Field(example="ok")++class Error(Schema):"""A generic error response."""diff --git a/mwmbl/crawler/env_vars.py b/mwmbl/crawler/env_vars.pyindex f5adeda..173503f 100644--- a/mwmbl/crawler/env_vars.py+++ b/mwmbl/crawler/env_vars.py@@ -22,6 +22,11 @@ MWMBL_CONTACT_INFO = os.environ.get("MWMBL_CONTACT_INFO", "[email protected]# directory - see mwmbl.crawler.device. The machine hostname is deliberately never used.MWMBL_DEVICE_NAME = os.environ.get("MWMBL_DEVICE_NAME", "")+# Whether to send the errors the crawl loops catch to the Mwmbl server, so we can see when+# crawlers are failing. Reports carry the exception and traceback, the crawler version and+# the device name - nothing about the machine beyond the file paths in the traceback.+CRAWL_REPORT_ERRORS = os.environ.get("CRAWL_REPORT_ERRORS", "true").lower() == "true"+# Bounds for running the crawler somewhere that is not a volunteer's machine - CI, mainly.# The defaults are the unbounded behaviour a volunteer crawler has always had.diff --git a/test/test_crawler_error_reports.py b/test/test_crawler_error_reports.pynew file mode 100644index 0000000..f579c47--- /dev/null+++ b/test/test_crawler_error_reports.py@@ -0,0 +1,135 @@+"""+Crawler errors reach us: the crawl loops catch and retry every exception, so without a+report a crawler failing on every pass looks the same as one that has stopped.++Covers the crawler's report_error and POST /api/v1/crawler/errors, which passes reports on+to Sentry.+"""++from unittest.mock import patch++import pytest+import requests+from django.contrib.auth import get_user_model+from django.test import Client++from mwmbl import crawl+from mwmbl.models import ApiKey, generate_api_key++User = get_user_model()++REPORT = {+ "context": "indexing",+ "exception_type": "requests.exceptions.HTTPError",+ "message": "502 Server Error: Bad Gateway",+ "frames": [+ {"filename": "mwmbl/crawl.py", "lineno": 400, "function": "run_indexing_continuously", "line": None},+ {+ "filename": "mwmbl/crawl.py",+ "lineno": 312,+ "function": "run_indexing",+ "line": "response.raise_for_status()",+ },+ ],+ "crawler_version": "1.2.0",+ "device_name": "crawler-3f7a1c9e",+}+++def make_api_key(scope):+ user = User.objects.create_user(username=f"user-{scope}", email=f"{scope}@example.com", password="pass")+ raw_key, key_hash = generate_api_key()+ ApiKey.objects.create(user=user, key=key_hash, name="Test key", scopes=[scope])+ return user, raw_key+++def post_report(raw_key):+ return Client().post("/api/v1/crawler/errors", content_type="application/json", data=REPORT, HTTP_X_API_KEY=raw_key)+++def test_a_report_goes_to_sentry_as_an_exception_with_its_traceback():+ user, raw_key = make_api_key(ApiKey.Scope.CRAWL)++ with patch("mwmbl.crawler.app.sentry_sdk.capture_event") as capture_event:+ response = post_report(raw_key)++ assert response.status_code == 200+ event = capture_event.call_args.args[0]+ exception = event["exception"]["values"][0]+ assert exception["type"] == "requests.exceptions.HTTPError"+ assert exception["value"] == "502 Server Error: Bad Gateway"+ assert [frame["function"] for frame in exception["stacktrace"]["frames"]] == [+ "run_indexing_continuously",+ "run_indexing",+ ]+ assert event["tags"]["crawler_version"] == "1.2.0"+ assert event["user"] == {"id": str(user.id)}++[email protected]("scope", [ApiKey.Scope.SEARCH, None])+def test_a_report_needs_a_crawl_key(scope):+ raw_key = make_api_key(scope)[1] if scope else ""++ with patch("mwmbl.crawler.app.sentry_sdk.capture_event") as capture_event:+ response = post_report(raw_key)++ assert response.status_code == 401+ capture_event.assert_not_called()+++def raise_and_catch(error):+ try:+ raise error+ except Exception as caught:+ return caught+++def reporting(monkeypatch):+ monkeypatch.setattr(crawl, "_error_last_reported_at", {})+ monkeypatch.setattr(crawl, "CRAWL_REPORT_ERRORS", True)+ monkeypatch.setattr(crawl, "CRAWL_SUBMIT_MODE", crawl.SUBMIT_MODE_DRY_RUN)+ monkeypatch.setattr(crawl, "get_device_name", lambda data_path: "crawler-test")+ with patch("mwmbl.crawl.requests.post") as post:+ yield post+++def test_report_error_posts_the_traceback(reporting):+ crawl.report_error("indexing", raise_and_catch(ValueError("bad page")))++ report = reporting.call_args.kwargs["json"]+ assert reporting.call_args.args[0] == f"{crawl.REMOTE_SERVER}/api/v1/crawler/errors"+ assert report["exception_type"] == "builtins.ValueError"+ assert report["message"] == "bad page"+ assert report["frames"][-1]["function"] == "raise_and_catch"+ assert report["device_name"] == "crawler-test"+++def test_the_same_error_is_reported_once_an_hour_not_every_retry(reporting):+ def fail():+ return raise_and_catch(ValueError("same place"))++ for _ in range(3):+ crawl.report_error("indexing", fail())+ crawl.report_error("batch", fail())++ assert reporting.call_count == 2+++ "setting, value", [("CRAWL_REPORT_ERRORS", False), ("CRAWL_SUBMIT_MODE", crawl.SUBMIT_MODE_OFF)]+)+def test_nothing_is_reported_when_turned_off(reporting, monkeypatch, setting, value):+ monkeypatch.setattr(crawl, setting, value)++ crawl.report_error("indexing", raise_and_catch(ValueError("bad page")))++ reporting.assert_not_called()+++def test_a_failed_report_does_not_raise(reporting):+ reporting.side_effect = requests.ConnectionError("server down")++ crawl.report_error("indexing", raise_and_catch(ValueError("bad page")))