Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Report errors the crawler loops catch to Sentry via the server

Claude5e0060e50ecf
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.py
index e68b08b..1136b9b 100644
--- a/mwmbl/crawl.py
+++ b/mwmbl/crawl.py
@@ -2,6 +2,7 @@ import argparse
import logging
import os
import time
+import traceback
from datetime import datetime, timedelta, timezone
from multiprocessing import Process
from pathlib import Path
@@ -29,10 +30,11 @@ print("Mwmbl crawling statistics: https://mwmbl.org/stats")
django.setup()
-from mwmbl.crawler.batch import HashedBatch, Result, Results
+from mwmbl.crawler.batch import CrawlerErrorReport, ErrorFrame, HashedBatch, Result, Results
from mwmbl.crawler.device import get_device_name
from mwmbl.crawler.env_vars import (
CRAWL_DELAY_SECONDS,
+ CRAWL_REPORT_ERRORS,
CRAWL_SUBMIT_MODE,
CRAWL_THREADS,
CRAWLER_WORKERS,
@@ -68,6 +70,10 @@ MIN_NEW_INDEX_ENTRIES = 1
# again often enough that the cache grew by around 800MB a day until the disk filled.
REMOTE_INDEX_CACHE_EXPIRY = timedelta(hours=1)
+# The server indexes submitted results before it answers, so allow it well over a typical
+# request's time - but not forever.
+RESULTS_POST_TIMEOUT_SECONDS = 120
+
def _fetch_curated_domains() -> set[str]:
global _curated_domains_cache, _curated_domains_fetched_at
@@ -90,6 +96,58 @@ def _fetch_curated_domains() -> set[str]:
return _curated_domains_cache
+# How long to wait before reporting the same error from the same process again. The loops
+# retry every ten seconds, so a persistent failure would otherwise send a report each time.
+ERROR_REPORT_INTERVAL_SECONDS = 3600.0
+_error_last_reported_at: dict[tuple[str, str, str, int | None], float] = {}
+
+
+def report_error(context: str, error: Exception):
+ """
+ Send an error the crawl loops caught to the Mwmbl server, which passes it on to Sentry.
+
+ The loops catch everything and retry, which keeps a volunteer's crawler running but means
+ a crawler failing on every pass looks, from our side, exactly like one that has stopped.
+ Reporting goes through our own endpoint rather than straight to Sentry so that volunteers'
+ machines only ever talk to Mwmbl, and so each report arrives tied to an API key.
+ """
+ if not CRAWL_REPORT_ERRORS or CRAWL_SUBMIT_MODE == SUBMIT_MODE_OFF:
+ return
+
+ frames = traceback.extract_tb(error.__traceback__)
+ exception_type = f"{type(error).__module__}.{type(error).__qualname__}"
+ innermost = frames[-1]
+ error_key = (context, exception_type, innermost.filename, innermost.lineno)
+ now = time.monotonic()
+ last_reported_at = _error_last_reported_at.get(error_key)
+ if last_reported_at is not None and now - last_reported_at < ERROR_REPORT_INTERVAL_SECONDS:
+ return
+ _error_last_reported_at[error_key] = now
+
+ report = CrawlerErrorReport(
+ context=context,
+ exception_type=exception_type,
+ message=str(error),
+ frames=[
+ ErrorFrame(filename=frame.filename, lineno=frame.lineno, function=frame.name, line=frame.line)
+ for frame in frames
+ ],
+ crawler_version=CRAWLER_VERSION,
+ device_name=get_device_name(data_path),
+ )
+ # Failing to report must not become a second error in the loop that caught the first.
+ try:
+ response = requests.post(
+ f"{REMOTE_SERVER}/api/v1/crawler/errors",
+ json=report.dict(),
+ headers={"X-API-Key": MWMBL_API_KEY, "User-Agent": USER_AGENT},
+ timeout=10,
+ )
+ response.raise_for_status()
+ except requests.RequestException:
+ logger.exception("Failed to report error to the Mwmbl server")
+
+
def count_new_index_entries(term: str, new_items: list[Document], remote_items: list[Document]) -> int:
"""
Estimate how many of our new local items the main index will actually keep for a term.
@@ -312,7 +370,14 @@ class Crawler:
if CRAWL_SUBMIT_MODE == SUBMIT_MODE_DRY_RUN:
results_url += "?dry_run=true"
logger.info(f"Posting {len(result_items)} results to {results_url}")
- response = requests.post(results_url, json=results.dict(), headers={"X-API-Key": MWMBL_API_KEY})
+ # A post with no timeout that never gets an answer hangs this loop for
+ # good: no exception, so nothing logged or reported, and no more syncing.
+ response = requests.post(
+ results_url,
+ json=results.dict(),
+ headers={"X-API-Key": MWMBL_API_KEY},
+ timeout=RESULTS_POST_TIMEOUT_SECONDS,
+ )
logger.info(f"Response: {response.text}")
response.raise_for_status()
@@ -349,6 +414,7 @@ class Crawler:
self.process_batch()
except Exception as err:
logger.exception(f"Error processing batch: '{err}'")
+ report_error("batch", err)
time.sleep(10)
def run_indexing_continuously(self):
@@ -365,6 +431,7 @@ class Crawler:
self.run_indexing()
except Exception as err:
logger.exception(f"Error running indexing: '{err}'")
+ report_error("indexing", err)
time.sleep(10)
def run_once(self):
diff --git a/mwmbl/crawler/README.md b/mwmbl/crawler/README.md
index 6391018..5cdf420 100644
--- a/mwmbl/crawler/README.md
+++ b/mwmbl/crawler/README.md
@@ -21,6 +21,9 @@ Batches crawled by different workers at the same time can share domains, since e
### Rate Limiting Configuration
- `CRAWL_DELAY_SECONDS` (default: 0.0) - Delay in seconds each crawl thread waits between the URLs it crawls. Includes 10% random fuzz (±10%) to avoid synchronized requests across workers. Set to 0 to disable delays.
+### Error Reporting
+- `CRAWL_REPORT_ERRORS` (default: true) - Send the errors the crawler catches and recovers from to the Mwmbl server (`POST /api/v1/crawler/errors`), so the maintainers can see when crawlers are failing. A report holds the exception, its traceback, the crawler version and the device name. Each distinct error is reported at most once an hour per process. Set to `false` to opt out. Nothing is reported when `CRAWL_SUBMIT_MODE` is `off`.
+
### Redis Configuration
- `REDIS_URL` (default: 'redis://127.0.0.1:6379') - Redis connection URL for URL queues and stats
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index dbfc58a..fef988f 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -8,17 +8,20 @@ from uuid import uuid4
import boto3
import requests
+import sentry_sdk
from django.conf import settings
from ninja import NinjaAPI, Router, Schema
from ninja.errors import HttpError
from redis import Redis
from mwmbl.crawler.batch import (
+ CrawlerErrorReport,
DatasetRequest,
Error,
HashedDataset,
PostResultsResponse,
Results,
+ StatusResponse,
)
from mwmbl.crawler.stats import MwmblStats, StatsManager
from mwmbl.devices import record_device
@@ -44,6 +47,8 @@ stats_manager = StatsManager(
logger = getLogger(__name__)
+INVALID_CRAWL_KEY_MESSAGE = "Invalid API key or insufficient scope (crawl scope required)."
+
# Module-level router used by the unified v1 API
router = Router(tags=["Crawler"])
@@ -226,17 +231,9 @@ def _register_routes(r: Router | NinjaAPI):
if not raw_key:
return 401, {"message": "API key required. Pass it in the X-API-Key header."}
- key_hash = hashlib.sha256(raw_key.encode()).hexdigest()
- api_key = (
- ApiKey.objects.filter(
- key=key_hash,
- scopes__contains=[ApiKey.Scope.CRAWL],
- )
- .select_related("user")
- .first()
- )
+ api_key = _get_crawl_api_key(raw_key)
if api_key is None:
- return 401, {"message": "Invalid API key or insufficient scope (crawl scope required)."}
+ return 401, {"message": INVALID_CRAWL_KEY_MESSAGE}
now = datetime.now(timezone.utc)
ApiKey.objects.filter(pk=api_key.pk).update(last_used=now)
@@ -280,6 +277,28 @@ def _register_routes(r: Router | NinjaAPI):
"url": f"{PUBLIC_URL_PREFIX}{filename}",
}
+ @r.post(
+ "/errors",
+ response={200: StatusResponse, 401: Error},
+ summary="Report a crawler error",
+ description=(
+ "Report an exception a crawler caught and recovered from. The crawler loops retry "
+ "after every error, so without this a crawler can fail on every pass - and stop "
+ "submitting results - with nothing to show for it but its own logs. Requires a "
+ "crawl-scoped API key in the `X-API-Key` request header."
+ ),
+ )
+ def post_error(request, report: CrawlerErrorReport):
+ api_key = _get_crawl_api_key(request.headers.get("X-API-Key", ""))
+ if api_key is None:
+ return 401, {"message": INVALID_CRAWL_KEY_MESSAGE}
+
+ logger.error(
+ f"Crawler error from user {api_key.user.id} ({report.context}): {report.exception_type}: {report.message}"
+ )
+ sentry_sdk.capture_event(_crawler_error_event(report, api_key.user.id))
+ return {"status": "ok"}
+
@r.post(
"/dataset",
summary="Submit Firefox extension dataset",
@@ -331,6 +350,60 @@ def create_router(version: str) -> NinjaAPI:
return api
+def _get_crawl_api_key(raw_key: str) -> ApiKey | None:
+ if not raw_key:
+ return None
+ key_hash = hashlib.sha256(raw_key.encode()).hexdigest()
+ return (
+ ApiKey.objects.filter(
+ key=key_hash,
+ scopes__contains=[ApiKey.Scope.CRAWL],
+ )
+ .select_related("user")
+ .first()
+ )
+
+
+def _crawler_error_event(report: CrawlerErrorReport, user_id: int) -> dict:
+ """
+ Build the Sentry event for a crawler's error as though Sentry had caught it itself.
+
+ Sending the traceback as a structured exception rather than a message is what lets Sentry
+ group reports by where they were raised, so a hundred crawlers hitting the same bug make
+ one issue, not a hundred.
+ """
+ frames = [
+ {
+ "filename": frame.filename,
+ "lineno": frame.lineno,
+ "function": frame.function,
+ "context_line": frame.line,
+ }
+ for frame in report.frames
+ ]
+ tags = {
+ "source": "crawler",
+ "crawler_context": report.context,
+ "crawler_version": report.crawler_version,
+ "crawler_device": report.device_name,
+ }
+ return {
+ "level": "error",
+ "logger": "mwmbl.crawler.remote",
+ "exception": {
+ "values": [
+ {
+ "type": report.exception_type,
+ "value": report.message,
+ "stacktrace": {"frames": frames},
+ }
+ ]
+ },
+ "tags": {tag: value for tag, value in tags.items() if value is not None},
+ "user": {"id": str(user_id)},
+ }
+
+
def _get_user_id_hash(batch: DatasetRequest):
return hashlib.sha3_256(batch.user_id.encode("utf8")).hexdigest()
diff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.py
index acb7aed..473f4eb 100644
--- a/mwmbl/crawler/batch.py
+++ b/mwmbl/crawler/batch.py
@@ -222,6 +222,36 @@ class PostResultsResponse(Schema):
)
+class ErrorFrame(Schema):
+ """One frame of the traceback of an exception a crawler caught."""
+
+ filename: str = Field(example="/venv/lib/python3.11/site-packages/mwmbl/crawl.py")
+ lineno: Optional[int] = Field(example=312)
+ function: str = Field(example="run_indexing")
+ line: Optional[str] = Field(default=None, example="response.raise_for_status()")
+
+
+class CrawlerErrorReport(Schema):
+ """An exception a crawler caught and recovered from, reported so it is not lost in local logs."""
+
+ context: str = Field(
+ description="Which crawler loop caught the exception.",
+ example="indexing",
+ )
+ exception_type: str = Field(example="requests.exceptions.HTTPError")
+ message: str = Field(example="502 Server Error: Bad Gateway")
+ frames: list[ErrorFrame] = Field(
+ description="The traceback, outermost frame first.",
+ max_length=200,
+ )
+ crawler_version: Optional[str] = Field(default=None, example="1.2.0")
+ device_name: Optional[str] = Field(default=None, example="crawler-3f7a1c9e")
+
+
+class StatusResponse(Schema):
+ status: str = Field(example="ok")
+
+
class Error(Schema):
"""A generic error response."""
diff --git a/mwmbl/crawler/env_vars.py b/mwmbl/crawler/env_vars.py
index f5adeda..173503f 100644
--- a/mwmbl/crawler/env_vars.py
+++ b/mwmbl/crawler/env_vars.py
@@ -22,6 +22,11 @@ MWMBL_CONTACT_INFO = os.environ.get("MWMBL_CONTACT_INFO", "[email protected]
# directory - see mwmbl.crawler.device. The machine hostname is deliberately never used.
MWMBL_DEVICE_NAME = os.environ.get("MWMBL_DEVICE_NAME", "")
+# Whether to send the errors the crawl loops catch to the Mwmbl server, so we can see when
+# crawlers are failing. Reports carry the exception and traceback, the crawler version and
+# the device name - nothing about the machine beyond the file paths in the traceback.
+CRAWL_REPORT_ERRORS = os.environ.get("CRAWL_REPORT_ERRORS", "true").lower() == "true"
+
# Bounds for running the crawler somewhere that is not a volunteer's machine - CI, mainly.
# The defaults are the unbounded behaviour a volunteer crawler has always had.
diff --git a/test/test_crawler_error_reports.py b/test/test_crawler_error_reports.py
new file mode 100644
index 0000000..f579c47
--- /dev/null
+++ b/test/test_crawler_error_reports.py
@@ -0,0 +1,135 @@
+"""
+Crawler errors reach us: the crawl loops catch and retry every exception, so without a
+report a crawler failing on every pass looks the same as one that has stopped.
+
+Covers the crawler's report_error and POST /api/v1/crawler/errors, which passes reports on
+to Sentry.
+"""
+
+from unittest.mock import patch
+
+import pytest
+import requests
+from django.contrib.auth import get_user_model
+from django.test import Client
+
+from mwmbl import crawl
+from mwmbl.models import ApiKey, generate_api_key
+
+User = get_user_model()
+
+REPORT = {
+ "context": "indexing",
+ "exception_type": "requests.exceptions.HTTPError",
+ "message": "502 Server Error: Bad Gateway",
+ "frames": [
+ {"filename": "mwmbl/crawl.py", "lineno": 400, "function": "run_indexing_continuously", "line": None},
+ {
+ "filename": "mwmbl/crawl.py",
+ "lineno": 312,
+ "function": "run_indexing",
+ "line": "response.raise_for_status()",
+ },
+ ],
+ "crawler_version": "1.2.0",
+ "device_name": "crawler-3f7a1c9e",
+}
+
+
+def make_api_key(scope):
+ user = User.objects.create_user(username=f"user-{scope}", email=f"{scope}@example.com", password="pass")
+ raw_key, key_hash = generate_api_key()
+ ApiKey.objects.create(user=user, key=key_hash, name="Test key", scopes=[scope])
+ return user, raw_key
+
+
+def post_report(raw_key):
+ return Client().post("/api/v1/crawler/errors", content_type="application/json", data=REPORT, HTTP_X_API_KEY=raw_key)
+
+
+def test_a_report_goes_to_sentry_as_an_exception_with_its_traceback():
+ user, raw_key = make_api_key(ApiKey.Scope.CRAWL)
+
+ with patch("mwmbl.crawler.app.sentry_sdk.capture_event") as capture_event:
+ response = post_report(raw_key)
+
+ assert response.status_code == 200
+ event = capture_event.call_args.args[0]
+ exception = event["exception"]["values"][0]
+ assert exception["type"] == "requests.exceptions.HTTPError"
+ assert exception["value"] == "502 Server Error: Bad Gateway"
+ assert [frame["function"] for frame in exception["stacktrace"]["frames"]] == [
+ "run_indexing_continuously",
+ "run_indexing",
+ ]
+ assert event["tags"]["crawler_version"] == "1.2.0"
+ assert event["user"] == {"id": str(user.id)}
+
+
[email protected]("scope", [ApiKey.Scope.SEARCH, None])
+def test_a_report_needs_a_crawl_key(scope):
+ raw_key = make_api_key(scope)[1] if scope else ""
+
+ with patch("mwmbl.crawler.app.sentry_sdk.capture_event") as capture_event:
+ response = post_report(raw_key)
+
+ assert response.status_code == 401
+ capture_event.assert_not_called()
+
+
+def raise_and_catch(error):
+ try:
+ raise error
+ except Exception as caught:
+ return caught
+
+
+def reporting(monkeypatch):
+ monkeypatch.setattr(crawl, "_error_last_reported_at", {})
+ monkeypatch.setattr(crawl, "CRAWL_REPORT_ERRORS", True)
+ monkeypatch.setattr(crawl, "CRAWL_SUBMIT_MODE", crawl.SUBMIT_MODE_DRY_RUN)
+ monkeypatch.setattr(crawl, "get_device_name", lambda data_path: "crawler-test")
+ with patch("mwmbl.crawl.requests.post") as post:
+ yield post
+
+
+def test_report_error_posts_the_traceback(reporting):
+ crawl.report_error("indexing", raise_and_catch(ValueError("bad page")))
+
+ report = reporting.call_args.kwargs["json"]
+ assert reporting.call_args.args[0] == f"{crawl.REMOTE_SERVER}/api/v1/crawler/errors"
+ assert report["exception_type"] == "builtins.ValueError"
+ assert report["message"] == "bad page"
+ assert report["frames"][-1]["function"] == "raise_and_catch"
+ assert report["device_name"] == "crawler-test"
+
+
+def test_the_same_error_is_reported_once_an_hour_not_every_retry(reporting):
+ def fail():
+ return raise_and_catch(ValueError("same place"))
+
+ for _ in range(3):
+ crawl.report_error("indexing", fail())
+ crawl.report_error("batch", fail())
+
+ assert reporting.call_count == 2
+
+
+ "setting, value", [("CRAWL_REPORT_ERRORS", False), ("CRAWL_SUBMIT_MODE", crawl.SUBMIT_MODE_OFF)]
+)
+def test_nothing_is_reported_when_turned_off(reporting, monkeypatch, setting, value):
+ monkeypatch.setattr(crawl, setting, value)
+
+ crawl.report_error("indexing", raise_and_catch(ValueError("bad page")))
+
+ reporting.assert_not_called()
+
+
+def test_a_failed_report_does_not_raise(reporting):
+ reporting.side_effect = requests.ConnectionError("server down")
+
+ crawl.report_error("indexing", raise_and_catch(ValueError("bad page")))