Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Stop a Redis that answers with an error killing the crawler

Daoud Clarkefb3050bbd65e
diff --git a/docker-compose.yml b/docker-compose.yml
index 63b8303..3016b11 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -41,7 +41,13 @@ services:
redis:
image: redis:alpine
restart: unless-stopped
- command: redis-server --maxmemory 2gb --maxmemory-policy allkeys-lru
+ # stop-writes-on-bgsave-error no: a failed snapshot otherwise makes Redis reject
+ # every write until someone intervenes, which stops the crawler dead. What is in
+ # here is a work queue that allkeys-lru is already free to evict, so a snapshot we
+ # could not write is worth losing, and carrying on is worth more than the snapshot.
+ command: >
+ redis-server --maxmemory 2gb --maxmemory-policy allkeys-lru
+ --stop-writes-on-bgsave-error no
volumes:
- redis_data:/data
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.py
index 6803be1..10b62b8 100644
--- a/mwmbl/crawl.py
+++ b/mwmbl/crawl.py
@@ -11,6 +11,7 @@ import django
import requests
from django.conf import settings
from redis import Redis
+from redis.exceptions import ConnectionError as RedisConnectionError
FORMAT = "%(process)d:%(levelname)s:%(name)s:%(message)s"
logging.basicConfig(level=logging.INFO, format=FORMAT)
@@ -170,11 +171,16 @@ class Crawler:
return self._url_queue
def check_redis(self):
- """Check Redis connection health."""
+ """Check Redis connection health.
+
+ redis.exceptions.ConnectionError is not the builtin of that name - it inherits from
+ RedisError - so catching only the builtin meant the friendly message never appeared
+ for the case it was written for: redis-py wraps a refused socket in its own class.
+ """
try:
self.redis.ping()
logger.debug("Redis ping successful")
- except ConnectionError:
+ except (ConnectionError, RedisConnectionError):
raise SystemExit(f"Cannot reach Redis at {self.redis_url}. Make sure your Redis server is running.")
def process_batch(self):
@@ -321,20 +327,35 @@ class Crawler:
logger.info(f"Page content: {new_page_content}")
def process_batch_continuously(self):
- """Continuously process batches with error handling."""
+ """Continuously process batches with error handling.
+
+ The health check runs inside the try for the reason everything else does: a Redis
+ that answers with an error is a condition to wait out, not to die of. Redis refusing
+ every write ("MISCONF ... unable to persist to disk", after a failed background save)
+ took out all four crawl workers and the indexing process this way, because
+ check_redis only converts a ConnectionError and let the rest through into a loop that
+ was not guarding it. An unreachable Redis still ends the process: check_redis raises
+ SystemExit for that, and SystemExit is not an Exception.
+ """
while True:
- self.check_redis()
try:
+ self.check_redis()
self.process_batch()
except Exception as err:
logger.exception(f"Error processing batch: '{err}'")
time.sleep(10)
def run_indexing_continuously(self):
- """Continuously run indexing with error handling."""
+ """Continuously run indexing with error handling.
+
+ The health check is inside the try - see process_batch_continuously. This loop is
+ the one that costs the whole crawler: run() gives up and exits once the indexing
+ process has died six times in an hour, so a Redis error escaping here stops
+ crawling, not just indexing.
+ """
while True:
- self.check_redis()
try:
+ self.check_redis()
self.run_indexing()
except Exception as err:
logger.exception(f"Error running indexing: '{err}'")
diff --git a/test/test_crawl_redis_errors.py b/test/test_crawl_redis_errors.py
new file mode 100644
index 0000000..4350abd
--- /dev/null
+++ b/test/test_crawl_redis_errors.py
@@ -0,0 +1,63 @@
+"""What a Redis that answers with an error costs the crawler.
+
+A Redis whose background save fails rejects every write with "MISCONF ... unable to
+persist to disk". check_redis converted a connection failure and let everything else
+through, and both continuous loops called it outside their try, so the error killed the
+process. run() exits the whole crawler once the indexing process has died six times in an
+hour, which is how one full disk on the Redis host stopped a crawler crawling.
+"""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+from redis.exceptions import ConnectionError as RedisConnectionError
+from redis.exceptions import ResponseError
+
+from mwmbl.crawl import Crawler
+
+MISCONF = ResponseError(
+ "MISCONF Redis is configured to save RDB snapshots, but it is currently unable to persist to disk."
+)
+
+
+class LoopStopped(BaseException):
+ """Breaks the while True once it has done what the test is watching for.
+
+ A BaseException so the loop's own `except Exception` cannot swallow it.
+ """
+
+
+def crawler_with_failing_redis(error):
+ crawler = Crawler()
+ crawler._redis = MagicMock()
+ crawler._redis.ping.side_effect = error
+ return crawler
+
+
+ "loop_name, work",
+ [("process_batch_continuously", "process_batch"), ("run_indexing_continuously", "run_indexing")],
+)
+def test_a_redis_error_is_waited_out_not_died_of(loop_name, work):
+ crawler = crawler_with_failing_redis(MISCONF)
+
+ with (
+ patch.object(Crawler, work) as do_work,
+ patch("mwmbl.crawl.time.sleep", side_effect=LoopStopped) as sleep,
+ ):
+ with pytest.raises(LoopStopped):
+ getattr(crawler, loop_name)()
+
+ # Reaching the sleep is the point: the loop caught the error and will try again.
+ sleep.assert_called_once_with(10)
+ do_work.assert_not_called()
+
+
[email protected]("loop_name", ["process_batch_continuously", "run_indexing_continuously"])
+def test_a_redis_that_cannot_be_reached_still_ends_the_process(loop_name):
+ """Retrying is for a Redis that answers. Nothing here can fix one that is not there."""
+ crawler = crawler_with_failing_redis(RedisConnectionError("Connection refused"))
+
+ with patch("mwmbl.crawl.time.sleep", side_effect=LoopStopped):
+ with pytest.raises(SystemExit):
+ getattr(crawler, loop_name)()