An open source, non-profit web search engine
Stop a Redis that answers with an error killing the crawler
Daoud Clarkefb3050bbd65e
diff --git a/docker-compose.yml b/docker-compose.ymlindex 63b8303..3016b11 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -41,7 +41,13 @@ services:redis:image: redis:alpinerestart: unless-stopped- command: redis-server --maxmemory 2gb --maxmemory-policy allkeys-lru+ # stop-writes-on-bgsave-error no: a failed snapshot otherwise makes Redis reject+ # every write until someone intervenes, which stops the crawler dead. What is in+ # here is a work queue that allkeys-lru is already free to evict, so a snapshot we+ # could not write is worth losing, and carrying on is worth more than the snapshot.+ command: >+ redis-server --maxmemory 2gb --maxmemory-policy allkeys-lru+ --stop-writes-on-bgsave-error novolumes:- redis_data:/datadiff --git a/mwmbl/crawl.py b/mwmbl/crawl.pyindex 6803be1..10b62b8 100644--- a/mwmbl/crawl.py+++ b/mwmbl/crawl.py@@ -11,6 +11,7 @@ import djangoimport requestsfrom django.conf import settingsfrom redis import Redis+from redis.exceptions import ConnectionError as RedisConnectionErrorFORMAT = "%(process)d:%(levelname)s:%(name)s:%(message)s"logging.basicConfig(level=logging.INFO, format=FORMAT)@@ -170,11 +171,16 @@ class Crawler:return self._url_queuedef check_redis(self):- """Check Redis connection health."""+ """Check Redis connection health.++ redis.exceptions.ConnectionError is not the builtin of that name - it inherits from+ RedisError - so catching only the builtin meant the friendly message never appeared+ for the case it was written for: redis-py wraps a refused socket in its own class.+ """try:self.redis.ping()logger.debug("Redis ping successful")- except ConnectionError:+ except (ConnectionError, RedisConnectionError):raise SystemExit(f"Cannot reach Redis at {self.redis_url}. Make sure your Redis server is running.")def process_batch(self):@@ -321,20 +327,35 @@ class Crawler:logger.info(f"Page content: {new_page_content}")def process_batch_continuously(self):- """Continuously process batches with error handling."""+ """Continuously process batches with error handling.++ The health check runs inside the try for the reason everything else does: a Redis+ that answers with an error is a condition to wait out, not to die of. Redis refusing+ every write ("MISCONF ... unable to persist to disk", after a failed background save)+ took out all four crawl workers and the indexing process this way, because+ check_redis only converts a ConnectionError and let the rest through into a loop that+ was not guarding it. An unreachable Redis still ends the process: check_redis raises+ SystemExit for that, and SystemExit is not an Exception.+ """while True:- self.check_redis()try:+ self.check_redis()self.process_batch()except Exception as err:logger.exception(f"Error processing batch: '{err}'")time.sleep(10)def run_indexing_continuously(self):- """Continuously run indexing with error handling."""+ """Continuously run indexing with error handling.++ The health check is inside the try - see process_batch_continuously. This loop is+ the one that costs the whole crawler: run() gives up and exits once the indexing+ process has died six times in an hour, so a Redis error escaping here stops+ crawling, not just indexing.+ """while True:- self.check_redis()try:+ self.check_redis()self.run_indexing()except Exception as err:logger.exception(f"Error running indexing: '{err}'")diff --git a/test/test_crawl_redis_errors.py b/test/test_crawl_redis_errors.pynew file mode 100644index 0000000..4350abd--- /dev/null+++ b/test/test_crawl_redis_errors.py@@ -0,0 +1,63 @@+"""What a Redis that answers with an error costs the crawler.++A Redis whose background save fails rejects every write with "MISCONF ... unable to+persist to disk". check_redis converted a connection failure and let everything else+through, and both continuous loops called it outside their try, so the error killed the+process. run() exits the whole crawler once the indexing process has died six times in an+hour, which is how one full disk on the Redis host stopped a crawler crawling.+"""++from unittest.mock import MagicMock, patch++import pytest+from redis.exceptions import ConnectionError as RedisConnectionError+from redis.exceptions import ResponseError++from mwmbl.crawl import Crawler++MISCONF = ResponseError(+ "MISCONF Redis is configured to save RDB snapshots, but it is currently unable to persist to disk."+)+++class LoopStopped(BaseException):+ """Breaks the while True once it has done what the test is watching for.++ A BaseException so the loop's own `except Exception` cannot swallow it.+ """+++def crawler_with_failing_redis(error):+ crawler = Crawler()+ crawler._redis = MagicMock()+ crawler._redis.ping.side_effect = error+ return crawler+++ "loop_name, work",+ [("process_batch_continuously", "process_batch"), ("run_indexing_continuously", "run_indexing")],+)+def test_a_redis_error_is_waited_out_not_died_of(loop_name, work):+ crawler = crawler_with_failing_redis(MISCONF)++ with (+ patch.object(Crawler, work) as do_work,+ patch("mwmbl.crawl.time.sleep", side_effect=LoopStopped) as sleep,+ ):+ with pytest.raises(LoopStopped):+ getattr(crawler, loop_name)()++ # Reaching the sleep is the point: the loop caught the error and will try again.+ sleep.assert_called_once_with(10)+ do_work.assert_not_called()++[email protected]("loop_name", ["process_batch_continuously", "run_indexing_continuously"])+def test_a_redis_that_cannot_be_reached_still_ends_the_process(loop_name):+ """Retrying is for a Redis that answers. Nothing here can fix one that is not there."""+ crawler = crawler_with_failing_redis(RedisConnectionError("Connection refused"))++ with patch("mwmbl.crawl.time.sleep", side_effect=LoopStopped):+ with pytest.raises(SystemExit):+ getattr(crawler, loop_name)()