Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix a memory leak on the Media Pipeline (Files and Images) (#3813)
Victor Torresf4f2b1695c4d
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 404bbf5bf..95dca9a3f 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -3,7 +3,7 @@ from __future__ import print_functionimport functoolsimport loggingfrom collections import defaultdict-from twisted.internet.defer import Deferred, DeferredList+from twisted.internet.defer import Deferred, DeferredList, _DefGen_Returnfrom twisted.python.failure import Failurefrom scrapy.settings import Settings@@ -139,6 +139,30 @@ class MediaPipeline(object):result.cleanFailure()result.frames = []result.stack = None++ # This code fixes a memory leak by avoiding to keep references to+ # the Request and Response objects on the Media Pipeline cache.+ #+ # Twisted inline callbacks pass return values using the function+ # twisted.internet.defer.returnValue, which encapsulates the return+ # value inside a _DefGen_Return base exception.+ #+ # What happens when the media_downloaded callback raises another+ # exception, for example a FileException('download-error') when+ # the Response status code is not 200 OK, is that it stores the+ # _DefGen_Return exception on the FileException context.+ #+ # To avoid keeping references to the Response and therefore Request+ # objects on the Media Pipeline cache, we should wipe the context of+ # the exception encapsulated by the Twisted Failure when its a+ # _DefGen_Return instance.+ #+ # This problem does not occur in Python 2.7 since we don't have+ # Exception Chaining (https://www.python.org/dev/peps/pep-3134/).+ context = getattr(result.value, '__context__', None)+ if isinstance(context, _DefGen_Return):+ setattr(result.value, '__context__', None)+info.downloading.remove(fp)info.downloaded[fp] = result # cache resultfor wad in info.waiting.pop(fp):diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 5f6a6d9e6..28e39cefa 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -1,15 +1,19 @@from __future__ import print_function++import sys+from testfixtures import LogCapturefrom twisted.trial import unittestfrom twisted.python.failure import Failurefrom twisted.internet import reactor-from twisted.internet.defer import Deferred, inlineCallbacks+from twisted.internet.defer import Deferred, inlineCallbacks, returnValuefrom scrapy.http import Request, Responsefrom scrapy.settings import Settingsfrom scrapy.spiders import Spiderfrom scrapy.utils.request import request_fingerprintfrom scrapy.pipelines.media import MediaPipeline+from scrapy.pipelines.files import FileExceptionfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.signal import disconnect_allfrom scrapy import signals@@ -90,6 +94,77 @@ class BaseMediaPipelineTestCase(unittest.TestCase):self.pipe._modify_media_request(request)assert request.meta == {'handle_httpstatus_all': True}+ def test_should_remove_req_res_references_before_caching_the_results(self):+ """Regression test case to prevent a memory leak in the Media Pipeline.++ The memory leak is triggered when an exception is raised when a Response+ scheduled by the Media Pipeline is being returned. For example, when a+ FileException('download-error') is raised because the Response status+ code is not 200 OK.++ It happens because we are keeping a reference to the Response object+ inside the FileException context. This is caused by the way Twisted+ return values from inline callbacks. It raises a custom exception+ encapsulating the original return value.++ The solution is to remove the exception context when this context is a+ _DefGen_Return instance, the BaseException used by Twisted to pass the+ returned value from those inline callbacks.++ Maybe there's a better and more reliable way to test the case described+ here, but it would be more complicated and involve running - or at least+ mocking - some async steps from the Media Pipeline. The current test+ case is simple and detects the problem very fast. On the other hand, it+ would not detect another kind of leak happening due to old object+ references being kept inside the Media Pipeline cache.++ This problem does not occur in Python 2.7 since we don't have Exception+ Chaining (https://www.python.org/dev/peps/pep-3134/).+ """+ # Create sample pair of Request and Response objects+ request = Request('http://url')+ response = Response('http://url', body=b'', request=request)++ # Simulate the Media Pipeline behavior to produce a Twisted Failure+ try:+ # Simulate a Twisted inline callback returning a Response+ # The returnValue method raises an exception encapsulating the value+ returnValue(response)+ except BaseException as exc:+ def_gen_return_exc = exc+ try:+ # Simulate the media_downloaded callback raising a FileException+ # This usually happens when the status code is not 200 OK+ raise FileException('download-error')+ except Exception as exc:+ file_exc = exc+ # Simulate Twisted capturing the FileException+ # It encapsulates the exception inside a Twisted Failure+ failure = Failure(file_exc)++ # The Failure should encapsulate a FileException ...+ self.assertEqual(failure.value, file_exc)+ # ... and if we're running on Python 3 ...+ if sys.version_info.major >= 3:+ # ... it should have the returnValue exception set as its context+ self.assertEqual(failure.value.__context__, def_gen_return_exc)++ # Let's calculate the request fingerprint and fake some runtime data...+ fp = request_fingerprint(request)+ info = self.pipe.spiderinfo+ info.downloading.add(fp)+ info.waiting[fp] = []++ # When calling the method that caches the Request's result ...+ self.pipe._cache_result_and_execute_waiters(failure, fp, info)+ # ... it should store the Twisted Failure ...+ self.assertEqual(info.downloaded[fp], failure)+ # ... encapsulating the original FileException ...+ self.assertEqual(info.downloaded[fp].value, file_exc)+ # ... but it should not store the returnValue exception on its context+ context = getattr(info.downloaded[fp].value, '__context__', None)+ self.assertIsNone(context)+class MockedMediaPipeline(MediaPipeline):