Scrapy, a fast high-level web crawling & scraping framework for Python.
Use collections.deque instead of list to store methods
Eugenio Lacuestaf97e3e90f25c
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex c3b23e284..f5e2fca63 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -26,9 +26,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):if hasattr(mw, 'process_request'):self.methods['process_request'].append(mw.process_request)if hasattr(mw, 'process_response'):- self.methods['process_response'].insert(0, mw.process_response)+ self.methods['process_response'].appendleft(mw.process_response)if hasattr(mw, 'process_exception'):- self.methods['process_exception'].insert(0, mw.process_exception)+ self.methods['process_exception'].appendleft(mw.process_exception)def download(self, download_func, request, spider):@defer.inlineCallbacksdiff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex a206e4b0c..16b8435ab 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -25,11 +25,11 @@ class SpiderMiddlewareManager(MiddlewareManager):if hasattr(mw, 'process_spider_input'):self.methods['process_spider_input'].append(mw.process_spider_input)if hasattr(mw, 'process_spider_output'):- self.methods['process_spider_output'].insert(0, mw.process_spider_output)+ self.methods['process_spider_output'].appendleft(mw.process_spider_output)if hasattr(mw, 'process_spider_exception'):- self.methods['process_spider_exception'].insert(0, mw.process_spider_exception)+ self.methods['process_spider_exception'].appendleft(mw.process_spider_exception)if hasattr(mw, 'process_start_requests'):- self.methods['process_start_requests'].insert(0, mw.process_start_requests)+ self.methods['process_start_requests'].appendleft(mw.process_start_requests)def scrape_response(self, scrape_func, response, request, spider):fname = lambda f:'%s.%s' % (diff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex f2240984c..1cfd8a782 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -1,4 +1,4 @@-from collections import defaultdict+from collections import defaultdict, dequeimport loggingimport pprint@@ -16,7 +16,7 @@ class MiddlewareManager(object):def __init__(self, *middlewares):self.middlewares = middlewares- self.methods = defaultdict(list)+ self.methods = defaultdict(deque)for mw in middlewares:self._add_middleware(mw)@@ -56,7 +56,7 @@ class MiddlewareManager(object):if hasattr(mw, 'open_spider'):self.methods['open_spider'].append(mw.open_spider)if hasattr(mw, 'close_spider'):- self.methods['close_spider'].insert(0, mw.close_spider)+ self.methods['close_spider'].appendleft(mw.close_spider)def _process_parallel(self, methodname, obj, *args):return process_parallel(self.methods[methodname], obj, *args)diff --git a/tests/test_middleware.py b/tests/test_middleware.pyindex b6d885330..aea0be825 100644--- a/tests/test_middleware.py+++ b/tests/test_middleware.py@@ -60,9 +60,9 @@ class MiddlewareManagerTest(unittest.TestCase):def test_init(self):m1, m2, m3 = M1(), M2(), M3()mwman = TestMiddlewareManager(m1, m2, m3)- self.assertEqual(mwman.methods['open_spider'], [m1.open_spider, m2.open_spider])- self.assertEqual(mwman.methods['close_spider'], [m2.close_spider, m1.close_spider])- self.assertEqual(mwman.methods['process'], [m1.process, m3.process])+ self.assertEqual(list(mwman.methods['open_spider']), [m1.open_spider, m2.open_spider])+ self.assertEqual(list(mwman.methods['close_spider']), [m2.close_spider, m1.close_spider])+ self.assertEqual(list(mwman.methods['process']), [m1.process, m3.process])def test_methods(self):mwman = TestMiddlewareManager(M1(), M2(), M3())