Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use collections.deque instead of list to store methods

Eugenio Lacuestaf97e3e90f25c
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index c3b23e284..f5e2fca63 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -26,9 +26,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
if hasattr(mw, 'process_request'):
self.methods['process_request'].append(mw.process_request)
if hasattr(mw, 'process_response'):
- self.methods['process_response'].insert(0, mw.process_response)
+ self.methods['process_response'].appendleft(mw.process_response)
if hasattr(mw, 'process_exception'):
- self.methods['process_exception'].insert(0, mw.process_exception)
+ self.methods['process_exception'].appendleft(mw.process_exception)
def download(self, download_func, request, spider):
@defer.inlineCallbacks
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index a206e4b0c..16b8435ab 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -25,11 +25,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
if hasattr(mw, 'process_spider_input'):
self.methods['process_spider_input'].append(mw.process_spider_input)
if hasattr(mw, 'process_spider_output'):
- self.methods['process_spider_output'].insert(0, mw.process_spider_output)
+ self.methods['process_spider_output'].appendleft(mw.process_spider_output)
if hasattr(mw, 'process_spider_exception'):
- self.methods['process_spider_exception'].insert(0, mw.process_spider_exception)
+ self.methods['process_spider_exception'].appendleft(mw.process_spider_exception)
if hasattr(mw, 'process_start_requests'):
- self.methods['process_start_requests'].insert(0, mw.process_start_requests)
+ self.methods['process_start_requests'].appendleft(mw.process_start_requests)
def scrape_response(self, scrape_func, response, request, spider):
fname = lambda f:'%s.%s' % (
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index f2240984c..1cfd8a782 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,4 +1,4 @@
-from collections import defaultdict
+from collections import defaultdict, deque
import logging
import pprint
@@ -16,7 +16,7 @@ class MiddlewareManager(object):
def __init__(self, *middlewares):
self.middlewares = middlewares
- self.methods = defaultdict(list)
+ self.methods = defaultdict(deque)
for mw in middlewares:
self._add_middleware(mw)
@@ -56,7 +56,7 @@ class MiddlewareManager(object):
if hasattr(mw, 'open_spider'):
self.methods['open_spider'].append(mw.open_spider)
if hasattr(mw, 'close_spider'):
- self.methods['close_spider'].insert(0, mw.close_spider)
+ self.methods['close_spider'].appendleft(mw.close_spider)
def _process_parallel(self, methodname, obj, *args):
return process_parallel(self.methods[methodname], obj, *args)
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index b6d885330..aea0be825 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -60,9 +60,9 @@ class MiddlewareManagerTest(unittest.TestCase):
def test_init(self):
m1, m2, m3 = M1(), M2(), M3()
mwman = TestMiddlewareManager(m1, m2, m3)
- self.assertEqual(mwman.methods['open_spider'], [m1.open_spider, m2.open_spider])
- self.assertEqual(mwman.methods['close_spider'], [m2.close_spider, m1.close_spider])
- self.assertEqual(mwman.methods['process'], [m1.process, m3.process])
+ self.assertEqual(list(mwman.methods['open_spider']), [m1.open_spider, m2.open_spider])
+ self.assertEqual(list(mwman.methods['close_spider']), [m2.close_spider, m1.close_spider])
+ self.assertEqual(list(mwman.methods['process']), [m1.process, m3.process])
def test_methods(self):
mwman = TestMiddlewareManager(M1(), M2(), M3())