Scrapy, a fast high-level web crawling & scraping framework for Python.
Add template for a downloader middleware
Paul Tremberthaaaa4da7a4e1
diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmplindex 292bf572e..1a4b0caa5 100644--- a/scrapy/templates/project/module/middlewares.py.tmpl+++ b/scrapy/templates/project/module/middlewares.py.tmpl@@ -54,3 +54,50 @@ class ${ProjectName}SpiderMiddleware(object):def spider_opened(self, spider):spider.logger.info('Spider opened: %s' % spider.name)+++class ${ProjectName}DownloaderMiddleware(object):+ # Not all methods need to be defined. If a method is not defined,+ # scrapy acts as if the downloader middleware does not modify the+ # passed objects.++ @classmethod+ def from_crawler(cls, crawler):+ # This method is used by Scrapy to create your spiders.+ s = cls()+ crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)+ return s++ def process_request(self, request, spider):+ # Called for each request that goes through the downloader+ # middleware.++ # Must either:+ # - return None: continue processing this request+ # - or return a Response object+ # - or return a Request object+ # - or raise IgnoreRequest: process_exception() methods of+ # installed downloader middleware will be called+ return None++ def process_response(self, request, response, spider):+ # Called with the response returned from the downloader.++ # Must either;+ # - return a Response object+ # - return a Request object+ # - or raise IgnoreRequest+ return response++ def process_exception(self, request, exception, spider):+ # Called when a download handler or a process_request()+ # (from other downloader middleware) raises an exception.++ # Must either:+ # - return None: continue processing this exception+ # - return a Response object: stops process_exception() chain+ # - return a Request object: stops process_exception() chain+ pass++ def spider_opened(self, spider):+ spider.logger.info('Spider opened: %s' % spider.name)diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmplindex 486df6b71..35a0f9a45 100644--- a/scrapy/templates/project/module/settings.py.tmpl+++ b/scrapy/templates/project/module/settings.py.tmpl@@ -53,7 +53,7 @@ ROBOTSTXT_OBEY = True# Enable or disable downloader middlewares# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#DOWNLOADER_MIDDLEWARES = {-# '$project_name.middlewares.MyCustomDownloaderMiddleware': 543,+# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,#}# Enable or disable extensions