Scrapy, a fast high-level web crawling & scraping framework for Python.
DOC Add info and example on errbacks
Paul Tremberthb3367c7acd8d
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 7b48c4d1c..c54685d9e 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -117,6 +117,8 @@ Request objectsraised while processing the request. This includes pages that failedwith 404 HTTP errors and such. It receives a `Twisted Failure`_ instanceas first parameter.+ For more information,+ see :ref:`topics-request-response-ref-errbacks` below.:type errback: callable.. attribute:: Request.url@@ -212,6 +214,69 @@ different fields from different pages::item['other_url'] = response.urlreturn item++.. _topics-request-response-ref-errbacks:++Using errbacks to catch exceptions in request processing+--------------------------------------------------------++The errback of a request is a function that will be called when an exception+is raise while processing it.++It receives a `Twisted Failure`_ instance as first parameter and can be+used to track connection establishment timeouts, DNS errors etc.++Here's an example spider logging all errors and catching some specific+errors if needed::++ import scrapy++ from scrapy.spidermiddlewares.httperror import HttpError+ from twisted.internet.error import DNSLookupError+ from twisted.internet.error import TimeoutError, TCPTimedOutError++ class ErrbackSpider(scrapy.Spider):+ name = "errback_example"+ start_urls = [+ "http://www.httpbin.org/", # HTTP 200 expected+ "http://www.httpbin.org/status/404", # Not found error+ "http://www.httpbin.org/status/500", # server issue+ "http://www.httpbin.org:12345/", # non-responding host, timeout expected+ "http://www.httphttpbinbin.org/", # DNS error expected+ ]++ def start_requests(self):+ for u in self.start_urls:+ yield scrapy.Request(u, callback=self.parse_httpbin,+ errback=self.errback_httpbin,+ dont_filter=True)++ def parse_httpbin(self, response):+ self.logger.info('Got successful response from {}'.format(response.url))+ # do something useful here...++ def errback_httpbin(self, failure):+ # log all failures+ self.logger.error(repr(failure))++ # in case you want to do something special for some errors,+ # you may need the failure's type:++ if failure.check(HttpError):+ # these exceptions come from HttpError spider middleware+ # you can get the non-200 response+ response = failure.value.response+ self.logger.error('HttpError on %s', response.url)++ elif failure.check(DNSLookupError):+ # this is the original request+ request = failure.request+ self.logger.error('DNSLookupError on %s', request.url)++ elif failure.check(TimeoutError, TCPTimedOutError):+ request = failure.request+ self.logger.error('TimeoutError on %s', request.url)+.. _topics-request-meta:Request.meta special keys