Scrapy, a fast high-level web crawling & scraping framework for Python.
Get server IP address for HTTP/1.1 responses
Eugenio Lacuesta80925ab845b7
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 8997a7f19..a4cc1a7d7 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -34,7 +34,7 @@ Request objects:type url: string:param callback: the function that will be called with the response of this- request (once its downloaded) as its first parameter. For more information+ request (once it's downloaded) as its first parameter. For more informationsee :ref:`topics-request-response-ref-request-callback-arguments` below.If a Request doesn't specify a callback, the spider's:meth:`~scrapy.spiders.Spider.parse` method will be used.@@ -611,6 +611,12 @@ Response objectsThis represents the :class:`Request` that generated this response.:type request: :class:`Request` object+ :param ip_address: The IP address of the server from which the Response originated.+ :type ip_address: :class:`ipaddress.IPv4Address` object++ .. FIXME: Add ipaddress.IPv6Address once it's supported++.. attribute:: Response.urlA string containing the URL of the response.@@ -679,6 +685,10 @@ Response objectsthey're shown on the string representation of the Response (`__str__`method) which is used by the engine for logging.+ .. attribute:: Response.ip_address++ The IP address of the server from which the Response originated.+.. method:: Response.copy()Returns a new Response which is a copy of this Response.diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 157dc3418..11c9dd908 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -172,7 +172,7 @@ class Downloader(object):return responsedfd.addCallback(_downloaded)- # 3. After response arrives, remove the request from transferring+ # 3. After response arrives, remove the request from transferring# state to free up the transferring slot so it can be used by the# following requests (perhaps those which came from the downloader# middleware itself)diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 5a5f6cf0a..b690f439f 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -4,6 +4,7 @@ import loggingimport reimport warningsfrom io import BytesIO+from ipaddress import ip_addressfrom time import timefrom urllib.parse import urldefrag@@ -382,7 +383,7 @@ class ScrapyAgent(object):def _cb_bodyready(self, txresponse, request):# deliverBody hangs for responses without bodyif txresponse.length == 0:- return txresponse, b'', None+ return txresponse, b'', None, Nonemaxsize = request.meta.get('download_maxsize', self._maxsize)warnsize = request.meta.get('download_warnsize', self._warnsize)@@ -418,11 +419,11 @@ class ScrapyAgent(object):return ddef _cb_bodydone(self, result, request, url):- txresponse, body, flags = result+ txresponse, body, flags, ip_address = resultstatus = int(txresponse.code)headers = Headers(txresponse.headers.getAllRawHeaders())respcls = responsetypes.from_args(headers=headers, url=url, body=body)- return respcls(url=url, status=status, headers=headers, body=body, flags=flags)+ return respcls(url=url, status=status, headers=headers, body=body, flags=flags, ip_address=ip_address)@implementer(IBodyProducer)@@ -456,6 +457,11 @@ class _ResponseReader(protocol.Protocol):self._fail_on_dataloss_warned = Falseself._reached_warnsize = Falseself._bytes_received = 0+ self._ip_address = None++ def connectionMade(self):+ if self._ip_address is None:+ self._ip_address = ip_address(self.transport._producer.getPeer().host)def dataReceived(self, bodyBytes):# This maybe called several times after cancel was called with buffered data.@@ -488,16 +494,16 @@ class _ResponseReader(protocol.Protocol):body = self._bodybuf.getvalue()if reason.check(ResponseDone):- self._finished.callback((self._txresponse, body, None))+ self._finished.callback((self._txresponse, body, None, self._ip_address))returnif reason.check(PotentialDataLoss):- self._finished.callback((self._txresponse, body, ['partial']))+ self._finished.callback((self._txresponse, body, ['partial'], self._ip_address))returnif reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):if not self._fail_on_dataloss:- self._finished.callback((self._txresponse, body, ['dataloss']))+ self._finished.callback((self._txresponse, body, ['dataloss'], self._ip_address))returnelif not self._fail_on_dataloss_warned:diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex f92d0901c..ca5ecc02c 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -17,13 +17,14 @@ from scrapy.utils.trackref import object_refclass Response(object_ref):- def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None):+ def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, ip_address=None):self.headers = Headers(headers or {})self.status = int(status)self._set_body(body)self._set_url(url)self.request = requestself.flags = [] if flags is None else list(flags)+ self.ip_address = ip_address@propertydef meta(self):@@ -76,7 +77,7 @@ class Response(object_ref):"""Create a new Response with the same attributes except for thosegiven new values."""- for x in ['url', 'status', 'headers', 'body', 'request', 'flags']:+ for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'ip_address']:kwargs.setdefault(x, getattr(self, x))cls = kwargs.pop('cls', self.__class__)return cls(*args, **kwargs)diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex f433fcea6..6281160ae 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -1,5 +1,7 @@import jsonimport logging+from ipaddress import IPv4Address+from urllib.parse import urlparsefrom testfixtures import LogCapturefrom twisted.internet import defer@@ -308,3 +310,28 @@ with multiples linesself.assertIn("[callback] status 201", str(log))self.assertIn("[errback] status 404", str(log))self.assertIn("[errback] status 500", str(log))++ @defer.inlineCallbacks+ def test_dns_server_ip_address(self):+ from socket import gethostbyname++ crawler = self.runner.create_crawler(SingleRequestSpider)+ url = 'https://example.org'+ yield crawler.crawl(seed=url)+ ip_address = crawler.spider.meta['responses'][0].ip_address+ self.assertIsInstance(ip_address, IPv4Address)+ self.assertEqual(str(ip_address), gethostbyname(urlparse(url).netloc))++ crawler = self.runner.create_crawler(SingleRequestSpider)+ url = self.mockserver.url('/status?n=200')+ yield crawler.crawl(seed=url, mockserver=self.mockserver)+ ip_address = crawler.spider.meta['responses'][0].ip_address+ self.assertIsNone(ip_address)++ crawler = self.runner.create_crawler(SingleRequestSpider)+ url = self.mockserver.url('/echo?body=test')+ expected_netloc, _ = urlparse(url).netloc.split(':')+ yield crawler.crawl(seed=url, mockserver=self.mockserver)+ ip_address = crawler.spider.meta['responses'][0].ip_address+ self.assertIsInstance(ip_address, IPv4Address)+ self.assertEqual(str(ip_address), gethostbyname(expected_netloc))