Scrapy, a fast high-level web crawling & scraping framework for Python.
Expose certificate for HTTPS responses (#4054)
elacuesta9d983c1b9962
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 672c0b3d6..f009facd6 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -609,7 +609,10 @@ Response objects:param request: the initial value of the :attr:`Response.request` attribute.This represents the :class:`Request` that generated this response.- :type request: :class:`Request` object+ :type request: scrapy.http.Request++ :param certificate: an object representing the server's SSL certificate.+ :type certificate: twisted.internet.ssl.Certificate.. attribute:: Response.url@@ -691,6 +694,13 @@ Response objectsthey're shown on the string representation of the Response (`__str__`method) which is used by the engine for logging.+ .. attribute:: Response.certificate++ A :class:`twisted.internet.ssl.Certificate` object representing+ the server's SSL certificate.++ Only populated for ``https`` responses, ``None`` otherwise.+.. method:: Response.copy()Returns a new Response which is a copy of this Response.diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 5a5f6cf0a..93951d3b5 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -3,11 +3,12 @@import loggingimport reimport warnings+from contextlib import suppressfrom io import BytesIOfrom time import timefrom urllib.parse import urldefrag-from twisted.internet import defer, protocol, reactor+from twisted.internet import defer, protocol, reactor, sslfrom twisted.internet.endpoints import TCP4ClientEndpointfrom twisted.internet.error import TimeoutErrorfrom twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI@@ -382,7 +383,7 @@ class ScrapyAgent(object):def _cb_bodyready(self, txresponse, request):# deliverBody hangs for responses without bodyif txresponse.length == 0:- return txresponse, b'', None+ return txresponse, b'', None, Nonemaxsize = request.meta.get('download_maxsize', self._maxsize)warnsize = request.meta.get('download_warnsize', self._warnsize)@@ -418,11 +419,12 @@ class ScrapyAgent(object):return ddef _cb_bodydone(self, result, request, url):- txresponse, body, flags = result+ txresponse, body, flags, certificate = resultstatus = int(txresponse.code)headers = Headers(txresponse.headers.getAllRawHeaders())respcls = responsetypes.from_args(headers=headers, url=url, body=body)- return respcls(url=url, status=status, headers=headers, body=body, flags=flags)+ return respcls(url=url, status=status, headers=headers, body=body,+ flags=flags, certificate=certificate)@implementer(IBodyProducer)@@ -456,6 +458,12 @@ class _ResponseReader(protocol.Protocol):self._fail_on_dataloss_warned = Falseself._reached_warnsize = Falseself._bytes_received = 0+ self._certificate = None++ def connectionMade(self):+ if self._certificate is None:+ with suppress(AttributeError):+ self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())def dataReceived(self, bodyBytes):# This maybe called several times after cancel was called with buffered data.@@ -488,16 +496,16 @@ class _ResponseReader(protocol.Protocol):body = self._bodybuf.getvalue()if reason.check(ResponseDone):- self._finished.callback((self._txresponse, body, None))+ self._finished.callback((self._txresponse, body, None, self._certificate))returnif reason.check(PotentialDataLoss):- self._finished.callback((self._txresponse, body, ['partial']))+ self._finished.callback((self._txresponse, body, ['partial'], self._certificate))returnif reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):if not self._fail_on_dataloss:- self._finished.callback((self._txresponse, body, ['dataloss']))+ self._finished.callback((self._txresponse, body, ['dataloss'], self._certificate))returnelif not self._fail_on_dataloss_warned:diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex f60d09608..119dd2f63 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -17,13 +17,14 @@ from scrapy.utils.trackref import object_refclass Response(object_ref):- def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None):+ def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, certificate=None):self.headers = Headers(headers or {})self.status = int(status)self._set_body(body)self._set_url(url)self.request = requestself.flags = [] if flags is None else list(flags)+ self.certificate = certificate@propertydef cb_kwargs(self):@@ -86,7 +87,7 @@ class Response(object_ref):"""Create a new Response with the same attributes except for thosegiven new values."""- for x in ['url', 'status', 'headers', 'body', 'request', 'flags']:+ for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'certificate']:kwargs.setdefault(x, getattr(self, x))cls = kwargs.pop('cls', self.__class__)return cls(*args, **kwargs)diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 64819acb6..bbe97d034 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -5,6 +5,7 @@ import sysfrom pytest import markfrom testfixtures import LogCapturefrom twisted.internet import defer+from twisted.internet.ssl import Certificatefrom twisted.trial.unittest import TestCasefrom scrapy import signals@@ -407,3 +408,31 @@ with multiples linesyield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)for req_id in range(3):self.assertIn("Got response 200, req_id %d" % req_id, str(log))++ @defer.inlineCallbacks+ def test_response_ssl_certificate_none(self):+ crawler = self.runner.create_crawler(SingleRequestSpider)+ url = self.mockserver.url("/echo?body=test", is_secure=False)+ yield crawler.crawl(seed=url, mockserver=self.mockserver)+ self.assertIsNone(crawler.spider.meta['responses'][0].certificate)++ @defer.inlineCallbacks+ def test_response_ssl_certificate(self):+ crawler = self.runner.create_crawler(SingleRequestSpider)+ url = self.mockserver.url("/echo?body=test", is_secure=True)+ yield crawler.crawl(seed=url, mockserver=self.mockserver)+ cert = crawler.spider.meta['responses'][0].certificate+ self.assertIsInstance(cert, Certificate)+ self.assertEqual(cert.getSubject().commonName, b"localhost")+ self.assertEqual(cert.getIssuer().commonName, b"localhost")++ @mark.xfail(reason="Responses with no body return early and contain no certificate")+ @defer.inlineCallbacks+ def test_response_ssl_certificate_empty_response(self):+ crawler = self.runner.create_crawler(SingleRequestSpider)+ url = self.mockserver.url("/status?n=200", is_secure=True)+ yield crawler.crawl(seed=url, mockserver=self.mockserver)+ cert = crawler.spider.meta['responses'][0].certificate+ self.assertIsInstance(cert, Certificate)+ self.assertEqual(cert.getSubject().commonName, b"localhost")+ self.assertEqual(cert.getIssuer().commonName, b"localhost")