Scrapy, a fast high-level web crawling & scraping framework for Python.
test: warnsize logs, no content header, dataloss
Aditya90a7007f8818
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 0f3730c9e..0a2719d23 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -1,26 +1,26 @@-# TODO: Add test cases for-# 1. No Content Length response header-# 2. Cancel Response Deferredimport jsonimport osimport random+import reimport shutilimport string+from h2.exceptions import InvalidBodyLengthErrorfrom twisted.internet import reactor-from twisted.internet.defer import inlineCallbacks, DeferredList+from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledErrorfrom twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpointfrom twisted.internet.protocol import Factoryfrom twisted.internet.ssl import optionsForClientTLS, PrivateCertificate+from twisted.python.failure import Failurefrom twisted.trial.unittest import TestCasefrom twisted.web.http import Request as TxRequest-from twisted.web.resource import Resource-from twisted.web.server import Site+from twisted.web.server import Site, NOT_DONE_YETfrom twisted.web.static import Filefrom scrapy.core.http2.protocol import H2ClientProtocolfrom scrapy.http import Request, Response, JsonRequest-from tests.mockserver import ssl_context_factory+from scrapy.utils.python import to_bytes, to_unicode+from tests.mockserver import ssl_context_factory, LeafResourcedef generate_random_string(size):@@ -35,7 +35,7 @@ def make_html_body(val):<h1>Hello from HTTP2<h1><p>{}</p></html>'''.format(val)- return bytes(response, 'utf-8')+ return to_bytes(response)class Data:@@ -54,9 +54,8 @@ class Data:JSON_SMALL = {'data': STR_SMALL}JSON_LARGE = {'data': STR_LARGE}--class LeafResource(Resource):- isLeaf = True+ DATALOSS = b'Dataloss Content'+ NO_CONTENT_LENGTH = b'This response do not have any content-length header'class GetDataHtmlSmall(LeafResource):@@ -80,9 +79,9 @@ class PostDataJsonMixin:'extra-data': extra_data}for k, v in request.requestHeaders.getAllRawHeaders():- response['request-headers'][k.decode('utf-8')] = v[0].decode('utf-8')+ response['request-headers'][to_unicode(k)] = to_unicode(v[0])- response_bytes = bytes(json.dumps(response), 'utf-8')+ response_bytes = to_bytes(json.dumps(response))request.setHeader('Content-Type', 'application/json')return response_bytes@@ -97,6 +96,31 @@ class PostDataJsonLarge(LeafResource, PostDataJsonMixin):return self.make_response(request, Data.EXTRA_LARGE)+class Dataloss(LeafResource):++ def render_GET(self, request: TxRequest):+ request.setHeader(b"Content-Length", b"1024")+ self.deferRequest(request, 0, self._delayed_render, request)+ return NOT_DONE_YET++ @staticmethod+ def _delayed_render(request: TxRequest):+ request.write(Data.DATALOSS)+ request.finish()+++class NoContentLengthHeader(LeafResource):+ def render_GET(self, request: TxRequest):+ request.requestHeaders.removeHeader('Content-Length')+ self.deferRequest(request, 0, self._delayed_render, request)+ return NOT_DONE_YET++ @staticmethod+ def _delayed_render(request: TxRequest):+ request.write(Data.NO_CONTENT_LENGTH)+ request.finish()++def get_client_certificate(key_file, certificate_file):with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate:pem = ''.join(key.readlines()) + ''.join(certificate.readlines())@@ -118,6 +142,9 @@ class Https2ClientProtocolTestCase(TestCase):r.putChild(b'post-data-json-small', PostDataJsonSmall())r.putChild(b'post-data-json-large', PostDataJsonLarge())++ r.putChild(b'dataloss', Dataloss())+ r.putChild(b'no-content-length-header', NoContentLengthHeader())return r@inlineCallbacks@@ -172,10 +199,10 @@ class Https2ClientProtocolTestCase(TestCase):return DeferredList(d_list, fireOnOneErrback=True)def _check_GET(- self,- request: Request,- expected_body,- expected_status+ self,+ request: Request,+ expected_body,+ expected_status):def check_response(response: Response):self.assertEqual(response.status, expected_status)@@ -220,11 +247,11 @@ class Https2ClientProtocolTestCase(TestCase):)def _check_POST_json(- self,- request: Request,- expected_request_body,- expected_extra_data,- expected_status: int+ self,+ request: Request,+ expected_request_body,+ expected_extra_data,+ expected_status: int):d = self.client.request(request)@@ -237,7 +264,7 @@ class Https2ClientProtocolTestCase(TestCase):self.assertEqual(len(response.body), content_length)# Parse the body- body = json.loads(response.body.decode('utf-8'))+ body = json.loads(to_unicode(response.body))self.assertIn('request-body', body)self.assertIn('extra-data', body)self.assertIn('request-headers', body)@@ -251,11 +278,12 @@ class Https2ClientProtocolTestCase(TestCase):# Check if headers were sent successfullyrequest_headers = body['request-headers']for k, v in request.headers.items():- k_str = k.decode('utf-8')+ k_str = to_unicode(k)self.assertIn(k_str, request_headers)- self.assertEqual(request_headers[k_str], v[0].decode('utf-8'))+ self.assertEqual(request_headers[k_str], to_unicode(v[0]))d.addCallback(assert_response)+ d.addErrback(self.fail)return ddef test_POST_small_json(self):@@ -310,6 +338,95 @@ class Https2ClientProtocolTestCase(TestCase):d = self.client.request(request)d.addCallback(assert_response)+ d.addErrback(self.fail)d.cancel()return d++ def test_download_maxsize_exceeded(self):+ request = Request(url=self.get_url('/get-data-html-large'), meta={'download_maxsize': 1000})++ def assert_cancelled_error(failure):+ self.assertIsInstance(failure.value, CancelledError)++ d = self.client.request(request)+ d.addCallback(self.fail)+ d.addErrback(assert_cancelled_error)+ return d++ # TODO: Test in multiple requests if one request fails due to dataloss+ # remaining request do not fail (change expected behaviour)+ # Can be done only when hyper-h2 don't terminate connection over+ # InvalidBodyLengthError check+ def test_received_dataloss_response(self):+ """In case when value of Header Content-Length != len(Received Data)+ ProtocolError is raised"""+ request = Request(url=self.get_url('/dataloss'))++ def assert_failure(failure: Failure):+ self.assertTrue(len(failure.value.reasons) > 0)+ self.assertTrue(any(+ isinstance(error, InvalidBodyLengthError)+ for error in failure.value.reasons+ ))++ d = self.client.request(request)+ d.addCallback(self.fail)+ d.addErrback(assert_failure)+ return d++ def test_missing_content_length_header(self):+ request = Request(url=self.get_url('/no-content-length-header'))++ def assert_content_length(response: Response):+ self.assertEqual(response.status, 200)+ self.assertEqual(response.body, Data.NO_CONTENT_LENGTH)+ self.assertEqual(response.request, request)+ self.assertEqual(response.url, request.url)+ self.assertIn('partial', response.flags)+ self.assertNotIn('Content-Length', response.headers)++ d = self.client.request(request)+ d.addCallback(assert_content_length)+ d.addErrback(self.fail)+ return d++ @inlineCallbacks+ def _check_log_warnsize(+ self,+ request,+ warn_pattern,+ expected_body+ ):+ with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm:+ response = yield self.client.request(request)+ self.assertEqual(response.status, 200)+ self.assertEqual(response.request, request)+ self.assertEqual(response.url, request.url)+ self.assertEqual(response.body, expected_body)++ # Check the warning is raised only once for this request+ self.assertEqual(sum(+ len(re.findall(warn_pattern, log))+ for log in cm.output+ ), 1)++ @inlineCallbacks+ def test_log_expected_warnsize(self):+ request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000})+ warn_pattern = re.compile(+ r'Expected response size \(\d*\) larger than '+ r'download warn size \(1000\) in request {}'.format(request)+ )++ yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE)++ @inlineCallbacks+ def test_log_received_warnsize(self):+ request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10})+ warn_pattern = re.compile(+ r'Received more \(\d*\) bytes than download '+ r'warn size \(10\) in request {}'.format(request)+ )++ yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH)