Scrapy, a fast high-level web crawling & scraping framework for Python.
Added test for http11 choosing response type without content-type header.
Matvei Nazarukb76b374648bc
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 45a806f2e..09a0950e8 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -27,6 +27,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandlerfrom scrapy.spiders import Spiderfrom scrapy.http import Request+from scrapy.http.response.text import TextResponsefrom scrapy.settings import Settingsfrom scrapy.utils.test import get_crawler, skip_if_no_botofrom scrapy.utils.python import to_bytes@@ -114,6 +115,16 @@ class ContentLengthHeaderResource(resource.Resource):return request.requestHeaders.getRawHeaders(b"content-length")[0]+class EmptyContentTypeHeaderResource(resource.Resource):+ """+ A testing resource which renders itself as the value of request body+ without content-type header in response.+ """+ def render(self, request):+ request.setHeader("content-type", "")+ return request.content.read()++class HttpTestCase(unittest.TestCase):scheme = 'http'@@ -136,6 +147,7 @@ class HttpTestCase(unittest.TestCase):r.putChild(b"payload", PayloadResource())r.putChild(b"broken", BrokenDownloadResource())r.putChild(b"contentlength", ContentLengthHeaderResource())+ r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())self.site = server.Site(r, timeout=None)self.wrapper = WrappingFactory(self.site)self.host = 'localhost'@@ -243,11 +255,6 @@ class HttpTestCase(unittest.TestCase):request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'})return self.download_request(request, Spider('foo')).addCallback(_test)- d = self.download_request(request, Spider('foo'))- d.addCallback(lambda r: r.body)- d.addCallback(self.assertEquals, b'0')- return d-def test_payload(self):body = b'1'*100 # PayloadResource requires body length to be 100request = Request(self.getURL('payload'), method='POST', body=body)@@ -284,6 +291,20 @@ class Http11TestCase(HttpTestCase):d.addCallback(self.assertEquals, b"0123456789")return d+ def test_response_class_choosing_request(self):+ """Tests choosing of correct response type+ in case of Content-Type is empty but body contains text.+ """+ body = b'Some plain text\ndata with tabs\t and null bytes\0'++ def _test_type(response):+ self.assertEquals(type(response), TextResponse)++ request = Request(self.getURL('nocontenttype'), body=body)+ d = self.download_request(request, Spider('foo'))+ d.addCallback(_test_type)+ return d+@defer.inlineCallbacksdef test_download_with_maxsize(self):request = Request(self.getURL('file'))