Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Added test for http11 choosing response type without content-type header.

Matvei Nazarukb76b374648bc
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 45a806f2e..09a0950e8 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -27,6 +27,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
from scrapy.spiders import Spider
from scrapy.http import Request
+from scrapy.http.response.text import TextResponse
from scrapy.settings import Settings
from scrapy.utils.test import get_crawler, skip_if_no_boto
from scrapy.utils.python import to_bytes
@@ -114,6 +115,16 @@ class ContentLengthHeaderResource(resource.Resource):
return request.requestHeaders.getRawHeaders(b"content-length")[0]
+class EmptyContentTypeHeaderResource(resource.Resource):
+ """
+ A testing resource which renders itself as the value of request body
+ without content-type header in response.
+ """
+ def render(self, request):
+ request.setHeader("content-type", "")
+ return request.content.read()
+
+
class HttpTestCase(unittest.TestCase):
scheme = 'http'
@@ -136,6 +147,7 @@ class HttpTestCase(unittest.TestCase):
r.putChild(b"payload", PayloadResource())
r.putChild(b"broken", BrokenDownloadResource())
r.putChild(b"contentlength", ContentLengthHeaderResource())
+ r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
self.site = server.Site(r, timeout=None)
self.wrapper = WrappingFactory(self.site)
self.host = 'localhost'
@@ -243,11 +255,6 @@ class HttpTestCase(unittest.TestCase):
request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'})
return self.download_request(request, Spider('foo')).addCallback(_test)
- d = self.download_request(request, Spider('foo'))
- d.addCallback(lambda r: r.body)
- d.addCallback(self.assertEquals, b'0')
- return d
-
def test_payload(self):
body = b'1'*100 # PayloadResource requires body length to be 100
request = Request(self.getURL('payload'), method='POST', body=body)
@@ -284,6 +291,20 @@ class Http11TestCase(HttpTestCase):
d.addCallback(self.assertEquals, b"0123456789")
return d
+ def test_response_class_choosing_request(self):
+ """Tests choosing of correct response type
+ in case of Content-Type is empty but body contains text.
+ """
+ body = b'Some plain text\ndata with tabs\t and null bytes\0'
+
+ def _test_type(response):
+ self.assertEquals(type(response), TextResponse)
+
+ request = Request(self.getURL('nocontenttype'), body=body)
+ d = self.download_request(request, Spider('foo'))
+ d.addCallback(_test_type)
+ return d
+
@defer.inlineCallbacks
def test_download_with_maxsize(self):
request = Request(self.getURL('file'))