Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Adding test cases for the zstd content encoding

P. Chen50e1f35d1fca
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 2247ed917..2eed2f5da 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -16,6 +16,7 @@ uvloop; platform_system != "Windows"
# optional for shell wrapper tests
bpython
-brotlipy
+brotlipy # optional for HTTP compress downloader middleware tests
+zstandard # optional for HTTP compress downloader middleware tests
ipython
pywin32; sys_platform == "win32"
diff --git a/tests/sample_data/compressed/html-zstd-static-content-size.bin b/tests/sample_data/compressed/html-zstd-static-content-size.bin
new file mode 100644
index 000000000..b5c2038e8
Binary files /dev/null and b/tests/sample_data/compressed/html-zstd-static-content-size.bin differ
diff --git a/tests/sample_data/compressed/html-zstd-static-no-content-size.bin b/tests/sample_data/compressed/html-zstd-static-no-content-size.bin
new file mode 100644
index 000000000..3d494192e
Binary files /dev/null and b/tests/sample_data/compressed/html-zstd-static-no-content-size.bin differ
diff --git a/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin b/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin
new file mode 100644
index 000000000..97bdbcae0
Binary files /dev/null and b/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin differ
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index a806f55ce..63a69f7af 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -20,6 +20,12 @@ FORMAT = {
'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
'br': ('html-br.bin', 'br'),
+ # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin
+ 'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'),
+ # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin
+ 'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'),
+ # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin
+ 'zstd-streaming-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'),
}
@@ -80,6 +86,26 @@ class HttpCompressionTest(TestCase):
assert newresponse.body.startswith(b"<!DOCTYPE")
assert 'Content-Encoding' not in newresponse.headers
+ def test_process_response_zstd(self):
+ try:
+ import zstandard # noqa: F401
+ except ImportError:
+ raise SkipTest("no zstd support (zstandard)")
+ raw_content = None
+ for check_key in FORMAT:
+ if not check_key.startswith('zstd-'):
+ continue
+ response = self._getresponse(check_key)
+ request = response.request
+ self.assertEqual(response.headers['Content-Encoding'], b'zstd')
+ newresponse = self.mw.process_response(request, response, self.spider)
+ if raw_content is None:
+ raw_content = newresponse.body
+ assert raw_content == newresponse.body
+ assert newresponse is not response
+ assert newresponse.body.startswith(b"<!DOCTYPE")
+ assert 'Content-Encoding' not in newresponse.headers
+
def test_process_response_rawdeflate(self):
response = self._getresponse('rawdeflate')
request = response.request