Scrapy, a fast high-level web crawling & scraping framework for Python.
Adding test cases for the zstd content encoding
P. Chen50e1f35d1fca
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txtindex 2247ed917..2eed2f5da 100644--- a/tests/requirements-py3.txt+++ b/tests/requirements-py3.txt@@ -16,6 +16,7 @@ uvloop; platform_system != "Windows"# optional for shell wrapper testsbpython-brotlipy+brotlipy # optional for HTTP compress downloader middleware tests+zstandard # optional for HTTP compress downloader middleware testsipythonpywin32; sys_platform == "win32"diff --git a/tests/sample_data/compressed/html-zstd-static-content-size.bin b/tests/sample_data/compressed/html-zstd-static-content-size.binnew file mode 100644index 000000000..b5c2038e8Binary files /dev/null and b/tests/sample_data/compressed/html-zstd-static-content-size.bin differdiff --git a/tests/sample_data/compressed/html-zstd-static-no-content-size.bin b/tests/sample_data/compressed/html-zstd-static-no-content-size.binnew file mode 100644index 000000000..3d494192eBinary files /dev/null and b/tests/sample_data/compressed/html-zstd-static-no-content-size.bin differdiff --git a/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin b/tests/sample_data/compressed/html-zstd-streaming-no-content-size.binnew file mode 100644index 000000000..97bdbcae0Binary files /dev/null and b/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin differdiff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex a806f55ce..63a69f7af 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -20,6 +20,12 @@ FORMAT = {'rawdeflate': ('html-rawdeflate.bin', 'deflate'),'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),'br': ('html-br.bin', 'br'),+ # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin+ 'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'),+ # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin+ 'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'),+ # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin+ 'zstd-streaming-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'),}@@ -80,6 +86,26 @@ class HttpCompressionTest(TestCase):assert newresponse.body.startswith(b"<!DOCTYPE")assert 'Content-Encoding' not in newresponse.headers+ def test_process_response_zstd(self):+ try:+ import zstandard # noqa: F401+ except ImportError:+ raise SkipTest("no zstd support (zstandard)")+ raw_content = None+ for check_key in FORMAT:+ if not check_key.startswith('zstd-'):+ continue+ response = self._getresponse(check_key)+ request = response.request+ self.assertEqual(response.headers['Content-Encoding'], b'zstd')+ newresponse = self.mw.process_response(request, response, self.spider)+ if raw_content is None:+ raw_content = newresponse.body+ assert raw_content == newresponse.body+ assert newresponse is not response+ assert newresponse.body.startswith(b"<!DOCTYPE")+ assert 'Content-Encoding' not in newresponse.headers+def test_process_response_rawdeflate(self):response = self._getresponse('rawdeflate')request = response.request