Scrapy, a fast high-level web crawling & scraping framework for Python.
Using the `zstandard` package than `zstd` for supporting frames both with and without the content size info
P. Chenda3171d4f71d
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 9645ed5fd..7c63a623d 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -685,13 +685,13 @@ HttpCompressionMiddlewaresent/received from web sites.This middleware also supports decoding `brotli-compressed`_ as well as- `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is+ `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ isinstalled, respectively... _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt.. _brotlipy: https://pypi.org/project/brotlipy/.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt-.. _zstd: https://pypi.org/project/zstd/+.. _zstandard: https://pypi.org/project/zstandard/HttpCompressionMiddleware Settings~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex b1abb8b5c..56421a6ba 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -1,4 +1,5 @@import zlib+import iofrom scrapy.utils.gz import gunzipfrom scrapy.http import Response, TextResponse@@ -15,7 +16,7 @@ except ImportError:passtry:- import zstd+ import zstandardACCEPTED_ENCODINGS.append(b'zstd')except ImportError:pass@@ -74,5 +75,8 @@ class HttpCompressionMiddleware:if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:body = brotli.decompress(body)if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS:- body = zstd.decompress(body)+ # Using its streaming API since its simple API could handle only cases+ # where there is content size data embedded in the frame+ reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))+ body = reader.read()return body