Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Using the `zstandard` package than `zstd` for supporting frames both with and without the content size info

P. Chenda3171d4f71d
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 9645ed5fd..7c63a623d 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -685,13 +685,13 @@ HttpCompressionMiddleware
sent/received from web sites.
This middleware also supports decoding `brotli-compressed`_ as well as
- `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is
+ `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
installed, respectively.
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
.. _brotlipy: https://pypi.org/project/brotlipy/
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
-.. _zstd: https://pypi.org/project/zstd/
+.. _zstandard: https://pypi.org/project/zstandard/
HttpCompressionMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index b1abb8b5c..56421a6ba 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,4 +1,5 @@
import zlib
+import io
from scrapy.utils.gz import gunzip
from scrapy.http import Response, TextResponse
@@ -15,7 +16,7 @@ except ImportError:
pass
try:
- import zstd
+ import zstandard
ACCEPTED_ENCODINGS.append(b'zstd')
except ImportError:
pass
@@ -74,5 +75,8 @@ class HttpCompressionMiddleware:
if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
body = brotli.decompress(body)
if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS:
- body = zstd.decompress(body)
+ # Using its streaming API since its simple API could handle only cases
+ # where there is content size data embedded in the frame
+ reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
+ body = reader.read()
return body