Scrapy, a fast high-level web crawling & scraping framework for Python.
Provide extras for optional dependencies (#7596)
Adriana54c438da192
diff --git a/docs/conf.py b/docs/conf.pyindex b950c4ee2..de722baac 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -158,6 +158,7 @@ scrapy_intersphinx_enable = ["itemloaders","parsel","pytest",+ "pypug","scrapy-lint","sphinx","tox",diff --git a/docs/intro/install.rst b/docs/intro/install.rstindex 67a84e538..cba8c15a1 100644--- a/docs/intro/install.rst+++ b/docs/intro/install.rst@@ -89,6 +89,56 @@ just like any other Python package.(See :ref:`platform-specific guides <intro-install-platform-notes>`below for non-Python dependencies that you may need to install beforehand).+.. _extras:++Optional extras+===============++Scrapy provides optional :ref:`extras <pypug:dependency-specifiers-extras>`+that install additional dependencies to enable specific features. To install+Scrapy with one or more extras, list them in square brackets:++.. code-block:: console++ pip install scrapy[s3,images]++The following extras are available:++.. list-table::+ :header-rows: 1++ * - Extra+ - Provides+ * - ``bpython``+ - :ref:`bpython shell <shell-config>`+ * - ``brotli``+ - :ref:`Brotli response decompression <http-compression>`+ * - ``gcs``+ - :ref:`Google Cloud Storage <topics-feed-storage-gcs>` for+ :ref:`feed exports <topics-feed-exports>` and+ :ref:`media pipelines <media-pipeline-gcs>`+ * - ``httpx``+ - :ref:`httpx-handler`, including its HTTP/2 and SOCKS proxy support+ * - ``images``+ - :ref:`Images pipeline <images-pipeline>`+ * - ``ipython``+ - :ref:`IPython shell <shell-config>`+ * - ``ptpython``+ - :ref:`ptpython shell <shell-config>`+ * - ``robotparser``+ - :ref:`Robotexclusionrulesparser robots.txt parsing <rerp-parser>`+ * - ``s3``+ - :ref:`Amazon S3 <topics-feed-storage-s3>` storage for+ :ref:`feed exports <topics-feed-exports>`,+ :ref:`media pipelines <media-pipelines-s3>`, and+ :ref:`S3 downloads <s3-handler>`+ * - ``twisted-http2``+ - :ref:`twisted-http2-handler`+ * - ``uvloop``+ - `uvloop <https://github.com/MagicStack/uvloop>`_ event loop+ * - ``zstd``+ - :ref:`Zstandard response decompression <http-compression>`+.. _intro-install-platform-notes:diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rstindex 5fbc5df06..33cbf185e 100644--- a/docs/topics/download-handlers.rst+++ b/docs/topics/download-handlers.rst@@ -173,6 +173,8 @@ of this package for more information.H2DownloadHandler-----------------+.. note:: Requires the :ref:`twisted-http2 <extras>` extra.+.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler| Supported scheme: ``https``.@@ -185,9 +187,6 @@ for them.It's implemented using :mod:`twisted.web.client` and the ``h2`` library.-For this handler to work you need to install the ``Twisted[http2]`` extra-dependency.-If you want to use this handler you need to replace the default one for the``https`` scheme:@@ -273,9 +272,13 @@ Other limitations:- HTTPS proxies to HTTPS destinations are not supported.+.. _httpx-handler:+HttpxDownloadHandler--------------------+.. note:: Requires the :ref:`httpx <extras>` extra.+.. versionadded:: 2.15.0.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler@@ -288,7 +291,9 @@ HttpxDownloadHandlerThis handler supports ``http://host/path`` and ``https://host/path`` URLs anduses the HTTP/1.1 or HTTP/2 protocol for them.-It's implemented using the ``httpx`` library and needs it to be installed.+It's implemented using the httpx_ library.++.. _httpx: https://www.python-httpx.org/If you want to use this handler you need to replace the default ones for the``http`` and ``https`` schemes:@@ -311,8 +316,8 @@ Features and limitations=========================== =======================================HTTP proxies Yes-SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``)-HTTP/2 Yes (requires ``httpx[http2]``)+SOCKS proxies Yes (SOCKS5)+HTTP/2 Yes``response.certificate`` DER bytesPer-request ``bindaddress`` No (not supported by the library)TLS implementation Standard library ``ssl``@@ -333,8 +338,7 @@ HTTPX_HTTP2_ENABLEDDefault: ``False``-Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra-needs to be installed if you want to enable this setting.+Whether to enable HTTP/2 support in this handler.Built-in non-HTTP download handlers reference=============================================@@ -378,9 +382,13 @@ This handler supports ``ftp://host/path`` FTP URIs.It's implemented using :mod:`twisted.protocols.ftp`.+.. _s3-handler:+S3DownloadHandler-----------------+.. note:: Requires the :ref:`s3 <extras>` extra.+.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler| Supported scheme: ``s3``.@@ -390,4 +398,6 @@ S3DownloadHandlerThis handler supports ``s3://bucket/path`` S3 URIs.-It's implemented using the ``botocore`` library and needs it to be installed.+It's implemented using the botocore_ library.++.. _botocore: https://github.com/boto/botocorediff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 096f87edd..934eb19ac 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -727,6 +727,8 @@ We assume that the spider will not issue Cache-Control directivesin requests unless it actually needs them, so directives in requests arenot filtered.+.. _http-compression:+HttpCompressionMiddleware-------------------------@@ -738,14 +740,12 @@ HttpCompressionMiddlewareThis middleware allows compressed (gzip, deflate) traffic to besent/received from web sites.- This middleware also supports decoding `brotli-compressed`_ as well as- `zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is- installed, respectively.+ This middleware also supports decoding `brotli-compressed`_ responses with+ the :ref:`brotli <extras>` extra, and `zstd-compressed`_+ responses with the :ref:`zstd <extras>` extra... _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt-.. _brotli: https://pypi.org/project/Brotli/.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt-.. _zstandard: https://pypi.org/project/zstandard/HttpCompressionMiddleware Settings@@ -1239,8 +1239,7 @@ Based on `Robotexclusionrulesparser <https://pypi.org/project/robotexclusionruleIn order to use this parser:-* Install ``Robotexclusionrulesparser`` by running- ``pip install robotexclusionrulesparser``+* Install the :ref:`robotparser <extras>` extra.* Set :setting:`ROBOTSTXT_PARSER` setting to``scrapy.robotstxt.RerpRobotParser``diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 51e142b04..66768c97b 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -92,7 +92,6 @@ Marshal- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`-.. _topics-feed-storage:Storages@@ -106,14 +105,13 @@ The storages backends supported out of the box are:- :ref:`topics-feed-storage-fs`- :ref:`topics-feed-storage-ftp`-- :ref:`topics-feed-storage-s3` (requires boto3_)-- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)+- :ref:`topics-feed-storage-s3` (requires the :ref:`s3 <extras>` extra)+- :ref:`topics-feed-storage-gcs` (requires the :ref:`gcs <extras>` extra)- :ref:`topics-feed-storage-stdout`-Some storage backends may be unavailable if the required external libraries are-not available. For example, the S3 backend is only available if the boto3_-library is installed.-+Some storage backends may be unavailable if the required :ref:`extras <extras>`+are not installed. For example, the S3 backend requires the :ref:`s3 <extras>`+extra... _topics-feed-uri-params:@@ -209,7 +207,7 @@ The feeds are stored on `Amazon S3`_.- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``-- Required external libraries: `boto3`_ >= 1.20.0+- Required extras: :ref:`s3 <extras>`The AWS credentials can be passed as user/password in the URI, or they can bepassed through the following settings:@@ -249,7 +247,7 @@ The feeds are stored on `Google Cloud Storage`_.- ``gs://mybucket/path/to/export.csv``-- Required external libraries: `google-cloud-storage`_.+- Required extras: :ref:`gcs <extras>`For more information about authentication, please refer to `Google Cloud documentation <https://docs.cloud.google.com/docs/authentication>`_.@@ -266,7 +264,6 @@ storage backend is: ``True``.This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.-.. _google-cloud-storage: https://docs.cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python.. _topics-feed-storage-stdout:@@ -794,6 +791,5 @@ source spider in the feed URI:.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier.. _Amazon S3: https://aws.amazon.com/s3/-.. _boto3: https://github.com/boto/boto3.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl.. _Google Cloud Storage: https://cloud.google.com/storage/diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex 56b53d18d..4ceb4732a 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -60,6 +60,8 @@ this:Using the Images Pipeline=========================+.. note:: Requires the :ref:`images <extras>` extra.+Using the :class:`ImagesPipeline` is a lot like using the :class:`FilesPipeline`,except the default field names used are different: you use ``image_urls`` forthe image URLs of an item and it will populate an ``images`` field for the information@@ -69,12 +71,6 @@ The advantage of using the :class:`ImagesPipeline` for image files is that youcan configure some extra functions like generating thumbnails and filteringthe images based on their size.-The Images Pipeline requires Pillow_ 8.3.2 or greater. It is used for-thumbnailing and normalizing images to JPEG/RGB format.--.. _Pillow: https://github.com/python-pillow/Pillow--.. _topics-media-pipeline-enabling:Enabling your Media Pipeline@@ -231,12 +227,13 @@ set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.Amazon S3 storage-----------------+.. note:: Requires the :ref:`s3 <extras>` extra.+.. setting:: FILES_STORE_S3_ACL.. setting:: IMAGES_STORE_S3_ACL-If botocore_ >= 1.13.45 is installed, :setting:`FILES_STORE` and-:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will-automatically upload the files to the bucket.+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3+bucket. Scrapy will automatically upload the files to the bucket.For example, this is a valid :setting:`IMAGES_STORE` value:@@ -271,7 +268,6 @@ For self-hosting you also might feel the need not to use SSL and not to verify SAWS_USE_SSL = False # or True (None by default)AWS_VERIFY = False # or True (None by default)-.. _botocore: https://github.com/boto/botocore.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl.. _Minio: https://github.com/minio/minio.. _Zenko CloudServer: https://www.zenko.io/cloudserver/@@ -282,13 +278,13 @@ For self-hosting you also might feel the need not to use SSL and not to verify SGoogle Cloud Storage---------------------+.. note:: Requires the :ref:`gcs <extras>` extra.+.. setting:: FILES_STORE_GCS_ACL.. setting:: IMAGES_STORE_GCS_ACL-:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage-bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ )--.. _google-cloud-storage: https://docs.cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud+Storage bucket. Scrapy will automatically upload the files to the bucket.For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings:diff --git a/docs/topics/shell.rst b/docs/topics/shell.rstindex ea97a22aa..6f7e67cf9 100644--- a/docs/topics/shell.rst+++ b/docs/topics/shell.rst@@ -17,32 +17,35 @@ spider, without having to run the spider to test every change.Once you get familiarized with the Scrapy shell, you'll see that it's aninvaluable tool for developing and debugging your spiders.+.. _shell-config:+Configuring the shell=====================-If you have `IPython`_ installed, the Scrapy shell will use it (instead of the-standard Python console). The `IPython`_ console is much more powerful and-provides smart auto-completion and colorized output, among other things.+With the :ref:`ptpython <extras>` extra, the Scrapy shell will use ptpython_+instead of the :term:`REPL`. ptpython provides syntax highlighting, smart+auto-completion, and more.-We highly recommend you install `IPython`_, especially if you're working on-Unix systems (where `IPython`_ excels). See the `IPython installation guide`_-for more info.+Failing that, with the :ref:`ipython <extras>` extra, the Scrapy shell will+use IPython_ instead. IPython provides smart auto-completion, colorized+output, and more.-Scrapy also has support for `bpython`_, and will try to use it where `IPython`_-is unavailable.+Scrapy also has support for `bpython`_ via the :ref:`bpython <extras>` extra,+and will try to use it where neither ptpython nor IPython is available.Through Scrapy's settings you can configure it to use any one of-``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which-are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment-variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`:+``ptpython``, ``ipython``, ``bpython`` or the standard ``python`` shell,+regardless of which are installed. This is done by setting the+``SCRAPY_PYTHON_SHELL`` environment variable; or by defining it in your+:ref:`scrapy.cfg <topics-config-settings>`:.. code-block:: ini[settings]shell = bpython+.. _ptpython: https://github.com/prompt-toolkit/ptpython.. _IPython: https://ipython.org/-.. _IPython installation guide: https://ipython.org/install/.. _bpython: https://bpython-interpreter.org/Launch the shelldiff --git a/pyproject.toml b/pyproject.tomlindex b24397c02..81d697c12 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -60,6 +60,25 @@ Source = "https://github.com/scrapy/scrapy"Tracker = "https://github.com/scrapy/scrapy/issues""Release notes" = "https://docs.scrapy.org/en/latest/news.html"+[project.optional-dependencies]+bpython = ["bpython>=0.7.1"]+brotli = [+ "brotli>=1.2.0; implementation_name != 'pypy'",+ "brotlicffi>=1.2.0.0; implementation_name == 'pypy'",+]+gcs = ["google-cloud-storage>=1.29.0"]+httpx = ["httpx[http2,socks]>=0.27.1"]+images = ["Pillow>=8.3.2"]+ipython = ["ipython>=7.1.0"]+ptpython = ["ptpython>=2.0.1"]+robotparser = ["robotexclusionrulesparser>=1.6.2"]+s3 = ["boto3>=1.20.0"]+twisted-http2 = ["Twisted[http2]>=21.7.0"]+uvloop = [+ "uvloop>=0.16.0; platform_system != 'Windows' and implementation_name != 'pypy'",+]+zstd = ["zstandard>=0.16.0; implementation_name != 'pypy'"]+[project.scripts]scrapy = "scrapy.cmdline:execute"diff --git a/scrapy/utils/console.py b/scrapy/utils/console.pyindex 93fbb9033..31a4bb32f 100644--- a/scrapy/utils/console.py+++ b/scrapy/utils/console.py@@ -60,7 +60,7 @@ def _embed_ptpython_shell(namespace: dict[str, Any] | None = None, banner: str = "") -> EmbedFuncT:"""Start a ptpython shell"""- import ptpython.repl # noqa: PLC0415 # pylint: disable=import-error+ import ptpython.repl # noqa: PLC0415@wraps(_embed_ptpython_shell)def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:diff --git a/tests/test_utils_console.py b/tests/test_utils_console.pyindex 05b272832..ad9aa3dff 100644--- a/tests/test_utils_console.py+++ b/tests/test_utils_console.py@@ -1,24 +1,11 @@from __future__ import annotations+from importlib.util import find_spec+import pytestfrom scrapy.utils.console import get_shell_embed_func-try:- import bpython-- bpy = True- del bpython-except ImportError:- bpy = False-try:- import IPython-- ipy = True- del IPython-except ImportError:- ipy = False-def test_get_shell_embed_func():shell = get_shell_embed_func(["invalid"])@@ -29,16 +16,46 @@ def test_get_shell_embed_func():assert shell.__name__ == "_embed_standard_shell"[email protected](not bpy, reason="bpython not available in testenv")+def test_get_shell_embed_func_python():+ # the standard shell is always available+ shell = get_shell_embed_func(["python"])+ assert callable(shell)+ assert shell.__name__ == "_embed_standard_shell"++def test_get_shell_embed_func_bpython():+ pytest.importorskip("bpython")shell = get_shell_embed_func(["bpython"])assert callable(shell)assert shell.__name__ == "_embed_bpython_shell"[email protected](not ipy, reason="IPython not available in testenv")def test_get_shell_embed_func_ipython():- # default shell should be 'ipython'- shell = get_shell_embed_func()+ pytest.importorskip("IPython")+ shell = get_shell_embed_func(["ipython"])assert shell is not Noneassert shell.__name__ == "_embed_ipython_shell"+++def test_get_shell_embed_func_ptpython():+ pytest.importorskip("ptpython")+ shell = get_shell_embed_func(["ptpython"])+ assert shell is not None+ assert shell.__name__ == "_embed_ptpython_shell"+++def test_get_shell_embed_func_default():+ # with no shells given, the first available shell in preference order+ # (ptpython, ipython, bpython, python) is returned; the standard shell+ # is always available, so the default is never None+ shell = get_shell_embed_func()+ assert shell is not None+ if find_spec("ptpython") is not None:+ expected = "_embed_ptpython_shell"+ elif find_spec("IPython") is not None:+ expected = "_embed_ipython_shell"+ elif find_spec("bpython") is not None:+ expected = "_embed_bpython_shell"+ else:+ expected = "_embed_standard_shell"+ assert shell.__name__ == expecteddiff --git a/tox.ini b/tox.iniindex ec7273b42..b2f3b0db1 100644--- a/tox.ini+++ b/tox.ini@@ -173,6 +173,7 @@ deps =google-cloud-storagehttpx[http2,socks]ipython+ ptpython # optional for shell wrapper testsrobotexclusionrulesparseruvloop; platform_system != "Windows" and implementation_name != "pypy"zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests@@ -188,8 +189,9 @@ deps =brotli==1.2.0; implementation_name != "pypy"brotlicffi==1.2.0.0; implementation_name == "pypy"google-cloud-storage==1.29.0- httpx==0.27.1+ httpx[http2,socks]==0.27.1ipython==7.1.0+ ptpython==2.0.1robotexclusionrulesparser==1.6.2uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"zstandard==0.16.0; implementation_name != "pypy"