Scrapy, a fast high-level web crawling & scraping framework for Python.
Add an UPLOAD_TIMEOUT setting (#8162)
Adrian3d2e0e1ca1cb
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 2bbb87e5c..51fb7c435 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -447,6 +447,7 @@ These are the settings used for configuring the feed exports:- :setting:`FEED_STORAGE_S3_ACL`- :setting:`FEED_EXPORTERS`- :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`+- :setting:`UPLOAD_TIMEOUT`.. setting:: FEEDSdiff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex ca2bfae64..a843e6f07 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -244,6 +244,9 @@ on the item in your spider, and read it back from the item inSupported Storage=================+Use :setting:`UPLOAD_TIMEOUT` to limit how long uploads to a remote storage+backend may take.+File system storage-------------------diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex c8f4da941..6acdfad32 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -2532,6 +2532,27 @@ For additional information, see :doc:`core/howto/choosing-reactor`... note:: This is a :ref:`reactor setting <reactor-settings>`.+.. setting:: UPLOAD_TIMEOUT++UPLOAD_TIMEOUT+--------------++.. versionadded:: VERSION++Default: ``None``++Number of seconds that uploads to a remote :ref:`feed storage backend+<topics-feed-storage-backends>` or :ref:`media pipeline storage backend+<topics-media-pipeline>` wait for a response before giving up.++Raise it if large uploads fail over a slow connection.++If ``None``, each backend keeps the default of the library it uses:+`botocore <https://docs.aws.amazon.com/botocore/latest/reference/config.html>`_+for Amazon S3, `google-cloud-storage+<https://docs.cloud.google.com/python/docs/reference/storage/latest/retry_timeout#configuring-timeouts>`_+for Google Cloud Storage, and :mod:`ftplib` for FTP, which waits indefinitely.+.. setting:: URLLENGTH_LIMITURLLENGTH_LIMITdiff --git a/pyproject.toml b/pyproject.tomlindex aeabe0c47..6d009005a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -70,7 +70,7 @@ Tracker = "https://github.com/scrapy/scrapy/issues"[project.optional-dependencies]bpython = ["bpython>=0.7.1"]color = ["colorlog>=6.6.0"]-gcs = ["google-cloud-storage>=1.29.0"]+gcs = ["google-cloud-storage>=1.30.0"]httpx = ["httpx2[http2,socks]>=2.0.0"]images = ["Pillow>=8.3.2"]ipython = ["ipython>=8.15.0"]diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 30c131d86..4342792cd 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -216,6 +216,7 @@ class S3FeedStorage(BlockingFeedStorage):session_token: str | None = None,region_name: str | None = None,max_pool_connections: int | None = None,+ upload_timeout: float | None = None,):try:import boto3.session # noqa: PLC0415@@ -234,6 +235,13 @@ class S3FeedStorage(BlockingFeedStorage):self.endpoint_url: str | None = endpoint_urlself.region_name: str | None = region_nameself.max_pool_connections: int | None = max_pool_connections+ self.upload_timeout: float | None = upload_timeout++ config_kwargs: dict[str, Any] = {}+ if max_pool_connections is not None:+ config_kwargs["max_pool_connections"] = max_pool_connections+ if upload_timeout is not None:+ config_kwargs["read_timeout"] = upload_timeoutboto3_session = boto3.session.Session()self.s3_client = boto3_session.client(@@ -243,11 +251,7 @@ class S3FeedStorage(BlockingFeedStorage):aws_session_token=self.session_token,endpoint_url=self.endpoint_url,region_name=self.region_name,- config=(- Config(max_pool_connections=self.max_pool_connections)- if self.max_pool_connections is not None- else None- ),+ config=Config(**config_kwargs),)if feed_options and feed_options.get("overwrite", True) is False:@@ -274,6 +278,7 @@ class S3FeedStorage(BlockingFeedStorage):endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,region_name=crawler.settings["AWS_REGION_NAME"] or None,max_pool_connections=_get_max_pool_connections(crawler.settings),+ upload_timeout=crawler.settings.getfloat("UPLOAD_TIMEOUT") or None,feed_options=feed_options,)@@ -305,9 +310,11 @@ class GCSFeedStorage(BlockingFeedStorage):acl: str | None,*,feed_options: dict[str, Any] | None = None,+ timeout: float | None = None,):self.project_id: str | None = project_idself.acl: str | None = acl+ self.timeout: float | None = timeoutu = urlparse(uri)assert u.hostnameself.bucket_name: str = u.hostname@@ -327,6 +334,7 @@ class GCSFeedStorage(BlockingFeedStorage):crawler.settings["GCS_PROJECT_ID"],crawler.settings["FEED_STORAGE_GCS_ACL"] or None,feed_options=feed_options,+ timeout=crawler.settings.getfloat("UPLOAD_TIMEOUT") or None,)def _store_in_thread(self, file: IO[bytes]) -> None:@@ -336,10 +344,11 @@ class GCSFeedStorage(BlockingFeedStorage):client = Client(project=self.project_id)bucket = client.bucket(self.bucket_name)- blob = None if self.overwrite else bucket.get_blob(self.blob_name)+ kwargs = {} if self.timeout is None else {"timeout": self.timeout}+ blob = None if self.overwrite else bucket.get_blob(self.blob_name, **kwargs)if blob is None:bucket.blob(self.blob_name).upload_from_file(- file, predefined_acl=self.acl+ file, predefined_acl=self.acl, **kwargs)return# Appending uploads the new data as a separate object and composes@@ -347,13 +356,13 @@ class GCSFeedStorage(BlockingFeedStorage):# untouched. Composition resolves its sources by name, so it always# appends to the latest version of the blob.part = bucket.blob(f"{self.blob_name}.{uuid4().hex}.part")- part.upload_from_file(file, predefined_acl=self.acl)+ part.upload_from_file(file, predefined_acl=self.acl, **kwargs)try:- blob.compose([blob, part], client=client)+ blob.compose([blob, part], client=client, **kwargs)finally:- part.delete(client=client)+ part.delete(client=client, **kwargs)if self.acl:- blob.acl.save_predefined(self.acl, client=client)+ blob.acl.save_predefined(self.acl, client=client, **kwargs)finally:file.close()@@ -365,6 +374,7 @@ class FTPFeedStorage(BlockingFeedStorage):use_active_mode: bool = False,*,feed_options: dict[str, Any] | None = None,+ timeout: float | None = None,):u = urlparse(uri)if not u.hostname:@@ -376,6 +386,7 @@ class FTPFeedStorage(BlockingFeedStorage):self.path: str = u.pathself.tls: bool = u.scheme == "ftps"self.use_active_mode: bool = use_active_mode+ self.timeout: float | None = timeoutself.overwrite: bool = not feed_options or feed_options.get("overwrite", True)@classmethod@@ -390,6 +401,7 @@ class FTPFeedStorage(BlockingFeedStorage):uri,use_active_mode=crawler.settings.getbool("FEED_STORAGE_FTP_ACTIVE"),feed_options=feed_options,+ timeout=crawler.settings.getfloat("UPLOAD_TIMEOUT") or None,)def _store_in_thread(self, file: IO[bytes]) -> None:@@ -403,6 +415,7 @@ class FTPFeedStorage(BlockingFeedStorage):use_active_mode=self.use_active_mode,overwrite=self.overwrite,tls=self.tls,+ timeout=self.timeout,)diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex c5f8ef9f9..d0fd32777 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -166,6 +166,7 @@ class S3FilesStore:# Overridden from settings.AWS_MAX_POOL_CONNECTIONS in# FilesPipeline.from_crawler(); None means the botocore defaultAWS_MAX_POOL_CONNECTIONS: int | None = None+ UPLOAD_TIMEOUT: float | None = NonePOLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler()HEADERS: ClassVar[dict[str, str]] = {@@ -178,11 +179,11 @@ class S3FilesStore:import botocore.session # noqa: PLC0415from botocore.config import Config # noqa: PLC0415- config = (- Config(max_pool_connections=self.AWS_MAX_POOL_CONNECTIONS)- if self.AWS_MAX_POOL_CONNECTIONS is not None- else None- )+ config_kwargs: dict[str, Any] = {}+ if self.AWS_MAX_POOL_CONNECTIONS is not None:+ config_kwargs["max_pool_connections"] = self.AWS_MAX_POOL_CONNECTIONS+ if self.UPLOAD_TIMEOUT is not None:+ config_kwargs["read_timeout"] = self.UPLOAD_TIMEOUTsession = botocore.session.get_session()self.s3_client = session.create_client("s3",@@ -193,7 +194,7 @@ class S3FilesStore:region_name=self.AWS_REGION_NAME,use_ssl=self.AWS_USE_SSL,verify=self.AWS_VERIFY,- config=config,+ config=Config(**config_kwargs),)if not uri.startswith("s3://"):raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")@@ -294,6 +295,7 @@ class S3FilesStore:class GCSFilesStore:GCS_PROJECT_ID = None+ UPLOAD_TIMEOUT: float | None = NoneCACHE_CONTROL = "max-age=172800"@@ -361,12 +363,15 @@ class GCSFilesStore:blob = self.bucket.blob(blob_path)blob.cache_control = self.CACHE_CONTROLblob.metadata = {k: str(v) for k, v in meta.items()} if meta else {}+ timeout = self.UPLOAD_TIMEOUT+ kwargs = {} if timeout is None else {"timeout": timeout}return deferred_from_coro(run_in_thread(blob.upload_from_string,data=buf.getvalue(),content_type=self._get_content_type(headers),predefined_acl=self.POLICY,+ **kwargs,))@@ -375,6 +380,7 @@ class FTPFilesStore:FTP_USERNAME: str | None = NoneFTP_PASSWORD: str | None = NoneUSE_ACTIVE_MODE: bool | None = None+ UPLOAD_TIMEOUT: float | None = Nonedef __init__(self, uri: str):if not uri.startswith("ftp://"):@@ -410,6 +416,7 @@ class FTPFilesStore:username=self.username,password=self.password,use_active_mode=bool(self.USE_ACTIVE_MODE),+ timeout=self.UPLOAD_TIMEOUT,))@@ -527,6 +534,8 @@ class FilesPipeline(MediaPipeline):@classmethoddef _update_stores(cls, settings: BaseSettings) -> None:+ upload_timeout: float | None = settings.getfloat("UPLOAD_TIMEOUT") or None+s3store: type[S3FilesStore] = cast("type[S3FilesStore]", cls.STORE_SCHEMES["s3"])@@ -539,12 +548,14 @@ class FilesPipeline(MediaPipeline):s3store.AWS_VERIFY = settings["AWS_VERIFY"]s3store.AWS_MAX_POOL_CONNECTIONS = _get_max_pool_connections(settings)s3store.POLICY = settings["FILES_STORE_S3_ACL"]+ s3store.UPLOAD_TIMEOUT = upload_timeoutgcs_store: type[GCSFilesStore] = cast("type[GCSFilesStore]", cls.STORE_SCHEMES["gs"])gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"]gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None+ gcs_store.UPLOAD_TIMEOUT = upload_timeoutftp_store: type[FTPFilesStore] = cast("type[FTPFilesStore]", cls.STORE_SCHEMES["ftp"]@@ -552,6 +563,7 @@ class FilesPipeline(MediaPipeline):ftp_store.FTP_USERNAME = settings["FTP_USER"]ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"]ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE")+ ftp_store.UPLOAD_TIMEOUT = upload_timeoutdef _get_store(self, uri: str) -> FilesStoreProtocol:# to support win32 paths like: C:\\some\dirdiff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex a03ee957e..58cea3609 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -226,6 +226,7 @@ __all__ = ["TWISTED_DNS_RESOLVER","TWISTED_REACTOR","TWISTED_REACTOR_ENABLED",+ "UPLOAD_TIMEOUT","URLLENGTH_LIMIT","USER_AGENT","WARN_ON_GENERATOR_RETURN_VALUE",@@ -620,6 +621,8 @@ TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver"TWISTED_REACTOR_ENABLED = TrueTWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"+UPLOAD_TIMEOUT = None+URLLENGTH_LIMIT = 2083USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)"diff --git a/scrapy/utils/_ftp.py b/scrapy/utils/_ftp.pyindex f08e5303d..5d7fce08a 100644--- a/scrapy/utils/_ftp.py+++ b/scrapy/utils/_ftp.py@@ -31,16 +31,23 @@ def ftp_store_file(use_active_mode: bool = False,overwrite: bool = True,tls: bool = False,+ timeout: float | None = None,) -> None:"""Opens a FTP connection with passed credentials, sets current directoryto the directory extracted from given path, then uploads the file to server.If *tls* is ``True``, the connection is secured with TLS (FTPS), and thecertificate of the server is verified.++ *timeout* is the number of seconds that socket operations wait before+ giving up. If ``None``, they wait indefinitely."""ftp = FTP_TLS(context=create_default_context()) if tls else FTP()with ftp, closing(file):- ftp.connect(host, port)+ if timeout is None:+ ftp.connect(host, port)+ else:+ ftp.connect(host, port, timeout)ftp.login(username, password)if isinstance(ftp, FTP_TLS):ftp.prot_p()diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.pyindex 3870ad6cc..c494c0e98 100644--- a/tests/test_feedexport_storages.py+++ b/tests/test_feedexport_storages.py@@ -172,6 +172,18 @@ class TestFTPFeedStorage:await self._store(url, b"bar", settings=settings)self._assert_stored(ftp_server.path / filename, b"bar")+ @coroutine_test+ async def test_upload_timeout(self):+ crawler = get_crawler(settings_dict={"UPLOAD_TIMEOUT": 300})+ with MockFTPServer() as ftp_server:+ url = ftp_server.url("file")+ storage = build_from_crawler(FTPFeedStorage, crawler, url)+ assert storage.timeout == 300+ file = storage.open(get_test_spider())+ file.write(b"foo")+ await maybe_deferred_to_future(storage.store(file))+ self._assert_stored(ftp_server.path / "file", b"foo")+@coroutine_testasync def test_missing_parent_directories(self):with MockFTPServer() as ftp_server:@@ -464,6 +476,20 @@ class TestS3FeedStorage:config: Any = storage.s3_client.meta.configassert config.max_pool_connections == expected+ def test_upload_timeout(self) -> None:+ crawler = get_crawler(settings_dict={"UPLOAD_TIMEOUT": 300})+ storage = build_from_crawler(S3FeedStorage, crawler, "s3://mybucket/export.csv")+ assert storage.upload_timeout == 300+ config: Any = storage.s3_client.meta.config+ assert config.read_timeout == 300++ def test_default_upload_timeout(self) -> None:+ crawler = get_crawler()+ storage = build_from_crawler(S3FeedStorage, crawler, "s3://mybucket/export.csv")+ assert storage.upload_timeout is None+ config: Any = storage.s3_client.meta.config+ assert config.read_timeout == 60+@coroutine_testasync def test_store_without_acl(self):storage = S3FeedStorage(@@ -526,9 +552,19 @@ class TestGCSFeedStorage:)assert storage.project_id == "123"assert storage.acl == "publicRead"+ assert storage.timeout is Noneassert storage.bucket_name == "mybucket"assert storage.blob_name == "export.csv"+ def test_parse_timeout(self):+ pytest.importorskip("google.cloud.storage")++ crawler = get_crawler(settings_dict={"UPLOAD_TIMEOUT": 300})+ storage = build_from_crawler(+ GCSFeedStorage, crawler, "gs://mybucket/export.csv"+ )+ assert storage.timeout == 300+def test_parse_empty_acl(self):pytest.importorskip("google.cloud.storage")@@ -568,6 +604,22 @@ class TestGCSFeedStorage:blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)f.close.assert_called_once_with()+ @coroutine_test+ async def test_store_timeout(self):+ pytest.importorskip("google.cloud.storage")++ (client_mock, _, blob_mock) = mock_google_cloud_storage()+ with mock.patch("google.cloud.storage.Client", return_value=client_mock):+ f = mock.Mock()+ storage = GCSFeedStorage(+ "gs://mybucket/export.csv", "myproject-123", None, timeout=300+ )+ await maybe_deferred_to_future(storage.store(f))++ blob_mock.upload_from_file.assert_called_once_with(+ f, predefined_acl=None, timeout=300+ )+@coroutine_testasync def test_store_closes_file_on_upload_error(self):pytest.importorskip("google.cloud.storage")@@ -653,6 +705,36 @@ class TestGCSFeedStorage:blob_mock.upload_from_file.assert_not_called()f.close.assert_called_once_with()+ @coroutine_test+ async def test_store_append_timeout(self) -> None:+ pytest.importorskip("google.cloud.storage")++ (client_mock, bucket_mock, part_mock) = mock_google_cloud_storage()+ blob_mock = mock_google_cloud_blob()+ bucket_mock.get_blob.return_value = blob_mock+ with mock.patch("google.cloud.storage.Client", return_value=client_mock):+ f = mock.Mock()+ storage = GCSFeedStorage(+ "gs://mybucket/export.csv",+ "myproject-123",+ "publicRead",+ feed_options={"overwrite": False},+ timeout=300,+ )+ await maybe_deferred_to_future(storage.store(f))++ bucket_mock.get_blob.assert_called_once_with("export.csv", timeout=300)+ part_mock.upload_from_file.assert_called_once_with(+ f, predefined_acl="publicRead", timeout=300+ )+ blob_mock.compose.assert_called_once_with(+ [blob_mock, part_mock], client=client_mock, timeout=300+ )+ part_mock.delete.assert_called_once_with(client=client_mock, timeout=300)+ blob_mock.acl.save_predefined.assert_called_once_with(+ "publicRead", client=client_mock, timeout=300+ )+@coroutine_testasync def test_store_append_without_acl(self) -> None:pytest.importorskip("google.cloud.storage")diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 2d08fdde7..1b8fbb662 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -989,6 +989,25 @@ class TestS3FilesStore:config: Any = store.s3_client.meta.configassert config.max_pool_connections == 10+ @pytest.mark.parametrize(+ ("settings", "expected"),+ [+ ({}, 60),+ ({"UPLOAD_TIMEOUT": 300}, 300),+ ],+ )+ def test_upload_timeout(+ self, monkeypatch: pytest.MonkeyPatch, settings: dict[str, Any], expected: float+ ) -> None:+ monkeypatch.setattr(S3FilesStore, "UPLOAD_TIMEOUT", None)+ crawler = get_crawler(+ settings_dict={"FILES_STORE": "s3://mybucket/prefix/", **settings}+ )+ store = build_from_crawler(FilesPipeline, crawler).store+ assert isinstance(store, S3FilesStore)+ config: Any = store.s3_client.meta.config+ assert config.read_timeout == expected+@pytest.mark.parametrize(("settings", "expected"),[@@ -1058,13 +1077,19 @@ class TestGCSFilesStore:def test_update_stores(self, monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr(GCSFilesStore, "GCS_PROJECT_ID", None)monkeypatch.setattr(GCSFilesStore, "POLICY", None)+ monkeypatch.setattr(GCSFilesStore, "UPLOAD_TIMEOUT", None)settings = Settings(- {"GCS_PROJECT_ID": "my-project", "FILES_STORE_GCS_ACL": "publicRead"}+ {+ "GCS_PROJECT_ID": "my-project",+ "FILES_STORE_GCS_ACL": "publicRead",+ "UPLOAD_TIMEOUT": 300,+ })FilesPipeline._update_stores(settings)assert GCSFilesStore.GCS_PROJECT_ID == "my-project"assert GCSFilesStore.POLICY == "publicRead"+ assert GCSFilesStore.UPLOAD_TIMEOUT == 300# An empty FILES_STORE_GCS_ACL is normalised to None.settings = Settings({"GCS_PROJECT_ID": "my-project", "FILES_STORE_GCS_ACL": ""})@@ -1109,6 +1134,17 @@ class TestGCSFilesStore:)assert blob.metadata == {}+ @coroutine_test+ async def test_persist_timeout(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(GCSFilesStore, "UPLOAD_TIMEOUT", 300)+ store, _, blob = self.build_gcs_files_store()+ await maybe_deferred_to_future(+ store.persist_file(+ "full/filename", BytesIO(b"hello"), info=DUMMY_SPIDER_INFO+ )+ )+ assert blob.upload_from_string.call_args.kwargs["timeout"] == 300+@coroutine_testasync def test_stat(self) -> None:store, bucket, blob = self.build_gcs_files_store()diff --git a/tox.ini b/tox.iniindex 633068ab2..2149805bf 100644--- a/tox.ini+++ b/tox.ini@@ -200,7 +200,7 @@ deps =boto3==1.20.0bpython==0.7.1colorlog==6.6.0- google-cloud-storage==1.29.0+ google-cloud-storage==1.30.0httpx2[http2,socks]==2.0.0ipython==8.15.0ptpython==3.0.23