Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add an UPLOAD_TIMEOUT setting (#8162)

Adrian3d2e0e1ca1cb
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2bbb87e5c..51fb7c435 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -447,6 +447,7 @@ These are the settings used for configuring the feed exports:
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`FEED_EXPORTERS`
- :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
+- :setting:`UPLOAD_TIMEOUT`
.. setting:: FEEDS
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index ca2bfae64..a843e6f07 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -244,6 +244,9 @@ on the item in your spider, and read it back from the item in
Supported Storage
=================
+Use :setting:`UPLOAD_TIMEOUT` to limit how long uploads to a remote storage
+backend may take.
+
File system storage
-------------------
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c8f4da941..6acdfad32 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -2532,6 +2532,27 @@ For additional information, see :doc:`core/howto/choosing-reactor`.
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
+.. setting:: UPLOAD_TIMEOUT
+
+UPLOAD_TIMEOUT
+--------------
+
+.. versionadded:: VERSION
+
+Default: ``None``
+
+Number of seconds that uploads to a remote :ref:`feed storage backend
+<topics-feed-storage-backends>` or :ref:`media pipeline storage backend
+<topics-media-pipeline>` wait for a response before giving up.
+
+Raise it if large uploads fail over a slow connection.
+
+If ``None``, each backend keeps the default of the library it uses:
+`botocore <https://docs.aws.amazon.com/botocore/latest/reference/config.html>`_
+for Amazon S3, `google-cloud-storage
+<https://docs.cloud.google.com/python/docs/reference/storage/latest/retry_timeout#configuring-timeouts>`_
+for Google Cloud Storage, and :mod:`ftplib` for FTP, which waits indefinitely.
+
.. setting:: URLLENGTH_LIMIT
URLLENGTH_LIMIT
diff --git a/pyproject.toml b/pyproject.toml
index aeabe0c47..6d009005a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -70,7 +70,7 @@ Tracker = "https://github.com/scrapy/scrapy/issues"
[project.optional-dependencies]
bpython = ["bpython>=0.7.1"]
color = ["colorlog>=6.6.0"]
-gcs = ["google-cloud-storage>=1.29.0"]
+gcs = ["google-cloud-storage>=1.30.0"]
httpx = ["httpx2[http2,socks]>=2.0.0"]
images = ["Pillow>=8.3.2"]
ipython = ["ipython>=8.15.0"]
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 30c131d86..4342792cd 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -216,6 +216,7 @@ class S3FeedStorage(BlockingFeedStorage):
session_token: str | None = None,
region_name: str | None = None,
max_pool_connections: int | None = None,
+ upload_timeout: float | None = None,
):
try:
import boto3.session # noqa: PLC0415
@@ -234,6 +235,13 @@ class S3FeedStorage(BlockingFeedStorage):
self.endpoint_url: str | None = endpoint_url
self.region_name: str | None = region_name
self.max_pool_connections: int | None = max_pool_connections
+ self.upload_timeout: float | None = upload_timeout
+
+ config_kwargs: dict[str, Any] = {}
+ if max_pool_connections is not None:
+ config_kwargs["max_pool_connections"] = max_pool_connections
+ if upload_timeout is not None:
+ config_kwargs["read_timeout"] = upload_timeout
boto3_session = boto3.session.Session()
self.s3_client = boto3_session.client(
@@ -243,11 +251,7 @@ class S3FeedStorage(BlockingFeedStorage):
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
region_name=self.region_name,
- config=(
- Config(max_pool_connections=self.max_pool_connections)
- if self.max_pool_connections is not None
- else None
- ),
+ config=Config(**config_kwargs),
)
if feed_options and feed_options.get("overwrite", True) is False:
@@ -274,6 +278,7 @@ class S3FeedStorage(BlockingFeedStorage):
endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,
region_name=crawler.settings["AWS_REGION_NAME"] or None,
max_pool_connections=_get_max_pool_connections(crawler.settings),
+ upload_timeout=crawler.settings.getfloat("UPLOAD_TIMEOUT") or None,
feed_options=feed_options,
)
@@ -305,9 +310,11 @@ class GCSFeedStorage(BlockingFeedStorage):
acl: str | None,
*,
feed_options: dict[str, Any] | None = None,
+ timeout: float | None = None,
):
self.project_id: str | None = project_id
self.acl: str | None = acl
+ self.timeout: float | None = timeout
u = urlparse(uri)
assert u.hostname
self.bucket_name: str = u.hostname
@@ -327,6 +334,7 @@ class GCSFeedStorage(BlockingFeedStorage):
crawler.settings["GCS_PROJECT_ID"],
crawler.settings["FEED_STORAGE_GCS_ACL"] or None,
feed_options=feed_options,
+ timeout=crawler.settings.getfloat("UPLOAD_TIMEOUT") or None,
)
def _store_in_thread(self, file: IO[bytes]) -> None:
@@ -336,10 +344,11 @@ class GCSFeedStorage(BlockingFeedStorage):
client = Client(project=self.project_id)
bucket = client.bucket(self.bucket_name)
- blob = None if self.overwrite else bucket.get_blob(self.blob_name)
+ kwargs = {} if self.timeout is None else {"timeout": self.timeout}
+ blob = None if self.overwrite else bucket.get_blob(self.blob_name, **kwargs)
if blob is None:
bucket.blob(self.blob_name).upload_from_file(
- file, predefined_acl=self.acl
+ file, predefined_acl=self.acl, **kwargs
)
return
# Appending uploads the new data as a separate object and composes
@@ -347,13 +356,13 @@ class GCSFeedStorage(BlockingFeedStorage):
# untouched. Composition resolves its sources by name, so it always
# appends to the latest version of the blob.
part = bucket.blob(f"{self.blob_name}.{uuid4().hex}.part")
- part.upload_from_file(file, predefined_acl=self.acl)
+ part.upload_from_file(file, predefined_acl=self.acl, **kwargs)
try:
- blob.compose([blob, part], client=client)
+ blob.compose([blob, part], client=client, **kwargs)
finally:
- part.delete(client=client)
+ part.delete(client=client, **kwargs)
if self.acl:
- blob.acl.save_predefined(self.acl, client=client)
+ blob.acl.save_predefined(self.acl, client=client, **kwargs)
finally:
file.close()
@@ -365,6 +374,7 @@ class FTPFeedStorage(BlockingFeedStorage):
use_active_mode: bool = False,
*,
feed_options: dict[str, Any] | None = None,
+ timeout: float | None = None,
):
u = urlparse(uri)
if not u.hostname:
@@ -376,6 +386,7 @@ class FTPFeedStorage(BlockingFeedStorage):
self.path: str = u.path
self.tls: bool = u.scheme == "ftps"
self.use_active_mode: bool = use_active_mode
+ self.timeout: float | None = timeout
self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)
@classmethod
@@ -390,6 +401,7 @@ class FTPFeedStorage(BlockingFeedStorage):
uri,
use_active_mode=crawler.settings.getbool("FEED_STORAGE_FTP_ACTIVE"),
feed_options=feed_options,
+ timeout=crawler.settings.getfloat("UPLOAD_TIMEOUT") or None,
)
def _store_in_thread(self, file: IO[bytes]) -> None:
@@ -403,6 +415,7 @@ class FTPFeedStorage(BlockingFeedStorage):
use_active_mode=self.use_active_mode,
overwrite=self.overwrite,
tls=self.tls,
+ timeout=self.timeout,
)
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index c5f8ef9f9..d0fd32777 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -166,6 +166,7 @@ class S3FilesStore:
# Overridden from settings.AWS_MAX_POOL_CONNECTIONS in
# FilesPipeline.from_crawler(); None means the botocore default
AWS_MAX_POOL_CONNECTIONS: int | None = None
+ UPLOAD_TIMEOUT: float | None = None
POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler()
HEADERS: ClassVar[dict[str, str]] = {
@@ -178,11 +179,11 @@ class S3FilesStore:
import botocore.session # noqa: PLC0415
from botocore.config import Config # noqa: PLC0415
- config = (
- Config(max_pool_connections=self.AWS_MAX_POOL_CONNECTIONS)
- if self.AWS_MAX_POOL_CONNECTIONS is not None
- else None
- )
+ config_kwargs: dict[str, Any] = {}
+ if self.AWS_MAX_POOL_CONNECTIONS is not None:
+ config_kwargs["max_pool_connections"] = self.AWS_MAX_POOL_CONNECTIONS
+ if self.UPLOAD_TIMEOUT is not None:
+ config_kwargs["read_timeout"] = self.UPLOAD_TIMEOUT
session = botocore.session.get_session()
self.s3_client = session.create_client(
"s3",
@@ -193,7 +194,7 @@ class S3FilesStore:
region_name=self.AWS_REGION_NAME,
use_ssl=self.AWS_USE_SSL,
verify=self.AWS_VERIFY,
- config=config,
+ config=Config(**config_kwargs),
)
if not uri.startswith("s3://"):
raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
@@ -294,6 +295,7 @@ class S3FilesStore:
class GCSFilesStore:
GCS_PROJECT_ID = None
+ UPLOAD_TIMEOUT: float | None = None
CACHE_CONTROL = "max-age=172800"
@@ -361,12 +363,15 @@ class GCSFilesStore:
blob = self.bucket.blob(blob_path)
blob.cache_control = self.CACHE_CONTROL
blob.metadata = {k: str(v) for k, v in meta.items()} if meta else {}
+ timeout = self.UPLOAD_TIMEOUT
+ kwargs = {} if timeout is None else {"timeout": timeout}
return deferred_from_coro(
run_in_thread(
blob.upload_from_string,
data=buf.getvalue(),
content_type=self._get_content_type(headers),
predefined_acl=self.POLICY,
+ **kwargs,
)
)
@@ -375,6 +380,7 @@ class FTPFilesStore:
FTP_USERNAME: str | None = None
FTP_PASSWORD: str | None = None
USE_ACTIVE_MODE: bool | None = None
+ UPLOAD_TIMEOUT: float | None = None
def __init__(self, uri: str):
if not uri.startswith("ftp://"):
@@ -410,6 +416,7 @@ class FTPFilesStore:
username=self.username,
password=self.password,
use_active_mode=bool(self.USE_ACTIVE_MODE),
+ timeout=self.UPLOAD_TIMEOUT,
)
)
@@ -527,6 +534,8 @@ class FilesPipeline(MediaPipeline):
@classmethod
def _update_stores(cls, settings: BaseSettings) -> None:
+ upload_timeout: float | None = settings.getfloat("UPLOAD_TIMEOUT") or None
+
s3store: type[S3FilesStore] = cast(
"type[S3FilesStore]", cls.STORE_SCHEMES["s3"]
)
@@ -539,12 +548,14 @@ class FilesPipeline(MediaPipeline):
s3store.AWS_VERIFY = settings["AWS_VERIFY"]
s3store.AWS_MAX_POOL_CONNECTIONS = _get_max_pool_connections(settings)
s3store.POLICY = settings["FILES_STORE_S3_ACL"]
+ s3store.UPLOAD_TIMEOUT = upload_timeout
gcs_store: type[GCSFilesStore] = cast(
"type[GCSFilesStore]", cls.STORE_SCHEMES["gs"]
)
gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"]
gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None
+ gcs_store.UPLOAD_TIMEOUT = upload_timeout
ftp_store: type[FTPFilesStore] = cast(
"type[FTPFilesStore]", cls.STORE_SCHEMES["ftp"]
@@ -552,6 +563,7 @@ class FilesPipeline(MediaPipeline):
ftp_store.FTP_USERNAME = settings["FTP_USER"]
ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"]
ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE")
+ ftp_store.UPLOAD_TIMEOUT = upload_timeout
def _get_store(self, uri: str) -> FilesStoreProtocol:
# to support win32 paths like: C:\\some\dir
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index a03ee957e..58cea3609 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -226,6 +226,7 @@ __all__ = [
"TWISTED_DNS_RESOLVER",
"TWISTED_REACTOR",
"TWISTED_REACTOR_ENABLED",
+ "UPLOAD_TIMEOUT",
"URLLENGTH_LIMIT",
"USER_AGENT",
"WARN_ON_GENERATOR_RETURN_VALUE",
@@ -620,6 +621,8 @@ TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver"
TWISTED_REACTOR_ENABLED = True
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+UPLOAD_TIMEOUT = None
+
URLLENGTH_LIMIT = 2083
USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)"
diff --git a/scrapy/utils/_ftp.py b/scrapy/utils/_ftp.py
index f08e5303d..5d7fce08a 100644
--- a/scrapy/utils/_ftp.py
+++ b/scrapy/utils/_ftp.py
@@ -31,16 +31,23 @@ def ftp_store_file(
use_active_mode: bool = False,
overwrite: bool = True,
tls: bool = False,
+ timeout: float | None = None,
) -> None:
"""Opens a FTP connection with passed credentials, sets current directory
to the directory extracted from given path, then uploads the file to server.
If *tls* is ``True``, the connection is secured with TLS (FTPS), and the
certificate of the server is verified.
+
+ *timeout* is the number of seconds that socket operations wait before
+ giving up. If ``None``, they wait indefinitely.
"""
ftp = FTP_TLS(context=create_default_context()) if tls else FTP()
with ftp, closing(file):
- ftp.connect(host, port)
+ if timeout is None:
+ ftp.connect(host, port)
+ else:
+ ftp.connect(host, port, timeout)
ftp.login(username, password)
if isinstance(ftp, FTP_TLS):
ftp.prot_p()
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index 3870ad6cc..c494c0e98 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -172,6 +172,18 @@ class TestFTPFeedStorage:
await self._store(url, b"bar", settings=settings)
self._assert_stored(ftp_server.path / filename, b"bar")
+ @coroutine_test
+ async def test_upload_timeout(self):
+ crawler = get_crawler(settings_dict={"UPLOAD_TIMEOUT": 300})
+ with MockFTPServer() as ftp_server:
+ url = ftp_server.url("file")
+ storage = build_from_crawler(FTPFeedStorage, crawler, url)
+ assert storage.timeout == 300
+ file = storage.open(get_test_spider())
+ file.write(b"foo")
+ await maybe_deferred_to_future(storage.store(file))
+ self._assert_stored(ftp_server.path / "file", b"foo")
+
@coroutine_test
async def test_missing_parent_directories(self):
with MockFTPServer() as ftp_server:
@@ -464,6 +476,20 @@ class TestS3FeedStorage:
config: Any = storage.s3_client.meta.config
assert config.max_pool_connections == expected
+ def test_upload_timeout(self) -> None:
+ crawler = get_crawler(settings_dict={"UPLOAD_TIMEOUT": 300})
+ storage = build_from_crawler(S3FeedStorage, crawler, "s3://mybucket/export.csv")
+ assert storage.upload_timeout == 300
+ config: Any = storage.s3_client.meta.config
+ assert config.read_timeout == 300
+
+ def test_default_upload_timeout(self) -> None:
+ crawler = get_crawler()
+ storage = build_from_crawler(S3FeedStorage, crawler, "s3://mybucket/export.csv")
+ assert storage.upload_timeout is None
+ config: Any = storage.s3_client.meta.config
+ assert config.read_timeout == 60
+
@coroutine_test
async def test_store_without_acl(self):
storage = S3FeedStorage(
@@ -526,9 +552,19 @@ class TestGCSFeedStorage:
)
assert storage.project_id == "123"
assert storage.acl == "publicRead"
+ assert storage.timeout is None
assert storage.bucket_name == "mybucket"
assert storage.blob_name == "export.csv"
+ def test_parse_timeout(self):
+ pytest.importorskip("google.cloud.storage")
+
+ crawler = get_crawler(settings_dict={"UPLOAD_TIMEOUT": 300})
+ storage = build_from_crawler(
+ GCSFeedStorage, crawler, "gs://mybucket/export.csv"
+ )
+ assert storage.timeout == 300
+
def test_parse_empty_acl(self):
pytest.importorskip("google.cloud.storage")
@@ -568,6 +604,22 @@ class TestGCSFeedStorage:
blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
f.close.assert_called_once_with()
+ @coroutine_test
+ async def test_store_timeout(self):
+ pytest.importorskip("google.cloud.storage")
+
+ (client_mock, _, blob_mock) = mock_google_cloud_storage()
+ with mock.patch("google.cloud.storage.Client", return_value=client_mock):
+ f = mock.Mock()
+ storage = GCSFeedStorage(
+ "gs://mybucket/export.csv", "myproject-123", None, timeout=300
+ )
+ await maybe_deferred_to_future(storage.store(f))
+
+ blob_mock.upload_from_file.assert_called_once_with(
+ f, predefined_acl=None, timeout=300
+ )
+
@coroutine_test
async def test_store_closes_file_on_upload_error(self):
pytest.importorskip("google.cloud.storage")
@@ -653,6 +705,36 @@ class TestGCSFeedStorage:
blob_mock.upload_from_file.assert_not_called()
f.close.assert_called_once_with()
+ @coroutine_test
+ async def test_store_append_timeout(self) -> None:
+ pytest.importorskip("google.cloud.storage")
+
+ (client_mock, bucket_mock, part_mock) = mock_google_cloud_storage()
+ blob_mock = mock_google_cloud_blob()
+ bucket_mock.get_blob.return_value = blob_mock
+ with mock.patch("google.cloud.storage.Client", return_value=client_mock):
+ f = mock.Mock()
+ storage = GCSFeedStorage(
+ "gs://mybucket/export.csv",
+ "myproject-123",
+ "publicRead",
+ feed_options={"overwrite": False},
+ timeout=300,
+ )
+ await maybe_deferred_to_future(storage.store(f))
+
+ bucket_mock.get_blob.assert_called_once_with("export.csv", timeout=300)
+ part_mock.upload_from_file.assert_called_once_with(
+ f, predefined_acl="publicRead", timeout=300
+ )
+ blob_mock.compose.assert_called_once_with(
+ [blob_mock, part_mock], client=client_mock, timeout=300
+ )
+ part_mock.delete.assert_called_once_with(client=client_mock, timeout=300)
+ blob_mock.acl.save_predefined.assert_called_once_with(
+ "publicRead", client=client_mock, timeout=300
+ )
+
@coroutine_test
async def test_store_append_without_acl(self) -> None:
pytest.importorskip("google.cloud.storage")
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 2d08fdde7..1b8fbb662 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -989,6 +989,25 @@ class TestS3FilesStore:
config: Any = store.s3_client.meta.config
assert config.max_pool_connections == 10
+ @pytest.mark.parametrize(
+ ("settings", "expected"),
+ [
+ ({}, 60),
+ ({"UPLOAD_TIMEOUT": 300}, 300),
+ ],
+ )
+ def test_upload_timeout(
+ self, monkeypatch: pytest.MonkeyPatch, settings: dict[str, Any], expected: float
+ ) -> None:
+ monkeypatch.setattr(S3FilesStore, "UPLOAD_TIMEOUT", None)
+ crawler = get_crawler(
+ settings_dict={"FILES_STORE": "s3://mybucket/prefix/", **settings}
+ )
+ store = build_from_crawler(FilesPipeline, crawler).store
+ assert isinstance(store, S3FilesStore)
+ config: Any = store.s3_client.meta.config
+ assert config.read_timeout == expected
+
@pytest.mark.parametrize(
("settings", "expected"),
[
@@ -1058,13 +1077,19 @@ class TestGCSFilesStore:
def test_update_stores(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(GCSFilesStore, "GCS_PROJECT_ID", None)
monkeypatch.setattr(GCSFilesStore, "POLICY", None)
+ monkeypatch.setattr(GCSFilesStore, "UPLOAD_TIMEOUT", None)
settings = Settings(
- {"GCS_PROJECT_ID": "my-project", "FILES_STORE_GCS_ACL": "publicRead"}
+ {
+ "GCS_PROJECT_ID": "my-project",
+ "FILES_STORE_GCS_ACL": "publicRead",
+ "UPLOAD_TIMEOUT": 300,
+ }
)
FilesPipeline._update_stores(settings)
assert GCSFilesStore.GCS_PROJECT_ID == "my-project"
assert GCSFilesStore.POLICY == "publicRead"
+ assert GCSFilesStore.UPLOAD_TIMEOUT == 300
# An empty FILES_STORE_GCS_ACL is normalised to None.
settings = Settings({"GCS_PROJECT_ID": "my-project", "FILES_STORE_GCS_ACL": ""})
@@ -1109,6 +1134,17 @@ class TestGCSFilesStore:
)
assert blob.metadata == {}
+ @coroutine_test
+ async def test_persist_timeout(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(GCSFilesStore, "UPLOAD_TIMEOUT", 300)
+ store, _, blob = self.build_gcs_files_store()
+ await maybe_deferred_to_future(
+ store.persist_file(
+ "full/filename", BytesIO(b"hello"), info=DUMMY_SPIDER_INFO
+ )
+ )
+ assert blob.upload_from_string.call_args.kwargs["timeout"] == 300
+
@coroutine_test
async def test_stat(self) -> None:
store, bucket, blob = self.build_gcs_files_store()
diff --git a/tox.ini b/tox.ini
index 633068ab2..2149805bf 100644
--- a/tox.ini
+++ b/tox.ini
@@ -200,7 +200,7 @@ deps =
boto3==1.20.0
bpython==0.7.1
colorlog==6.6.0
- google-cloud-storage==1.29.0
+ google-cloud-storage==1.30.0
httpx2[http2,socks]==2.0.0
ipython==8.15.0
ptpython==3.0.23