Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Allow empty port or creds in FTPFilesStore. (#8490)

Andrey Rakhmatullin1bdb3488f86d
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index a28958f29..8ddb34b06 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -499,15 +499,11 @@ class FTPFilesStore:
if not uri.startswith("ftp://"):
raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'")
u = urlparse(uri)
- assert u.port
assert u.hostname
- self.port: int = u.port
self.host: str = u.hostname
- self.port = int(u.port or 21)
- assert self.FTP_USERNAME
- assert self.FTP_PASSWORD
- self.username: str = u.username or self.FTP_USERNAME
- self.password: str = u.password or self.FTP_PASSWORD
+ self.port: int = u.port or 21
+ self.username: str = u.username or self.FTP_USERNAME or ""
+ self.password: str = u.password or self.FTP_PASSWORD or ""
self.basedir: str = u.path.rstrip("/")
def persist_file(
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index edca2ac46..f9be4d418 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1811,6 +1811,34 @@ class TestFTPFileStore:
stat = yield store.stat_file(path, info=DUMMY_SPIDER_INFO)
assert stat["checksum"] == "ff1575649a39a27c13faa0d37c84bab3"
+ @pytest.mark.parametrize(
+ ("uri", "settings", "expected"),
+ [
+ ("ftp://example.com/files", {}, (21, "anonymous", "guest")),
+ (
+ "ftp://user:[email protected]:2121/files",
+ {"FTP_USER": "", "FTP_PASSWORD": ""},
+ (2121, "user", "pass"),
+ ),
+ (
+ "ftp://example.com/files",
+ {"FTP_USER": None, "FTP_PASSWORD": None},
+ (21, "", ""),
+ ),
+ ],
+ )
+ def test_uri(
+ self,
+ uri: str,
+ settings: dict[str, str | None],
+ expected: tuple[int, str, str],
+ ) -> None:
+ crawler = get_crawler(settings_dict={"FILES_STORE": uri, **settings})
+ store = build_from_crawler(FilesPipeline, crawler).store
+ assert isinstance(store, FTPFilesStore)
+ assert store.host == "example.com"
+ assert (store.port, store.username, store.password) == expected
+
def test_wrong_uri_scheme(self):
with pytest.raises(
ValueError,