Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support Path Objects Issue #5739

Alexb1dd893fbb4d
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 51aedafe8..4b0c96b25 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -14,7 +14,7 @@ from contextlib import suppress
from ftplib import FTP
from io import BytesIO
from pathlib import Path
-from typing import DefaultDict, Optional, Set
+from typing import DefaultDict, Optional, Set, Union
from urllib.parse import urlparse
from itemadapter import ItemAdapter
@@ -41,7 +41,8 @@ class FileException(Exception):
class FSFilesStore:
- def __init__(self, basedir: str):
+ def __init__(self, basedir: Union[str, os.PathLike]):
+ basedir = str(basedir) # support Path object
if '://' in basedir:
basedir = basedir.split('://', 1)[1]
self.basedir = basedir
@@ -65,8 +66,8 @@ class FSFilesStore:
return {'last_modified': last_modified, 'checksum': checksum}
- def _get_filesystem_path(self, path: str) -> Path:
- path_comps = path.split('/')
+ def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path:
+ path_comps = str(path).split('/')
return Path(self.basedir, *path_comps)
def _mkdir(self, dirname: Path, domain: Optional[str] = None):
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 4acd29bf7..5280ab0d2 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -405,6 +405,23 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
self.assertEqual(getattr(pipeline_cls, pipe_inst_attr),
expected_value)
+ def test_file_pipeline_using_pathlike_objects(self):
+
+ class CustomFilesPipelineWithPathLikeDir(FilesPipeline):
+ def file_path(self, request, response=None, info=None, *, item=None):
+ return Path('subdir') / Path(request.url).name
+
+ pipeline = CustomFilesPipelineWithPathLikeDir.from_settings(
+ Settings({'FILES_STORE': Path('./Temp')})
+ )
+ request = Request("http://example.com/image01.jpg")
+ self.assertEqual(pipeline.file_path(request), Path('subdir/image01.jpg'))
+
+ def test_files_store_constructor_with_pathlike_object(self):
+ path = Path('./FileDir')
+ fs_store = FSFilesStore(path)
+ self.assertEqual(fs_store.basedir, str(path))
+
class TestS3FilesStore(unittest.TestCase):