Scrapy, a fast high-level web crawling & scraping framework for Python.
Support Path Objects Issue #5739
Alexb1dd893fbb4d
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 51aedafe8..4b0c96b25 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -14,7 +14,7 @@ from contextlib import suppressfrom ftplib import FTPfrom io import BytesIOfrom pathlib import Path-from typing import DefaultDict, Optional, Set+from typing import DefaultDict, Optional, Set, Unionfrom urllib.parse import urlparsefrom itemadapter import ItemAdapter@@ -41,7 +41,8 @@ class FileException(Exception):class FSFilesStore:- def __init__(self, basedir: str):+ def __init__(self, basedir: Union[str, os.PathLike]):+ basedir = str(basedir) # support Path objectif '://' in basedir:basedir = basedir.split('://', 1)[1]self.basedir = basedir@@ -65,8 +66,8 @@ class FSFilesStore:return {'last_modified': last_modified, 'checksum': checksum}- def _get_filesystem_path(self, path: str) -> Path:- path_comps = path.split('/')+ def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path:+ path_comps = str(path).split('/')return Path(self.basedir, *path_comps)def _mkdir(self, dirname: Path, domain: Optional[str] = None):diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 4acd29bf7..5280ab0d2 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -405,6 +405,23 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):self.assertEqual(getattr(pipeline_cls, pipe_inst_attr),expected_value)+ def test_file_pipeline_using_pathlike_objects(self):++ class CustomFilesPipelineWithPathLikeDir(FilesPipeline):+ def file_path(self, request, response=None, info=None, *, item=None):+ return Path('subdir') / Path(request.url).name++ pipeline = CustomFilesPipelineWithPathLikeDir.from_settings(+ Settings({'FILES_STORE': Path('./Temp')})+ )+ request = Request("http://example.com/image01.jpg")+ self.assertEqual(pipeline.file_path(request), Path('subdir/image01.jpg'))++ def test_files_store_constructor_with_pathlike_object(self):+ path = Path('./FileDir')+ fs_store = FSFilesStore(path)+ self.assertEqual(fs_store.basedir, str(path))+class TestS3FilesStore(unittest.TestCase):