Scrapy, a fast high-level web crawling & scraping framework for Python.
Enable PTH Ruff rules.
Andrey Rakhmatullin273620488ced
diff --git a/pyproject.toml b/pyproject.tomlindex a0b37b966..08b4b09b2 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -240,6 +240,8 @@ extend-select = ["PIE",# pylint"PL",+ # flake8-use-pathlib+ "PTH",# flake8-pyi"PYI",# flake8-quotesdiff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex d7dc104c2..2a1dea997 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -154,7 +154,7 @@ class Command(ScrapyCommand):spiders_dir = Path(spiders_module.__file__).parent.resolve()else:spiders_module = None- spiders_dir = Path(".")+ spiders_dir = Path()spider_file = f"{spiders_dir / module}.py"shutil.copyfile(template_file, spider_file)render_templatefile(spider_file, **tvars)diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.pyindex 5cb73f0d2..e0c004580 100644--- a/scrapy/commands/startproject.py+++ b/scrapy/commands/startproject.py@@ -1,6 +1,5 @@from __future__ import annotations-import osimport reimport stringfrom importlib.util import find_spec@@ -28,9 +27,9 @@ TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = (IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn")-def _make_writable(path: str | os.PathLike) -> None:- current_permissions = os.stat(path).st_mode- os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)+def _make_writable(path: Path) -> None:+ current_permissions = path.stat().st_mode+ path.chmod(current_permissions | OWNER_WRITE_PERMISSION)class Command(ScrapyCommand):diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.pyindex 598659b4d..0ad10baff 100644--- a/scrapy/core/downloader/handlers/ftp.py+++ b/scrapy/core/downloader/handlers/ftp.py@@ -32,6 +32,7 @@ from __future__ import annotationsimport refrom io import BytesIO+from pathlib import Pathfrom typing import TYPE_CHECKING, Any, BinaryIOfrom urllib.parse import unquote@@ -56,9 +57,11 @@ if TYPE_CHECKING:class ReceivedDataProtocol(Protocol):- def __init__(self, filename: str | None = None):- self.__filename: str | None = filename- self.body: BinaryIO = open(filename, "wb") if filename else BytesIO()+ def __init__(self, filename: bytes | None = None):+ self.__filename: bytes | None = filename+ self.body: BinaryIO = (+ Path(filename.decode()).open("wb") if filename else BytesIO()+ )self.size: int = 0def dataReceived(self, data: bytes) -> None:@@ -66,7 +69,7 @@ class ReceivedDataProtocol(Protocol):self.size += len(data)@property- def filename(self) -> str | None:+ def filename(self) -> bytes | None:return self.__filenamedef close(self) -> None:@@ -128,8 +131,8 @@ class FTPDownloadHandler:) -> Response:self.result = resultprotocol.close()- headers = {"local filename": protocol.filename or "", "size": protocol.size}- body = to_bytes(protocol.filename or protocol.body.read())+ headers = {"local filename": protocol.filename or b"", "size": protocol.size}+ body = protocol.filename or protocol.body.read()respcls = responsetypes.from_args(url=request.url, body=body)# hints for Headers-related types may need to be fixed to not use AnyStrreturn respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type]diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.pyindex 3b1035eab..85d7c940f 100644--- a/scrapy/utils/testproc.py+++ b/scrapy/utils/testproc.py@@ -17,7 +17,7 @@ if TYPE_CHECKING:class ProcessTest:command: str | None = Noneprefix = [sys.executable, "-m", "scrapy.cmdline"]- cwd = os.getcwd() # trial chdirs to temp dir+ cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109def execute(self,diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 8b3a6eeca..6c3fe96b0 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -1,5 +1,4 @@import logging-import osimport platformimport reimport signal@@ -643,7 +642,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase):class ScriptRunnerMixin:script_dir: Path- cwd = os.getcwd()def get_script_args(self, script_name: str, *script_args: str) -> list[str]:script_path = self.script_dir / script_namediff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 8ecba41bf..05b64e704 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -116,7 +116,7 @@ class FileTestCase(unittest.TestCase):def tearDown(self):os.close(self.fd)- os.remove(self.tmpname)+ Path(self.tmpname).unlink()def test_download(self):def _test(response):diff --git a/tests/test_utils_project.py b/tests/test_utils_project.pyindex 3831f4c21..1d149d48d 100644--- a/tests/test_utils_project.py+++ b/tests/test_utils_project.py@@ -12,7 +12,7 @@ from scrapy.utils.project import data_path, get_project_settings@contextlib.contextmanagerdef inside_a_project():- prev_dir = os.getcwd()+ prev_dir = Path.cwd()project_dir = tempfile.mkdtemp()try: