Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Enable PTH Ruff rules.

Andrey Rakhmatullin273620488ced
diff --git a/pyproject.toml b/pyproject.toml
index a0b37b966..08b4b09b2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -240,6 +240,8 @@ extend-select = [
"PIE",
# pylint
"PL",
+ # flake8-use-pathlib
+ "PTH",
# flake8-pyi
"PYI",
# flake8-quotes
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index d7dc104c2..2a1dea997 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -154,7 +154,7 @@ class Command(ScrapyCommand):
spiders_dir = Path(spiders_module.__file__).parent.resolve()
else:
spiders_module = None
- spiders_dir = Path(".")
+ spiders_dir = Path()
spider_file = f"{spiders_dir / module}.py"
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 5cb73f0d2..e0c004580 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import os
import re
import string
from importlib.util import find_spec
@@ -28,9 +27,9 @@ TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = (
IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn")
-def _make_writable(path: str | os.PathLike) -> None:
- current_permissions = os.stat(path).st_mode
- os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
+def _make_writable(path: Path) -> None:
+ current_permissions = path.stat().st_mode
+ path.chmod(current_permissions | OWNER_WRITE_PERMISSION)
class Command(ScrapyCommand):
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 598659b4d..0ad10baff 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -32,6 +32,7 @@ from __future__ import annotations
import re
from io import BytesIO
+from pathlib import Path
from typing import TYPE_CHECKING, Any, BinaryIO
from urllib.parse import unquote
@@ -56,9 +57,11 @@ if TYPE_CHECKING:
class ReceivedDataProtocol(Protocol):
- def __init__(self, filename: str | None = None):
- self.__filename: str | None = filename
- self.body: BinaryIO = open(filename, "wb") if filename else BytesIO()
+ def __init__(self, filename: bytes | None = None):
+ self.__filename: bytes | None = filename
+ self.body: BinaryIO = (
+ Path(filename.decode()).open("wb") if filename else BytesIO()
+ )
self.size: int = 0
def dataReceived(self, data: bytes) -> None:
@@ -66,7 +69,7 @@ class ReceivedDataProtocol(Protocol):
self.size += len(data)
@property
- def filename(self) -> str | None:
+ def filename(self) -> bytes | None:
return self.__filename
def close(self) -> None:
@@ -128,8 +131,8 @@ class FTPDownloadHandler:
) -> Response:
self.result = result
protocol.close()
- headers = {"local filename": protocol.filename or "", "size": protocol.size}
- body = to_bytes(protocol.filename or protocol.body.read())
+ headers = {"local filename": protocol.filename or b"", "size": protocol.size}
+ body = protocol.filename or protocol.body.read()
respcls = responsetypes.from_args(url=request.url, body=body)
# hints for Headers-related types may need to be fixed to not use AnyStr
return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type]
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index 3b1035eab..85d7c940f 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
class ProcessTest:
command: str | None = None
prefix = [sys.executable, "-m", "scrapy.cmdline"]
- cwd = os.getcwd() # trial chdirs to temp dir
+ cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109
def execute(
self,
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8b3a6eeca..6c3fe96b0 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,5 +1,4 @@
import logging
-import os
import platform
import re
import signal
@@ -643,7 +642,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
class ScriptRunnerMixin:
script_dir: Path
- cwd = os.getcwd()
def get_script_args(self, script_name: str, *script_args: str) -> list[str]:
script_path = self.script_dir / script_name
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 8ecba41bf..05b64e704 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -116,7 +116,7 @@ class FileTestCase(unittest.TestCase):
def tearDown(self):
os.close(self.fd)
- os.remove(self.tmpname)
+ Path(self.tmpname).unlink()
def test_download(self):
def _test(response):
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 3831f4c21..1d149d48d 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -12,7 +12,7 @@ from scrapy.utils.project import data_path, get_project_settings
@contextlib.contextmanager
def inside_a_project():
- prev_dir = os.getcwd()
+ prev_dir = Path.cwd()
project_dir = tempfile.mkdtemp()
try: