Scrapy, a fast high-level web crawling & scraping framework for Python.
use pathlib
Matthew Donoughe40d9ca3bdd6f
diff --git a/conftest.py b/conftest.pyindex d7fe80321..7c1da3556 100644--- a/conftest.py+++ b/conftest.py@@ -21,7 +21,7 @@ collect_ignore = [*_py_files("tests/CrawlerRunner"),]-with open('tests/ignores.txt') as reader:+with Path('tests/ignores.txt').open() as reader:for line in reader:file_path = line.strip()if file_path and file_path[0] != '#':diff --git a/docs/Makefile b/docs/Makefileindex 87d5d3047..596cb6cef 100644--- a/docs/Makefile+++ b/docs/Makefile@@ -86,8 +86,8 @@ coverage: BUILDER = coveragecoverage: buildhtmlview: html- $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \- os.path.realpath('build/html/index.html'))"+ $(PYTHON) -c "import webbrowser; from pathlib import Path; \+ webbrowser.open('file://' + Path('build/html/index.html').resolve())"clean:-rm -rf build/*diff --git a/docs/conf.py b/docs/conf.pyindex 3241295af..d2a77003e 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -11,13 +11,12 @@import sysfrom datetime import datetime-from os import path+from pathlib import Path# If your extensions are in another directory, add it here. If the directory-# is relative to the documentation root, use os.path.abspath to make it-# absolute, like shown here.-sys.path.append(path.join(path.dirname(__file__), "_ext"))-sys.path.insert(0, path.dirname(path.dirname(__file__)))+# is relative to the documentation root, use Path.absolute to make it absolute.+sys.path.append(str(Path(__file__).parent / "_ext"))+sys.path.insert(0, str(Path(__file__).parent.parent))# General configurationdiff --git a/docs/conftest.py b/docs/conftest.pyindex a0636f8ac..24a72a4b6 100644--- a/docs/conftest.py+++ b/docs/conftest.py@@ -1,5 +1,5 @@-import osfrom doctest import ELLIPSIS, NORMALIZE_WHITESPACE+from pathlib import Pathfrom scrapy.http.response.html import HtmlResponsefrom sybil import Sybil@@ -12,10 +12,9 @@ from sybil.parsers.doctest import DocTestParserfrom sybil.parsers.skip import skip-def load_response(url, filename):- input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)- with open(input_path, 'rb') as input_file:- return HtmlResponse(url, body=input_file.read())+def load_response(url: str, filename: str) -> HtmlResponse:+ input_path = Path(__file__).parent / '_tests' / filename+ return HtmlResponse(url, body=input_path.read_bytes())def setup(namespace):diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rstindex 092123d1d..901a170b4 100644--- a/docs/intro/tutorial.rst+++ b/docs/intro/tutorial.rst@@ -85,6 +85,8 @@ page content to extract data.This is the code for our first Spider. Save it in a file named``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::+ from pathlib import Path+import scrapy@@ -102,8 +104,7 @@ This is the code for our first Spider. Save it in a file nameddef parse(self, response):page = response.url.split("/")[-2]filename = f'quotes-{page}.html'- with open(filename, 'wb') as f:- f.write(response.body)+ Path(filename).write_bytes(response.body)self.log(f'Saved file {filename}')@@ -178,6 +179,8 @@ with a list of URLs. This list will then be used by the default implementationof :meth:`~scrapy.Spider.start_requests` to create the initial requestsfor your spider::+ from pathlib import Path+import scrapy@@ -191,8 +194,7 @@ for your spider::def parse(self, response):page = response.url.split("/")[-2]filename = f'quotes-{page}.html'- with open(filename, 'wb') as f:- f.write(response.body)+ Path(filename).write_bytes(response.body)The :meth:`~scrapy.Spider.parse` method will be called to handle eachof the requests for those URLs, even though we haven't explicitly told Scrapydiff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rstindex af294f52c..1672ccbcc 100644--- a/docs/topics/item-pipeline.rst+++ b/docs/topics/item-pipeline.rst@@ -186,6 +186,7 @@ item.::import hashlib+ from pathlib import Pathfrom urllib.parse import quoteimport scrapy@@ -214,8 +215,7 @@ item.url = adapter["url"]url_hash = hashlib.md5(url.encode("utf8")).hexdigest()filename = f"{url_hash}.png"- with open(filename, "wb") as f:- f.write(response.body)+ Path(filename).write_bytes(response.body)# Store filename in item.adapter["screenshot_filename"] = filenamediff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex 0925e6bb5..a528746b0 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -156,7 +156,6 @@ By overriding ``file_path`` like this:.. code-block:: pythonimport hashlib- from os.path import splitextdef file_path(self, request, response=None, info=None, *, item=None):image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)@@ -498,7 +497,7 @@ See here the methods that you can override in your custom Files Pipeline:approach to download all files into the ``files`` folder with theiroriginal filenames (e.g. ``files/foo.png``)::- import os+ from pathlib import PurePosixPathfrom urllib.parse import urlparsefrom scrapy.pipelines.files import FilesPipeline@@ -506,7 +505,7 @@ See here the methods that you can override in your custom Files Pipeline:class MyFilesPipeline(FilesPipeline):def file_path(self, request, response=None, info=None, *, item=None):- return 'files/' + os.path.basename(urlparse(request.url).path)+ return 'files/' + PurePosixPath(urlparse(request.url).path).nameSimilarly, you can use the ``item`` to determine the file path based on some itemproperty.@@ -637,7 +636,7 @@ See here the methods that you can override in your custom Images Pipeline:approach to download all files into the ``files`` folder with theiroriginal filenames (e.g. ``files/foo.png``)::- import os+ from pathlib import PurePosixPathfrom urllib.parse import urlparsefrom scrapy.pipelines.images import ImagesPipeline@@ -645,7 +644,7 @@ See here the methods that you can override in your custom Images Pipeline:class MyImagesPipeline(ImagesPipeline):def file_path(self, request, response=None, info=None, *, item=None):- return 'files/' + os.path.basename(urlparse(request.url).path)+ return 'files/' + PurePosixPath(urlparse(request.url).path).nameSimilarly, you can use the ``item`` to determine the file path based on some itemproperty.diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.pyindex 95a3f17d5..7a0c5288c 100755--- a/docs/utils/linkfix.py+++ b/docs/utils/linkfix.py@@ -13,6 +13,7 @@ Author: dufferzafar"""import re+from pathlib import Pathdef main():@@ -27,7 +28,7 @@ def main():# Read lines from the linkcheck output filetry:- with open("build/linkcheck/output.txt") as out:+ with Path("build/linkcheck/output.txt").open() as out:output_lines = out.readlines()except IOError:print("linkcheck output not found; please run linkcheck first.")@@ -51,14 +52,12 @@ def main():# Update the previous fileif _filename:- with open(_filename, "w") as _file:- _file.write(_contents)+ Path(_filename).write_text(_contents)_filename = newfilename# Read the new file to memory- with open(_filename) as _file:- _contents = _file.read()+ _contents = Path(_filename).read_text()_contents = _contents.replace(match.group(3), match.group(4))else:diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex fb304b8c0..8570d90bd 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -3,6 +3,7 @@ Base class for Scrapy commands"""import osimport argparse+from pathlib import Pathfrom typing import Any, Dictfrom twisted.python import failure@@ -93,8 +94,7 @@ class ScrapyCommand:self.settings.set('LOG_ENABLED', False, priority='cmdline')if opts.pidfile:- with open(opts.pidfile, "w") as f:- f.write(str(os.getpid()) + os.linesep)+ Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep)if opts.pdb:failure.startDebugMode()diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex ed5f588e9..01b4a0dbd 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -2,8 +2,9 @@ import osimport shutilimport string+from pathlib import Pathfrom importlib import import_module-from os.path import join, dirname, abspath, exists, splitext+from typing import Optionalfrom urllib.parse import urlparseimport scrapy@@ -62,8 +63,7 @@ class Command(ScrapyCommand):if opts.dump:template_file = self._find_template(opts.dump)if template_file:- with open(template_file, "r") as f:- print(f.read())+ print(template_file.read_text())returnif len(args) != 2:raise UsageError()@@ -98,11 +98,11 @@ class Command(ScrapyCommand):}if self.settings.get('NEWSPIDER_MODULE'):spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])- spiders_dir = abspath(dirname(spiders_module.__file__))+ spiders_dir = Path(spiders_module.__file__).parent.resolve()else:spiders_module = None- spiders_dir = "."- spider_file = f"{join(spiders_dir, module)}.py"+ spiders_dir = Path(".")+ spider_file = f"{spiders_dir / module}.py"shutil.copyfile(template_file, spider_file)render_templatefile(spider_file, **tvars)print(f"Created spider {name!r} using template {template_name!r} ",@@ -110,24 +110,25 @@ class Command(ScrapyCommand):if spiders_module:print(f"in module:\n {spiders_module.__name__}.{module}")- def _find_template(self, template):- template_file = join(self.templates_dir, f'{template}.tmpl')- if exists(template_file):+ def _find_template(self, template: str) -> Optional[Path]:+ template_file = Path(self.templates_dir, f'{template}.tmpl')+ if template_file.exists():return template_fileprint(f"Unable to find template: {template}\n")print('Use "scrapy genspider --list" to see all available templates.')def _list_templates(self):print("Available templates:")- for filename in sorted(os.listdir(self.templates_dir)):- if filename.endswith('.tmpl'):- print(f" {splitext(filename)[0]}")+ for file in sorted(Path(self.templates_dir).iterdir()):+ if file.suffix == '.tmpl':+ print(f" {file.stem}")- def _spider_exists(self, name):+ def _spider_exists(self, name: str) -> bool:if not self.settings.get('NEWSPIDER_MODULE'):# if run as a standalone command and file with same filename already exists- if exists(name + ".py"):- print(f"{abspath(name + '.py')} already exists")+ path = Path(name + ".py")+ if path.exists():+ print(f"{path.resolve()} already exists")return Truereturn False@@ -143,17 +144,18 @@ class Command(ScrapyCommand):# a file with the same name exists in the target directoryspiders_module = import_module(self.settings['NEWSPIDER_MODULE'])- spiders_dir = dirname(spiders_module.__file__)- spiders_dir_abs = abspath(spiders_dir)- if exists(join(spiders_dir_abs, name + ".py")):- print(f"{join(spiders_dir_abs, (name + '.py'))} already exists")+ spiders_dir = Path(spiders_module.__file__).parent+ spiders_dir_abs = spiders_dir.resolve()+ path = spiders_dir_abs / (name + ".py")+ if path.exists():+ print(f"{path} already exists")return Truereturn False@property- def templates_dir(self):- return join(- self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),+ def templates_dir(self) -> str:+ return str(Path(+ self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),'spiders'- )+ ))diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.pyindex b957c29fb..c41135508 100644--- a/scrapy/commands/runspider.py+++ b/scrapy/commands/runspider.py@@ -1,22 +1,23 @@import sys-import os+from os import PathLike+from pathlib import Pathfrom importlib import import_module+from types import ModuleTypefrom scrapy.utils.spider import iter_spider_classesfrom scrapy.exceptions import UsageErrorfrom scrapy.commands import BaseRunSpiderCommand-def _import_file(filepath):- abspath = os.path.abspath(filepath)- dirname, file = os.path.split(abspath)- fname, fext = os.path.splitext(file)- if fext not in ('.py', '.pyw'):+def _import_file(filepath: str | PathLike[str]) -> ModuleType:+ abspath = Path(filepath).resolve()+ dirname = str(abspath.parent)+ if abspath.suffix not in ('.py', '.pyw'):raise ValueError(f"Not a Python source file: {abspath}")if dirname:sys.path = [dirname] + sys.pathtry:- module = import_module(fname)+ module = import_module(abspath.stem)finally:if dirname:sys.path.pop(0)@@ -40,13 +41,13 @@ class Command(BaseRunSpiderCommand):def run(self, args, opts):if len(args) != 1:raise UsageError()- filename = args[0]- if not os.path.exists(filename):+ filename = Path(args[0])+ if not filename.exists():raise UsageError(f"File not found: {filename}\n")try:module = _import_file(filename)except (ImportError, ValueError) as e:- raise UsageError(f"Unable to load {filename!r}: {e}\n")+ raise UsageError(f"Unable to load {str(filename)!r}: {e}\n")spclasses = list(iter_spider_classes(module))if not spclasses:raise UsageError(f"No spider found in file: {filename}\n")diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.pyindex 1b6374c39..4323cdb53 100644--- a/scrapy/commands/startproject.py+++ b/scrapy/commands/startproject.py@@ -2,7 +2,7 @@ import reimport osimport stringfrom importlib.util import find_spec-from os.path import join, exists, abspath+from pathlib import Pathfrom shutil import ignore_patterns, move, copy2, copystatfrom stat import S_IWUSR as OWNER_WRITE_PERMISSION@@ -54,7 +54,7 @@ class Command(ScrapyCommand):return Truereturn False- def _copytree(self, src, dst):+ def _copytree(self, src: Path, dst: Path):"""Since the original function always creates the directory, to resolvethe issue a new function had to be created. It's a simple copy and@@ -64,19 +64,19 @@ class Command(ScrapyCommand):https://github.com/scrapy/scrapy/pull/2005"""ignore = IGNORE- names = os.listdir(src)+ names = [x.name for x in src.iterdir()]ignored_names = ignore(src, names)- if not os.path.exists(dst):- os.makedirs(dst)+ if not dst.exists():+ dst.mkdir(parents=True)for name in names:if name in ignored_names:continue- srcname = os.path.join(src, name)- dstname = os.path.join(dst, name)- if os.path.isdir(srcname):+ srcname = src / name+ dstname = dst / name+ if srcname.is_dir():self._copytree(srcname, dstname)else:copy2(srcname, dstname)@@ -90,36 +90,36 @@ class Command(ScrapyCommand):raise UsageError()project_name = args[0]- project_dir = args[0]if len(args) == 2:- project_dir = args[1]+ project_dir = Path(args[1])+ else:+ project_dir = Path(args[0])- if exists(join(project_dir, 'scrapy.cfg')):+ if (project_dir / 'scrapy.cfg').exists():self.exitcode = 1- print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}')+ print(f'Error: scrapy.cfg already exists in {project_dir.resolve()}')returnif not self._is_valid_name(project_name):self.exitcode = 1return- self._copytree(self.templates_dir, abspath(project_dir))- move(join(project_dir, 'module'), join(project_dir, project_name))+ self._copytree(Path(self.templates_dir), project_dir.resolve())+ move(project_dir / 'module', project_dir / project_name)for paths in TEMPLATES_TO_RENDER:- path = join(*paths)- tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))- render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))+ tplfile = Path(project_dir, *(string.Template(s).substitute(project_name=project_name) for s in paths))+ render_templatefile(str(tplfile), project_name=project_name, ProjectName=string_camelcase(project_name))print(f"New Scrapy project '{project_name}', using template directory "f"'{self.templates_dir}', created in:")- print(f" {abspath(project_dir)}\n")+ print(f" {project_dir.resolve()}\n")print("You can start your first spider with:")print(f" cd {project_dir}")print(" scrapy genspider example example.com")@property- def templates_dir(self):- return join(- self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),+ def templates_dir(self) -> str:+ return str(Path(+ self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),'project'- )+ ))diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.pyindex 0d94e3df0..4824167da 100644--- a/scrapy/core/downloader/handlers/file.py+++ b/scrapy/core/downloader/handlers/file.py@@ -1,3 +1,5 @@+from pathlib import Path+from w3lib.url import file_uri_to_pathfrom scrapy.responsetypes import responsetypes@@ -10,7 +12,6 @@ class FileDownloadHandler:@defersdef download_request(self, request, spider):filepath = file_uri_to_path(request.url)- with open(filepath, 'rb') as fo:- body = fo.read()+ body = Path(filepath).read_bytes()respcls = responsetypes.from_args(filename=filepath, body=body)return respcls(url=request.url, body=body)diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex 5ba0fb63b..366449f51 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -1,8 +1,7 @@import jsonimport logging-import osfrom abc import abstractmethod-from os.path import exists, join+from pathlib import Pathfrom typing import Optional, Type, TypeVarfrom twisted.internet.defer import Deferred@@ -324,19 +323,19 @@ class Scheduler(BaseScheduler):def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:""" Return a folder name to keep disk queue state at """if jobdir is not None:- dqdir = join(jobdir, 'requests.queue')- if not exists(dqdir):- os.makedirs(dqdir)- return dqdir+ dqdir = Path(jobdir, 'requests.queue')+ if not dqdir.exists():+ dqdir.mkdir(parents=True)+ return str(dqdir)return Nonedef _read_dqs_state(self, dqdir: str) -> list:- path = join(dqdir, 'active.json')- if not exists(path):+ path = Path(dqdir, 'active.json')+ if not path.exists():return []- with open(path) as f:+ with path.open() as f:return json.load(f)def _write_dqs_state(self, dqdir: str, state: list) -> None:- with open(join(dqdir, 'active.json'), 'w') as f:+ with Path(dqdir, 'active.json').open('w') as f:json.dump(state, f)diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex e768bca12..b7108cdcc 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -1,7 +1,10 @@+from __future__ import annotations+import loggingimport pprintimport signalimport warnings+from typing import TYPE_CHECKINGfrom twisted.internet import deferfrom zope.interface.exceptions import DoesNotImplement@@ -33,6 +36,9 @@ from scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.ossignal import install_shutdown_handlers, signal_namesfrom scrapy.utils.reactor import install_reactor, verify_installed_reactor+if TYPE_CHECKING:+ from scrapy.utils.request import RequestFingerprinter+logger = logging.getLogger(__name__)@@ -72,7 +78,7 @@ class Crawler:lf_cls = load_object(self.settings['LOG_FORMATTER'])self.logformatter = lf_cls.from_crawler(self)- self.request_fingerprinter = create_instance(+ self.request_fingerprinter: RequestFingerprinter = create_instance(load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),settings=self.settings,crawler=self,diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex d1b0559ef..2b8b09614 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -1,5 +1,5 @@import logging-import os+from pathlib import Pathfrom typing import Optional, Set, Type, TypeVarfrom warnings import warn@@ -55,7 +55,7 @@ class RFPDupeFilter(BaseDupeFilter):self.debug = debugself.logger = logging.getLogger(__name__)if path:- self.file = open(os.path.join(path, 'requests.seen'), 'a+')+ self.file = Path(path, 'requests.seen').open('a+')self.file.seek(0)self.fingerprints.update(x.rstrip() for x in self.file)diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex e7097b7a1..0aa27e417 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -5,13 +5,13 @@ See documentation in docs/topics/feed-exports.rst"""import logging-import osimport reimport sysimport warningsfrom datetime import datetime+from pathlib import Pathfrom tempfile import NamedTemporaryFile-from typing import Any, Callable, Optional, Tuple, Union+from typing import IO, Any, Callable, Optional, Tuple, Unionfrom urllib.parse import unquote, urlparsefrom twisted.internet import defer, threads@@ -101,7 +101,7 @@ class BlockingFeedStorage:def open(self, spider):path = spider.crawler.settings['FEED_TEMPDIR']- if path and not os.path.isdir(path):+ if path and not Path(path).is_dir():raise OSError('Not a Directory: ' + str(path))return NamedTemporaryFile(prefix='feed-', dir=path)@@ -141,11 +141,11 @@ class FileFeedStorage:feed_options = feed_options or {}self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'- def open(self, spider):- dirname = os.path.dirname(self.path)- if dirname and not os.path.exists(dirname):- os.makedirs(dirname)- return open(self.path, self.write_mode)+ def open(self, spider) -> IO[Any]:+ dirname = Path(self.path).parent+ if dirname and not dirname.exists():+ dirname.mkdir(parents=True)+ return Path(self.path).open(self.write_mode)def store(self, file):file.close()diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex 843e14812..3057bf157 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -1,16 +1,18 @@import gzipimport logging-import osimport picklefrom email.utils import mktime_tz, parsedate_tzfrom importlib import import_module+from pathlib import Pathfrom time import timefrom weakref import WeakKeyDictionaryfrom w3lib.http import headers_raw_to_dict, headers_dict_to_rawfrom scrapy.http import Headers, Response+from scrapy.http.request import Requestfrom scrapy.responsetypes import responsetypes+from scrapy.spiders import Spiderfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.project import data_pathfrom scrapy.utils.python import to_bytes, to_unicode@@ -221,9 +223,9 @@ class DbmCacheStorage:self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE'])self.db = None- def open_spider(self, spider):- dbpath = os.path.join(self.cachedir, f'{spider.name}.db')- self.db = self.dbmodule.open(dbpath, 'c')+ def open_spider(self, spider: Spider):+ dbpath = Path(self.cachedir, f'{spider.name}.db')+ self.db = self.dbmodule.open(str(dbpath), 'c')logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})@@ -277,7 +279,7 @@ class FilesystemCacheStorage:self.use_gzip = settings.getbool('HTTPCACHE_GZIP')self._open = gzip.open if self.use_gzip else open- def open_spider(self, spider):+ def open_spider(self, spider: Spider):logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},extra={'spider': spider})@@ -286,15 +288,15 @@ class FilesystemCacheStorage:def close_spider(self, spider):pass- def retrieve_response(self, spider, request):+ def retrieve_response(self, spider: Spider, request: Request):"""Return response if present in cache, or None otherwise."""metadata = self._read_meta(spider, request)if metadata is None:return # not cachedrpath = self._get_request_path(spider, request)- with self._open(os.path.join(rpath, 'response_body'), 'rb') as f:+ with self._open(rpath / 'response_body', 'rb') as f:body = f.read()- with self._open(os.path.join(rpath, 'response_headers'), 'rb') as f:+ with self._open(rpath / 'response_headers', 'rb') as f:rawheaders = f.read()url = metadata.get('response_url')status = metadata['status']@@ -303,11 +305,11 @@ class FilesystemCacheStorage:response = respcls(url=url, headers=headers, status=status, body=body)return response- def store_response(self, spider, request, response):+ def store_response(self, spider: Spider, request: Request, response):"""Store the given response in the cache."""rpath = self._get_request_path(spider, request)- if not os.path.exists(rpath):- os.makedirs(rpath)+ if not rpath.exists():+ rpath.mkdir(parents=True)metadata = {'url': request.url,'method': request.method,@@ -315,29 +317,29 @@ class FilesystemCacheStorage:'response_url': response.url,'timestamp': time(),}- with self._open(os.path.join(rpath, 'meta'), 'wb') as f:+ with self._open(rpath / 'meta', 'wb') as f:f.write(to_bytes(repr(metadata)))- with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:+ with self._open(rpath / 'pickled_meta', 'wb') as f:pickle.dump(metadata, f, protocol=4)- with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f:+ with self._open(rpath / 'response_headers', 'wb') as f:f.write(headers_dict_to_raw(response.headers))- with self._open(os.path.join(rpath, 'response_body'), 'wb') as f:+ with self._open(rpath / 'response_body', 'wb') as f:f.write(response.body)- with self._open(os.path.join(rpath, 'request_headers'), 'wb') as f:+ with self._open(rpath / 'request_headers', 'wb') as f:f.write(headers_dict_to_raw(request.headers))- with self._open(os.path.join(rpath, 'request_body'), 'wb') as f:+ with self._open(rpath / 'request_body', 'wb') as f:f.write(request.body)- def _get_request_path(self, spider, request):+ def _get_request_path(self, spider: Spider, request: Request) -> Path:key = self._fingerprinter.fingerprint(request).hex()- return os.path.join(self.cachedir, spider.name, key[0:2], key)+ return Path(self.cachedir, spider.name, key[0:2], key)- def _read_meta(self, spider, request):+ def _read_meta(self, spider: Spider, request: Request):rpath = self._get_request_path(spider, request)- metapath = os.path.join(rpath, 'pickled_meta')- if not os.path.exists(metapath):+ metapath = rpath / 'pickled_meta'+ if not metapath.exists():return # not found- mtime = os.stat(metapath).st_mtime+ mtime = metapath.stat().st_mtimeif 0 < self.expiration_secs < time() - mtime:return # expiredwith self._open(metapath, 'rb') as f:diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.pyindex bea00596e..e9c8b1d6a 100644--- a/scrapy/extensions/spiderstate.py+++ b/scrapy/extensions/spiderstate.py@@ -1,5 +1,5 @@-import osimport pickle+from pathlib import Pathfrom scrapy import signalsfrom scrapy.exceptions import NotConfigured@@ -25,16 +25,16 @@ class SpiderState:def spider_closed(self, spider):if self.jobdir:- with open(self.statefn, 'wb') as f:+ with Path(self.statefn).open('wb') as f:pickle.dump(spider.state, f, protocol=4)def spider_opened(self, spider):- if self.jobdir and os.path.exists(self.statefn):- with open(self.statefn, 'rb') as f:+ if self.jobdir and Path(self.statefn).exists():+ with Path(self.statefn).open('rb') as f:spider.state = pickle.load(f)else:spider.state = {}@property- def statefn(self):- return os.path.join(self.jobdir, 'spider.state')+ def statefn(self) -> str:+ return str(Path(self.jobdir, 'spider.state'))diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex 4de6c9b5b..9359cc7c8 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -32,7 +32,7 @@ class Response(object_ref):def __init__(self,- url,+ url: str,status=200,headers=None,body=b"",@@ -75,7 +75,7 @@ class Response(object_ref):def _get_url(self):return self._url- def _set_url(self, url):+ def _set_url(self, url: str):if isinstance(url, str):self._url = urlelse:diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 906e7eb24..ffb12d910 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -13,6 +13,8 @@ from collections import defaultdictfrom contextlib import suppressfrom ftplib import FTPfrom io import BytesIO+from pathlib import Path+from typing import Optionalfrom urllib.parse import urlparsefrom itemadapter import ItemAdapter@@ -39,41 +41,40 @@ class FileException(Exception):class FSFilesStore:- def __init__(self, basedir):+ def __init__(self, basedir: str):if '://' in basedir:basedir = basedir.split('://', 1)[1]self.basedir = basedir- self._mkdir(self.basedir)- self.created_directories = defaultdict(set)+ self._mkdir(Path(self.basedir))+ self.created_directories: defaultdict[str, set[str]] = defaultdict(set)- def persist_file(self, path, buf, info, meta=None, headers=None):+ def persist_file(self, path: str, buf, info, meta=None, headers=None):absolute_path = self._get_filesystem_path(path)- self._mkdir(os.path.dirname(absolute_path), info)- with open(absolute_path, 'wb') as f:- f.write(buf.getvalue())+ self._mkdir(absolute_path.parent, info)+ absolute_path.write_bytes(buf.getvalue())- def stat_file(self, path, info):+ def stat_file(self, path: str, info):absolute_path = self._get_filesystem_path(path)try:- last_modified = os.path.getmtime(absolute_path)+ last_modified = absolute_path.stat().st_mtimeexcept os.error:return {}- with open(absolute_path, 'rb') as f:+ with absolute_path.open('rb') as f:checksum = md5sum(f)return {'last_modified': last_modified, 'checksum': checksum}- def _get_filesystem_path(self, path):+ def _get_filesystem_path(self, path: str) -> Path:path_comps = path.split('/')- return os.path.join(self.basedir, *path_comps)+ return Path(self.basedir, *path_comps)- def _mkdir(self, dirname, domain=None):+ def _mkdir(self, dirname: Path, domain: Optional[str] = None):seen = self.created_directories[domain] if domain else set()- if dirname not in seen:- if not os.path.exists(dirname):- os.makedirs(dirname)- seen.add(dirname)+ if str(dirname) not in seen:+ if not dirname.exists():+ dirname.mkdir(parents=True)+ seen.add(str(dirname))class S3FilesStore:@@ -374,8 +375,8 @@ class FilesPipeline(MediaPipeline):store_uri = settings['FILES_STORE']return cls(store_uri, settings=settings)- def _get_store(self, uri):- if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir+ def _get_store(self, uri: str):+ if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dirscheme = 'file'else:scheme = urlparse(uri).scheme@@ -510,7 +511,7 @@ class FilesPipeline(MediaPipeline):def file_path(self, request, response=None, info=None, *, item=None):media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()- media_ext = os.path.splitext(request.url)[1]+ media_ext = Path(request.url).suffix# Handles empty and wild extensions by trying to guess the# mime type then extension or default to empty string otherwiseif media_ext not in mimetypes.types_map:diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex ff86af125..84e0a94d2 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -15,7 +15,7 @@ Scrapy developers, if you add a setting here remember to:import sysfrom importlib import import_module-from os.path import join, abspath, dirname+from pathlib import PathAJAXCRAWL_ENABLED = False@@ -288,7 +288,7 @@ STATS_DUMP = TrueSTATSMAILER_RCPTS = []-TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))+TEMPLATES_DIR = str((Path(__file__).parent / '..' / 'templates').resolve())URLLENGTH_LIMIT = 2083diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex d8248c606..9a97e7801 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -3,14 +3,19 @@ Base class for Scrapy spidersSee documentation in docs/topics/spiders.rst"""+from __future__ import annotations+import logging-from typing import Optional+from typing import TYPE_CHECKING, Optionalfrom scrapy import signalsfrom scrapy.http import Requestfrom scrapy.utils.trackref import object_reffrom scrapy.utils.url import url_is_from_spider+if TYPE_CHECKING:+ from scrapy.crawler import Crawler+class Spider(object_ref):"""Base class for scrapy spiders. All spiders must inherit from this@@ -49,7 +54,7 @@ class Spider(object_ref):spider._set_crawler(crawler)return spider- def _set_crawler(self, crawler):+ def _set_crawler(self, crawler: Crawler):self.crawler = crawlerself.settings = crawler.settingscrawler.signals.connect(self.close, signals.spider_closed)diff --git a/scrapy/squeues.py b/scrapy/squeues.pyindex dff9b1350..1f2dee55f 100644--- a/scrapy/squeues.py+++ b/scrapy/squeues.py@@ -3,8 +3,9 @@ Scheduler queues"""import marshal-import osimport pickle+from os import PathLike+from pathlib import Pathfrom queuelib import queue@@ -16,10 +17,10 @@ def _with_mkdir(queue_class):class DirectoriesCreated(queue_class):- def __init__(self, path, *args, **kwargs):- dirname = os.path.dirname(path)- if not os.path.exists(dirname):- os.makedirs(dirname, exist_ok=True)+ def __init__(self, path: str | PathLike[str], *args, **kwargs):+ dirname = Path(path).parent+ if not dirname.exists():+ dirname.mkdir(parents=True, exist_ok=True)super().__init__(path, *args, **kwargs)return DirectoriesCreateddiff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex 00cc53725..e247f5999 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -4,6 +4,8 @@ import sysimport warningsfrom configparser import ConfigParserfrom operator import itemgetter+from pathlib import Path+from typing import Optionalfrom scrapy.exceptions import ScrapyDeprecationWarning, UsageError@@ -65,17 +67,17 @@ def arglist_to_dict(arglist):return dict(x.split('=', 1) for x in arglist)-def closest_scrapy_cfg(path='.', prevpath=None):+def closest_scrapy_cfg(path: str | os.PathLike[str] = '.', prevpath: Optional[str | os.PathLike] = None) -> str:"""Return the path to the closest scrapy.cfg file by traversing the currentdirectory and its parents"""- if path == prevpath:+ if prevpath is not None and str(path) == str(prevpath):return ''- path = os.path.abspath(path)- cfgfile = os.path.join(path, 'scrapy.cfg')- if os.path.exists(cfgfile):- return cfgfile- return closest_scrapy_cfg(os.path.dirname(path), path)+ path = Path(path).resolve()+ cfgfile = path / 'scrapy.cfg'+ if cfgfile.exists():+ return str(cfgfile)+ return closest_scrapy_cfg(path.parent, path)def init_env(project='default', set_syspath=True):@@ -88,7 +90,7 @@ def init_env(project='default', set_syspath=True):os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project)closest = closest_scrapy_cfg()if closest:- projdir = os.path.dirname(closest)+ projdir = str(Path(closest).parent)if set_syspath and projdir not in sys.path:sys.path.append(projdir)@@ -101,13 +103,13 @@ def get_config(use_closest=True):return cfg-def get_sources(use_closest=True):- xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config')+def get_sources(use_closest=True) -> list[str]:+ xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or Path('~/.config').expanduser()sources = ['/etc/scrapy.cfg',r'c:\scrapy\scrapy.cfg',- xdg_config_home + '/scrapy.cfg',- os.path.expanduser('~/.scrapy.cfg'),+ str(Path(xdg_config_home) / 'scrapy.cfg'),+ str(Path('~/.scrapy.cfg').expanduser()),]if use_closest:sources.append(closest_scrapy_cfg())@@ -129,8 +131,8 @@ def feed_complete_default_values_from_settings(feed, settings):return out-def feed_process_params_from_cli(settings, output, output_format=None,- overwrite_output=None):+def feed_process_params_from_cli(settings, output: list[str], output_format=None,+ overwrite_output: Optional[list[str]] = None):"""Receives feed export params (from the 'crawl' or 'runspider' commands),checks for inconsistencies in their quantities and returns a dictionary@@ -180,7 +182,7 @@ def feed_process_params_from_cli(settings, output, output_format=None,feed_uri, feed_format = element.rsplit(':', 1)except ValueError:feed_uri = element- feed_format = os.path.splitext(element)[1].replace('.', '')+ feed_format = Path(element).suffix.replace('.', '')else:if feed_uri == '-':feed_uri = 'stdout:'diff --git a/scrapy/utils/job.py b/scrapy/utils/job.pyindex c92ef36f5..a65f92e95 100644--- a/scrapy/utils/job.py+++ b/scrapy/utils/job.py@@ -1,4 +1,4 @@-import os+from pathlib import Pathfrom typing import Optionalfrom scrapy.settings import BaseSettings@@ -6,6 +6,6 @@ from scrapy.settings import BaseSettingsdef job_dir(settings: BaseSettings) -> Optional[str]:path = settings['JOBDIR']- if path and not os.path.exists(path):- os.makedirs(path)+ if path and not Path(path).exists():+ Path(path).mkdir(parents=True)return pathdiff --git a/scrapy/utils/project.py b/scrapy/utils/project.pyindex c66af497e..e54b71d45 100644--- a/scrapy/utils/project.py+++ b/scrapy/utils/project.py@@ -2,7 +2,7 @@ import osimport warningsfrom importlib import import_module-from os.path import join, dirname, abspath, isabs, exists+from pathlib import Pathfrom scrapy.utils.conf import closest_scrapy_cfg, get_config, init_envfrom scrapy.settings import Settings@@ -25,36 +25,37 @@ def inside_project():return bool(closest_scrapy_cfg())-def project_data_dir(project='default'):+def project_data_dir(project='default') -> str:"""Return the current project data dir, creating it if it doesn't exist"""if not inside_project():raise NotConfigured("Not inside a project")cfg = get_config()if cfg.has_option(DATADIR_CFG_SECTION, project):- d = cfg.get(DATADIR_CFG_SECTION, project)+ d = Path(cfg.get(DATADIR_CFG_SECTION, project))else:scrapy_cfg = closest_scrapy_cfg()if not scrapy_cfg:raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir")- d = abspath(join(dirname(scrapy_cfg), '.scrapy'))- if not exists(d):- os.makedirs(d)- return d+ d = (Path(scrapy_cfg).parent / '.scrapy').resolve()+ if not d.exists():+ d.mkdir(parents=True)+ return str(d)-def data_path(path, createdir=False):+def data_path(path: str, createdir=False) -> str:"""Return the given path joined with the .scrapy data directory.If given an absolute path, return it unmodified."""- if not isabs(path):+ path_obj = Path(path)+ if not path_obj.is_absolute():if inside_project():- path = join(project_data_dir(), path)+ path_obj = Path(project_data_dir(), path)else:- path = join('.scrapy', path)- if createdir and not exists(path):- os.makedirs(path)- return path+ path_obj = Path('.scrapy', path)+ if createdir and not path_obj.exists():+ path_obj.mkdir(parents=True)+ return str(path_obj)def get_project_settings():diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex cf33317ce..545d489be 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -264,7 +264,7 @@ class RequestFingerprinter:f'and \'VERSION\'.')- def fingerprint(self, request):+ def fingerprint(self, request: Request):return self._fingerprint(request)diff --git a/scrapy/utils/template.py b/scrapy/utils/template.pyindex f068be737..8075902b3 100644--- a/scrapy/utils/template.py+++ b/scrapy/utils/template.py@@ -1,23 +1,21 @@"""Helper functions for working with templates"""-import osimport reimport string+from pathlib import Path-def render_templatefile(path, **kwargs):- with open(path, 'rb') as fp:- raw = fp.read().decode('utf8')+def render_templatefile(path: str, **kwargs):+ raw = Path(path).read_text('utf8')content = string.Template(raw).substitute(**kwargs)render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else pathif path.endswith('.tmpl'):- os.rename(path, render_path)+ Path(path).rename(render_path)- with open(render_path, 'wb') as fp:- fp.write(content.encode('utf8'))+ Path(render_path).write_text(content, 'utf8')CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 0b828f7c0..4d01f1ef1 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -4,6 +4,7 @@ This module contains some assorted functions used in testsimport asyncioimport os+from pathlib import Pathfrom posixpath import splitfrom unittest import mock@@ -71,11 +72,11 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):return runner.create_crawler(spidercls or Spider)-def get_pythonpath():+def get_pythonpath() -> str:"""Return a PYTHONPATH suitable to use in processes so that they find thisinstallation of Scrapy"""scrapy_path = import_module('scrapy').__path__[0]- return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '')+ return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get('PYTHONPATH', '')def get_testenv():diff --git a/setup.py b/setup.pyindex a43cf08c8..e413ea6e4 100644--- a/setup.py+++ b/setup.py@@ -1,10 +1,9 @@-from os.path import dirname, join+from pathlib import Pathfrom pkg_resources import parse_versionfrom setuptools import setup, find_packages, __version__ as setuptools_version-with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f:- version = f.read().decode('ascii').strip()+version = (Path(__file__).parent / 'scrapy/VERSION').read_text('ascii').strip()def has_environment_marker_platform_impl_support():diff --git a/tests/__init__.py b/tests/__init__.pyindex bb62851dc..be263fa16 100644--- a/tests/__init__.py+++ b/tests/__init__.py@@ -6,6 +6,7 @@ see https://docs.scrapy.org/en/latest/contributing.html#running-testsimport osimport socket+from pathlib import Path# ignore system-wide proxies for tests# which would send requests to a totally unsuspecting server@@ -16,14 +17,12 @@ os.environ['ftp_proxy'] = ''# Absolutize paths to coverage config and output file because tests that# spawn subprocesses also changes current working directory.-_sourceroot = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+_sourceroot = Path(__file__).resolve().parent.parentif 'COV_CORE_CONFIG' in os.environ:- os.environ['COVERAGE_FILE'] = os.path.join(_sourceroot, '.coverage')- os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot,- os.environ['COV_CORE_CONFIG'])+ os.environ['COVERAGE_FILE'] = str(_sourceroot / '.coverage')+ os.environ['COV_CORE_CONFIG'] = str(_sourceroot / os.environ['COV_CORE_CONFIG'])-tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),- 'sample_data')+tests_datadir = str(Path(__file__).parent.resolve() / 'sample_data')# In some environments accessing a non-existing host doesn't raise an@@ -35,8 +34,6 @@ except socket.gaierror:NON_EXISTING_RESOLVABLE = False-def get_testdata(*paths):+def get_testdata(*paths: str) -> bytes:"""Return test data"""- path = os.path.join(tests_datadir, *paths)- with open(path, 'rb') as f:- return f.read()+ return Path(tests_datadir, *paths).read_bytes()diff --git a/tests/keys/__init__.py b/tests/keys/__init__.pyindex bb4a8e5af..3a41b3a3e 100644--- a/tests/keys/__init__.py+++ b/tests/keys/__init__.py@@ -1,5 +1,5 @@-import osfrom datetime import datetime, timedelta+from pathlib import Pathfrom cryptography.hazmat.backends import default_backendfrom cryptography.hazmat.primitives.asymmetric import rsa@@ -22,21 +22,20 @@ from cryptography.x509.oid import NameOID# https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificatedef generate_keys():- folder = os.path.dirname(__file__)+ folder = Path(__file__).parentkey = rsa.generate_private_key(public_exponent=65537,key_size=2048,backend=default_backend(),)- with open(os.path.join(folder, 'localhost.key'), "wb") as f:- f.write(- key.private_bytes(- encoding=Encoding.PEM,- format=PrivateFormat.TraditionalOpenSSL,- encryption_algorithm=NoEncryption(),- )- )+ (folder / 'localhost.key').write_bytes(+ key.private_bytes(+ encoding=Encoding.PEM,+ format=PrivateFormat.TraditionalOpenSSL,+ encryption_algorithm=NoEncryption(),+ ),+ )subject = issuer = Name([@@ -59,5 +58,4 @@ def generate_keys():).sign(key, SHA256(), default_backend()))- with open(os.path.join(folder, 'localhost.crt'), "wb") as f:- f.write(cert.public_bytes(Encoding.PEM))+ (folder / 'localhost.crt').write_bytes(cert.public_bytes(Encoding.PEM))diff --git a/tests/mockserver.py b/tests/mockserver.pyindex 72d7e0241..7916798f7 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -1,6 +1,5 @@import argparseimport json-import osimport randomimport sysfrom pathlib import Path@@ -253,7 +252,7 @@ class Root(resource.Resource):self.putChild(b"alpayload", ArbitraryLengthPayloadResource())try:from tests import tests_datadir- self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/')))+ self.putChild(b"files", File(str(Path(tests_datadir, 'test_site/files/'))))except Exception:passself.putChild(b"redirect-to", RedirectTo())@@ -346,8 +345,8 @@ class MockFTPServer:def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):factory = ssl.DefaultOpenSSLContextFactory(- os.path.join(os.path.dirname(__file__), keyfile),- os.path.join(os.path.dirname(__file__), certfile),+ str(Path(__file__).parent / keyfile),+ str(Path(__file__).parent / certfile),)if cipher_string:ctx = factory.getContext()diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.pyindex 8233e0101..da73a4c45 100644--- a/tests/test_cmdline/__init__.py+++ b/tests/test_cmdline/__init__.py@@ -1,11 +1,11 @@import json-import osimport pstatsimport shutilimport sysimport tempfileimport unittestfrom io import StringIO+from pathlib import Pathfrom subprocess import Popen, PIPEfrom scrapy.utils.test import get_testenv@@ -36,17 +36,17 @@ class CmdlineTest(unittest.TestCase):self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override')def test_profiling(self):- path = tempfile.mkdtemp()- filename = os.path.join(path, 'res.prof')+ path = Path(tempfile.mkdtemp())+ filename = path / 'res.prof'try:- self._execute('version', '--profile', filename)- self.assertTrue(os.path.exists(filename))+ self._execute('version', '--profile', str(filename))+ self.assertTrue(filename.exists())out = StringIO()- stats = pstats.Stats(filename, stream=out)+ stats = pstats.Stats(str(filename), stream=out)stats.print_stats()out.seek(0)stats = out.read()- self.assertIn(os.path.join('scrapy', 'commands', 'version.py'),+ self.assertIn(str(Path('scrapy', 'commands', 'version.py')),stats)self.assertIn('tottime', stats)finally:diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.pyindex d341888d3..fcafcef68 100644--- a/tests/test_cmdline_crawl_with_pipeline/__init__.py+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py@@ -1,6 +1,6 @@-import osimport sysimport unittest+from pathlib import Pathfrom subprocess import Popen, PIPE@@ -8,7 +8,7 @@ class CmdlineCrawlPipelineTest(unittest.TestCase):def _execute(self, spname):args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname)- cwd = os.path.dirname(os.path.abspath(__file__))+ cwd = Path(__file__).resolve().parentproc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)proc.communicate()return proc.returncodediff --git a/tests/test_command_check.py b/tests/test_command_check.pyindex c3d705194..4077a9bce 100644--- a/tests/test_command_check.py+++ b/tests/test_command_check.py@@ -1,5 +1,3 @@-from os.path import join, abspath-from tests.test_commands import CommandTest@@ -10,11 +8,10 @@ class CheckCommandTest(CommandTest):def setUp(self):super(CheckCommandTest, self).setUp()self.spider_name = 'check_spider'- self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py'))+ self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve()def _write_contract(self, contracts, parse_def):- with open(self.spider, 'w') as file:- file.write(f"""+ self.spider.write_text(f"""import scrapyclass CheckSpider(scrapy.Spider):@@ -27,7 +24,7 @@ class CheckSpider(scrapy.Spider):{contracts}\"\"\"{parse_def}- """)+ """)def _test_contract(self, contracts='', parse_def='pass'):self._write_contract(contracts, parse_def)diff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex 0d992be56..154287d74 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -1,6 +1,6 @@import osimport argparse-from os.path import join, abspath, isfile, exists+from pathlib import Pathfrom twisted.internet import deferfrom scrapy.commands import parse@@ -23,9 +23,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):def setUp(self):super().setUp()self.spider_name = 'parse_spider'- fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))- with open(fname, 'w') as f:- f.write(f"""+ (self.proj_mod_path / 'spiders' / 'myspider.py').write_text(f"""import scrapyfrom scrapy.linkextractors import LinkExtractorfrom scrapy.spiders import CrawlSpider, Rule@@ -88,9 +86,7 @@ class MyBadCrawlSpider(CrawlSpider):return [scrapy.Item(), dict(foo='bar')]""")- fname = abspath(join(self.proj_mod_path, 'pipelines.py'))- with open(fname, 'w') as f:- f.write("""+ (self.proj_mod_path / 'pipelines.py').write_text("""import loggingclass MyPipeline:@@ -101,8 +97,7 @@ class MyPipeline:return item""")- fname = abspath(join(self.proj_mod_path, 'settings.py'))- with open(fname, 'a') as f:+ with (self.proj_mod_path / 'settings.py').open("a") as f:f.write(f"""ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}""")@@ -234,7 +229,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}correct format containing correct data in it."""file_name = 'data.json'- file_path = join(self.proj_path, file_name)+ file_path = Path(self.proj_path, file_name)yield self.execute(['--spider', self.spider_name,'-c', 'parse',@@ -242,12 +237,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}self.url('/html')])- self.assertTrue(exists(file_path))- self.assertTrue(isfile(file_path))+ self.assertTrue(file_path.exists())+ self.assertTrue(file_path.is_file())content = '[\n{},\n{"foo": "bar"}\n]'- with open(file_path, 'r') as f:- self.assertEqual(f.read(), content)+ self.assertEqual(file_path.read_text(), content)def test_parse_add_options(self):command = parse.Command()diff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex 33189e9be..33c98ad69 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -1,4 +1,4 @@-from os.path import join+from pathlib import Pathfrom twisted.trial import unittestfrom twisted.internet import defer@@ -96,8 +96,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):@defer.inlineCallbacksdef test_local_file(self):- filepath = join(tests_datadir, 'test_site', 'index.html')- _, out, _ = yield self.execute([filepath, '-c', 'item'])+ filepath = Path(tests_datadir, 'test_site', 'index.html')+ _, out, _ = yield self.execute([str(filepath), '-c', 'item'])assert b'{}' in out@defer.inlineCallbacksdiff --git a/tests/test_commands.py b/tests/test_commands.pyindex 76d5f3935..39f718cce 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -9,12 +9,12 @@ import sysimport tempfilefrom contextlib import contextmanagerfrom itertools import chain-from os.path import exists, join, abspath, getmtimefrom pathlib import Pathfrom shutil import rmtree, copytreefrom stat import S_IWRITE as ANYONE_WRITE_PERMISSIONfrom tempfile import mkdtempfrom threading import Timer+from typing import Generator, Optionalfrom unittest import skipIffrom pytest import mark@@ -66,8 +66,8 @@ class ProjectTest(unittest.TestCase):def setUp(self):self.temp_path = mkdtemp()self.cwd = self.temp_path- self.proj_path = join(self.temp_path, self.project_name)- self.proj_mod_path = join(self.proj_path, self.project_name)+ self.proj_path = Path(self.temp_path, self.project_name)+ self.proj_mod_path = self.proj_path / self.project_nameself.env = get_testenv()def tearDown(self):@@ -104,10 +104,10 @@ class ProjectTest(unittest.TestCase):return p, to_unicode(stdout), to_unicode(stderr)- def find_in_file(self, filename, regex):+ def find_in_file(self, filename: str | os.PathLike[str], regex) -> Optional[re.Match]:"""Find first pattern occurrence in file"""pattern = re.compile(regex)- with open(filename, "r") as f:+ with Path(filename).open("r") as f:for line in f:match = pattern.search(line)if match is not None:@@ -122,13 +122,13 @@ class StartprojectTest(ProjectTest):print(err, file=sys.stderr)self.assertEqual(p.returncode, 0)- assert exists(join(self.proj_path, 'scrapy.cfg'))- assert exists(join(self.proj_path, 'testproject'))- assert exists(join(self.proj_mod_path, '__init__.py'))- assert exists(join(self.proj_mod_path, 'items.py'))- assert exists(join(self.proj_mod_path, 'pipelines.py'))- assert exists(join(self.proj_mod_path, 'settings.py'))- assert exists(join(self.proj_mod_path, 'spiders', '__init__.py'))+ assert Path(self.proj_path, 'scrapy.cfg').exists()+ assert Path(self.proj_path, 'testproject').exists()+ assert Path(self.proj_mod_path, '__init__.py').exists()+ assert Path(self.proj_mod_path, 'items.py').exists()+ assert Path(self.proj_mod_path, 'pipelines.py').exists()+ assert Path(self.proj_mod_path, 'settings.py').exists()+ assert Path(self.proj_mod_path, 'spiders', '__init__.py').exists()self.assertEqual(1, self.call('startproject', self.project_name))self.assertEqual(1, self.call('startproject', 'wrong---project---name'))@@ -138,13 +138,13 @@ class StartprojectTest(ProjectTest):project_dir = mkdtemp()self.assertEqual(0, self.call('startproject', self.project_name, project_dir))- assert exists(join(abspath(project_dir), 'scrapy.cfg'))- assert exists(join(abspath(project_dir), 'testproject'))- assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py'))- assert exists(join(join(abspath(project_dir), self.project_name), 'items.py'))- assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py'))- assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py'))- assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py'))+ assert Path(project_dir, 'scrapy.cfg').exists()+ assert Path(project_dir, 'testproject').exists()+ assert Path(project_dir, self.project_name, '__init__.py').exists()+ assert Path(project_dir, self.project_name, 'items.py').exists()+ assert Path(project_dir, self.project_name, 'pipelines.py').exists()+ assert Path(project_dir, self.project_name, 'settings.py').exists()+ assert Path(project_dir, self.project_name, 'spiders', '__init__.py').exists()self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2'))@@ -158,40 +158,42 @@ class StartprojectTest(ProjectTest):def test_existing_project_dir(self):project_dir = mkdtemp()project_name = self.project_name + '_existing'- project_path = os.path.join(project_dir, project_name)- os.mkdir(project_path)+ project_path = Path(project_dir, project_name)+ project_path.mkdir()p, out, err = self.proc('startproject', project_name, cwd=project_dir)print(out)print(err, file=sys.stderr)self.assertEqual(p.returncode, 0)- assert exists(join(abspath(project_path), 'scrapy.cfg'))- assert exists(join(abspath(project_path), project_name))- assert exists(join(join(abspath(project_path), project_name), '__init__.py'))- assert exists(join(join(abspath(project_path), project_name), 'items.py'))- assert exists(join(join(abspath(project_path), project_name), 'pipelines.py'))- assert exists(join(join(abspath(project_path), project_name), 'settings.py'))- assert exists(join(join(abspath(project_path), project_name), 'spiders', '__init__.py'))+ assert Path(project_path, 'scrapy.cfg').exists()+ assert Path(project_path, project_name).exists()+ assert Path(project_path, project_name, '__init__.py').exists()+ assert Path(project_path, project_name, 'items.py').exists()+ assert Path(project_path, project_name, 'pipelines.py').exists()+ assert Path(project_path, project_name, 'settings.py').exists()+ assert Path(project_path, project_name, 'spiders', '__init__.py').exists()-def get_permissions_dict(path, renamings=None, ignore=None):+def get_permissions_dict(path: str | os.PathLike[str], renamings=None, ignore=None) -> dict[str, str]:- def get_permissions(path):- return oct(os.stat(path).st_mode)+ def get_permissions(path: Path) -> str:+ return oct(path.stat().st_mode)++ path_obj = Path(path)renamings = renamings or tuple()permissions_dict = {- '.': get_permissions(path),+ '.': get_permissions(path_obj),}- for root, dirs, files in os.walk(path):+ for root, dirs, files in os.walk(path_obj):nodes = list(chain(dirs, files))if ignore:ignored_names = ignore(root, nodes)nodes = [node for node in nodes if node not in ignored_names]for node in nodes:- absolute_path = os.path.join(root, node)- relative_path = os.path.relpath(absolute_path, path)+ absolute_path = Path(root, node)+ relative_path = str(absolute_path.relative_to(path))for search_string, replacement in renamings:relative_path = relative_path.replace(search_string,@@ -208,28 +210,27 @@ class StartprojectTemplatesTest(ProjectTest):def setUp(self):super().setUp()- self.tmpl = join(self.temp_path, 'templates')- self.tmpl_proj = join(self.tmpl, 'project')+ self.tmpl = str(Path(self.temp_path, 'templates'))+ self.tmpl_proj = str(Path(self.tmpl, 'project'))def test_startproject_template_override(self):- copytree(join(scrapy.__path__[0], 'templates'), self.tmpl)- with open(join(self.tmpl_proj, 'root_template'), 'w'):- pass- assert exists(join(self.tmpl_proj, 'root_template'))+ copytree(Path(scrapy.__path__[0], 'templates'), self.tmpl)+ Path(self.tmpl_proj, 'root_template').write_bytes(b"")+ assert Path(self.tmpl_proj, 'root_template').exists()args = ['--set', f'TEMPLATES_DIR={self.tmpl}']p, out, err = self.proc('startproject', self.project_name, *args)self.assertIn(f"New Scrapy project '{self.project_name}', ""using template directory", out)self.assertIn(self.tmpl_proj, out)- assert exists(join(self.proj_path, 'root_template'))+ assert Path(self.proj_path, 'root_template').exists()def test_startproject_permissions_from_writable(self):"""Check that generated files have the right permissions when thetemplate folder has the same permissions as in the project, i.e.everything is writable."""scrapy_path = scrapy.__path__[0]- project_template = os.path.join(scrapy_path, 'templates', 'project')+ project_template = Path(scrapy_path, 'templates', 'project')project_name = 'startproject1'renamings = (('module', project_name),@@ -255,7 +256,7 @@ class StartprojectTemplatesTest(ProjectTest):)process.wait()- project_dir = os.path.join(destination, project_name)+ project_dir = Path(destination, project_name)actual_permissions = get_permissions_dict(project_dir)self.assertEqual(actual_permissions, expected_permissions)@@ -268,8 +269,8 @@ class StartprojectTemplatesTest(ProjectTest):See https://github.com/scrapy/scrapy/pull/4604"""scrapy_path = scrapy.__path__[0]- templates_dir = os.path.join(scrapy_path, 'templates')- project_template = os.path.join(templates_dir, 'project')+ templates_dir = Path(scrapy_path, 'templates')+ project_template = Path(templates_dir, 'project')project_name = 'startproject2'renamings = (('module', project_name),@@ -281,16 +282,16 @@ class StartprojectTemplatesTest(ProjectTest):IGNORE,)- def _make_read_only(path):- current_permissions = os.stat(path).st_mode- os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION)+ def _make_read_only(path: Path):+ current_permissions = path.stat().st_mode+ path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION)read_only_templates_dir = str(Path(mkdtemp()) / 'templates')copytree(templates_dir, read_only_templates_dir)for root, dirs, files in os.walk(read_only_templates_dir):for node in chain(dirs, files):- _make_read_only(os.path.join(root, node))+ _make_read_only(Path(root, node))destination = mkdtemp()process = subprocess.Popen(@@ -308,7 +309,7 @@ class StartprojectTemplatesTest(ProjectTest):)process.wait()- project_dir = os.path.join(destination, project_name)+ project_dir = Path(destination, project_name)actual_permissions = get_permissions_dict(project_dir)self.assertEqual(actual_permissions, expected_permissions)@@ -317,7 +318,7 @@ class StartprojectTemplatesTest(ProjectTest):"""Check that pre-existing folders and files in the destination folderdo not see their permissions modified."""scrapy_path = scrapy.__path__[0]- project_template = os.path.join(scrapy_path, 'templates', 'project')+ project_template = Path(scrapy_path, 'templates', 'project')project_name = 'startproject3'renamings = (('module', project_name),@@ -330,7 +331,7 @@ class StartprojectTemplatesTest(ProjectTest):)destination = mkdtemp()- project_dir = os.path.join(destination, project_name)+ project_dir = Path(destination, project_name)existing_nodes = {oct(permissions)[2:] + extension: permissions@@ -339,10 +340,9 @@ class StartprojectTemplatesTest(ProjectTest):0o444, 0o555, 0o644, 0o666, 0o755, 0o777,)}- os.mkdir(project_dir)- project_dir_path = Path(project_dir)+ project_dir.mkdir()for node, permissions in existing_nodes.items():- path = project_dir_path / node+ path = project_dir / nodeif node.endswith('.d'):path.mkdir(mode=permissions)else:@@ -378,7 +378,7 @@ class StartprojectTemplatesTest(ProjectTest):os.umask(cur_mask)scrapy_path = scrapy.__path__[0]- project_template = os.path.join(+ project_template = Path(scrapy_path,'templates','project'@@ -409,7 +409,7 @@ class StartprojectTemplatesTest(ProjectTest):)process.wait()- project_dir = os.path.join(destination, project_name)+ project_dir = Path(destination, project_name)actual_permissions = get_permissions_dict(project_dir)self.assertEqual(actual_permissions, expected_permissions)@@ -420,7 +420,7 @@ class CommandTest(ProjectTest):def setUp(self):super().setUp()self.call('startproject', self.project_name)- self.cwd = join(self.temp_path, self.project_name)+ self.cwd = Path(self.temp_path, self.project_name)self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings'@@ -429,10 +429,10 @@ class GenspiderCommandTest(CommandTest):def test_arguments(self):# only pass one argument. spider script shouldn't be createdself.assertEqual(2, self.call('genspider', 'test_name'))- assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py'))+ assert not Path(self.proj_mod_path, 'spiders', 'test_name.py').exists()# pass two arguments <name> <domain>. spider script should be createdself.assertEqual(0, self.call('genspider', 'test_name', 'test.com'))- assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py'))+ assert Path(self.proj_mod_path, 'spiders', 'test_name.py').exists()def test_template(self, tplname='crawl'):args = [f'--template={tplname}'] if tplname else []@@ -440,11 +440,11 @@ class GenspiderCommandTest(CommandTest):spmodule = f"{self.project_name}.spiders.{spname}"p, out, err = self.proc('genspider', spname, 'test.com', *args)self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out)- self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))- modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))+ self.assertTrue(Path(self.proj_mod_path, 'spiders', 'test_spider.py').exists())+ modify_time_before = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtimep, out, err = self.proc('genspider', spname, 'test.com', *args)self.assertIn(f"Spider {spname!r} already exists in module", out)- modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))+ modify_time_after = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtimeself.assertEqual(modify_time_after, modify_time_before)def test_template_basic(self):@@ -465,37 +465,37 @@ class GenspiderCommandTest(CommandTest):def test_same_name_as_project(self):self.assertEqual(2, self.call('genspider', self.project_name))- assert not exists(join(self.proj_mod_path, 'spiders', f'{self.project_name}.py'))+ assert not Path(self.proj_mod_path, 'spiders', f'{self.project_name}.py').exists()def test_same_filename_as_existing_spider(self, force=False):file_name = 'example'- file_path = join(self.proj_mod_path, 'spiders', f'{file_name}.py')+ file_path = Path(self.proj_mod_path, 'spiders', f'{file_name}.py')self.assertEqual(0, self.call('genspider', file_name, 'example.com'))- assert exists(file_path)+ assert file_path.exists()# change name of spider but not its file name- with open(file_path, 'r+') as spider_file:+ with file_path.open('r+') as spider_file:file_data = spider_file.read()file_data = file_data.replace("name = \'example\'", "name = \'renamed\'")spider_file.seek(0)spider_file.write(file_data)spider_file.truncate()- modify_time_before = getmtime(file_path)+ modify_time_before = file_path.stat().st_mtimefile_contents_before = file_dataif force:p, out, err = self.proc('genspider', '--force', file_name, 'example.com')self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out)- modify_time_after = getmtime(file_path)+ modify_time_after = file_path.stat().st_mtimeself.assertNotEqual(modify_time_after, modify_time_before)- file_contents_after = open(file_path, 'r').read()+ file_contents_after = file_path.read_text()self.assertNotEqual(file_contents_after, file_contents_before)else:p, out, err = self.proc('genspider', file_name, 'example.com')self.assertIn(f"{file_path} already exists", out)- modify_time_after = getmtime(file_path)+ modify_time_after = file_path.stat().st_mtimeself.assertEqual(modify_time_after, modify_time_before)- file_contents_after = open(file_path, 'r').read()+ file_contents_after = file_path.read_text()self.assertEqual(file_contents_after, file_contents_before)def test_same_filename_as_existing_spider_force(self):@@ -504,11 +504,11 @@ class GenspiderCommandTest(CommandTest):def test_url(self, url='test.com', domain="test.com"):self.assertEqual(0, self.call('genspider', '--force', 'test_name', url))self.assertEqual(domain,- self.find_in_file(join(self.proj_mod_path,+ self.find_in_file(Path(self.proj_mod_path,'spiders', 'test_name.py'),r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1))self.assertEqual(f'http://{domain}/',- self.find_in_file(join(self.proj_mod_path,+ self.find_in_file(Path(self.proj_mod_path,'spiders', 'test_name.py'),r'start_urls\s*=\s*\[\'(.+)\'\]').group(1))@@ -526,31 +526,31 @@ class GenspiderStandaloneCommandTest(ProjectTest):def test_generate_standalone_spider(self):self.call('genspider', 'example', 'example.com')- assert exists(join(self.temp_path, 'example.py'))+ assert Path(self.temp_path, 'example.py').exists()def test_same_name_as_existing_file(self, force=False):file_name = 'example'- file_path = join(self.temp_path, file_name + '.py')+ file_path = Path(self.temp_path, file_name + '.py')p, out, err = self.proc('genspider', file_name, 'example.com')self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out)- assert exists(file_path)- modify_time_before = getmtime(file_path)- file_contents_before = open(file_path, 'r').read()+ assert file_path.exists()+ modify_time_before = file_path.stat().st_mtime+ file_contents_before = file_path.read_text()if force:# use different template to ensure contents were changedp, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com')self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out)- modify_time_after = getmtime(file_path)+ modify_time_after = file_path.stat().st_mtimeself.assertNotEqual(modify_time_after, modify_time_before)- file_contents_after = open(file_path, 'r').read()+ file_contents_after = file_path.read_text()self.assertNotEqual(file_contents_after, file_contents_before)else:p, out, err = self.proc('genspider', file_name, 'example.com')- self.assertIn(f"{join(self.temp_path, file_name + '.py')} already exists", out)- modify_time_after = getmtime(file_path)+ self.assertIn(f"{Path(self.temp_path, file_name + '.py')} already exists", out)+ modify_time_after = file_path.stat().st_mtimeself.assertEqual(modify_time_after, modify_time_before)- file_contents_after = open(file_path, 'r').read()+ file_contents_after = file_path.read_text()self.assertEqual(file_contents_after, file_contents_before)def test_same_name_as_existing_file_force(self):@@ -588,17 +588,16 @@ class BadSpider(scrapy.Spider):"""@contextmanager- def _create_file(self, content, name=None):- tmpdir = self.mktemp()- os.mkdir(tmpdir)+ def _create_file(self, content, name=None) -> Generator[str, None, None]:+ tmpdir = Path(self.mktemp())+ tmpdir.mkdir()if name:- fname = abspath(join(tmpdir, name))+ fname = (tmpdir / name).resolve()else:- fname = abspath(join(tmpdir, self.spider_filename))- with open(fname, 'w') as f:- f.write(content)+ fname = (tmpdir / self.spider_filename).resolve()+ fname.write_text(content)try:- yield fname+ yield str(fname)finally:rmtree(tmpdir)@@ -747,12 +746,11 @@ class MySpider(scrapy.Spider):)return []"""- with open(os.path.join(self.cwd, "example.json"), "w") as f1:- f1.write("not empty")+ Path(self.cwd, "example.json").write_text("not empty")args = ['-O', 'example.json']log = self.get_log(spider_code, args=args)self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)- with open(os.path.join(self.cwd, "example.json")) as f2:+ with Path(self.cwd, "example.json").open() as f2:first_line = f2.readline()self.assertNotEqual(first_line, "not empty")@@ -854,9 +852,7 @@ class ViewCommandTest(CommandTest):class CrawlCommandTest(CommandTest):def crawl(self, code, args=()):- fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))- with open(fname, 'w') as f:- f.write(code)+ Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code)return self.proc('crawl', 'myspider', *args)def get_log(self, code, args=()):@@ -908,12 +904,11 @@ class MySpider(scrapy.Spider):)return []"""- with open(os.path.join(self.cwd, "example.json"), "w") as f1:- f1.write("not empty")+ Path(self.cwd, "example.json").write_text("not empty")args = ['-O', 'example.json']log = self.get_log(spider_code, args=args)self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)- with open(os.path.join(self.cwd, "example.json")) as f2:+ with Path(self.cwd, "example.json").open() as f2:first_line = f2.readline()self.assertNotEqual(first_line, "not empty")diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex cf15ba9b9..19f4229a3 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -1,9 +1,9 @@import logging-import osimport platformimport subprocessimport sysimport warnings+from pathlib import Pathfrom pytest import raises, markfrom twisted import version as twisted_version@@ -100,15 +100,14 @@ class CrawlerLoggingTestCase(unittest.TestCase):assert get_scrapy_root_handler() is Nonedef test_spider_custom_settings_log_level(self):- log_file = self.mktemp()- with open(log_file, 'wb') as fo:- fo.write('previous message\n'.encode('utf-8'))+ log_file = Path(self.mktemp())+ log_file.write_text('previous message\n', encoding='utf-8')class MySpider(scrapy.Spider):name = 'spider'custom_settings = {'LOG_LEVEL': 'INFO',- 'LOG_FILE': log_file,+ 'LOG_FILE': str(log_file),# settings to avoid extra warnings'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION','TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,@@ -124,8 +123,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):logging.warning('warning message')logging.error('error message')- with open(log_file, 'rb') as fo:- logged = fo.read().decode('utf-8')+ logged = log_file.read_text(encoding='utf-8')self.assertIn('previous message', logged)self.assertNotIn('debug message', logged)@@ -139,14 +137,13 @@ class CrawlerLoggingTestCase(unittest.TestCase):self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)def test_spider_custom_settings_log_append(self):- log_file = self.mktemp()- with open(log_file, 'wb') as fo:- fo.write('previous message\n'.encode('utf-8'))+ log_file = Path(self.mktemp())+ log_file.write_text('previous message\n', encoding='utf-8')class MySpider(scrapy.Spider):name = 'spider'custom_settings = {- 'LOG_FILE': log_file,+ 'LOG_FILE': str(log_file),'LOG_FILE_APPEND': False,# disable telnet if not available to avoid an extra warning'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,@@ -156,8 +153,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):get_crawler(MySpider)logging.debug('debug message')- with open(log_file, 'rb') as fo:- logged = fo.read().decode('utf-8')+ logged = log_file.read_text(encoding='utf-8')self.assertNotIn('previous message', logged)self.assertIn('debug message', logged)@@ -296,9 +292,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase):class ScriptRunnerMixin:- def run_script(self, script_name, *script_args):- script_path = os.path.join(self.script_dir, script_name)- args = [sys.executable, script_path] + list(script_args)+ def run_script(self, script_name: str, *script_args):+ script_path = self.script_dir / script_name+ args = [sys.executable, str(script_path)] + list(script_args)p = subprocess.Popen(args, env=get_testenv(),stdout=subprocess.PIPE, stderr=subprocess.PIPE)stdout, stderr = p.communicate()@@ -306,7 +302,7 @@ class ScriptRunnerMixin:class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):- script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')+ script_dir = Path(__file__).parent.resolve() / 'CrawlerProcess'def test_simple(self):log = self.run_script('simple.py')@@ -463,7 +459,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):- script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')+ script_dir = Path(__file__).parent.resolve() / 'CrawlerRunner'def test_response_ip_address(self):log = self.run_script("ip_address.py")diff --git a/tests/test_dependencies.py b/tests/test_dependencies.pyindex 5e63ebffb..2558e4f91 100644--- a/tests/test_dependencies.py+++ b/tests/test_dependencies.py@@ -2,6 +2,7 @@ import osimport refrom configparser import ConfigParserfrom importlib import import_module+from pathlib import Pathfrom twisted import version as twisted_versionfrom twisted.trial import unittest@@ -29,11 +30,7 @@ class ScrapyUtilsTest(unittest.TestCase):if not os.environ.get('_SCRAPY_PINNED', None):self.skipTest('Not in a pinned environment')- tox_config_file_path = os.path.join(- os.path.dirname(__file__),- '..',- 'tox.ini',- )+ tox_config_file_path = Path(__file__) / '..' / 'tox.ini'config_parser = ConfigParser()config_parser.read(tox_config_file_path)pattern = r'Twisted\[http2\]==([\d.]+)'diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 883960084..29ff8c2dc 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -3,6 +3,7 @@ import osimport shutilimport sysimport tempfile+from pathlib import Pathfrom typing import Optional, Typefrom unittest import mock, SkipTest@@ -10,7 +11,6 @@ from testfixtures import LogCapturefrom twisted.cred import checkers, credentials, portalfrom twisted.internet import defer, error, reactorfrom twisted.protocols.policies import WrappingFactory-from twisted.python.filepath import FilePathfrom twisted.trial import unittestfrom twisted.web import resource, server, static, utilfrom twisted.web._newclient import ResponseFailed@@ -108,14 +108,13 @@ class LoadTestCase(unittest.TestCase):class FileTestCase(unittest.TestCase):def setUp(self):- self.tmpname = self.mktemp()- with open(self.tmpname + '^', 'w') as f:- f.write('0123456789')+ self.tmpname = Path(self.mktemp() + '^')+ Path(self.tmpname).write_text('0123456789')handler = create_instance(FileDownloadHandler, None, get_crawler())self.download_request = handler.download_requestdef tearDown(self):- os.unlink(self.tmpname + '^')+ self.tmpname.unlink()def test_download(self):def _test(response):@@ -124,7 +123,7 @@ class FileTestCase(unittest.TestCase):self.assertEqual(response.body, b'0123456789')self.assertEqual(response.protocol, None)- request = Request(path_to_file_uri(self.tmpname + '^'))+ request = Request(path_to_file_uri(str(self.tmpname)))assert request.url.upper().endswith('%5E')return self.download_request(request, Spider('foo')).addCallback(_test)@@ -223,10 +222,10 @@ class HttpTestCase(unittest.TestCase):certfile = 'keys/localhost.crt'def setUp(self):- self.tmpname = self.mktemp()- os.mkdir(self.tmpname)- FilePath(self.tmpname).child("file").setContent(b"0123456789")- r = static.File(self.tmpname)+ self.tmpname = Path(self.mktemp())+ self.tmpname.mkdir()+ (self.tmpname / "file").write_bytes(b"0123456789")+ r = static.File(str(self.tmpname))r.putChild(b"redirect", util.Redirect(b"/file"))r.putChild(b"wait", ForeverTakingResource())r.putChild(b"hang-after-headers", ForeverTakingResource(write=True))@@ -626,10 +625,10 @@ class Https11CustomCiphers(unittest.TestCase):certfile = 'keys/localhost.crt'def setUp(self):- self.tmpname = self.mktemp()- os.mkdir(self.tmpname)- FilePath(self.tmpname).child("file").setContent(b"0123456789")- r = static.File(self.tmpname)+ self.tmpname = Path(self.mktemp())+ self.tmpname.mkdir()+ (self.tmpname / "file").write_bytes(b"0123456789")+ r = static.File(str(self.tmpname))self.site = server.Site(r, timeout=None)self.host = 'localhost'self.port = reactor.listenSSL(@@ -1002,16 +1001,15 @@ class BaseFTPTestCase(unittest.TestCase):from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler# setup dirs and test file- self.directory = self.mktemp()- os.mkdir(self.directory)- userdir = os.path.join(self.directory, self.username)- os.mkdir(userdir)- fp = FilePath(userdir)+ self.directory = Path(self.mktemp())+ self.directory.mkdir()+ userdir = self.directory / self.username+ userdir.mkdir()for filename, content in self.test_files:- fp.child(filename).setContent(content)+ (userdir / filename).write_bytes(content)# setup server- realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory)+ realm = FTPRealm(anonymousRoot=str(self.directory), userHome=str(self.directory))p = portal.Portal(realm)users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()users_checker.addUser(self.username, self.password)@@ -1076,28 +1074,28 @@ class BaseFTPTestCase(unittest.TestCase):def test_ftp_local_filename(self):f, local_fname = tempfile.mkstemp()- local_fname = to_bytes(local_fname)+ fname_bytes = to_bytes(local_fname)+ local_fname = Path(local_fname)os.close(f)- meta = {"ftp_local_filename": local_fname}+ meta = {"ftp_local_filename": fname_bytes}meta.update(self.req_meta)request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt",meta=meta)d = self.download_handler.download_request(request, None)def _test(r):- self.assertEqual(r.body, local_fname)- self.assertEqual(r.headers, {b'Local Filename': [local_fname],+ self.assertEqual(r.body, fname_bytes)+ self.assertEqual(r.headers, {b'Local Filename': [fname_bytes],b'Size': [b'17']})- self.assertTrue(os.path.exists(local_fname))- with open(local_fname, "rb") as f:- self.assertEqual(f.read(), b"I have the power!")- os.remove(local_fname)+ self.assertTrue(local_fname.exists())+ self.assertEqual(local_fname.read_bytes(), b"I have the power!")+ local_fname.unlink()return self._add_test_callbacks(d, _test)def _test_response_class(self, filename, response_class):f, local_fname = tempfile.mkstemp()- local_fname = to_bytes(local_fname)+ local_fname = Path(local_fname)os.close(f)meta = {}meta.update(self.req_meta)@@ -1107,7 +1105,7 @@ class BaseFTPTestCase(unittest.TestCase):def _test(r):self.assertEqual(type(r), response_class)- os.remove(local_fname)+ local_fname.unlink()return self._add_test_callbacks(d, _test)def test_response_class_from_url(self):@@ -1147,15 +1145,14 @@ class AnonymousFTPTestCase(BaseFTPTestCase):from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler# setup dir and test file- self.directory = self.mktemp()- os.mkdir(self.directory)+ self.directory = Path(self.mktemp())+ self.directory.mkdir()- fp = FilePath(self.directory)for filename, content in self.test_files:- fp.child(filename).setContent(content)+ (self.directory / filename).write_bytes(content)# setup server for anonymous access- realm = FTPRealm(anonymousRoot=self.directory)+ realm = FTPRealm(anonymousRoot=str(self.directory))p = portal.Portal(realm)p.registerChecker(checkers.AllowAnonymousAccess(),credentials.IAnonymous)diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 40e9f3a96..6f4e217e6 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -1,6 +1,6 @@from gzip import GzipFilefrom io import BytesIO-from os.path import join+from pathlib import Pathfrom unittest import TestCase, SkipTestfrom warnings import catch_warnings@@ -15,7 +15,7 @@ from tests import tests_datadirfrom w3lib.encoding import resolve_encoding-SAMPLEDIR = join(tests_datadir, 'compressed')+SAMPLEDIR = Path(tests_datadir, 'compressed')FORMAT = {'gzip': ('html-gzip.bin', 'gzip'),@@ -46,8 +46,7 @@ class HttpCompressionTest(TestCase):samplefile, contentencoding = FORMAT[coding]- with open(join(SAMPLEDIR, samplefile), 'rb') as sample:- body = sample.read()+ body = (SAMPLEDIR / samplefile).read_bytes()headers = {'Server': 'Yaws/1.49 Yet Another Web Server',diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex 8a37a8ebe..911d23069 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -2,8 +2,8 @@ import hashlibimport tempfileimport unittestimport shutil-import osimport sys+from pathlib import Pathfrom testfixtures import LogCapturefrom scrapy.dupefilters import RFPDupeFilter@@ -157,7 +157,7 @@ class RFPDupeFilterTest(unittest.TestCase):df.request_seen(r1)df.close('finished')- with open(os.path.join(path, 'requests.seen'), 'rb') as seen_file:+ with Path(path, 'requests.seen').open('rb') as seen_file:line = next(seen_file).decode()assert not line.endswith('\r\r\n')if sys.platform == 'win32':diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 5677052f6..aa3313659 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -10,11 +10,11 @@ module with the ``runserver`` argument::python test_engine.py runserver"""-import osimport reimport subprocessimport sysfrom collections import defaultdict+from pathlib import Pathfrom threading import Timerfrom urllib.parse import urlparsefrom dataclasses import dataclass@@ -127,8 +127,8 @@ class ChangeCloseReasonSpider(TestSpider):def start_test_site(debug=False):- root_dir = os.path.join(tests_datadir, "test_site")- r = static.File(root_dir)+ root_dir = Path(tests_datadir, "test_site")+ r = static.File(str(root_dir))r.putChild(b"redirect", util.Redirect(b"/redirected"))r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))numbers = [str(x).encode("utf8") for x in range(2**18)]diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex ad2383018..98905d2c0 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -3,7 +3,6 @@ import csvimport gzipimport jsonimport lzma-import osimport randomimport shutilimport string@@ -15,6 +14,7 @@ from collections import defaultdictfrom contextlib import ExitStackfrom io import BytesIOfrom logging import getLogger+from os import PathLikefrom pathlib import Pathfrom string import ascii_letters, digitsfrom unittest import mock@@ -63,41 +63,41 @@ def printf_escape(string):return string.replace('%', '%%')-def build_url(path):- if path[0] != '/':- path = '/' + path- return urljoin('file:', path)+def build_url(path: str | PathLike[str]) -> str:+ path_str = str(path)+ if path_str[0] != '/':+ path_str = '/' + path_str+ return urljoin('file:', path_str)class FileFeedStorageTest(unittest.TestCase):def test_store_file_uri(self):- path = os.path.abspath(self.mktemp())- uri = path_to_file_uri(path)+ path = Path(self.mktemp()).resolve()+ uri = path_to_file_uri(str(path))return self._assert_stores(FileFeedStorage(uri), path)def test_store_file_uri_makedirs(self):- path = os.path.abspath(self.mktemp())- path = os.path.join(path, 'more', 'paths', 'file.txt')- uri = path_to_file_uri(path)+ path = Path(self.mktemp()).resolve() / 'more' / 'paths' / 'file.txt'+ uri = path_to_file_uri(str(path))return self._assert_stores(FileFeedStorage(uri), path)def test_store_direct_path(self):- path = os.path.abspath(self.mktemp())- return self._assert_stores(FileFeedStorage(path), path)+ path = Path(self.mktemp()).resolve()+ return self._assert_stores(FileFeedStorage(str(path)), path)def test_store_direct_path_relative(self):- path = self.mktemp()- return self._assert_stores(FileFeedStorage(path), path)+ path = Path(self.mktemp())+ return self._assert_stores(FileFeedStorage(str(path)), path)def test_interface(self):path = self.mktemp()st = FileFeedStorage(path)verifyObject(IFeedStorage, st)- def _store(self, feed_options=None):- path = os.path.abspath(self.mktemp())- storage = FileFeedStorage(path, feed_options=feed_options)+ def _store(self, feed_options=None) -> Path:+ path = Path(self.mktemp()).resolve()+ storage = FileFeedStorage(str(path), feed_options=feed_options)spider = scrapy.Spider("default")file = storage.open(spider)file.write(b"content")@@ -106,27 +106,26 @@ class FileFeedStorageTest(unittest.TestCase):def test_append(self):path = self._store()- return self._assert_stores(FileFeedStorage(path), path, b"contentcontent")+ return self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent")def test_overwrite(self):path = self._store({"overwrite": True})return self._assert_stores(- FileFeedStorage(path, feed_options={"overwrite": True}),+ FileFeedStorage(str(path), feed_options={"overwrite": True}),path)@defer.inlineCallbacks- def _assert_stores(self, storage, path, expected_content=b"content"):+ def _assert_stores(self, storage, path: Path, expected_content=b"content"):spider = scrapy.Spider("default")file = storage.open(spider)file.write(b"content")yield storage.store(file)- self.assertTrue(os.path.exists(path))+ self.assertTrue(path.exists())try:- with open(path, 'rb') as fp:- self.assertEqual(fp.read(), expected_content)+ self.assertEqual(path.read_bytes(), expected_content)finally:- os.unlink(path)+ path.unlink()class FTPFeedStorageTest(unittest.TestCase):@@ -152,13 +151,12 @@ class FTPFeedStorageTest(unittest.TestCase):file.write(content)return storage.store(file)- def _assert_stored(self, path, content):+ def _assert_stored(self, path: Path, content):self.assertTrue(path.exists())try:- with path.open('rb') as fp:- self.assertEqual(fp.read(), content)+ self.assertEqual(path.read_bytes(), content)finally:- os.unlink(str(path))+ path.unlink()@defer.inlineCallbacksdef test_append(self):@@ -221,24 +219,24 @@ class BlockingFeedStorageTest(unittest.TestCase):b = BlockingFeedStorage()tmp = b.open(self.get_test_spider())- tmp_path = os.path.dirname(tmp.name)- self.assertEqual(tmp_path, tempfile.gettempdir())+ tmp_path = Path(tmp.name).parent+ self.assertEqual(str(tmp_path), tempfile.gettempdir())def test_temp_file(self):b = BlockingFeedStorage()- tests_path = os.path.dirname(os.path.abspath(__file__))- spider = self.get_test_spider({'FEED_TEMPDIR': tests_path})+ tests_path = Path(__file__).resolve().parent+ spider = self.get_test_spider({'FEED_TEMPDIR': str(tests_path)})tmp = b.open(spider)- tmp_path = os.path.dirname(tmp.name)+ tmp_path = Path(tmp.name).parentself.assertEqual(tmp_path, tests_path)def test_invalid_folder(self):b = BlockingFeedStorage()- tests_path = os.path.dirname(os.path.abspath(__file__))- invalid_path = os.path.join(tests_path, 'invalid_path')- spider = self.get_test_spider({'FEED_TEMPDIR': invalid_path})+ tests_path = Path(__file__).resolve().parent+ invalid_path = tests_path / 'invalid_path'+ spider = self.get_test_spider({'FEED_TEMPDIR': str(invalid_path)})self.assertRaises(OSError, b.open, spider=spider)@@ -564,13 +562,13 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):class DummyBlockingFeedStorage(BlockingFeedStorage):def __init__(self, uri, *args, feed_options=None):- self.path = file_uri_to_path(uri)+ self.path = Path(file_uri_to_path(uri))def _store_in_thread(self, file):- dirname = os.path.dirname(self.path)- if dirname and not os.path.exists(dirname):- os.makedirs(dirname)- with open(self.path, 'ab') as output_file:+ dirname = self.path.parent+ if dirname and not dirname.exists():+ dirname.mkdir(parents=True)+ with self.path.open('ab') as output_file:output_file.write(file.read())file.close()@@ -613,10 +611,10 @@ class FeedExportTestBase(ABC, unittest.TestCase):foo = scrapy.Field()hello = scrapy.Field()- def _random_temp_filename(self, inter_dir=''):+ def _random_temp_filename(self, inter_dir='') -> Path:chars = [random.choice(ascii_letters + digits) for _ in range(15)]filename = ''.join(chars)- return os.path.join(self.temp_dir, inter_dir, filename)+ return Path(self.temp_dir, inter_dir, filename)def setUp(self):self.temp_dir = tempfile.mkdtemp()@@ -702,18 +700,17 @@ class FeedExportTest(FeedExportTestBase):yield crawler.crawl()for file_path, feed_options in FEEDS.items():- if not os.path.exists(str(file_path)):+ if not Path(file_path).exists():continue- with open(str(file_path), 'rb') as f:- content[feed_options['format']] = f.read()+ content[feed_options['format']] = Path(file_path).read_bytes()finally:for file_path in FEEDS.keys():- if not os.path.exists(str(file_path)):+ if not Path(file_path).exists():continue- os.remove(str(file_path))+ Path(file_path).unlink()return content@@ -808,7 +805,7 @@ class FeedExportTest(FeedExportTestBase):def test_stats_file_success(self):settings = {"FEEDS": {- printf_escape(path_to_url(self._random_temp_filename())): {+ printf_escape(path_to_url(str(self._random_temp_filename()))): {"format": "json",}},@@ -823,7 +820,7 @@ class FeedExportTest(FeedExportTestBase):def test_stats_file_failed(self):settings = {"FEEDS": {- printf_escape(path_to_url(self._random_temp_filename())): {+ printf_escape(path_to_url(str(self._random_temp_filename()))): {"format": "json",}},@@ -846,7 +843,7 @@ class FeedExportTest(FeedExportTestBase):'AWS_ACCESS_KEY_ID': 'access_key','AWS_SECRET_ACCESS_KEY': 'secret_key',"FEEDS": {- printf_escape(path_to_url(self._random_temp_filename())): {+ printf_escape(path_to_url(str(self._random_temp_filename()))): {"format": "json",},"s3://bucket/key/foo.csv": {@@ -1427,12 +1424,11 @@ class FeedExportTest(FeedExportTestBase):self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler)@defer.inlineCallbacks- def test_pathlib_uri(self):- feed_path = Path(self._random_temp_filename())+ def test_str_uri(self):settings = {'FEED_STORE_EMPTY': True,'FEEDS': {- feed_path: {'format': 'csv'}+ str(self._random_temp_filename()): {'format': 'csv'}},}data = yield self.exported_no_data(settings)@@ -1538,8 +1534,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):def close(self):self.file.close()- def _named_tempfile(self, name):- return os.path.join(self.temp_dir, name)+ def _named_tempfile(self, name) -> str:+ return str(Path(self.temp_dir, name))@defer.inlineCallbacksdef run_and_export(self, spider_cls, settings):@@ -1559,18 +1555,17 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):yield crawler.crawl()for file_path, feed_options in FEEDS.items():- if not os.path.exists(str(file_path)):+ if not Path(file_path).exists():continue- with open(str(file_path), 'rb') as f:- content[str(file_path)] = f.read()+ content[str(file_path)] = Path(file_path).read_bytes()finally:for file_path in FEEDS.keys():- if not os.path.exists(str(file_path)):+ if not Path(file_path).exists():continue- os.remove(str(file_path))+ Path(file_path).unlink()return content@@ -2031,11 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase):yield crawler.crawl()for path, feed in FEEDS.items():- dir_name = os.path.dirname(path)- for file in sorted(os.listdir(dir_name)):- with open(os.path.join(dir_name, file), 'rb') as f:- data = f.read()- content[feed['format']].append(data)+ dir_name = Path(path).parent+ for file in sorted(dir_name.iterdir()):+ content[feed['format']].append(file.read_bytes())finally:self.tearDown()defer.returnValue(content)@@ -2045,7 +2038,7 @@ class BatchDeliveriesTest(FeedExportTestBase):settings = settings or {}settings.update({'FEEDS': {- os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},+ self._random_temp_filename() / 'jl' / self._file_mark: {'format': 'jl'},},})batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')@@ -2061,7 +2054,7 @@ class BatchDeliveriesTest(FeedExportTestBase):settings = settings or {}settings.update({'FEEDS': {- os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},+ self._random_temp_filename() / 'csv' / self._file_mark: {'format': 'csv'},},})batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')@@ -2077,7 +2070,7 @@ class BatchDeliveriesTest(FeedExportTestBase):settings = settings or {}settings.update({'FEEDS': {- os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},+ self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'},},})batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')@@ -2094,8 +2087,8 @@ class BatchDeliveriesTest(FeedExportTestBase):settings = settings or {}settings.update({'FEEDS': {- os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},- os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},+ self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'},+ self._random_temp_filename() / 'json' / self._file_mark: {'format': 'json'},},})batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')@@ -2120,7 +2113,7 @@ class BatchDeliveriesTest(FeedExportTestBase):settings = settings or {}settings.update({'FEEDS': {- os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},+ self._random_temp_filename() / 'pickle' / self._file_mark: {'format': 'pickle'},},})batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')@@ -2137,7 +2130,7 @@ class BatchDeliveriesTest(FeedExportTestBase):settings = settings or {}settings.update({'FEEDS': {- os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},+ self._random_temp_filename() / 'marshal' / self._file_mark: {'format': 'marshal'},},})batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')@@ -2184,7 +2177,7 @@ class BatchDeliveriesTest(FeedExportTestBase):for fmt in ('json', 'jsonlines', 'xml', 'csv'):settings = {'FEEDS': {- os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},+ self._random_temp_filename() / fmt / self._file_mark: {'format': fmt},},'FEED_EXPORT_BATCH_ITEM_COUNT': 1}@@ -2204,7 +2197,7 @@ class BatchDeliveriesTest(FeedExportTestBase):for fmt, expctd in formats:settings = {'FEEDS': {- os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},+ self._random_temp_filename() / fmt / self._file_mark: {'format': fmt},},'FEED_STORE_EMPTY': True,'FEED_EXPORT_INDENT': None,@@ -2237,19 +2230,19 @@ class BatchDeliveriesTest(FeedExportTestBase):settings = {'FEEDS': {- os.path.join(self._random_temp_filename(), 'json', self._file_mark): {+ self._random_temp_filename() / 'json' / self._file_mark: {'format': 'json','indent': 0,'fields': ['bar'],'encoding': 'utf-8',},- os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {+ self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml','indent': 2,'fields': ['foo'],'encoding': 'latin-1',},- os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {+ self._random_temp_filename() / 'csv' / self._file_mark: {'format': 'csv','indent': None,'fields': ['foo', 'bar'],@@ -2272,7 +2265,7 @@ class BatchDeliveriesTest(FeedExportTestBase):}settings = {'FEEDS': {- os.path.join(self._random_temp_filename(), 'json', self._file_mark): {+ self._random_temp_filename() / 'json' / self._file_mark: {'format': 'json','indent': None,'encoding': 'utf-8',@@ -2299,7 +2292,7 @@ class BatchDeliveriesTest(FeedExportTestBase):]settings = {'FEEDS': {- os.path.join(self._random_temp_filename(), '%(batch_time)s'): {+ self._random_temp_filename() / '%(batch_time)s': {'format': 'json',},},@@ -2312,7 +2305,7 @@ class BatchDeliveriesTest(FeedExportTestBase):def test_stats_batch_file_success(self):settings = {"FEEDS": {- build_url(os.path.join(self._random_temp_filename(), "json", self._file_mark)): {+ build_url(str(self._random_temp_filename() / "json" / self._file_mark)): {"format": "json",}},diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 49c83132f..402348cf9 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -1,10 +1,10 @@import json-import osimport randomimport reimport shutilimport stringfrom ipaddress import IPv4Address+from pathlib import Pathfrom unittest import mock, skipIffrom urllib.parse import urlencode@@ -163,9 +163,8 @@ class RequestHeaders(LeafResource):return bytes(json.dumps(headers), 'utf-8')-def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:- with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate:- pem = ''.join(key.readlines()) + ''.join(certificate.readlines())+def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate:+ pem = key_file.read_text() + certificate_file.read_text()return PrivateCertificate.loadPEM(pem)@@ -173,12 +172,12 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")class Https2ClientProtocolTestCase(TestCase):scheme = 'https'- key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key')- certificate_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.crt')+ key_file = Path(__file__).parent / 'keys' / 'localhost.key'+ certificate_file = Path(__file__).parent / 'keys' / 'localhost.crt'def _init_resource(self):self.temp_directory = self.mktemp()- os.mkdir(self.temp_directory)+ Path(self.temp_directory).mkdir()r = File(self.temp_directory)r.putChild(b'get-data-html-small', GetDataHtmlSmall())r.putChild(b'get-data-html-large', GetDataHtmlLarge())@@ -202,7 +201,7 @@ class Https2ClientProtocolTestCase(TestCase):# Start server for testingself.hostname = 'localhost'- context_factory = ssl_context_factory(self.key_file, self.certificate_file)+ context_factory = ssl_context_factory(str(self.key_file), str(self.certificate_file))server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname)self.server = yield server_endpoint.listen(self.site)diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex e46532a1c..5d6547279 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -1,5 +1,5 @@-import osimport shutil+from pathlib import Pathfrom testfixtures import LogCapturefrom twisted.internet import defer@@ -61,12 +61,12 @@ class FileDownloadCrawlTestCase(TestCase):self.mockserver.__enter__()# prepare a directory for storing files- self.tmpmediastore = self.mktemp()- os.mkdir(self.tmpmediastore)+ self.tmpmediastore = Path(self.mktemp())+ self.tmpmediastore.mkdir()self.settings = {'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION','ITEM_PIPELINES': {self.pipeline_class: 1},- self.store_setting_key: self.tmpmediastore,+ self.store_setting_key: str(self.tmpmediastore),}self.runner = CrawlerRunner(self.settings)self.items = []@@ -111,9 +111,7 @@ class FileDownloadCrawlTestCase(TestCase):# check that the image files where actually written to the media storefor item in items:for i in item[self.media_key]:- self.assertTrue(- os.path.exists(- os.path.join(self.tmpmediastore, i['path'])))+ self.assertTrue((self.tmpmediastore / i['path']).exists())def _assert_files_download_failure(self, crawler, items, code, logs):@@ -133,7 +131,7 @@ class FileDownloadCrawlTestCase(TestCase):self.assertEqual(logs.count(file_dl_failure), 3)# check that no files were written to the media store- self.assertEqual(os.listdir(self.tmpmediastore), [])+ self.assertEqual([x for x in self.tmpmediastore.iterdir()], [])@defer.inlineCallbacksdef test_download_media(self):diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex d641e7a43..4acd29bf7 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -3,6 +3,7 @@ import randomimport timefrom datetime import datetimefrom io import BytesIO+from pathlib import Pathfrom shutil import rmtreefrom tempfile import mkdtempfrom unittest import mock@@ -89,7 +90,7 @@ class FilesPipelineTestCase(unittest.TestCase):self.assertEqual(self.pipeline.store.basedir, self.tempdir)path = 'some/image/key.jpg'- fullpath = os.path.join(self.tempdir, 'some', 'image', 'key.jpg')+ fullpath = Path(self.tempdir, 'some', 'image', 'key.jpg')self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath)@defer.inlineCallbacksdiff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pyindex afdfb2578..ea7701b5d 100644--- a/tests/test_proxy_connect.py+++ b/tests/test_proxy_connect.py@@ -2,6 +2,7 @@ import jsonimport osimport reimport sys+from pathlib import Pathfrom subprocess import Popen, PIPEfrom urllib.parse import urlsplit, urlunsplitfrom testfixtures import LogCapture@@ -27,14 +28,13 @@ from mitmproxy.tools.main import mitmdumpsys.argv[0] = "mitmdump"sys.exit(mitmdump())"""- cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),- 'keys', 'mitmproxy-ca.pem')+ cert_path = Path(__file__).parent.resolve() / 'keys' / 'mitmproxy-ca.pem'self.proc = Popen([sys.executable,'-c', script,'--listen-host', '127.0.0.1','--listen-port', '0','--proxyauth', f'{self.auth_user}:{self.auth_pass}',- '--certs', cert_path,+ '--certs', str(cert_path),'--ssl-insecure',],stdout=PIPE, env=get_testenv())diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.pyindex 3719c7c9f..b9025cc14 100644--- a/tests/test_spiderloader/__init__.py+++ b/tests/test_spiderloader/__init__.py@@ -1,7 +1,7 @@import sys-import osimport shutilimport warnings+from pathlib import Pathfrom zope.interface.verify import verifyObjectfrom twisted.trial import unittest@@ -17,10 +17,10 @@ from scrapy.settings import Settingsfrom scrapy.http import Requestfrom scrapy.crawler import CrawlerRunner-module_dir = os.path.dirname(os.path.abspath(__file__))+module_dir = Path(__file__).resolve().parent-def _copytree(source, target):+def _copytree(source: Path, target: Path):try:shutil.copytree(source, target)except shutil.Error:@@ -30,18 +30,18 @@ def _copytree(source, target):class SpiderLoaderTest(unittest.TestCase):def setUp(self):- orig_spiders_dir = os.path.join(module_dir, 'test_spiders')- self.tmpdir = tempfile.mkdtemp()- self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')+ orig_spiders_dir = module_dir / 'test_spiders'+ self.tmpdir = Path(tempfile.mkdtemp())+ self.spiders_dir = self.tmpdir / 'test_spiders_xxx'_copytree(orig_spiders_dir, self.spiders_dir)- sys.path.append(self.tmpdir)+ sys.path.append(str(self.tmpdir))settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})self.spider_loader = SpiderLoader.from_settings(settings)def tearDown(self):del self.spider_loaderdel sys.modules['test_spiders_xxx']- sys.path.remove(self.tmpdir)+ sys.path.remove(str(self.tmpdir))def test_interface(self):verifyObject(ISpiderLoader, self.spider_loader)@@ -135,22 +135,22 @@ class SpiderLoaderTest(unittest.TestCase):class DuplicateSpiderNameLoaderTest(unittest.TestCase):def setUp(self):- orig_spiders_dir = os.path.join(module_dir, 'test_spiders')- self.tmpdir = self.mktemp()- os.mkdir(self.tmpdir)- self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')+ orig_spiders_dir = module_dir / 'test_spiders'+ self.tmpdir = Path(self.mktemp())+ self.tmpdir.mkdir()+ self.spiders_dir = self.tmpdir / 'test_spiders_xxx'_copytree(orig_spiders_dir, self.spiders_dir)- sys.path.append(self.tmpdir)+ sys.path.append(str(self.tmpdir))self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})def tearDown(self):del sys.modules['test_spiders_xxx']- sys.path.remove(self.tmpdir)+ sys.path.remove(str(self.tmpdir))def test_dupename_warning(self):# copy 1 spider module so as to have duplicate spider name- shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'),- os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py'))+ shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider3.py',+ self.tmpdir / 'test_spiders_xxx' / 'spider3dupe.py')with warnings.catch_warnings(record=True) as w:spider_loader = SpiderLoader.from_settings(self.settings)@@ -171,10 +171,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):def test_multiple_dupename_warning(self):# copy 2 spider modules so as to have duplicate spider name# This should issue 2 warning, 1 for each duplicate spider name- shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'),- os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py'))- shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'),- os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py'))+ shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider1.py',+ self.tmpdir / 'test_spiders_xxx' / 'spider1dupe.py')+ shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider2.py',+ self.tmpdir / 'test_spiders_xxx' / 'spider2dupe.py')with warnings.catch_warnings(record=True) as w:spider_loader = SpiderLoader.from_settings(self.settings)diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.pyindex 383fadfeb..ab215576e 100644--- a/tests/test_spiderstate.py+++ b/tests/test_spiderstate.py@@ -1,5 +1,5 @@-import osfrom datetime import datetime+from pathlib import Pathimport shutilfrom twisted.trial import unittest@@ -13,7 +13,7 @@ class SpiderStateTest(unittest.TestCase):def test_store_load(self):jobdir = self.mktemp()- os.mkdir(jobdir)+ Path(jobdir).mkdir()try:spider = Spider(name='default')dt = datetime.now()diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.pyindex 4943731cb..ca98bff21 100644--- a/tests/test_utils_gz.py+++ b/tests/test_utils_gz.py@@ -1,5 +1,5 @@import unittest-from os.path import join+from pathlib import Pathfrom w3lib.encoding import html_to_unicode@@ -8,46 +8,40 @@ from scrapy.http import Responsefrom tests import tests_datadir-SAMPLEDIR = join(tests_datadir, 'compressed')+SAMPLEDIR = Path(tests_datadir, 'compressed')class GunzipTest(unittest.TestCase):def test_gunzip_basic(self):- with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f:- r1 = Response("http://www.example.com", body=f.read())- self.assertTrue(gzip_magic_number(r1))+ r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'feed-sample1.xml.gz').read_bytes())+ self.assertTrue(gzip_magic_number(r1))- r2 = Response("http://www.example.com", body=gunzip(r1.body))- self.assertFalse(gzip_magic_number(r2))- self.assertEqual(len(r2.body), 9950)+ r2 = Response("http://www.example.com", body=gunzip(r1.body))+ self.assertFalse(gzip_magic_number(r2))+ self.assertEqual(len(r2.body), 9950)def test_gunzip_truncated(self):- with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f:- text = gunzip(f.read())- assert text.endswith(b'</html')+ text = gunzip((SAMPLEDIR / 'truncated-crc-error.gz').read_bytes())+ assert text.endswith(b'</html')def test_gunzip_no_gzip_file_raises(self):- with open(join(SAMPLEDIR, 'feed-sample1.xml'), 'rb') as f:- self.assertRaises(IOError, gunzip, f.read())+ self.assertRaises(IOError, gunzip, (SAMPLEDIR / 'feed-sample1.xml').read_bytes())def test_gunzip_truncated_short(self):- with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f:- r1 = Response("http://www.example.com", body=f.read())- self.assertTrue(gzip_magic_number(r1))+ r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'truncated-crc-error-short.gz').read_bytes())+ self.assertTrue(gzip_magic_number(r1))- r2 = Response("http://www.example.com", body=gunzip(r1.body))- assert r2.body.endswith(b'</html>')- self.assertFalse(gzip_magic_number(r2))+ r2 = Response("http://www.example.com", body=gunzip(r1.body))+ assert r2.body.endswith(b'</html>')+ self.assertFalse(gzip_magic_number(r2))def test_is_gzipped_empty(self):r1 = Response("http://www.example.com")self.assertFalse(gzip_magic_number(r1))def test_gunzip_illegal_eof(self):- with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f:- text = html_to_unicode('charset=cp1252', gunzip(f.read()))[1]- with open(join(SAMPLEDIR, 'unexpected-eof-output.txt'), 'rb') as o:- expected_text = o.read().decode("utf-8")- self.assertEqual(len(text), len(expected_text))- self.assertEqual(text, expected_text)+ text = html_to_unicode('charset=cp1252', gunzip((SAMPLEDIR / 'unexpected-eof.gz').read_bytes()))[1]+ expected_text = (SAMPLEDIR / 'unexpected-eof-output.txt').read_text(encoding="utf-8")+ self.assertEqual(len(text), len(expected_text))+ self.assertEqual(text, expected_text)diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.pyindex f84cb2956..ba3136b96 100644--- a/tests/test_utils_iterators.py+++ b/tests/test_utils_iterators.py@@ -1,5 +1,3 @@-import os-from pytest import markfrom twisted.trial import unittest@@ -303,11 +301,6 @@ class LxmlXmliterTestCase(XmliterTestCase):class UtilsCsvTestCase(unittest.TestCase):- sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')- sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv')- sample_feed2_path = os.path.join(sample_feeds_dir, 'feed-sample4.csv')- sample_feed3_path = os.path.join(sample_feeds_dir, 'feed-sample5.csv')-def test_csviter_defaults(self):body = get_testdata('feeds', 'feed-sample3.csv')response = TextResponse(url="http://example.com/", body=body)diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.pyindex b83c1d6f0..dc5b9e123 100644--- a/tests/test_utils_misc/__init__.py+++ b/tests/test_utils_misc/__init__.py@@ -1,6 +1,7 @@import sysimport osimport unittest+from pathlib import Pathfrom unittest import mockfrom scrapy.item import Item, Field@@ -55,7 +56,7 @@ class UtilsMiscTestCase(unittest.TestCase):self.assertRaises(ImportError, walk_modules, 'nomodule999')def test_walk_modules_egg(self):- egg = os.path.join(os.path.dirname(__file__), 'test.egg')+ egg = str(Path(__file__).parent / 'test.egg')sys.path.append(egg)try:mods = walk_modules('testegg')diff --git a/tests/test_utils_project.py b/tests/test_utils_project.pyindex 46452415a..f35f039a9 100644--- a/tests/test_utils_project.py+++ b/tests/test_utils_project.py@@ -4,6 +4,7 @@ import tempfileimport shutilimport contextlibimport warnings+from pathlib import Pathfrom pytest import warns@@ -18,9 +19,7 @@ def inside_a_project():try:os.chdir(project_dir)- with open('scrapy.cfg', 'w') as f:- # create an empty scrapy.cfg- f.close()+ Path('scrapy.cfg').touch()yield project_dirfinally:@@ -31,20 +30,20 @@ def inside_a_project():class ProjectUtilsTest(unittest.TestCase):def test_data_path_outside_project(self):self.assertEqual(- os.path.join('.scrapy', 'somepath'),+ str(Path('.scrapy', 'somepath')),data_path('somepath'))- abspath = os.path.join(os.path.sep, 'absolute', 'path')+ abspath = str(Path(os.path.sep, 'absolute', 'path'))self.assertEqual(abspath, data_path(abspath))def test_data_path_inside_project(self):with inside_a_project() as proj_path:- expected = os.path.join(proj_path, '.scrapy', 'somepath')+ expected = Path(proj_path, '.scrapy', 'somepath')self.assertEqual(- os.path.realpath(expected),- os.path.realpath(data_path('somepath'))+ expected.resolve(),+ Path(data_path('somepath')).resolve())- abspath = os.path.join(os.path.sep, 'absolute', 'path')+ abspath = str(Path(os.path.sep, 'absolute', 'path').resolve())self.assertEqual(abspath, data_path(abspath))diff --git a/tests/test_utils_response.py b/tests/test_utils_response.pyindex d20852e62..cdf972933 100644--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@ -1,6 +1,6 @@-import osimport unittestimport warnings+from pathlib import Pathfrom urllib.parse import urlparsefrom scrapy.exceptions import ScrapyDeprecationWarning@@ -39,10 +39,9 @@ class ResponseUtilsTest(unittest.TestCase):def browser_open(burl):path = urlparse(burl).path- if not os.path.exists(path):+ if not path or not Path(path).exists():path = burl.replace('file://', '')- with open(path, "rb") as f:- bbody = f.read()+ bbody = Path(path).read_bytes()self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody)return Trueresponse = HtmlResponse(url, body=body)@@ -98,10 +97,9 @@ class ResponseUtilsTest(unittest.TestCase):def check_base_url(burl):path = urlparse(burl).path- if not os.path.exists(path):+ if not path or not Path(path).exists():path = burl.replace('file://', '')- with open(path, "rb") as f:- bbody = f.read()+ bbody = Path(path).read_bytes()self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)return Truediff --git a/tests/test_utils_template.py b/tests/test_utils_template.pyindex 1d5e63363..b1aca5ed3 100644--- a/tests/test_utils_template.py+++ b/tests/test_utils_template.py@@ -1,4 +1,4 @@-import os+from pathlib import Pathfrom shutil import rmtreefrom tempfile import mkdtempimport unittest@@ -22,21 +22,19 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):template = 'from ${project_name}.spiders.${name} import ${classname}'rendered = 'from proj.spiders.spi import TheSpider'- template_path = os.path.join(self.tmp_path, 'templ.py.tmpl')- render_path = os.path.join(self.tmp_path, 'templ.py')+ template_path = Path(self.tmp_path, 'templ.py.tmpl')+ render_path = Path(self.tmp_path, 'templ.py')- with open(template_path, 'wb') as tmpl_file:- tmpl_file.write(template.encode('utf8'))- assert os.path.isfile(template_path) # Failure of test itself+ template_path.write_text(template, encoding='utf8')+ assert template_path.is_file() # Failure of test itself- render_templatefile(template_path, **context)+ render_templatefile(str(template_path), **context)- self.assertFalse(os.path.exists(template_path))- with open(render_path, 'rb') as result:- self.assertEqual(result.read().decode('utf8'), rendered)+ self.assertFalse(template_path.exists())+ self.assertEqual(render_path.read_text(encoding='utf8'), rendered)- os.remove(render_path)- assert not os.path.exists(render_path) # Failure of test itself+ render_path.unlink()+ assert not render_path.exists() # Failure of test itselfif '__main__' == __name__:diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex 0d5827339..69d9a9e3a 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -2,8 +2,8 @@from twisted.internet import deferTests borrowed from the twisted.web.client tests."""-import osimport shutil+from pathlib import Pathimport OpenSSL.SSLfrom twisted.trial import unittest@@ -15,7 +15,6 @@ except ImportError:# deprecated in Twisted 19.7.0# (remove once we bump our requirement past that version)from twisted.test.proto_helpers import StringTransport-from twisted.python.filepath import FilePathfrom twisted.protocols.policies import WrappingFactoryfrom twisted.internet.defer import inlineCallbacks@@ -230,10 +229,10 @@ class WebClientTestCase(unittest.TestCase):return reactor.listenTCP(0, site, interface="127.0.0.1")def setUp(self):- self.tmpname = self.mktemp()- os.mkdir(self.tmpname)- FilePath(self.tmpname).child("file").setContent(b"0123456789")- r = static.File(self.tmpname)+ self.tmpname = Path(self.mktemp())+ self.tmpname.mkdir()+ (self.tmpname / "file").write_bytes(b"0123456789")+ r = static.File(str(self.tmpname))r.putChild(b"redirect", util.Redirect(b"/file"))r.putChild(b"wait", ForeverTakingResource())r.putChild(b"error", ErrorResource())@@ -379,10 +378,10 @@ class WebClientSSLTestCase(unittest.TestCase):return f"https://127.0.0.1:{self.portno}/{path}"def setUp(self):- self.tmpname = self.mktemp()- os.mkdir(self.tmpname)- FilePath(self.tmpname).child("file").setContent(b"0123456789")- r = static.File(self.tmpname)+ self.tmpname = Path(self.mktemp())+ self.tmpname.mkdir()+ (self.tmpname / "file").write_bytes(b"0123456789")+ r = static.File(str(self.tmpname))r.putChild(b"payload", PayloadResource())self.site = server.Site(r, timeout=None)self.wrapper = WrappingFactory(self.site)