Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

use pathlib

Matthew Donoughe40d9ca3bdd6f
diff --git a/conftest.py b/conftest.py
index d7fe80321..7c1da3556 100644
--- a/conftest.py
+++ b/conftest.py
@@ -21,7 +21,7 @@ collect_ignore = [
*_py_files("tests/CrawlerRunner"),
]
-with open('tests/ignores.txt') as reader:
+with Path('tests/ignores.txt').open() as reader:
for line in reader:
file_path = line.strip()
if file_path and file_path[0] != '#':
diff --git a/docs/Makefile b/docs/Makefile
index 87d5d3047..596cb6cef 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -86,8 +86,8 @@ coverage: BUILDER = coverage
coverage: build
htmlview: html
- $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \
- os.path.realpath('build/html/index.html'))"
+ $(PYTHON) -c "import webbrowser; from pathlib import Path; \
+ webbrowser.open('file://' + Path('build/html/index.html').resolve())"
clean:
-rm -rf build/*
diff --git a/docs/conf.py b/docs/conf.py
index 3241295af..d2a77003e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -11,13 +11,12 @@
import sys
from datetime import datetime
-from os import path
+from pathlib import Path
# If your extensions are in another directory, add it here. If the directory
-# is relative to the documentation root, use os.path.abspath to make it
-# absolute, like shown here.
-sys.path.append(path.join(path.dirname(__file__), "_ext"))
-sys.path.insert(0, path.dirname(path.dirname(__file__)))
+# is relative to the documentation root, use Path.absolute to make it absolute.
+sys.path.append(str(Path(__file__).parent / "_ext"))
+sys.path.insert(0, str(Path(__file__).parent.parent))
# General configuration
diff --git a/docs/conftest.py b/docs/conftest.py
index a0636f8ac..24a72a4b6 100644
--- a/docs/conftest.py
+++ b/docs/conftest.py
@@ -1,5 +1,5 @@
-import os
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
+from pathlib import Path
from scrapy.http.response.html import HtmlResponse
from sybil import Sybil
@@ -12,10 +12,9 @@ from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
-def load_response(url, filename):
- input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
- with open(input_path, 'rb') as input_file:
- return HtmlResponse(url, body=input_file.read())
+def load_response(url: str, filename: str) -> HtmlResponse:
+ input_path = Path(__file__).parent / '_tests' / filename
+ return HtmlResponse(url, body=input_path.read_bytes())
def setup(namespace):
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 092123d1d..901a170b4 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -85,6 +85,8 @@ page content to extract data.
This is the code for our first Spider. Save it in a file named
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
+ from pathlib import Path
+
import scrapy
@@ -102,8 +104,7 @@ This is the code for our first Spider. Save it in a file named
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
- with open(filename, 'wb') as f:
- f.write(response.body)
+ Path(filename).write_bytes(response.body)
self.log(f'Saved file {filename}')
@@ -178,6 +179,8 @@ with a list of URLs. This list will then be used by the default implementation
of :meth:`~scrapy.Spider.start_requests` to create the initial requests
for your spider::
+ from pathlib import Path
+
import scrapy
@@ -191,8 +194,7 @@ for your spider::
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
- with open(filename, 'wb') as f:
- f.write(response.body)
+ Path(filename).write_bytes(response.body)
The :meth:`~scrapy.Spider.parse` method will be called to handle each
of the requests for those URLs, even though we haven't explicitly told Scrapy
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index af294f52c..1672ccbcc 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -186,6 +186,7 @@ item.
::
import hashlib
+ from pathlib import Path
from urllib.parse import quote
import scrapy
@@ -214,8 +215,7 @@ item.
url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = f"{url_hash}.png"
- with open(filename, "wb") as f:
- f.write(response.body)
+ Path(filename).write_bytes(response.body)
# Store filename in item.
adapter["screenshot_filename"] = filename
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 0925e6bb5..a528746b0 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -156,7 +156,6 @@ By overriding ``file_path`` like this:
.. code-block:: python
import hashlib
- from os.path import splitext
def file_path(self, request, response=None, info=None, *, item=None):
image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)
@@ -498,7 +497,7 @@ See here the methods that you can override in your custom Files Pipeline:
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
- import os
+ from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.files import FilesPipeline
@@ -506,7 +505,7 @@ See here the methods that you can override in your custom Files Pipeline:
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
- return 'files/' + os.path.basename(urlparse(request.url).path)
+ return 'files/' + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.
@@ -637,7 +636,7 @@ See here the methods that you can override in your custom Images Pipeline:
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
- import os
+ from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.images import ImagesPipeline
@@ -645,7 +644,7 @@ See here the methods that you can override in your custom Images Pipeline:
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
- return 'files/' + os.path.basename(urlparse(request.url).path)
+ return 'files/' + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.
diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index 95a3f17d5..7a0c5288c 100755
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -13,6 +13,7 @@ Author: dufferzafar
"""
import re
+from pathlib import Path
def main():
@@ -27,7 +28,7 @@ def main():
# Read lines from the linkcheck output file
try:
- with open("build/linkcheck/output.txt") as out:
+ with Path("build/linkcheck/output.txt").open() as out:
output_lines = out.readlines()
except IOError:
print("linkcheck output not found; please run linkcheck first.")
@@ -51,14 +52,12 @@ def main():
# Update the previous file
if _filename:
- with open(_filename, "w") as _file:
- _file.write(_contents)
+ Path(_filename).write_text(_contents)
_filename = newfilename
# Read the new file to memory
- with open(_filename) as _file:
- _contents = _file.read()
+ _contents = Path(_filename).read_text()
_contents = _contents.replace(match.group(3), match.group(4))
else:
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index fb304b8c0..8570d90bd 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -3,6 +3,7 @@ Base class for Scrapy commands
"""
import os
import argparse
+from pathlib import Path
from typing import Any, Dict
from twisted.python import failure
@@ -93,8 +94,7 @@ class ScrapyCommand:
self.settings.set('LOG_ENABLED', False, priority='cmdline')
if opts.pidfile:
- with open(opts.pidfile, "w") as f:
- f.write(str(os.getpid()) + os.linesep)
+ Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep)
if opts.pdb:
failure.startDebugMode()
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index ed5f588e9..01b4a0dbd 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -2,8 +2,9 @@ import os
import shutil
import string
+from pathlib import Path
from importlib import import_module
-from os.path import join, dirname, abspath, exists, splitext
+from typing import Optional
from urllib.parse import urlparse
import scrapy
@@ -62,8 +63,7 @@ class Command(ScrapyCommand):
if opts.dump:
template_file = self._find_template(opts.dump)
if template_file:
- with open(template_file, "r") as f:
- print(f.read())
+ print(template_file.read_text())
return
if len(args) != 2:
raise UsageError()
@@ -98,11 +98,11 @@ class Command(ScrapyCommand):
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
- spiders_dir = abspath(dirname(spiders_module.__file__))
+ spiders_dir = Path(spiders_module.__file__).parent.resolve()
else:
spiders_module = None
- spiders_dir = "."
- spider_file = f"{join(spiders_dir, module)}.py"
+ spiders_dir = Path(".")
+ spider_file = f"{spiders_dir / module}.py"
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print(f"Created spider {name!r} using template {template_name!r} ",
@@ -110,24 +110,25 @@ class Command(ScrapyCommand):
if spiders_module:
print(f"in module:\n {spiders_module.__name__}.{module}")
- def _find_template(self, template):
- template_file = join(self.templates_dir, f'{template}.tmpl')
- if exists(template_file):
+ def _find_template(self, template: str) -> Optional[Path]:
+ template_file = Path(self.templates_dir, f'{template}.tmpl')
+ if template_file.exists():
return template_file
print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.')
def _list_templates(self):
print("Available templates:")
- for filename in sorted(os.listdir(self.templates_dir)):
- if filename.endswith('.tmpl'):
- print(f" {splitext(filename)[0]}")
+ for file in sorted(Path(self.templates_dir).iterdir()):
+ if file.suffix == '.tmpl':
+ print(f" {file.stem}")
- def _spider_exists(self, name):
+ def _spider_exists(self, name: str) -> bool:
if not self.settings.get('NEWSPIDER_MODULE'):
# if run as a standalone command and file with same filename already exists
- if exists(name + ".py"):
- print(f"{abspath(name + '.py')} already exists")
+ path = Path(name + ".py")
+ if path.exists():
+ print(f"{path.resolve()} already exists")
return True
return False
@@ -143,17 +144,18 @@ class Command(ScrapyCommand):
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
- spiders_dir = dirname(spiders_module.__file__)
- spiders_dir_abs = abspath(spiders_dir)
- if exists(join(spiders_dir_abs, name + ".py")):
- print(f"{join(spiders_dir_abs, (name + '.py'))} already exists")
+ spiders_dir = Path(spiders_module.__file__).parent
+ spiders_dir_abs = spiders_dir.resolve()
+ path = spiders_dir_abs / (name + ".py")
+ if path.exists():
+ print(f"{path} already exists")
return True
return False
@property
- def templates_dir(self):
- return join(
- self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
+ def templates_dir(self) -> str:
+ return str(Path(
+ self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),
'spiders'
- )
+ ))
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index b957c29fb..c41135508 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -1,22 +1,23 @@
import sys
-import os
+from os import PathLike
+from pathlib import Path
from importlib import import_module
+from types import ModuleType
from scrapy.utils.spider import iter_spider_classes
from scrapy.exceptions import UsageError
from scrapy.commands import BaseRunSpiderCommand
-def _import_file(filepath):
- abspath = os.path.abspath(filepath)
- dirname, file = os.path.split(abspath)
- fname, fext = os.path.splitext(file)
- if fext not in ('.py', '.pyw'):
+def _import_file(filepath: str | PathLike[str]) -> ModuleType:
+ abspath = Path(filepath).resolve()
+ dirname = str(abspath.parent)
+ if abspath.suffix not in ('.py', '.pyw'):
raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path
try:
- module = import_module(fname)
+ module = import_module(abspath.stem)
finally:
if dirname:
sys.path.pop(0)
@@ -40,13 +41,13 @@ class Command(BaseRunSpiderCommand):
def run(self, args, opts):
if len(args) != 1:
raise UsageError()
- filename = args[0]
- if not os.path.exists(filename):
+ filename = Path(args[0])
+ if not filename.exists():
raise UsageError(f"File not found: {filename}\n")
try:
module = _import_file(filename)
except (ImportError, ValueError) as e:
- raise UsageError(f"Unable to load {filename!r}: {e}\n")
+ raise UsageError(f"Unable to load {str(filename)!r}: {e}\n")
spclasses = list(iter_spider_classes(module))
if not spclasses:
raise UsageError(f"No spider found in file: {filename}\n")
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 1b6374c39..4323cdb53 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -2,7 +2,7 @@ import re
import os
import string
from importlib.util import find_spec
-from os.path import join, exists, abspath
+from pathlib import Path
from shutil import ignore_patterns, move, copy2, copystat
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
@@ -54,7 +54,7 @@ class Command(ScrapyCommand):
return True
return False
- def _copytree(self, src, dst):
+ def _copytree(self, src: Path, dst: Path):
"""
Since the original function always creates the directory, to resolve
the issue a new function had to be created. It's a simple copy and
@@ -64,19 +64,19 @@ class Command(ScrapyCommand):
https://github.com/scrapy/scrapy/pull/2005
"""
ignore = IGNORE
- names = os.listdir(src)
+ names = [x.name for x in src.iterdir()]
ignored_names = ignore(src, names)
- if not os.path.exists(dst):
- os.makedirs(dst)
+ if not dst.exists():
+ dst.mkdir(parents=True)
for name in names:
if name in ignored_names:
continue
- srcname = os.path.join(src, name)
- dstname = os.path.join(dst, name)
- if os.path.isdir(srcname):
+ srcname = src / name
+ dstname = dst / name
+ if srcname.is_dir():
self._copytree(srcname, dstname)
else:
copy2(srcname, dstname)
@@ -90,36 +90,36 @@ class Command(ScrapyCommand):
raise UsageError()
project_name = args[0]
- project_dir = args[0]
if len(args) == 2:
- project_dir = args[1]
+ project_dir = Path(args[1])
+ else:
+ project_dir = Path(args[0])
- if exists(join(project_dir, 'scrapy.cfg')):
+ if (project_dir / 'scrapy.cfg').exists():
self.exitcode = 1
- print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}')
+ print(f'Error: scrapy.cfg already exists in {project_dir.resolve()}')
return
if not self._is_valid_name(project_name):
self.exitcode = 1
return
- self._copytree(self.templates_dir, abspath(project_dir))
- move(join(project_dir, 'module'), join(project_dir, project_name))
+ self._copytree(Path(self.templates_dir), project_dir.resolve())
+ move(project_dir / 'module', project_dir / project_name)
for paths in TEMPLATES_TO_RENDER:
- path = join(*paths)
- tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
- render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
+ tplfile = Path(project_dir, *(string.Template(s).substitute(project_name=project_name) for s in paths))
+ render_templatefile(str(tplfile), project_name=project_name, ProjectName=string_camelcase(project_name))
print(f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:")
- print(f" {abspath(project_dir)}\n")
+ print(f" {project_dir.resolve()}\n")
print("You can start your first spider with:")
print(f" cd {project_dir}")
print(" scrapy genspider example example.com")
@property
- def templates_dir(self):
- return join(
- self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
+ def templates_dir(self) -> str:
+ return str(Path(
+ self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),
'project'
- )
+ ))
diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py
index 0d94e3df0..4824167da 100644
--- a/scrapy/core/downloader/handlers/file.py
+++ b/scrapy/core/downloader/handlers/file.py
@@ -1,3 +1,5 @@
+from pathlib import Path
+
from w3lib.url import file_uri_to_path
from scrapy.responsetypes import responsetypes
@@ -10,7 +12,6 @@ class FileDownloadHandler:
@defers
def download_request(self, request, spider):
filepath = file_uri_to_path(request.url)
- with open(filepath, 'rb') as fo:
- body = fo.read()
+ body = Path(filepath).read_bytes()
respcls = responsetypes.from_args(filename=filepath, body=body)
return respcls(url=request.url, body=body)
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 5ba0fb63b..366449f51 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -1,8 +1,7 @@
import json
import logging
-import os
from abc import abstractmethod
-from os.path import exists, join
+from pathlib import Path
from typing import Optional, Type, TypeVar
from twisted.internet.defer import Deferred
@@ -324,19 +323,19 @@ class Scheduler(BaseScheduler):
def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
""" Return a folder name to keep disk queue state at """
if jobdir is not None:
- dqdir = join(jobdir, 'requests.queue')
- if not exists(dqdir):
- os.makedirs(dqdir)
- return dqdir
+ dqdir = Path(jobdir, 'requests.queue')
+ if not dqdir.exists():
+ dqdir.mkdir(parents=True)
+ return str(dqdir)
return None
def _read_dqs_state(self, dqdir: str) -> list:
- path = join(dqdir, 'active.json')
- if not exists(path):
+ path = Path(dqdir, 'active.json')
+ if not path.exists():
return []
- with open(path) as f:
+ with path.open() as f:
return json.load(f)
def _write_dqs_state(self, dqdir: str, state: list) -> None:
- with open(join(dqdir, 'active.json'), 'w') as f:
+ with Path(dqdir, 'active.json').open('w') as f:
json.dump(state, f)
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index e768bca12..b7108cdcc 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,7 +1,10 @@
+from __future__ import annotations
+
import logging
import pprint
import signal
import warnings
+from typing import TYPE_CHECKING
from twisted.internet import defer
from zope.interface.exceptions import DoesNotImplement
@@ -33,6 +36,9 @@ from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.reactor import install_reactor, verify_installed_reactor
+if TYPE_CHECKING:
+ from scrapy.utils.request import RequestFingerprinter
+
logger = logging.getLogger(__name__)
@@ -72,7 +78,7 @@ class Crawler:
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
- self.request_fingerprinter = create_instance(
+ self.request_fingerprinter: RequestFingerprinter = create_instance(
load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
settings=self.settings,
crawler=self,
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index d1b0559ef..2b8b09614 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,5 +1,5 @@
import logging
-import os
+from pathlib import Path
from typing import Optional, Set, Type, TypeVar
from warnings import warn
@@ -55,7 +55,7 @@ class RFPDupeFilter(BaseDupeFilter):
self.debug = debug
self.logger = logging.getLogger(__name__)
if path:
- self.file = open(os.path.join(path, 'requests.seen'), 'a+')
+ self.file = Path(path, 'requests.seen').open('a+')
self.file.seek(0)
self.fingerprints.update(x.rstrip() for x in self.file)
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e7097b7a1..0aa27e417 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -5,13 +5,13 @@ See documentation in docs/topics/feed-exports.rst
"""
import logging
-import os
import re
import sys
import warnings
from datetime import datetime
+from pathlib import Path
from tempfile import NamedTemporaryFile
-from typing import Any, Callable, Optional, Tuple, Union
+from typing import IO, Any, Callable, Optional, Tuple, Union
from urllib.parse import unquote, urlparse
from twisted.internet import defer, threads
@@ -101,7 +101,7 @@ class BlockingFeedStorage:
def open(self, spider):
path = spider.crawler.settings['FEED_TEMPDIR']
- if path and not os.path.isdir(path):
+ if path and not Path(path).is_dir():
raise OSError('Not a Directory: ' + str(path))
return NamedTemporaryFile(prefix='feed-', dir=path)
@@ -141,11 +141,11 @@ class FileFeedStorage:
feed_options = feed_options or {}
self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'
- def open(self, spider):
- dirname = os.path.dirname(self.path)
- if dirname and not os.path.exists(dirname):
- os.makedirs(dirname)
- return open(self.path, self.write_mode)
+ def open(self, spider) -> IO[Any]:
+ dirname = Path(self.path).parent
+ if dirname and not dirname.exists():
+ dirname.mkdir(parents=True)
+ return Path(self.path).open(self.write_mode)
def store(self, file):
file.close()
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 843e14812..3057bf157 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,16 +1,18 @@
import gzip
import logging
-import os
import pickle
from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
+from pathlib import Path
from time import time
from weakref import WeakKeyDictionary
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
from scrapy.http import Headers, Response
+from scrapy.http.request import Request
from scrapy.responsetypes import responsetypes
+from scrapy.spiders import Spider
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
@@ -221,9 +223,9 @@ class DbmCacheStorage:
self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE'])
self.db = None
- def open_spider(self, spider):
- dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
- self.db = self.dbmodule.open(dbpath, 'c')
+ def open_spider(self, spider: Spider):
+ dbpath = Path(self.cachedir, f'{spider.name}.db')
+ self.db = self.dbmodule.open(str(dbpath), 'c')
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
@@ -277,7 +279,7 @@ class FilesystemCacheStorage:
self.use_gzip = settings.getbool('HTTPCACHE_GZIP')
self._open = gzip.open if self.use_gzip else open
- def open_spider(self, spider):
+ def open_spider(self, spider: Spider):
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
extra={'spider': spider})
@@ -286,15 +288,15 @@ class FilesystemCacheStorage:
def close_spider(self, spider):
pass
- def retrieve_response(self, spider, request):
+ def retrieve_response(self, spider: Spider, request: Request):
"""Return response if present in cache, or None otherwise."""
metadata = self._read_meta(spider, request)
if metadata is None:
return # not cached
rpath = self._get_request_path(spider, request)
- with self._open(os.path.join(rpath, 'response_body'), 'rb') as f:
+ with self._open(rpath / 'response_body', 'rb') as f:
body = f.read()
- with self._open(os.path.join(rpath, 'response_headers'), 'rb') as f:
+ with self._open(rpath / 'response_headers', 'rb') as f:
rawheaders = f.read()
url = metadata.get('response_url')
status = metadata['status']
@@ -303,11 +305,11 @@ class FilesystemCacheStorage:
response = respcls(url=url, headers=headers, status=status, body=body)
return response
- def store_response(self, spider, request, response):
+ def store_response(self, spider: Spider, request: Request, response):
"""Store the given response in the cache."""
rpath = self._get_request_path(spider, request)
- if not os.path.exists(rpath):
- os.makedirs(rpath)
+ if not rpath.exists():
+ rpath.mkdir(parents=True)
metadata = {
'url': request.url,
'method': request.method,
@@ -315,29 +317,29 @@ class FilesystemCacheStorage:
'response_url': response.url,
'timestamp': time(),
}
- with self._open(os.path.join(rpath, 'meta'), 'wb') as f:
+ with self._open(rpath / 'meta', 'wb') as f:
f.write(to_bytes(repr(metadata)))
- with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
+ with self._open(rpath / 'pickled_meta', 'wb') as f:
pickle.dump(metadata, f, protocol=4)
- with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f:
+ with self._open(rpath / 'response_headers', 'wb') as f:
f.write(headers_dict_to_raw(response.headers))
- with self._open(os.path.join(rpath, 'response_body'), 'wb') as f:
+ with self._open(rpath / 'response_body', 'wb') as f:
f.write(response.body)
- with self._open(os.path.join(rpath, 'request_headers'), 'wb') as f:
+ with self._open(rpath / 'request_headers', 'wb') as f:
f.write(headers_dict_to_raw(request.headers))
- with self._open(os.path.join(rpath, 'request_body'), 'wb') as f:
+ with self._open(rpath / 'request_body', 'wb') as f:
f.write(request.body)
- def _get_request_path(self, spider, request):
+ def _get_request_path(self, spider: Spider, request: Request) -> Path:
key = self._fingerprinter.fingerprint(request).hex()
- return os.path.join(self.cachedir, spider.name, key[0:2], key)
+ return Path(self.cachedir, spider.name, key[0:2], key)
- def _read_meta(self, spider, request):
+ def _read_meta(self, spider: Spider, request: Request):
rpath = self._get_request_path(spider, request)
- metapath = os.path.join(rpath, 'pickled_meta')
- if not os.path.exists(metapath):
+ metapath = rpath / 'pickled_meta'
+ if not metapath.exists():
return # not found
- mtime = os.stat(metapath).st_mtime
+ mtime = metapath.stat().st_mtime
if 0 < self.expiration_secs < time() - mtime:
return # expired
with self._open(metapath, 'rb') as f:
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index bea00596e..e9c8b1d6a 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -1,5 +1,5 @@
-import os
import pickle
+from pathlib import Path
from scrapy import signals
from scrapy.exceptions import NotConfigured
@@ -25,16 +25,16 @@ class SpiderState:
def spider_closed(self, spider):
if self.jobdir:
- with open(self.statefn, 'wb') as f:
+ with Path(self.statefn).open('wb') as f:
pickle.dump(spider.state, f, protocol=4)
def spider_opened(self, spider):
- if self.jobdir and os.path.exists(self.statefn):
- with open(self.statefn, 'rb') as f:
+ if self.jobdir and Path(self.statefn).exists():
+ with Path(self.statefn).open('rb') as f:
spider.state = pickle.load(f)
else:
spider.state = {}
@property
- def statefn(self):
- return os.path.join(self.jobdir, 'spider.state')
+ def statefn(self) -> str:
+ return str(Path(self.jobdir, 'spider.state'))
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 4de6c9b5b..9359cc7c8 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -32,7 +32,7 @@ class Response(object_ref):
def __init__(
self,
- url,
+ url: str,
status=200,
headers=None,
body=b"",
@@ -75,7 +75,7 @@ class Response(object_ref):
def _get_url(self):
return self._url
- def _set_url(self, url):
+ def _set_url(self, url: str):
if isinstance(url, str):
self._url = url
else:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 906e7eb24..ffb12d910 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -13,6 +13,8 @@ from collections import defaultdict
from contextlib import suppress
from ftplib import FTP
from io import BytesIO
+from pathlib import Path
+from typing import Optional
from urllib.parse import urlparse
from itemadapter import ItemAdapter
@@ -39,41 +41,40 @@ class FileException(Exception):
class FSFilesStore:
- def __init__(self, basedir):
+ def __init__(self, basedir: str):
if '://' in basedir:
basedir = basedir.split('://', 1)[1]
self.basedir = basedir
- self._mkdir(self.basedir)
- self.created_directories = defaultdict(set)
+ self._mkdir(Path(self.basedir))
+ self.created_directories: defaultdict[str, set[str]] = defaultdict(set)
- def persist_file(self, path, buf, info, meta=None, headers=None):
+ def persist_file(self, path: str, buf, info, meta=None, headers=None):
absolute_path = self._get_filesystem_path(path)
- self._mkdir(os.path.dirname(absolute_path), info)
- with open(absolute_path, 'wb') as f:
- f.write(buf.getvalue())
+ self._mkdir(absolute_path.parent, info)
+ absolute_path.write_bytes(buf.getvalue())
- def stat_file(self, path, info):
+ def stat_file(self, path: str, info):
absolute_path = self._get_filesystem_path(path)
try:
- last_modified = os.path.getmtime(absolute_path)
+ last_modified = absolute_path.stat().st_mtime
except os.error:
return {}
- with open(absolute_path, 'rb') as f:
+ with absolute_path.open('rb') as f:
checksum = md5sum(f)
return {'last_modified': last_modified, 'checksum': checksum}
- def _get_filesystem_path(self, path):
+ def _get_filesystem_path(self, path: str) -> Path:
path_comps = path.split('/')
- return os.path.join(self.basedir, *path_comps)
+ return Path(self.basedir, *path_comps)
- def _mkdir(self, dirname, domain=None):
+ def _mkdir(self, dirname: Path, domain: Optional[str] = None):
seen = self.created_directories[domain] if domain else set()
- if dirname not in seen:
- if not os.path.exists(dirname):
- os.makedirs(dirname)
- seen.add(dirname)
+ if str(dirname) not in seen:
+ if not dirname.exists():
+ dirname.mkdir(parents=True)
+ seen.add(str(dirname))
class S3FilesStore:
@@ -374,8 +375,8 @@ class FilesPipeline(MediaPipeline):
store_uri = settings['FILES_STORE']
return cls(store_uri, settings=settings)
- def _get_store(self, uri):
- if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir
+ def _get_store(self, uri: str):
+ if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir
scheme = 'file'
else:
scheme = urlparse(uri).scheme
@@ -510,7 +511,7 @@ class FilesPipeline(MediaPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
- media_ext = os.path.splitext(request.url)[1]
+ media_ext = Path(request.url).suffix
# Handles empty and wild extensions by trying to guess the
# mime type then extension or default to empty string otherwise
if media_ext not in mimetypes.types_map:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index ff86af125..84e0a94d2 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -15,7 +15,7 @@ Scrapy developers, if you add a setting here remember to:
import sys
from importlib import import_module
-from os.path import join, abspath, dirname
+from pathlib import Path
AJAXCRAWL_ENABLED = False
@@ -288,7 +288,7 @@ STATS_DUMP = True
STATSMAILER_RCPTS = []
-TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))
+TEMPLATES_DIR = str((Path(__file__).parent / '..' / 'templates').resolve())
URLLENGTH_LIMIT = 2083
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index d8248c606..9a97e7801 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -3,14 +3,19 @@ Base class for Scrapy spiders
See documentation in docs/topics/spiders.rst
"""
+from __future__ import annotations
+
import logging
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
from scrapy import signals
from scrapy.http import Request
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
+
class Spider(object_ref):
"""Base class for scrapy spiders. All spiders must inherit from this
@@ -49,7 +54,7 @@ class Spider(object_ref):
spider._set_crawler(crawler)
return spider
- def _set_crawler(self, crawler):
+ def _set_crawler(self, crawler: Crawler):
self.crawler = crawler
self.settings = crawler.settings
crawler.signals.connect(self.close, signals.spider_closed)
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index dff9b1350..1f2dee55f 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -3,8 +3,9 @@ Scheduler queues
"""
import marshal
-import os
import pickle
+from os import PathLike
+from pathlib import Path
from queuelib import queue
@@ -16,10 +17,10 @@ def _with_mkdir(queue_class):
class DirectoriesCreated(queue_class):
- def __init__(self, path, *args, **kwargs):
- dirname = os.path.dirname(path)
- if not os.path.exists(dirname):
- os.makedirs(dirname, exist_ok=True)
+ def __init__(self, path: str | PathLike[str], *args, **kwargs):
+ dirname = Path(path).parent
+ if not dirname.exists():
+ dirname.mkdir(parents=True, exist_ok=True)
super().__init__(path, *args, **kwargs)
return DirectoriesCreated
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 00cc53725..e247f5999 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -4,6 +4,8 @@ import sys
import warnings
from configparser import ConfigParser
from operator import itemgetter
+from pathlib import Path
+from typing import Optional
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
@@ -65,17 +67,17 @@ def arglist_to_dict(arglist):
return dict(x.split('=', 1) for x in arglist)
-def closest_scrapy_cfg(path='.', prevpath=None):
+def closest_scrapy_cfg(path: str | os.PathLike[str] = '.', prevpath: Optional[str | os.PathLike] = None) -> str:
"""Return the path to the closest scrapy.cfg file by traversing the current
directory and its parents
"""
- if path == prevpath:
+ if prevpath is not None and str(path) == str(prevpath):
return ''
- path = os.path.abspath(path)
- cfgfile = os.path.join(path, 'scrapy.cfg')
- if os.path.exists(cfgfile):
- return cfgfile
- return closest_scrapy_cfg(os.path.dirname(path), path)
+ path = Path(path).resolve()
+ cfgfile = path / 'scrapy.cfg'
+ if cfgfile.exists():
+ return str(cfgfile)
+ return closest_scrapy_cfg(path.parent, path)
def init_env(project='default', set_syspath=True):
@@ -88,7 +90,7 @@ def init_env(project='default', set_syspath=True):
os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project)
closest = closest_scrapy_cfg()
if closest:
- projdir = os.path.dirname(closest)
+ projdir = str(Path(closest).parent)
if set_syspath and projdir not in sys.path:
sys.path.append(projdir)
@@ -101,13 +103,13 @@ def get_config(use_closest=True):
return cfg
-def get_sources(use_closest=True):
- xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config')
+def get_sources(use_closest=True) -> list[str]:
+ xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or Path('~/.config').expanduser()
sources = [
'/etc/scrapy.cfg',
r'c:\scrapy\scrapy.cfg',
- xdg_config_home + '/scrapy.cfg',
- os.path.expanduser('~/.scrapy.cfg'),
+ str(Path(xdg_config_home) / 'scrapy.cfg'),
+ str(Path('~/.scrapy.cfg').expanduser()),
]
if use_closest:
sources.append(closest_scrapy_cfg())
@@ -129,8 +131,8 @@ def feed_complete_default_values_from_settings(feed, settings):
return out
-def feed_process_params_from_cli(settings, output, output_format=None,
- overwrite_output=None):
+def feed_process_params_from_cli(settings, output: list[str], output_format=None,
+ overwrite_output: Optional[list[str]] = None):
"""
Receives feed export params (from the 'crawl' or 'runspider' commands),
checks for inconsistencies in their quantities and returns a dictionary
@@ -180,7 +182,7 @@ def feed_process_params_from_cli(settings, output, output_format=None,
feed_uri, feed_format = element.rsplit(':', 1)
except ValueError:
feed_uri = element
- feed_format = os.path.splitext(element)[1].replace('.', '')
+ feed_format = Path(element).suffix.replace('.', '')
else:
if feed_uri == '-':
feed_uri = 'stdout:'
diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py
index c92ef36f5..a65f92e95 100644
--- a/scrapy/utils/job.py
+++ b/scrapy/utils/job.py
@@ -1,4 +1,4 @@
-import os
+from pathlib import Path
from typing import Optional
from scrapy.settings import BaseSettings
@@ -6,6 +6,6 @@ from scrapy.settings import BaseSettings
def job_dir(settings: BaseSettings) -> Optional[str]:
path = settings['JOBDIR']
- if path and not os.path.exists(path):
- os.makedirs(path)
+ if path and not Path(path).exists():
+ Path(path).mkdir(parents=True)
return path
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index c66af497e..e54b71d45 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -2,7 +2,7 @@ import os
import warnings
from importlib import import_module
-from os.path import join, dirname, abspath, isabs, exists
+from pathlib import Path
from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
from scrapy.settings import Settings
@@ -25,36 +25,37 @@ def inside_project():
return bool(closest_scrapy_cfg())
-def project_data_dir(project='default'):
+def project_data_dir(project='default') -> str:
"""Return the current project data dir, creating it if it doesn't exist"""
if not inside_project():
raise NotConfigured("Not inside a project")
cfg = get_config()
if cfg.has_option(DATADIR_CFG_SECTION, project):
- d = cfg.get(DATADIR_CFG_SECTION, project)
+ d = Path(cfg.get(DATADIR_CFG_SECTION, project))
else:
scrapy_cfg = closest_scrapy_cfg()
if not scrapy_cfg:
raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir")
- d = abspath(join(dirname(scrapy_cfg), '.scrapy'))
- if not exists(d):
- os.makedirs(d)
- return d
+ d = (Path(scrapy_cfg).parent / '.scrapy').resolve()
+ if not d.exists():
+ d.mkdir(parents=True)
+ return str(d)
-def data_path(path, createdir=False):
+def data_path(path: str, createdir=False) -> str:
"""
Return the given path joined with the .scrapy data directory.
If given an absolute path, return it unmodified.
"""
- if not isabs(path):
+ path_obj = Path(path)
+ if not path_obj.is_absolute():
if inside_project():
- path = join(project_data_dir(), path)
+ path_obj = Path(project_data_dir(), path)
else:
- path = join('.scrapy', path)
- if createdir and not exists(path):
- os.makedirs(path)
- return path
+ path_obj = Path('.scrapy', path)
+ if createdir and not path_obj.exists():
+ path_obj.mkdir(parents=True)
+ return str(path_obj)
def get_project_settings():
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index cf33317ce..545d489be 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -264,7 +264,7 @@ class RequestFingerprinter:
f'and \'VERSION\'.'
)
- def fingerprint(self, request):
+ def fingerprint(self, request: Request):
return self._fingerprint(request)
diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index f068be737..8075902b3 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -1,23 +1,21 @@
"""Helper functions for working with templates"""
-import os
import re
import string
+from pathlib import Path
-def render_templatefile(path, **kwargs):
- with open(path, 'rb') as fp:
- raw = fp.read().decode('utf8')
+def render_templatefile(path: str, **kwargs):
+ raw = Path(path).read_text('utf8')
content = string.Template(raw).substitute(**kwargs)
render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path
if path.endswith('.tmpl'):
- os.rename(path, render_path)
+ Path(path).rename(render_path)
- with open(render_path, 'wb') as fp:
- fp.write(content.encode('utf8'))
+ Path(render_path).write_text(content, 'utf8')
CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 0b828f7c0..4d01f1ef1 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -4,6 +4,7 @@ This module contains some assorted functions used in tests
import asyncio
import os
+from pathlib import Path
from posixpath import split
from unittest import mock
@@ -71,11 +72,11 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
return runner.create_crawler(spidercls or Spider)
-def get_pythonpath():
+def get_pythonpath() -> str:
"""Return a PYTHONPATH suitable to use in processes so that they find this
installation of Scrapy"""
scrapy_path = import_module('scrapy').__path__[0]
- return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '')
+ return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get('PYTHONPATH', '')
def get_testenv():
diff --git a/setup.py b/setup.py
index a43cf08c8..e413ea6e4 100644
--- a/setup.py
+++ b/setup.py
@@ -1,10 +1,9 @@
-from os.path import dirname, join
+from pathlib import Path
from pkg_resources import parse_version
from setuptools import setup, find_packages, __version__ as setuptools_version
-with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f:
- version = f.read().decode('ascii').strip()
+version = (Path(__file__).parent / 'scrapy/VERSION').read_text('ascii').strip()
def has_environment_marker_platform_impl_support():
diff --git a/tests/__init__.py b/tests/__init__.py
index bb62851dc..be263fa16 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -6,6 +6,7 @@ see https://docs.scrapy.org/en/latest/contributing.html#running-tests
import os
import socket
+from pathlib import Path
# ignore system-wide proxies for tests
# which would send requests to a totally unsuspecting server
@@ -16,14 +17,12 @@ os.environ['ftp_proxy'] = ''
# Absolutize paths to coverage config and output file because tests that
# spawn subprocesses also changes current working directory.
-_sourceroot = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_sourceroot = Path(__file__).resolve().parent.parent
if 'COV_CORE_CONFIG' in os.environ:
- os.environ['COVERAGE_FILE'] = os.path.join(_sourceroot, '.coverage')
- os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot,
- os.environ['COV_CORE_CONFIG'])
+ os.environ['COVERAGE_FILE'] = str(_sourceroot / '.coverage')
+ os.environ['COV_CORE_CONFIG'] = str(_sourceroot / os.environ['COV_CORE_CONFIG'])
-tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
- 'sample_data')
+tests_datadir = str(Path(__file__).parent.resolve() / 'sample_data')
# In some environments accessing a non-existing host doesn't raise an
@@ -35,8 +34,6 @@ except socket.gaierror:
NON_EXISTING_RESOLVABLE = False
-def get_testdata(*paths):
+def get_testdata(*paths: str) -> bytes:
"""Return test data"""
- path = os.path.join(tests_datadir, *paths)
- with open(path, 'rb') as f:
- return f.read()
+ return Path(tests_datadir, *paths).read_bytes()
diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py
index bb4a8e5af..3a41b3a3e 100644
--- a/tests/keys/__init__.py
+++ b/tests/keys/__init__.py
@@ -1,5 +1,5 @@
-import os
from datetime import datetime, timedelta
+from pathlib import Path
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa
@@ -22,21 +22,20 @@ from cryptography.x509.oid import NameOID
# https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate
def generate_keys():
- folder = os.path.dirname(__file__)
+ folder = Path(__file__).parent
key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend(),
)
- with open(os.path.join(folder, 'localhost.key'), "wb") as f:
- f.write(
- key.private_bytes(
- encoding=Encoding.PEM,
- format=PrivateFormat.TraditionalOpenSSL,
- encryption_algorithm=NoEncryption(),
- )
- )
+ (folder / 'localhost.key').write_bytes(
+ key.private_bytes(
+ encoding=Encoding.PEM,
+ format=PrivateFormat.TraditionalOpenSSL,
+ encryption_algorithm=NoEncryption(),
+ ),
+ )
subject = issuer = Name(
[
@@ -59,5 +58,4 @@ def generate_keys():
)
.sign(key, SHA256(), default_backend())
)
- with open(os.path.join(folder, 'localhost.crt'), "wb") as f:
- f.write(cert.public_bytes(Encoding.PEM))
+ (folder / 'localhost.crt').write_bytes(cert.public_bytes(Encoding.PEM))
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 72d7e0241..7916798f7 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,6 +1,5 @@
import argparse
import json
-import os
import random
import sys
from pathlib import Path
@@ -253,7 +252,7 @@ class Root(resource.Resource):
self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
try:
from tests import tests_datadir
- self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/')))
+ self.putChild(b"files", File(str(Path(tests_datadir, 'test_site/files/'))))
except Exception:
pass
self.putChild(b"redirect-to", RedirectTo())
@@ -346,8 +345,8 @@ class MockFTPServer:
def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
factory = ssl.DefaultOpenSSLContextFactory(
- os.path.join(os.path.dirname(__file__), keyfile),
- os.path.join(os.path.dirname(__file__), certfile),
+ str(Path(__file__).parent / keyfile),
+ str(Path(__file__).parent / certfile),
)
if cipher_string:
ctx = factory.getContext()
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 8233e0101..da73a4c45 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,11 +1,11 @@
import json
-import os
import pstats
import shutil
import sys
import tempfile
import unittest
from io import StringIO
+from pathlib import Path
from subprocess import Popen, PIPE
from scrapy.utils.test import get_testenv
@@ -36,17 +36,17 @@ class CmdlineTest(unittest.TestCase):
self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override')
def test_profiling(self):
- path = tempfile.mkdtemp()
- filename = os.path.join(path, 'res.prof')
+ path = Path(tempfile.mkdtemp())
+ filename = path / 'res.prof'
try:
- self._execute('version', '--profile', filename)
- self.assertTrue(os.path.exists(filename))
+ self._execute('version', '--profile', str(filename))
+ self.assertTrue(filename.exists())
out = StringIO()
- stats = pstats.Stats(filename, stream=out)
+ stats = pstats.Stats(str(filename), stream=out)
stats.print_stats()
out.seek(0)
stats = out.read()
- self.assertIn(os.path.join('scrapy', 'commands', 'version.py'),
+ self.assertIn(str(Path('scrapy', 'commands', 'version.py')),
stats)
self.assertIn('tottime', stats)
finally:
diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py
index d341888d3..fcafcef68 100644
--- a/tests/test_cmdline_crawl_with_pipeline/__init__.py
+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py
@@ -1,6 +1,6 @@
-import os
import sys
import unittest
+from pathlib import Path
from subprocess import Popen, PIPE
@@ -8,7 +8,7 @@ class CmdlineCrawlPipelineTest(unittest.TestCase):
def _execute(self, spname):
args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname)
- cwd = os.path.dirname(os.path.abspath(__file__))
+ cwd = Path(__file__).resolve().parent
proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
proc.communicate()
return proc.returncode
diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index c3d705194..4077a9bce 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -1,5 +1,3 @@
-from os.path import join, abspath
-
from tests.test_commands import CommandTest
@@ -10,11 +8,10 @@ class CheckCommandTest(CommandTest):
def setUp(self):
super(CheckCommandTest, self).setUp()
self.spider_name = 'check_spider'
- self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py'))
+ self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve()
def _write_contract(self, contracts, parse_def):
- with open(self.spider, 'w') as file:
- file.write(f"""
+ self.spider.write_text(f"""
import scrapy
class CheckSpider(scrapy.Spider):
@@ -27,7 +24,7 @@ class CheckSpider(scrapy.Spider):
{contracts}
\"\"\"
{parse_def}
- """)
+ """)
def _test_contract(self, contracts='', parse_def='pass'):
self._write_contract(contracts, parse_def)
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 0d992be56..154287d74 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,6 +1,6 @@
import os
import argparse
-from os.path import join, abspath, isfile, exists
+from pathlib import Path
from twisted.internet import defer
from scrapy.commands import parse
@@ -23,9 +23,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
def setUp(self):
super().setUp()
self.spider_name = 'parse_spider'
- fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
- with open(fname, 'w') as f:
- f.write(f"""
+ (self.proj_mod_path / 'spiders' / 'myspider.py').write_text(f"""
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
@@ -88,9 +86,7 @@ class MyBadCrawlSpider(CrawlSpider):
return [scrapy.Item(), dict(foo='bar')]
""")
- fname = abspath(join(self.proj_mod_path, 'pipelines.py'))
- with open(fname, 'w') as f:
- f.write("""
+ (self.proj_mod_path / 'pipelines.py').write_text("""
import logging
class MyPipeline:
@@ -101,8 +97,7 @@ class MyPipeline:
return item
""")
- fname = abspath(join(self.proj_mod_path, 'settings.py'))
- with open(fname, 'a') as f:
+ with (self.proj_mod_path / 'settings.py').open("a") as f:
f.write(f"""
ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
""")
@@ -234,7 +229,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
correct format containing correct data in it.
"""
file_name = 'data.json'
- file_path = join(self.proj_path, file_name)
+ file_path = Path(self.proj_path, file_name)
yield self.execute([
'--spider', self.spider_name,
'-c', 'parse',
@@ -242,12 +237,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
self.url('/html')
])
- self.assertTrue(exists(file_path))
- self.assertTrue(isfile(file_path))
+ self.assertTrue(file_path.exists())
+ self.assertTrue(file_path.is_file())
content = '[\n{},\n{"foo": "bar"}\n]'
- with open(file_path, 'r') as f:
- self.assertEqual(f.read(), content)
+ self.assertEqual(file_path.read_text(), content)
def test_parse_add_options(self):
command = parse.Command()
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 33189e9be..33c98ad69 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -1,4 +1,4 @@
-from os.path import join
+from pathlib import Path
from twisted.trial import unittest
from twisted.internet import defer
@@ -96,8 +96,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_local_file(self):
- filepath = join(tests_datadir, 'test_site', 'index.html')
- _, out, _ = yield self.execute([filepath, '-c', 'item'])
+ filepath = Path(tests_datadir, 'test_site', 'index.html')
+ _, out, _ = yield self.execute([str(filepath), '-c', 'item'])
assert b'{}' in out
@defer.inlineCallbacks
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 76d5f3935..39f718cce 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -9,12 +9,12 @@ import sys
import tempfile
from contextlib import contextmanager
from itertools import chain
-from os.path import exists, join, abspath, getmtime
from pathlib import Path
from shutil import rmtree, copytree
from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
from tempfile import mkdtemp
from threading import Timer
+from typing import Generator, Optional
from unittest import skipIf
from pytest import mark
@@ -66,8 +66,8 @@ class ProjectTest(unittest.TestCase):
def setUp(self):
self.temp_path = mkdtemp()
self.cwd = self.temp_path
- self.proj_path = join(self.temp_path, self.project_name)
- self.proj_mod_path = join(self.proj_path, self.project_name)
+ self.proj_path = Path(self.temp_path, self.project_name)
+ self.proj_mod_path = self.proj_path / self.project_name
self.env = get_testenv()
def tearDown(self):
@@ -104,10 +104,10 @@ class ProjectTest(unittest.TestCase):
return p, to_unicode(stdout), to_unicode(stderr)
- def find_in_file(self, filename, regex):
+ def find_in_file(self, filename: str | os.PathLike[str], regex) -> Optional[re.Match]:
"""Find first pattern occurrence in file"""
pattern = re.compile(regex)
- with open(filename, "r") as f:
+ with Path(filename).open("r") as f:
for line in f:
match = pattern.search(line)
if match is not None:
@@ -122,13 +122,13 @@ class StartprojectTest(ProjectTest):
print(err, file=sys.stderr)
self.assertEqual(p.returncode, 0)
- assert exists(join(self.proj_path, 'scrapy.cfg'))
- assert exists(join(self.proj_path, 'testproject'))
- assert exists(join(self.proj_mod_path, '__init__.py'))
- assert exists(join(self.proj_mod_path, 'items.py'))
- assert exists(join(self.proj_mod_path, 'pipelines.py'))
- assert exists(join(self.proj_mod_path, 'settings.py'))
- assert exists(join(self.proj_mod_path, 'spiders', '__init__.py'))
+ assert Path(self.proj_path, 'scrapy.cfg').exists()
+ assert Path(self.proj_path, 'testproject').exists()
+ assert Path(self.proj_mod_path, '__init__.py').exists()
+ assert Path(self.proj_mod_path, 'items.py').exists()
+ assert Path(self.proj_mod_path, 'pipelines.py').exists()
+ assert Path(self.proj_mod_path, 'settings.py').exists()
+ assert Path(self.proj_mod_path, 'spiders', '__init__.py').exists()
self.assertEqual(1, self.call('startproject', self.project_name))
self.assertEqual(1, self.call('startproject', 'wrong---project---name'))
@@ -138,13 +138,13 @@ class StartprojectTest(ProjectTest):
project_dir = mkdtemp()
self.assertEqual(0, self.call('startproject', self.project_name, project_dir))
- assert exists(join(abspath(project_dir), 'scrapy.cfg'))
- assert exists(join(abspath(project_dir), 'testproject'))
- assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py'))
- assert exists(join(join(abspath(project_dir), self.project_name), 'items.py'))
- assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py'))
- assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py'))
- assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py'))
+ assert Path(project_dir, 'scrapy.cfg').exists()
+ assert Path(project_dir, 'testproject').exists()
+ assert Path(project_dir, self.project_name, '__init__.py').exists()
+ assert Path(project_dir, self.project_name, 'items.py').exists()
+ assert Path(project_dir, self.project_name, 'pipelines.py').exists()
+ assert Path(project_dir, self.project_name, 'settings.py').exists()
+ assert Path(project_dir, self.project_name, 'spiders', '__init__.py').exists()
self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2'))
@@ -158,40 +158,42 @@ class StartprojectTest(ProjectTest):
def test_existing_project_dir(self):
project_dir = mkdtemp()
project_name = self.project_name + '_existing'
- project_path = os.path.join(project_dir, project_name)
- os.mkdir(project_path)
+ project_path = Path(project_dir, project_name)
+ project_path.mkdir()
p, out, err = self.proc('startproject', project_name, cwd=project_dir)
print(out)
print(err, file=sys.stderr)
self.assertEqual(p.returncode, 0)
- assert exists(join(abspath(project_path), 'scrapy.cfg'))
- assert exists(join(abspath(project_path), project_name))
- assert exists(join(join(abspath(project_path), project_name), '__init__.py'))
- assert exists(join(join(abspath(project_path), project_name), 'items.py'))
- assert exists(join(join(abspath(project_path), project_name), 'pipelines.py'))
- assert exists(join(join(abspath(project_path), project_name), 'settings.py'))
- assert exists(join(join(abspath(project_path), project_name), 'spiders', '__init__.py'))
+ assert Path(project_path, 'scrapy.cfg').exists()
+ assert Path(project_path, project_name).exists()
+ assert Path(project_path, project_name, '__init__.py').exists()
+ assert Path(project_path, project_name, 'items.py').exists()
+ assert Path(project_path, project_name, 'pipelines.py').exists()
+ assert Path(project_path, project_name, 'settings.py').exists()
+ assert Path(project_path, project_name, 'spiders', '__init__.py').exists()
-def get_permissions_dict(path, renamings=None, ignore=None):
+def get_permissions_dict(path: str | os.PathLike[str], renamings=None, ignore=None) -> dict[str, str]:
- def get_permissions(path):
- return oct(os.stat(path).st_mode)
+ def get_permissions(path: Path) -> str:
+ return oct(path.stat().st_mode)
+
+ path_obj = Path(path)
renamings = renamings or tuple()
permissions_dict = {
- '.': get_permissions(path),
+ '.': get_permissions(path_obj),
}
- for root, dirs, files in os.walk(path):
+ for root, dirs, files in os.walk(path_obj):
nodes = list(chain(dirs, files))
if ignore:
ignored_names = ignore(root, nodes)
nodes = [node for node in nodes if node not in ignored_names]
for node in nodes:
- absolute_path = os.path.join(root, node)
- relative_path = os.path.relpath(absolute_path, path)
+ absolute_path = Path(root, node)
+ relative_path = str(absolute_path.relative_to(path))
for search_string, replacement in renamings:
relative_path = relative_path.replace(
search_string,
@@ -208,28 +210,27 @@ class StartprojectTemplatesTest(ProjectTest):
def setUp(self):
super().setUp()
- self.tmpl = join(self.temp_path, 'templates')
- self.tmpl_proj = join(self.tmpl, 'project')
+ self.tmpl = str(Path(self.temp_path, 'templates'))
+ self.tmpl_proj = str(Path(self.tmpl, 'project'))
def test_startproject_template_override(self):
- copytree(join(scrapy.__path__[0], 'templates'), self.tmpl)
- with open(join(self.tmpl_proj, 'root_template'), 'w'):
- pass
- assert exists(join(self.tmpl_proj, 'root_template'))
+ copytree(Path(scrapy.__path__[0], 'templates'), self.tmpl)
+ Path(self.tmpl_proj, 'root_template').write_bytes(b"")
+ assert Path(self.tmpl_proj, 'root_template').exists()
args = ['--set', f'TEMPLATES_DIR={self.tmpl}']
p, out, err = self.proc('startproject', self.project_name, *args)
self.assertIn(f"New Scrapy project '{self.project_name}', "
"using template directory", out)
self.assertIn(self.tmpl_proj, out)
- assert exists(join(self.proj_path, 'root_template'))
+ assert Path(self.proj_path, 'root_template').exists()
def test_startproject_permissions_from_writable(self):
"""Check that generated files have the right permissions when the
template folder has the same permissions as in the project, i.e.
everything is writable."""
scrapy_path = scrapy.__path__[0]
- project_template = os.path.join(scrapy_path, 'templates', 'project')
+ project_template = Path(scrapy_path, 'templates', 'project')
project_name = 'startproject1'
renamings = (
('module', project_name),
@@ -255,7 +256,7 @@ class StartprojectTemplatesTest(ProjectTest):
)
process.wait()
- project_dir = os.path.join(destination, project_name)
+ project_dir = Path(destination, project_name)
actual_permissions = get_permissions_dict(project_dir)
self.assertEqual(actual_permissions, expected_permissions)
@@ -268,8 +269,8 @@ class StartprojectTemplatesTest(ProjectTest):
See https://github.com/scrapy/scrapy/pull/4604
"""
scrapy_path = scrapy.__path__[0]
- templates_dir = os.path.join(scrapy_path, 'templates')
- project_template = os.path.join(templates_dir, 'project')
+ templates_dir = Path(scrapy_path, 'templates')
+ project_template = Path(templates_dir, 'project')
project_name = 'startproject2'
renamings = (
('module', project_name),
@@ -281,16 +282,16 @@ class StartprojectTemplatesTest(ProjectTest):
IGNORE,
)
- def _make_read_only(path):
- current_permissions = os.stat(path).st_mode
- os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION)
+ def _make_read_only(path: Path):
+ current_permissions = path.stat().st_mode
+ path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION)
read_only_templates_dir = str(Path(mkdtemp()) / 'templates')
copytree(templates_dir, read_only_templates_dir)
for root, dirs, files in os.walk(read_only_templates_dir):
for node in chain(dirs, files):
- _make_read_only(os.path.join(root, node))
+ _make_read_only(Path(root, node))
destination = mkdtemp()
process = subprocess.Popen(
@@ -308,7 +309,7 @@ class StartprojectTemplatesTest(ProjectTest):
)
process.wait()
- project_dir = os.path.join(destination, project_name)
+ project_dir = Path(destination, project_name)
actual_permissions = get_permissions_dict(project_dir)
self.assertEqual(actual_permissions, expected_permissions)
@@ -317,7 +318,7 @@ class StartprojectTemplatesTest(ProjectTest):
"""Check that pre-existing folders and files in the destination folder
do not see their permissions modified."""
scrapy_path = scrapy.__path__[0]
- project_template = os.path.join(scrapy_path, 'templates', 'project')
+ project_template = Path(scrapy_path, 'templates', 'project')
project_name = 'startproject3'
renamings = (
('module', project_name),
@@ -330,7 +331,7 @@ class StartprojectTemplatesTest(ProjectTest):
)
destination = mkdtemp()
- project_dir = os.path.join(destination, project_name)
+ project_dir = Path(destination, project_name)
existing_nodes = {
oct(permissions)[2:] + extension: permissions
@@ -339,10 +340,9 @@ class StartprojectTemplatesTest(ProjectTest):
0o444, 0o555, 0o644, 0o666, 0o755, 0o777,
)
}
- os.mkdir(project_dir)
- project_dir_path = Path(project_dir)
+ project_dir.mkdir()
for node, permissions in existing_nodes.items():
- path = project_dir_path / node
+ path = project_dir / node
if node.endswith('.d'):
path.mkdir(mode=permissions)
else:
@@ -378,7 +378,7 @@ class StartprojectTemplatesTest(ProjectTest):
os.umask(cur_mask)
scrapy_path = scrapy.__path__[0]
- project_template = os.path.join(
+ project_template = Path(
scrapy_path,
'templates',
'project'
@@ -409,7 +409,7 @@ class StartprojectTemplatesTest(ProjectTest):
)
process.wait()
- project_dir = os.path.join(destination, project_name)
+ project_dir = Path(destination, project_name)
actual_permissions = get_permissions_dict(project_dir)
self.assertEqual(actual_permissions, expected_permissions)
@@ -420,7 +420,7 @@ class CommandTest(ProjectTest):
def setUp(self):
super().setUp()
self.call('startproject', self.project_name)
- self.cwd = join(self.temp_path, self.project_name)
+ self.cwd = Path(self.temp_path, self.project_name)
self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings'
@@ -429,10 +429,10 @@ class GenspiderCommandTest(CommandTest):
def test_arguments(self):
# only pass one argument. spider script shouldn't be created
self.assertEqual(2, self.call('genspider', 'test_name'))
- assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py'))
+ assert not Path(self.proj_mod_path, 'spiders', 'test_name.py').exists()
# pass two arguments <name> <domain>. spider script should be created
self.assertEqual(0, self.call('genspider', 'test_name', 'test.com'))
- assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py'))
+ assert Path(self.proj_mod_path, 'spiders', 'test_name.py').exists()
def test_template(self, tplname='crawl'):
args = [f'--template={tplname}'] if tplname else []
@@ -440,11 +440,11 @@ class GenspiderCommandTest(CommandTest):
spmodule = f"{self.project_name}.spiders.{spname}"
p, out, err = self.proc('genspider', spname, 'test.com', *args)
self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out)
- self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
- modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
+ self.assertTrue(Path(self.proj_mod_path, 'spiders', 'test_spider.py').exists())
+ modify_time_before = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime
p, out, err = self.proc('genspider', spname, 'test.com', *args)
self.assertIn(f"Spider {spname!r} already exists in module", out)
- modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
+ modify_time_after = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime
self.assertEqual(modify_time_after, modify_time_before)
def test_template_basic(self):
@@ -465,37 +465,37 @@ class GenspiderCommandTest(CommandTest):
def test_same_name_as_project(self):
self.assertEqual(2, self.call('genspider', self.project_name))
- assert not exists(join(self.proj_mod_path, 'spiders', f'{self.project_name}.py'))
+ assert not Path(self.proj_mod_path, 'spiders', f'{self.project_name}.py').exists()
def test_same_filename_as_existing_spider(self, force=False):
file_name = 'example'
- file_path = join(self.proj_mod_path, 'spiders', f'{file_name}.py')
+ file_path = Path(self.proj_mod_path, 'spiders', f'{file_name}.py')
self.assertEqual(0, self.call('genspider', file_name, 'example.com'))
- assert exists(file_path)
+ assert file_path.exists()
# change name of spider but not its file name
- with open(file_path, 'r+') as spider_file:
+ with file_path.open('r+') as spider_file:
file_data = spider_file.read()
file_data = file_data.replace("name = \'example\'", "name = \'renamed\'")
spider_file.seek(0)
spider_file.write(file_data)
spider_file.truncate()
- modify_time_before = getmtime(file_path)
+ modify_time_before = file_path.stat().st_mtime
file_contents_before = file_data
if force:
p, out, err = self.proc('genspider', '--force', file_name, 'example.com')
self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out)
- modify_time_after = getmtime(file_path)
+ modify_time_after = file_path.stat().st_mtime
self.assertNotEqual(modify_time_after, modify_time_before)
- file_contents_after = open(file_path, 'r').read()
+ file_contents_after = file_path.read_text()
self.assertNotEqual(file_contents_after, file_contents_before)
else:
p, out, err = self.proc('genspider', file_name, 'example.com')
self.assertIn(f"{file_path} already exists", out)
- modify_time_after = getmtime(file_path)
+ modify_time_after = file_path.stat().st_mtime
self.assertEqual(modify_time_after, modify_time_before)
- file_contents_after = open(file_path, 'r').read()
+ file_contents_after = file_path.read_text()
self.assertEqual(file_contents_after, file_contents_before)
def test_same_filename_as_existing_spider_force(self):
@@ -504,11 +504,11 @@ class GenspiderCommandTest(CommandTest):
def test_url(self, url='test.com', domain="test.com"):
self.assertEqual(0, self.call('genspider', '--force', 'test_name', url))
self.assertEqual(domain,
- self.find_in_file(join(self.proj_mod_path,
+ self.find_in_file(Path(self.proj_mod_path,
'spiders', 'test_name.py'),
r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1))
self.assertEqual(f'http://{domain}/',
- self.find_in_file(join(self.proj_mod_path,
+ self.find_in_file(Path(self.proj_mod_path,
'spiders', 'test_name.py'),
r'start_urls\s*=\s*\[\'(.+)\'\]').group(1))
@@ -526,31 +526,31 @@ class GenspiderStandaloneCommandTest(ProjectTest):
def test_generate_standalone_spider(self):
self.call('genspider', 'example', 'example.com')
- assert exists(join(self.temp_path, 'example.py'))
+ assert Path(self.temp_path, 'example.py').exists()
def test_same_name_as_existing_file(self, force=False):
file_name = 'example'
- file_path = join(self.temp_path, file_name + '.py')
+ file_path = Path(self.temp_path, file_name + '.py')
p, out, err = self.proc('genspider', file_name, 'example.com')
self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out)
- assert exists(file_path)
- modify_time_before = getmtime(file_path)
- file_contents_before = open(file_path, 'r').read()
+ assert file_path.exists()
+ modify_time_before = file_path.stat().st_mtime
+ file_contents_before = file_path.read_text()
if force:
# use different template to ensure contents were changed
p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com')
self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out)
- modify_time_after = getmtime(file_path)
+ modify_time_after = file_path.stat().st_mtime
self.assertNotEqual(modify_time_after, modify_time_before)
- file_contents_after = open(file_path, 'r').read()
+ file_contents_after = file_path.read_text()
self.assertNotEqual(file_contents_after, file_contents_before)
else:
p, out, err = self.proc('genspider', file_name, 'example.com')
- self.assertIn(f"{join(self.temp_path, file_name + '.py')} already exists", out)
- modify_time_after = getmtime(file_path)
+ self.assertIn(f"{Path(self.temp_path, file_name + '.py')} already exists", out)
+ modify_time_after = file_path.stat().st_mtime
self.assertEqual(modify_time_after, modify_time_before)
- file_contents_after = open(file_path, 'r').read()
+ file_contents_after = file_path.read_text()
self.assertEqual(file_contents_after, file_contents_before)
def test_same_name_as_existing_file_force(self):
@@ -588,17 +588,16 @@ class BadSpider(scrapy.Spider):
"""
@contextmanager
- def _create_file(self, content, name=None):
- tmpdir = self.mktemp()
- os.mkdir(tmpdir)
+ def _create_file(self, content, name=None) -> Generator[str, None, None]:
+ tmpdir = Path(self.mktemp())
+ tmpdir.mkdir()
if name:
- fname = abspath(join(tmpdir, name))
+ fname = (tmpdir / name).resolve()
else:
- fname = abspath(join(tmpdir, self.spider_filename))
- with open(fname, 'w') as f:
- f.write(content)
+ fname = (tmpdir / self.spider_filename).resolve()
+ fname.write_text(content)
try:
- yield fname
+ yield str(fname)
finally:
rmtree(tmpdir)
@@ -747,12 +746,11 @@ class MySpider(scrapy.Spider):
)
return []
"""
- with open(os.path.join(self.cwd, "example.json"), "w") as f1:
- f1.write("not empty")
+ Path(self.cwd, "example.json").write_text("not empty")
args = ['-O', 'example.json']
log = self.get_log(spider_code, args=args)
self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
- with open(os.path.join(self.cwd, "example.json")) as f2:
+ with Path(self.cwd, "example.json").open() as f2:
first_line = f2.readline()
self.assertNotEqual(first_line, "not empty")
@@ -854,9 +852,7 @@ class ViewCommandTest(CommandTest):
class CrawlCommandTest(CommandTest):
def crawl(self, code, args=()):
- fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
- with open(fname, 'w') as f:
- f.write(code)
+ Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code)
return self.proc('crawl', 'myspider', *args)
def get_log(self, code, args=()):
@@ -908,12 +904,11 @@ class MySpider(scrapy.Spider):
)
return []
"""
- with open(os.path.join(self.cwd, "example.json"), "w") as f1:
- f1.write("not empty")
+ Path(self.cwd, "example.json").write_text("not empty")
args = ['-O', 'example.json']
log = self.get_log(spider_code, args=args)
self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
- with open(os.path.join(self.cwd, "example.json")) as f2:
+ with Path(self.cwd, "example.json").open() as f2:
first_line = f2.readline()
self.assertNotEqual(first_line, "not empty")
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index cf15ba9b9..19f4229a3 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,9 +1,9 @@
import logging
-import os
import platform
import subprocess
import sys
import warnings
+from pathlib import Path
from pytest import raises, mark
from twisted import version as twisted_version
@@ -100,15 +100,14 @@ class CrawlerLoggingTestCase(unittest.TestCase):
assert get_scrapy_root_handler() is None
def test_spider_custom_settings_log_level(self):
- log_file = self.mktemp()
- with open(log_file, 'wb') as fo:
- fo.write('previous message\n'.encode('utf-8'))
+ log_file = Path(self.mktemp())
+ log_file.write_text('previous message\n', encoding='utf-8')
class MySpider(scrapy.Spider):
name = 'spider'
custom_settings = {
'LOG_LEVEL': 'INFO',
- 'LOG_FILE': log_file,
+ 'LOG_FILE': str(log_file),
# settings to avoid extra warnings
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
@@ -124,8 +123,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
logging.warning('warning message')
logging.error('error message')
- with open(log_file, 'rb') as fo:
- logged = fo.read().decode('utf-8')
+ logged = log_file.read_text(encoding='utf-8')
self.assertIn('previous message', logged)
self.assertNotIn('debug message', logged)
@@ -139,14 +137,13 @@ class CrawlerLoggingTestCase(unittest.TestCase):
self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)
def test_spider_custom_settings_log_append(self):
- log_file = self.mktemp()
- with open(log_file, 'wb') as fo:
- fo.write('previous message\n'.encode('utf-8'))
+ log_file = Path(self.mktemp())
+ log_file.write_text('previous message\n', encoding='utf-8')
class MySpider(scrapy.Spider):
name = 'spider'
custom_settings = {
- 'LOG_FILE': log_file,
+ 'LOG_FILE': str(log_file),
'LOG_FILE_APPEND': False,
# disable telnet if not available to avoid an extra warning
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
@@ -156,8 +153,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
get_crawler(MySpider)
logging.debug('debug message')
- with open(log_file, 'rb') as fo:
- logged = fo.read().decode('utf-8')
+ logged = log_file.read_text(encoding='utf-8')
self.assertNotIn('previous message', logged)
self.assertIn('debug message', logged)
@@ -296,9 +292,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
class ScriptRunnerMixin:
- def run_script(self, script_name, *script_args):
- script_path = os.path.join(self.script_dir, script_name)
- args = [sys.executable, script_path] + list(script_args)
+ def run_script(self, script_name: str, *script_args):
+ script_path = self.script_dir / script_name
+ args = [sys.executable, str(script_path)] + list(script_args)
p = subprocess.Popen(args, env=get_testenv(),
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
@@ -306,7 +302,7 @@ class ScriptRunnerMixin:
class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
- script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
+ script_dir = Path(__file__).parent.resolve() / 'CrawlerProcess'
def test_simple(self):
log = self.run_script('simple.py')
@@ -463,7 +459,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
- script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')
+ script_dir = Path(__file__).parent.resolve() / 'CrawlerRunner'
def test_response_ip_address(self):
log = self.run_script("ip_address.py")
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index 5e63ebffb..2558e4f91 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -2,6 +2,7 @@ import os
import re
from configparser import ConfigParser
from importlib import import_module
+from pathlib import Path
from twisted import version as twisted_version
from twisted.trial import unittest
@@ -29,11 +30,7 @@ class ScrapyUtilsTest(unittest.TestCase):
if not os.environ.get('_SCRAPY_PINNED', None):
self.skipTest('Not in a pinned environment')
- tox_config_file_path = os.path.join(
- os.path.dirname(__file__),
- '..',
- 'tox.ini',
- )
+ tox_config_file_path = Path(__file__) / '..' / 'tox.ini'
config_parser = ConfigParser()
config_parser.read(tox_config_file_path)
pattern = r'Twisted\[http2\]==([\d.]+)'
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 883960084..29ff8c2dc 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -3,6 +3,7 @@ import os
import shutil
import sys
import tempfile
+from pathlib import Path
from typing import Optional, Type
from unittest import mock, SkipTest
@@ -10,7 +11,6 @@ from testfixtures import LogCapture
from twisted.cred import checkers, credentials, portal
from twisted.internet import defer, error, reactor
from twisted.protocols.policies import WrappingFactory
-from twisted.python.filepath import FilePath
from twisted.trial import unittest
from twisted.web import resource, server, static, util
from twisted.web._newclient import ResponseFailed
@@ -108,14 +108,13 @@ class LoadTestCase(unittest.TestCase):
class FileTestCase(unittest.TestCase):
def setUp(self):
- self.tmpname = self.mktemp()
- with open(self.tmpname + '^', 'w') as f:
- f.write('0123456789')
+ self.tmpname = Path(self.mktemp() + '^')
+ Path(self.tmpname).write_text('0123456789')
handler = create_instance(FileDownloadHandler, None, get_crawler())
self.download_request = handler.download_request
def tearDown(self):
- os.unlink(self.tmpname + '^')
+ self.tmpname.unlink()
def test_download(self):
def _test(response):
@@ -124,7 +123,7 @@ class FileTestCase(unittest.TestCase):
self.assertEqual(response.body, b'0123456789')
self.assertEqual(response.protocol, None)
- request = Request(path_to_file_uri(self.tmpname + '^'))
+ request = Request(path_to_file_uri(str(self.tmpname)))
assert request.url.upper().endswith('%5E')
return self.download_request(request, Spider('foo')).addCallback(_test)
@@ -223,10 +222,10 @@ class HttpTestCase(unittest.TestCase):
certfile = 'keys/localhost.crt'
def setUp(self):
- self.tmpname = self.mktemp()
- os.mkdir(self.tmpname)
- FilePath(self.tmpname).child("file").setContent(b"0123456789")
- r = static.File(self.tmpname)
+ self.tmpname = Path(self.mktemp())
+ self.tmpname.mkdir()
+ (self.tmpname / "file").write_bytes(b"0123456789")
+ r = static.File(str(self.tmpname))
r.putChild(b"redirect", util.Redirect(b"/file"))
r.putChild(b"wait", ForeverTakingResource())
r.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
@@ -626,10 +625,10 @@ class Https11CustomCiphers(unittest.TestCase):
certfile = 'keys/localhost.crt'
def setUp(self):
- self.tmpname = self.mktemp()
- os.mkdir(self.tmpname)
- FilePath(self.tmpname).child("file").setContent(b"0123456789")
- r = static.File(self.tmpname)
+ self.tmpname = Path(self.mktemp())
+ self.tmpname.mkdir()
+ (self.tmpname / "file").write_bytes(b"0123456789")
+ r = static.File(str(self.tmpname))
self.site = server.Site(r, timeout=None)
self.host = 'localhost'
self.port = reactor.listenSSL(
@@ -1002,16 +1001,15 @@ class BaseFTPTestCase(unittest.TestCase):
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
# setup dirs and test file
- self.directory = self.mktemp()
- os.mkdir(self.directory)
- userdir = os.path.join(self.directory, self.username)
- os.mkdir(userdir)
- fp = FilePath(userdir)
+ self.directory = Path(self.mktemp())
+ self.directory.mkdir()
+ userdir = self.directory / self.username
+ userdir.mkdir()
for filename, content in self.test_files:
- fp.child(filename).setContent(content)
+ (userdir / filename).write_bytes(content)
# setup server
- realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory)
+ realm = FTPRealm(anonymousRoot=str(self.directory), userHome=str(self.directory))
p = portal.Portal(realm)
users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
users_checker.addUser(self.username, self.password)
@@ -1076,28 +1074,28 @@ class BaseFTPTestCase(unittest.TestCase):
def test_ftp_local_filename(self):
f, local_fname = tempfile.mkstemp()
- local_fname = to_bytes(local_fname)
+ fname_bytes = to_bytes(local_fname)
+ local_fname = Path(local_fname)
os.close(f)
- meta = {"ftp_local_filename": local_fname}
+ meta = {"ftp_local_filename": fname_bytes}
meta.update(self.req_meta)
request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt",
meta=meta)
d = self.download_handler.download_request(request, None)
def _test(r):
- self.assertEqual(r.body, local_fname)
- self.assertEqual(r.headers, {b'Local Filename': [local_fname],
+ self.assertEqual(r.body, fname_bytes)
+ self.assertEqual(r.headers, {b'Local Filename': [fname_bytes],
b'Size': [b'17']})
- self.assertTrue(os.path.exists(local_fname))
- with open(local_fname, "rb") as f:
- self.assertEqual(f.read(), b"I have the power!")
- os.remove(local_fname)
+ self.assertTrue(local_fname.exists())
+ self.assertEqual(local_fname.read_bytes(), b"I have the power!")
+ local_fname.unlink()
return self._add_test_callbacks(d, _test)
def _test_response_class(self, filename, response_class):
f, local_fname = tempfile.mkstemp()
- local_fname = to_bytes(local_fname)
+ local_fname = Path(local_fname)
os.close(f)
meta = {}
meta.update(self.req_meta)
@@ -1107,7 +1105,7 @@ class BaseFTPTestCase(unittest.TestCase):
def _test(r):
self.assertEqual(type(r), response_class)
- os.remove(local_fname)
+ local_fname.unlink()
return self._add_test_callbacks(d, _test)
def test_response_class_from_url(self):
@@ -1147,15 +1145,14 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
# setup dir and test file
- self.directory = self.mktemp()
- os.mkdir(self.directory)
+ self.directory = Path(self.mktemp())
+ self.directory.mkdir()
- fp = FilePath(self.directory)
for filename, content in self.test_files:
- fp.child(filename).setContent(content)
+ (self.directory / filename).write_bytes(content)
# setup server for anonymous access
- realm = FTPRealm(anonymousRoot=self.directory)
+ realm = FTPRealm(anonymousRoot=str(self.directory))
p = portal.Portal(realm)
p.registerChecker(checkers.AllowAnonymousAccess(),
credentials.IAnonymous)
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 40e9f3a96..6f4e217e6 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -1,6 +1,6 @@
from gzip import GzipFile
from io import BytesIO
-from os.path import join
+from pathlib import Path
from unittest import TestCase, SkipTest
from warnings import catch_warnings
@@ -15,7 +15,7 @@ from tests import tests_datadir
from w3lib.encoding import resolve_encoding
-SAMPLEDIR = join(tests_datadir, 'compressed')
+SAMPLEDIR = Path(tests_datadir, 'compressed')
FORMAT = {
'gzip': ('html-gzip.bin', 'gzip'),
@@ -46,8 +46,7 @@ class HttpCompressionTest(TestCase):
samplefile, contentencoding = FORMAT[coding]
- with open(join(SAMPLEDIR, samplefile), 'rb') as sample:
- body = sample.read()
+ body = (SAMPLEDIR / samplefile).read_bytes()
headers = {
'Server': 'Yaws/1.49 Yet Another Web Server',
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 8a37a8ebe..911d23069 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -2,8 +2,8 @@ import hashlib
import tempfile
import unittest
import shutil
-import os
import sys
+from pathlib import Path
from testfixtures import LogCapture
from scrapy.dupefilters import RFPDupeFilter
@@ -157,7 +157,7 @@ class RFPDupeFilterTest(unittest.TestCase):
df.request_seen(r1)
df.close('finished')
- with open(os.path.join(path, 'requests.seen'), 'rb') as seen_file:
+ with Path(path, 'requests.seen').open('rb') as seen_file:
line = next(seen_file).decode()
assert not line.endswith('\r\r\n')
if sys.platform == 'win32':
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 5677052f6..aa3313659 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -10,11 +10,11 @@ module with the ``runserver`` argument::
python test_engine.py runserver
"""
-import os
import re
import subprocess
import sys
from collections import defaultdict
+from pathlib import Path
from threading import Timer
from urllib.parse import urlparse
from dataclasses import dataclass
@@ -127,8 +127,8 @@ class ChangeCloseReasonSpider(TestSpider):
def start_test_site(debug=False):
- root_dir = os.path.join(tests_datadir, "test_site")
- r = static.File(root_dir)
+ root_dir = Path(tests_datadir, "test_site")
+ r = static.File(str(root_dir))
r.putChild(b"redirect", util.Redirect(b"/redirected"))
r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
numbers = [str(x).encode("utf8") for x in range(2**18)]
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index ad2383018..98905d2c0 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -3,7 +3,6 @@ import csv
import gzip
import json
import lzma
-import os
import random
import shutil
import string
@@ -15,6 +14,7 @@ from collections import defaultdict
from contextlib import ExitStack
from io import BytesIO
from logging import getLogger
+from os import PathLike
from pathlib import Path
from string import ascii_letters, digits
from unittest import mock
@@ -63,41 +63,41 @@ def printf_escape(string):
return string.replace('%', '%%')
-def build_url(path):
- if path[0] != '/':
- path = '/' + path
- return urljoin('file:', path)
+def build_url(path: str | PathLike[str]) -> str:
+ path_str = str(path)
+ if path_str[0] != '/':
+ path_str = '/' + path_str
+ return urljoin('file:', path_str)
class FileFeedStorageTest(unittest.TestCase):
def test_store_file_uri(self):
- path = os.path.abspath(self.mktemp())
- uri = path_to_file_uri(path)
+ path = Path(self.mktemp()).resolve()
+ uri = path_to_file_uri(str(path))
return self._assert_stores(FileFeedStorage(uri), path)
def test_store_file_uri_makedirs(self):
- path = os.path.abspath(self.mktemp())
- path = os.path.join(path, 'more', 'paths', 'file.txt')
- uri = path_to_file_uri(path)
+ path = Path(self.mktemp()).resolve() / 'more' / 'paths' / 'file.txt'
+ uri = path_to_file_uri(str(path))
return self._assert_stores(FileFeedStorage(uri), path)
def test_store_direct_path(self):
- path = os.path.abspath(self.mktemp())
- return self._assert_stores(FileFeedStorage(path), path)
+ path = Path(self.mktemp()).resolve()
+ return self._assert_stores(FileFeedStorage(str(path)), path)
def test_store_direct_path_relative(self):
- path = self.mktemp()
- return self._assert_stores(FileFeedStorage(path), path)
+ path = Path(self.mktemp())
+ return self._assert_stores(FileFeedStorage(str(path)), path)
def test_interface(self):
path = self.mktemp()
st = FileFeedStorage(path)
verifyObject(IFeedStorage, st)
- def _store(self, feed_options=None):
- path = os.path.abspath(self.mktemp())
- storage = FileFeedStorage(path, feed_options=feed_options)
+ def _store(self, feed_options=None) -> Path:
+ path = Path(self.mktemp()).resolve()
+ storage = FileFeedStorage(str(path), feed_options=feed_options)
spider = scrapy.Spider("default")
file = storage.open(spider)
file.write(b"content")
@@ -106,27 +106,26 @@ class FileFeedStorageTest(unittest.TestCase):
def test_append(self):
path = self._store()
- return self._assert_stores(FileFeedStorage(path), path, b"contentcontent")
+ return self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent")
def test_overwrite(self):
path = self._store({"overwrite": True})
return self._assert_stores(
- FileFeedStorage(path, feed_options={"overwrite": True}),
+ FileFeedStorage(str(path), feed_options={"overwrite": True}),
path
)
@defer.inlineCallbacks
- def _assert_stores(self, storage, path, expected_content=b"content"):
+ def _assert_stores(self, storage, path: Path, expected_content=b"content"):
spider = scrapy.Spider("default")
file = storage.open(spider)
file.write(b"content")
yield storage.store(file)
- self.assertTrue(os.path.exists(path))
+ self.assertTrue(path.exists())
try:
- with open(path, 'rb') as fp:
- self.assertEqual(fp.read(), expected_content)
+ self.assertEqual(path.read_bytes(), expected_content)
finally:
- os.unlink(path)
+ path.unlink()
class FTPFeedStorageTest(unittest.TestCase):
@@ -152,13 +151,12 @@ class FTPFeedStorageTest(unittest.TestCase):
file.write(content)
return storage.store(file)
- def _assert_stored(self, path, content):
+ def _assert_stored(self, path: Path, content):
self.assertTrue(path.exists())
try:
- with path.open('rb') as fp:
- self.assertEqual(fp.read(), content)
+ self.assertEqual(path.read_bytes(), content)
finally:
- os.unlink(str(path))
+ path.unlink()
@defer.inlineCallbacks
def test_append(self):
@@ -221,24 +219,24 @@ class BlockingFeedStorageTest(unittest.TestCase):
b = BlockingFeedStorage()
tmp = b.open(self.get_test_spider())
- tmp_path = os.path.dirname(tmp.name)
- self.assertEqual(tmp_path, tempfile.gettempdir())
+ tmp_path = Path(tmp.name).parent
+ self.assertEqual(str(tmp_path), tempfile.gettempdir())
def test_temp_file(self):
b = BlockingFeedStorage()
- tests_path = os.path.dirname(os.path.abspath(__file__))
- spider = self.get_test_spider({'FEED_TEMPDIR': tests_path})
+ tests_path = Path(__file__).resolve().parent
+ spider = self.get_test_spider({'FEED_TEMPDIR': str(tests_path)})
tmp = b.open(spider)
- tmp_path = os.path.dirname(tmp.name)
+ tmp_path = Path(tmp.name).parent
self.assertEqual(tmp_path, tests_path)
def test_invalid_folder(self):
b = BlockingFeedStorage()
- tests_path = os.path.dirname(os.path.abspath(__file__))
- invalid_path = os.path.join(tests_path, 'invalid_path')
- spider = self.get_test_spider({'FEED_TEMPDIR': invalid_path})
+ tests_path = Path(__file__).resolve().parent
+ invalid_path = tests_path / 'invalid_path'
+ spider = self.get_test_spider({'FEED_TEMPDIR': str(invalid_path)})
self.assertRaises(OSError, b.open, spider=spider)
@@ -564,13 +562,13 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
class DummyBlockingFeedStorage(BlockingFeedStorage):
def __init__(self, uri, *args, feed_options=None):
- self.path = file_uri_to_path(uri)
+ self.path = Path(file_uri_to_path(uri))
def _store_in_thread(self, file):
- dirname = os.path.dirname(self.path)
- if dirname and not os.path.exists(dirname):
- os.makedirs(dirname)
- with open(self.path, 'ab') as output_file:
+ dirname = self.path.parent
+ if dirname and not dirname.exists():
+ dirname.mkdir(parents=True)
+ with self.path.open('ab') as output_file:
output_file.write(file.read())
file.close()
@@ -613,10 +611,10 @@ class FeedExportTestBase(ABC, unittest.TestCase):
foo = scrapy.Field()
hello = scrapy.Field()
- def _random_temp_filename(self, inter_dir=''):
+ def _random_temp_filename(self, inter_dir='') -> Path:
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
filename = ''.join(chars)
- return os.path.join(self.temp_dir, inter_dir, filename)
+ return Path(self.temp_dir, inter_dir, filename)
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
@@ -702,18 +700,17 @@ class FeedExportTest(FeedExportTestBase):
yield crawler.crawl()
for file_path, feed_options in FEEDS.items():
- if not os.path.exists(str(file_path)):
+ if not Path(file_path).exists():
continue
- with open(str(file_path), 'rb') as f:
- content[feed_options['format']] = f.read()
+ content[feed_options['format']] = Path(file_path).read_bytes()
finally:
for file_path in FEEDS.keys():
- if not os.path.exists(str(file_path)):
+ if not Path(file_path).exists():
continue
- os.remove(str(file_path))
+ Path(file_path).unlink()
return content
@@ -808,7 +805,7 @@ class FeedExportTest(FeedExportTestBase):
def test_stats_file_success(self):
settings = {
"FEEDS": {
- printf_escape(path_to_url(self._random_temp_filename())): {
+ printf_escape(path_to_url(str(self._random_temp_filename()))): {
"format": "json",
}
},
@@ -823,7 +820,7 @@ class FeedExportTest(FeedExportTestBase):
def test_stats_file_failed(self):
settings = {
"FEEDS": {
- printf_escape(path_to_url(self._random_temp_filename())): {
+ printf_escape(path_to_url(str(self._random_temp_filename()))): {
"format": "json",
}
},
@@ -846,7 +843,7 @@ class FeedExportTest(FeedExportTestBase):
'AWS_ACCESS_KEY_ID': 'access_key',
'AWS_SECRET_ACCESS_KEY': 'secret_key',
"FEEDS": {
- printf_escape(path_to_url(self._random_temp_filename())): {
+ printf_escape(path_to_url(str(self._random_temp_filename()))): {
"format": "json",
},
"s3://bucket/key/foo.csv": {
@@ -1427,12 +1424,11 @@ class FeedExportTest(FeedExportTestBase):
self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler)
@defer.inlineCallbacks
- def test_pathlib_uri(self):
- feed_path = Path(self._random_temp_filename())
+ def test_str_uri(self):
settings = {
'FEED_STORE_EMPTY': True,
'FEEDS': {
- feed_path: {'format': 'csv'}
+ str(self._random_temp_filename()): {'format': 'csv'}
},
}
data = yield self.exported_no_data(settings)
@@ -1538,8 +1534,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
def close(self):
self.file.close()
- def _named_tempfile(self, name):
- return os.path.join(self.temp_dir, name)
+ def _named_tempfile(self, name) -> str:
+ return str(Path(self.temp_dir, name))
@defer.inlineCallbacks
def run_and_export(self, spider_cls, settings):
@@ -1559,18 +1555,17 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
yield crawler.crawl()
for file_path, feed_options in FEEDS.items():
- if not os.path.exists(str(file_path)):
+ if not Path(file_path).exists():
continue
- with open(str(file_path), 'rb') as f:
- content[str(file_path)] = f.read()
+ content[str(file_path)] = Path(file_path).read_bytes()
finally:
for file_path in FEEDS.keys():
- if not os.path.exists(str(file_path)):
+ if not Path(file_path).exists():
continue
- os.remove(str(file_path))
+ Path(file_path).unlink()
return content
@@ -2031,11 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
yield crawler.crawl()
for path, feed in FEEDS.items():
- dir_name = os.path.dirname(path)
- for file in sorted(os.listdir(dir_name)):
- with open(os.path.join(dir_name, file), 'rb') as f:
- data = f.read()
- content[feed['format']].append(data)
+ dir_name = Path(path).parent
+ for file in sorted(dir_name.iterdir()):
+ content[feed['format']].append(file.read_bytes())
finally:
self.tearDown()
defer.returnValue(content)
@@ -2045,7 +2038,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {}
settings.update({
'FEEDS': {
- os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
+ self._random_temp_filename() / 'jl' / self._file_mark: {'format': 'jl'},
},
})
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@@ -2061,7 +2054,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {}
settings.update({
'FEEDS': {
- os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
+ self._random_temp_filename() / 'csv' / self._file_mark: {'format': 'csv'},
},
})
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@@ -2077,7 +2070,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {}
settings.update({
'FEEDS': {
- os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
+ self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'},
},
})
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@@ -2094,8 +2087,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {}
settings.update({
'FEEDS': {
- os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
- os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
+ self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'},
+ self._random_temp_filename() / 'json' / self._file_mark: {'format': 'json'},
},
})
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@@ -2120,7 +2113,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {}
settings.update({
'FEEDS': {
- os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
+ self._random_temp_filename() / 'pickle' / self._file_mark: {'format': 'pickle'},
},
})
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@@ -2137,7 +2130,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {}
settings.update({
'FEEDS': {
- os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
+ self._random_temp_filename() / 'marshal' / self._file_mark: {'format': 'marshal'},
},
})
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@@ -2184,7 +2177,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
for fmt in ('json', 'jsonlines', 'xml', 'csv'):
settings = {
'FEEDS': {
- os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
+ self._random_temp_filename() / fmt / self._file_mark: {'format': fmt},
},
'FEED_EXPORT_BATCH_ITEM_COUNT': 1
}
@@ -2204,7 +2197,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
for fmt, expctd in formats:
settings = {
'FEEDS': {
- os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
+ self._random_temp_filename() / fmt / self._file_mark: {'format': fmt},
},
'FEED_STORE_EMPTY': True,
'FEED_EXPORT_INDENT': None,
@@ -2237,19 +2230,19 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = {
'FEEDS': {
- os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
+ self._random_temp_filename() / 'json' / self._file_mark: {
'format': 'json',
'indent': 0,
'fields': ['bar'],
'encoding': 'utf-8',
},
- os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {
+ self._random_temp_filename() / 'xml' / self._file_mark: {
'format': 'xml',
'indent': 2,
'fields': ['foo'],
'encoding': 'latin-1',
},
- os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
+ self._random_temp_filename() / 'csv' / self._file_mark: {
'format': 'csv',
'indent': None,
'fields': ['foo', 'bar'],
@@ -2272,7 +2265,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
}
settings = {
'FEEDS': {
- os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
+ self._random_temp_filename() / 'json' / self._file_mark: {
'format': 'json',
'indent': None,
'encoding': 'utf-8',
@@ -2299,7 +2292,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
]
settings = {
'FEEDS': {
- os.path.join(self._random_temp_filename(), '%(batch_time)s'): {
+ self._random_temp_filename() / '%(batch_time)s': {
'format': 'json',
},
},
@@ -2312,7 +2305,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
def test_stats_batch_file_success(self):
settings = {
"FEEDS": {
- build_url(os.path.join(self._random_temp_filename(), "json", self._file_mark)): {
+ build_url(str(self._random_temp_filename() / "json" / self._file_mark)): {
"format": "json",
}
},
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 49c83132f..402348cf9 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -1,10 +1,10 @@
import json
-import os
import random
import re
import shutil
import string
from ipaddress import IPv4Address
+from pathlib import Path
from unittest import mock, skipIf
from urllib.parse import urlencode
@@ -163,9 +163,8 @@ class RequestHeaders(LeafResource):
return bytes(json.dumps(headers), 'utf-8')
-def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
- with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate:
- pem = ''.join(key.readlines()) + ''.join(certificate.readlines())
+def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate:
+ pem = key_file.read_text() + certificate_file.read_text()
return PrivateCertificate.loadPEM(pem)
@@ -173,12 +172,12 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2ClientProtocolTestCase(TestCase):
scheme = 'https'
- key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key')
- certificate_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.crt')
+ key_file = Path(__file__).parent / 'keys' / 'localhost.key'
+ certificate_file = Path(__file__).parent / 'keys' / 'localhost.crt'
def _init_resource(self):
self.temp_directory = self.mktemp()
- os.mkdir(self.temp_directory)
+ Path(self.temp_directory).mkdir()
r = File(self.temp_directory)
r.putChild(b'get-data-html-small', GetDataHtmlSmall())
r.putChild(b'get-data-html-large', GetDataHtmlLarge())
@@ -202,7 +201,7 @@ class Https2ClientProtocolTestCase(TestCase):
# Start server for testing
self.hostname = 'localhost'
- context_factory = ssl_context_factory(self.key_file, self.certificate_file)
+ context_factory = ssl_context_factory(str(self.key_file), str(self.certificate_file))
server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname)
self.server = yield server_endpoint.listen(self.site)
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index e46532a1c..5d6547279 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -1,5 +1,5 @@
-import os
import shutil
+from pathlib import Path
from testfixtures import LogCapture
from twisted.internet import defer
@@ -61,12 +61,12 @@ class FileDownloadCrawlTestCase(TestCase):
self.mockserver.__enter__()
# prepare a directory for storing files
- self.tmpmediastore = self.mktemp()
- os.mkdir(self.tmpmediastore)
+ self.tmpmediastore = Path(self.mktemp())
+ self.tmpmediastore.mkdir()
self.settings = {
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
'ITEM_PIPELINES': {self.pipeline_class: 1},
- self.store_setting_key: self.tmpmediastore,
+ self.store_setting_key: str(self.tmpmediastore),
}
self.runner = CrawlerRunner(self.settings)
self.items = []
@@ -111,9 +111,7 @@ class FileDownloadCrawlTestCase(TestCase):
# check that the image files where actually written to the media store
for item in items:
for i in item[self.media_key]:
- self.assertTrue(
- os.path.exists(
- os.path.join(self.tmpmediastore, i['path'])))
+ self.assertTrue((self.tmpmediastore / i['path']).exists())
def _assert_files_download_failure(self, crawler, items, code, logs):
@@ -133,7 +131,7 @@ class FileDownloadCrawlTestCase(TestCase):
self.assertEqual(logs.count(file_dl_failure), 3)
# check that no files were written to the media store
- self.assertEqual(os.listdir(self.tmpmediastore), [])
+ self.assertEqual([x for x in self.tmpmediastore.iterdir()], [])
@defer.inlineCallbacks
def test_download_media(self):
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index d641e7a43..4acd29bf7 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -3,6 +3,7 @@ import random
import time
from datetime import datetime
from io import BytesIO
+from pathlib import Path
from shutil import rmtree
from tempfile import mkdtemp
from unittest import mock
@@ -89,7 +90,7 @@ class FilesPipelineTestCase(unittest.TestCase):
self.assertEqual(self.pipeline.store.basedir, self.tempdir)
path = 'some/image/key.jpg'
- fullpath = os.path.join(self.tempdir, 'some', 'image', 'key.jpg')
+ fullpath = Path(self.tempdir, 'some', 'image', 'key.jpg')
self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath)
@defer.inlineCallbacks
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index afdfb2578..ea7701b5d 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -2,6 +2,7 @@ import json
import os
import re
import sys
+from pathlib import Path
from subprocess import Popen, PIPE
from urllib.parse import urlsplit, urlunsplit
from testfixtures import LogCapture
@@ -27,14 +28,13 @@ from mitmproxy.tools.main import mitmdump
sys.argv[0] = "mitmdump"
sys.exit(mitmdump())
"""
- cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
- 'keys', 'mitmproxy-ca.pem')
+ cert_path = Path(__file__).parent.resolve() / 'keys' / 'mitmproxy-ca.pem'
self.proc = Popen([sys.executable,
'-c', script,
'--listen-host', '127.0.0.1',
'--listen-port', '0',
'--proxyauth', f'{self.auth_user}:{self.auth_pass}',
- '--certs', cert_path,
+ '--certs', str(cert_path),
'--ssl-insecure',
],
stdout=PIPE, env=get_testenv())
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 3719c7c9f..b9025cc14 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -1,7 +1,7 @@
import sys
-import os
import shutil
import warnings
+from pathlib import Path
from zope.interface.verify import verifyObject
from twisted.trial import unittest
@@ -17,10 +17,10 @@ from scrapy.settings import Settings
from scrapy.http import Request
from scrapy.crawler import CrawlerRunner
-module_dir = os.path.dirname(os.path.abspath(__file__))
+module_dir = Path(__file__).resolve().parent
-def _copytree(source, target):
+def _copytree(source: Path, target: Path):
try:
shutil.copytree(source, target)
except shutil.Error:
@@ -30,18 +30,18 @@ def _copytree(source, target):
class SpiderLoaderTest(unittest.TestCase):
def setUp(self):
- orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
- self.tmpdir = tempfile.mkdtemp()
- self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
+ orig_spiders_dir = module_dir / 'test_spiders'
+ self.tmpdir = Path(tempfile.mkdtemp())
+ self.spiders_dir = self.tmpdir / 'test_spiders_xxx'
_copytree(orig_spiders_dir, self.spiders_dir)
- sys.path.append(self.tmpdir)
+ sys.path.append(str(self.tmpdir))
settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
self.spider_loader = SpiderLoader.from_settings(settings)
def tearDown(self):
del self.spider_loader
del sys.modules['test_spiders_xxx']
- sys.path.remove(self.tmpdir)
+ sys.path.remove(str(self.tmpdir))
def test_interface(self):
verifyObject(ISpiderLoader, self.spider_loader)
@@ -135,22 +135,22 @@ class SpiderLoaderTest(unittest.TestCase):
class DuplicateSpiderNameLoaderTest(unittest.TestCase):
def setUp(self):
- orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
- self.tmpdir = self.mktemp()
- os.mkdir(self.tmpdir)
- self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
+ orig_spiders_dir = module_dir / 'test_spiders'
+ self.tmpdir = Path(self.mktemp())
+ self.tmpdir.mkdir()
+ self.spiders_dir = self.tmpdir / 'test_spiders_xxx'
_copytree(orig_spiders_dir, self.spiders_dir)
- sys.path.append(self.tmpdir)
+ sys.path.append(str(self.tmpdir))
self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
def tearDown(self):
del sys.modules['test_spiders_xxx']
- sys.path.remove(self.tmpdir)
+ sys.path.remove(str(self.tmpdir))
def test_dupename_warning(self):
# copy 1 spider module so as to have duplicate spider name
- shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'),
- os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py'))
+ shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider3.py',
+ self.tmpdir / 'test_spiders_xxx' / 'spider3dupe.py')
with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings)
@@ -171,10 +171,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
def test_multiple_dupename_warning(self):
# copy 2 spider modules so as to have duplicate spider name
# This should issue 2 warning, 1 for each duplicate spider name
- shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'),
- os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py'))
- shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'),
- os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py'))
+ shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider1.py',
+ self.tmpdir / 'test_spiders_xxx' / 'spider1dupe.py')
+ shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider2.py',
+ self.tmpdir / 'test_spiders_xxx' / 'spider2dupe.py')
with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings)
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index 383fadfeb..ab215576e 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -1,5 +1,5 @@
-import os
from datetime import datetime
+from pathlib import Path
import shutil
from twisted.trial import unittest
@@ -13,7 +13,7 @@ class SpiderStateTest(unittest.TestCase):
def test_store_load(self):
jobdir = self.mktemp()
- os.mkdir(jobdir)
+ Path(jobdir).mkdir()
try:
spider = Spider(name='default')
dt = datetime.now()
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index 4943731cb..ca98bff21 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -1,5 +1,5 @@
import unittest
-from os.path import join
+from pathlib import Path
from w3lib.encoding import html_to_unicode
@@ -8,46 +8,40 @@ from scrapy.http import Response
from tests import tests_datadir
-SAMPLEDIR = join(tests_datadir, 'compressed')
+SAMPLEDIR = Path(tests_datadir, 'compressed')
class GunzipTest(unittest.TestCase):
def test_gunzip_basic(self):
- with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f:
- r1 = Response("http://www.example.com", body=f.read())
- self.assertTrue(gzip_magic_number(r1))
+ r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'feed-sample1.xml.gz').read_bytes())
+ self.assertTrue(gzip_magic_number(r1))
- r2 = Response("http://www.example.com", body=gunzip(r1.body))
- self.assertFalse(gzip_magic_number(r2))
- self.assertEqual(len(r2.body), 9950)
+ r2 = Response("http://www.example.com", body=gunzip(r1.body))
+ self.assertFalse(gzip_magic_number(r2))
+ self.assertEqual(len(r2.body), 9950)
def test_gunzip_truncated(self):
- with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f:
- text = gunzip(f.read())
- assert text.endswith(b'</html')
+ text = gunzip((SAMPLEDIR / 'truncated-crc-error.gz').read_bytes())
+ assert text.endswith(b'</html')
def test_gunzip_no_gzip_file_raises(self):
- with open(join(SAMPLEDIR, 'feed-sample1.xml'), 'rb') as f:
- self.assertRaises(IOError, gunzip, f.read())
+ self.assertRaises(IOError, gunzip, (SAMPLEDIR / 'feed-sample1.xml').read_bytes())
def test_gunzip_truncated_short(self):
- with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f:
- r1 = Response("http://www.example.com", body=f.read())
- self.assertTrue(gzip_magic_number(r1))
+ r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'truncated-crc-error-short.gz').read_bytes())
+ self.assertTrue(gzip_magic_number(r1))
- r2 = Response("http://www.example.com", body=gunzip(r1.body))
- assert r2.body.endswith(b'</html>')
- self.assertFalse(gzip_magic_number(r2))
+ r2 = Response("http://www.example.com", body=gunzip(r1.body))
+ assert r2.body.endswith(b'</html>')
+ self.assertFalse(gzip_magic_number(r2))
def test_is_gzipped_empty(self):
r1 = Response("http://www.example.com")
self.assertFalse(gzip_magic_number(r1))
def test_gunzip_illegal_eof(self):
- with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f:
- text = html_to_unicode('charset=cp1252', gunzip(f.read()))[1]
- with open(join(SAMPLEDIR, 'unexpected-eof-output.txt'), 'rb') as o:
- expected_text = o.read().decode("utf-8")
- self.assertEqual(len(text), len(expected_text))
- self.assertEqual(text, expected_text)
+ text = html_to_unicode('charset=cp1252', gunzip((SAMPLEDIR / 'unexpected-eof.gz').read_bytes()))[1]
+ expected_text = (SAMPLEDIR / 'unexpected-eof-output.txt').read_text(encoding="utf-8")
+ self.assertEqual(len(text), len(expected_text))
+ self.assertEqual(text, expected_text)
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index f84cb2956..ba3136b96 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,5 +1,3 @@
-import os
-
from pytest import mark
from twisted.trial import unittest
@@ -303,11 +301,6 @@ class LxmlXmliterTestCase(XmliterTestCase):
class UtilsCsvTestCase(unittest.TestCase):
- sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')
- sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv')
- sample_feed2_path = os.path.join(sample_feeds_dir, 'feed-sample4.csv')
- sample_feed3_path = os.path.join(sample_feeds_dir, 'feed-sample5.csv')
-
def test_csviter_defaults(self):
body = get_testdata('feeds', 'feed-sample3.csv')
response = TextResponse(url="http://example.com/", body=body)
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index b83c1d6f0..dc5b9e123 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -1,6 +1,7 @@
import sys
import os
import unittest
+from pathlib import Path
from unittest import mock
from scrapy.item import Item, Field
@@ -55,7 +56,7 @@ class UtilsMiscTestCase(unittest.TestCase):
self.assertRaises(ImportError, walk_modules, 'nomodule999')
def test_walk_modules_egg(self):
- egg = os.path.join(os.path.dirname(__file__), 'test.egg')
+ egg = str(Path(__file__).parent / 'test.egg')
sys.path.append(egg)
try:
mods = walk_modules('testegg')
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 46452415a..f35f039a9 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -4,6 +4,7 @@ import tempfile
import shutil
import contextlib
import warnings
+from pathlib import Path
from pytest import warns
@@ -18,9 +19,7 @@ def inside_a_project():
try:
os.chdir(project_dir)
- with open('scrapy.cfg', 'w') as f:
- # create an empty scrapy.cfg
- f.close()
+ Path('scrapy.cfg').touch()
yield project_dir
finally:
@@ -31,20 +30,20 @@ def inside_a_project():
class ProjectUtilsTest(unittest.TestCase):
def test_data_path_outside_project(self):
self.assertEqual(
- os.path.join('.scrapy', 'somepath'),
+ str(Path('.scrapy', 'somepath')),
data_path('somepath')
)
- abspath = os.path.join(os.path.sep, 'absolute', 'path')
+ abspath = str(Path(os.path.sep, 'absolute', 'path'))
self.assertEqual(abspath, data_path(abspath))
def test_data_path_inside_project(self):
with inside_a_project() as proj_path:
- expected = os.path.join(proj_path, '.scrapy', 'somepath')
+ expected = Path(proj_path, '.scrapy', 'somepath')
self.assertEqual(
- os.path.realpath(expected),
- os.path.realpath(data_path('somepath'))
+ expected.resolve(),
+ Path(data_path('somepath')).resolve()
)
- abspath = os.path.join(os.path.sep, 'absolute', 'path')
+ abspath = str(Path(os.path.sep, 'absolute', 'path').resolve())
self.assertEqual(abspath, data_path(abspath))
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index d20852e62..cdf972933 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,6 +1,6 @@
-import os
import unittest
import warnings
+from pathlib import Path
from urllib.parse import urlparse
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -39,10 +39,9 @@ class ResponseUtilsTest(unittest.TestCase):
def browser_open(burl):
path = urlparse(burl).path
- if not os.path.exists(path):
+ if not path or not Path(path).exists():
path = burl.replace('file://', '')
- with open(path, "rb") as f:
- bbody = f.read()
+ bbody = Path(path).read_bytes()
self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody)
return True
response = HtmlResponse(url, body=body)
@@ -98,10 +97,9 @@ class ResponseUtilsTest(unittest.TestCase):
def check_base_url(burl):
path = urlparse(burl).path
- if not os.path.exists(path):
+ if not path or not Path(path).exists():
path = burl.replace('file://', '')
- with open(path, "rb") as f:
- bbody = f.read()
+ bbody = Path(path).read_bytes()
self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
return True
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 1d5e63363..b1aca5ed3 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -1,4 +1,4 @@
-import os
+from pathlib import Path
from shutil import rmtree
from tempfile import mkdtemp
import unittest
@@ -22,21 +22,19 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
template = 'from ${project_name}.spiders.${name} import ${classname}'
rendered = 'from proj.spiders.spi import TheSpider'
- template_path = os.path.join(self.tmp_path, 'templ.py.tmpl')
- render_path = os.path.join(self.tmp_path, 'templ.py')
+ template_path = Path(self.tmp_path, 'templ.py.tmpl')
+ render_path = Path(self.tmp_path, 'templ.py')
- with open(template_path, 'wb') as tmpl_file:
- tmpl_file.write(template.encode('utf8'))
- assert os.path.isfile(template_path) # Failure of test itself
+ template_path.write_text(template, encoding='utf8')
+ assert template_path.is_file() # Failure of test itself
- render_templatefile(template_path, **context)
+ render_templatefile(str(template_path), **context)
- self.assertFalse(os.path.exists(template_path))
- with open(render_path, 'rb') as result:
- self.assertEqual(result.read().decode('utf8'), rendered)
+ self.assertFalse(template_path.exists())
+ self.assertEqual(render_path.read_text(encoding='utf8'), rendered)
- os.remove(render_path)
- assert not os.path.exists(render_path) # Failure of test itself
+ render_path.unlink()
+ assert not render_path.exists() # Failure of test itself
if '__main__' == __name__:
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 0d5827339..69d9a9e3a 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -2,8 +2,8 @@
from twisted.internet import defer
Tests borrowed from the twisted.web.client tests.
"""
-import os
import shutil
+from pathlib import Path
import OpenSSL.SSL
from twisted.trial import unittest
@@ -15,7 +15,6 @@ except ImportError:
# deprecated in Twisted 19.7.0
# (remove once we bump our requirement past that version)
from twisted.test.proto_helpers import StringTransport
-from twisted.python.filepath import FilePath
from twisted.protocols.policies import WrappingFactory
from twisted.internet.defer import inlineCallbacks
@@ -230,10 +229,10 @@ class WebClientTestCase(unittest.TestCase):
return reactor.listenTCP(0, site, interface="127.0.0.1")
def setUp(self):
- self.tmpname = self.mktemp()
- os.mkdir(self.tmpname)
- FilePath(self.tmpname).child("file").setContent(b"0123456789")
- r = static.File(self.tmpname)
+ self.tmpname = Path(self.mktemp())
+ self.tmpname.mkdir()
+ (self.tmpname / "file").write_bytes(b"0123456789")
+ r = static.File(str(self.tmpname))
r.putChild(b"redirect", util.Redirect(b"/file"))
r.putChild(b"wait", ForeverTakingResource())
r.putChild(b"error", ErrorResource())
@@ -379,10 +378,10 @@ class WebClientSSLTestCase(unittest.TestCase):
return f"https://127.0.0.1:{self.portno}/{path}"
def setUp(self):
- self.tmpname = self.mktemp()
- os.mkdir(self.tmpname)
- FilePath(self.tmpname).child("file").setContent(b"0123456789")
- r = static.File(self.tmpname)
+ self.tmpname = Path(self.mktemp())
+ self.tmpname.mkdir()
+ (self.tmpname / "file").write_bytes(b"0123456789")
+ r = static.File(str(self.tmpname))
r.putChild(b"payload", PayloadResource())
self.site = server.Site(r, timeout=None)
self.wrapper = WrappingFactory(self.site)