Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into pathlib
Andrey Rahmatullind19a216e10ba
diff --cc scrapy/crawler.pyindex b7108cdcc,65174d846..04c6891e3--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@@ -34,11 -31,13 +34,16 @@@ from scrapy.utils.log import)from scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.ossignal import install_shutdown_handlers, signal_names- from scrapy.utils.reactor import install_reactor, verify_installed_reactor+ from scrapy.utils.reactor import (+ install_reactor,+ is_asyncio_reactor_installed,+ verify_installed_asyncio_event_loop,+ verify_installed_reactor,+ )+if TYPE_CHECKING:+ from scrapy.utils.request import RequestFingerprinter+logger = logging.getLogger(__name__)diff --cc scrapy/utils/request.pyindex 545d489be,fbddc41fb..d1df8d335--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@@ -260,11 -260,11 +260,11 @@@ class RequestFingerprinterraise ValueError(f'Got an invalid value on setting 'f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '- f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) '- f'and \'VERSION\'.'+ f'{implementation!r}. Valid values are \'2.6\' (deprecated) '+ f'and \'2.7\'.')- def fingerprint(self, request):+ def fingerprint(self, request: Request):return self._fingerprint(request)diff --cc tests/test_cmdline/__init__.pyindex da73a4c45,802f5c198..87ab1ee81--- a/tests/test_cmdline/__init__.py+++ b/tests/test_cmdline/__init__.py@@@ -31,18 -31,14 +31,14 @@@ class CmdlineTest(unittest.TestCase)self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s','TEST1=override'), 'override')- def test_override_settings_using_envvar(self):- self.env['SCRAPY_TEST1'] = 'override'- self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override')-def test_profiling(self):- path = tempfile.mkdtemp()- filename = os.path.join(path, 'res.prof')+ path = Path(tempfile.mkdtemp())+ filename = path / 'res.prof'try:- self._execute('version', '--profile', filename)- self.assertTrue(os.path.exists(filename))+ self._execute('version', '--profile', str(filename))+ self.assertTrue(filename.exists())out = StringIO()- stats = pstats.Stats(filename, stream=out)+ stats = pstats.Stats(str(filename), stream=out)stats.print_stats()out.seek(0)stats = out.read()diff --cc tests/test_crawler.pyindex a0703ad47,da6024c2b..c09f1a6f2--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@@ -107,9 -108,9 +107,9 @@@ class CrawlerLoggingTestCase(unittest.Tname = 'spider'custom_settings = {'LOG_LEVEL': 'INFO',- 'LOG_FILE': log_file,+ 'LOG_FILE': str(log_file),# settings to avoid extra warnings- 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7','TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,}diff --cc tests/test_downloader_handlers.pyindex 29ff8c2dc,8835267fe..0334be743--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@@ -108,8 -108,10 +108,9 @@@ class LoadTestCase(unittest.TestCase)class FileTestCase(unittest.TestCase):def setUp(self):- self.tmpname = self.mktemp()+ # add a special char to check that they are handled correctly- with open(self.tmpname + '^', 'w') as f:- f.write('0123456789')+ self.tmpname = Path(self.mktemp() + '^')+ Path(self.tmpname).write_text('0123456789')handler = create_instance(FileDownloadHandler, None, get_crawler())self.download_request = handler.download_requestdiff --cc tests/test_pipeline_crawl.pyindex 5d6547279,0e174cd34..39bca7bf2--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@@ -61,12 -61,12 +61,12 @@@ class FileDownloadCrawlTestCase(TestCasself.mockserver.__enter__()# prepare a directory for storing files- self.tmpmediastore = self.mktemp()- os.mkdir(self.tmpmediastore)+ self.tmpmediastore = Path(self.mktemp())+ self.tmpmediastore.mkdir()self.settings = {- 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7','ITEM_PIPELINES': {self.pipeline_class: 1},- self.store_setting_key: self.tmpmediastore,+ self.store_setting_key: str(self.tmpmediastore),}self.runner = CrawlerRunner(self.settings)self.items = []diff --cc tests/test_utils_project.pyindex f35f039a9,e77ffa18b..867e12103--- a/tests/test_utils_project.py+++ b/tests/test_utils_project.py@@@ -4,11 -4,7 +4,8 @@@ import tempfilimport shutilimport contextlibimport warnings+from pathlib import Path- from pytest import warns-- from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.project import data_path, get_project_settings