Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into pathlib

Andrey Rahmatullind19a216e10ba
diff --cc scrapy/crawler.py
index b7108cdcc,65174d846..04c6891e3
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@@ -34,11 -31,13 +34,16 @@@ from scrapy.utils.log import
)
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
- from scrapy.utils.reactor import install_reactor, verify_installed_reactor
+ from scrapy.utils.reactor import (
+ install_reactor,
+ is_asyncio_reactor_installed,
+ verify_installed_asyncio_event_loop,
+ verify_installed_reactor,
+ )
+if TYPE_CHECKING:
+ from scrapy.utils.request import RequestFingerprinter
+
logger = logging.getLogger(__name__)
diff --cc scrapy/utils/request.py
index 545d489be,fbddc41fb..d1df8d335
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@@ -260,11 -260,11 +260,11 @@@ class RequestFingerprinter
raise ValueError(
f'Got an invalid value on setting '
f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '
- f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) '
- f'and \'VERSION\'.'
+ f'{implementation!r}. Valid values are \'2.6\' (deprecated) '
+ f'and \'2.7\'.'
)
- def fingerprint(self, request):
+ def fingerprint(self, request: Request):
return self._fingerprint(request)
diff --cc tests/test_cmdline/__init__.py
index da73a4c45,802f5c198..87ab1ee81
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@@ -31,18 -31,14 +31,14 @@@ class CmdlineTest(unittest.TestCase)
self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s',
'TEST1=override'), 'override')
- def test_override_settings_using_envvar(self):
- self.env['SCRAPY_TEST1'] = 'override'
- self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override')
-
def test_profiling(self):
- path = tempfile.mkdtemp()
- filename = os.path.join(path, 'res.prof')
+ path = Path(tempfile.mkdtemp())
+ filename = path / 'res.prof'
try:
- self._execute('version', '--profile', filename)
- self.assertTrue(os.path.exists(filename))
+ self._execute('version', '--profile', str(filename))
+ self.assertTrue(filename.exists())
out = StringIO()
- stats = pstats.Stats(filename, stream=out)
+ stats = pstats.Stats(str(filename), stream=out)
stats.print_stats()
out.seek(0)
stats = out.read()
diff --cc tests/test_crawler.py
index a0703ad47,da6024c2b..c09f1a6f2
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@@ -107,9 -108,9 +107,9 @@@ class CrawlerLoggingTestCase(unittest.T
name = 'spider'
custom_settings = {
'LOG_LEVEL': 'INFO',
- 'LOG_FILE': log_file,
+ 'LOG_FILE': str(log_file),
# settings to avoid extra warnings
- 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
}
diff --cc tests/test_downloader_handlers.py
index 29ff8c2dc,8835267fe..0334be743
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@@ -108,8 -108,10 +108,9 @@@ class LoadTestCase(unittest.TestCase)
class FileTestCase(unittest.TestCase):
def setUp(self):
- self.tmpname = self.mktemp()
+ # add a special char to check that they are handled correctly
- with open(self.tmpname + '^', 'w') as f:
- f.write('0123456789')
+ self.tmpname = Path(self.mktemp() + '^')
+ Path(self.tmpname).write_text('0123456789')
handler = create_instance(FileDownloadHandler, None, get_crawler())
self.download_request = handler.download_request
diff --cc tests/test_pipeline_crawl.py
index 5d6547279,0e174cd34..39bca7bf2
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@@ -61,12 -61,12 +61,12 @@@ class FileDownloadCrawlTestCase(TestCas
self.mockserver.__enter__()
# prepare a directory for storing files
- self.tmpmediastore = self.mktemp()
- os.mkdir(self.tmpmediastore)
+ self.tmpmediastore = Path(self.mktemp())
+ self.tmpmediastore.mkdir()
self.settings = {
- 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
'ITEM_PIPELINES': {self.pipeline_class: 1},
- self.store_setting_key: self.tmpmediastore,
+ self.store_setting_key: str(self.tmpmediastore),
}
self.runner = CrawlerRunner(self.settings)
self.items = []
diff --cc tests/test_utils_project.py
index f35f039a9,e77ffa18b..867e12103
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@@ -4,11 -4,7 +4,8 @@@ import tempfil
import shutil
import contextlib
import warnings
+from pathlib import Path
- from pytest import warns
-
- from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.project import data_path, get_project_settings