Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into change-init-order
Andrey Rakhmatullin69282c53187e
diff --cc scrapy/crawler.pyindex 0754276f3,25823b6ac..ec853e0d8--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@@ -26,49 -69,72 +69,72 @@@ class Crawlerif isinstance(settings, dict) or settings is None:settings = Settings(settings)- self.spidercls = spidercls- self.settings = settings.copy()+ self.spidercls: Type[Spider] = spidercls+ self.settings: Settings = settings.copy()- self.spidercls.update_settings(self.settings)++ self.addons: AddonManager = AddonManager(self)+ self.addons.load_settings(self.settings)++ self.signals: SignalManager = SignalManager(self)- self.signals = SignalManager(self)- self.stats = load_object(self.settings['STATS_CLASS'])(self)+ self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self)- handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))+ handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))logging.root.addHandler(handler)++ d = dict(overridden_settings(self.settings))+ logger.info(+ "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}+ )++ if get_scrapy_root_handler() is not None:+ # scrapy root handler already installed: update it with new settings+ install_scrapy_root_handler(self.settings)# lambda is assigned to Crawler attribute because this way it is not# garbage collected after leaving __init__ scopeself.__remove_handler = lambda: logging.root.removeHandler(handler)self.signals.connect(self.__remove_handler, signals.engine_stopped)- lf_cls = load_object(self.settings['LOG_FORMATTER'])- self.logformatter = lf_cls.from_crawler(self)-- self.crawling = False- self.spider = None- self.engine = None-- @property- def spiders(self):- if not hasattr(self, '_spiders'):- warnings.warn("Crawler.spiders is deprecated, use "- "CrawlerRunner.spider_loader or instantiate "- "scrapy.spiderloader.SpiderLoader with your "- "settings.",- category=ScrapyDeprecationWarning, stacklevel=2)- self._spiders = _get_spider_loader(self.settings.frozencopy())- return self._spiders-- @defer.inlineCallbacks- def crawl(self, *args, **kwargs):- assert not self.crawling, "Crawling already taking place"+ lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])+ self.logformatter: LogFormatter = lf_cls.from_crawler(self)++ self.request_fingerprinter: RequestFingerprinter = create_instance(+ load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),+ settings=self.settings,+ crawler=self,+ )++ reactor_class: str = self.settings["TWISTED_REACTOR"]+ event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]+ if init_reactor:+ # this needs to be done after the spider settings are merged,+ # but before something imports twisted.internet.reactor+ if reactor_class:+ install_reactor(reactor_class, event_loop)+ else:+ from twisted.internet import reactor # noqa: F401+ log_reactor_info()+ if reactor_class:+ verify_installed_reactor(reactor_class)+ if is_asyncio_reactor_installed() and event_loop:+ verify_installed_asyncio_event_loop(event_loop)+- self.extensions: ExtensionManager = ExtensionManager.from_crawler(self)-- self.settings.freeze()+ self.crawling: bool = False+ self.spider: Optional[Spider] = None+ self.engine: Optional[ExecutionEngine] = None++ @inlineCallbacks+ def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:+ if self.crawling:+ raise RuntimeError("Crawling already taking place")self.crawling = Truetry:self.spider = self._create_spider(*args, **kwargs)+ self.spider.update_settings(self.settings)+ self.settings.freeze()- self.extensions = ExtensionManager.from_crawler(self)++ self.extensions: ExtensionManager = ExtensionManager.from_crawler(self)+self.engine = self._create_engine()start_requests = iter(self.spider.start_requests())yield self.engine.open_spider(self.spider, start_requests)diff --cc scrapy/spiders/__init__.pyindex 4b2771415,e16d71727..590158d1d--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@@ -65,21 -65,30 +65,29 @@@ class Spider(object_ref)self.settings = crawler.settingscrawler.signals.connect(self.close, signals.spider_closed)- def start_requests(self):+ def start_requests(self) -> Iterable[Request]:+ if not self.start_urls and hasattr(self, "start_url"):+ raise AttributeError(+ "Crawling could not start: 'start_urls' not found "+ "or empty (but found 'start_url' attribute instead, "+ "did you miss an 's'?)"+ )for url in self.start_urls:- yield self.make_requests_from_url(url)+ yield Request(url, dont_filter=True)- def make_requests_from_url(self, url):- return Request(url, dont_filter=True)+ def _parse(self, response: Response, **kwargs: Any) -> Any:+ return self.parse(response, **kwargs)- def parse(self, response):- raise NotImplementedError+ def parse(self, response: Response, **kwargs: Any) -> Any:+ raise NotImplementedError(+ f"{self.__class__.__name__}.parse callback is not defined"+ )- def update_settings(self, settings):- settings.setdict(self.custom_settings or {}, priority='spider')- @classmethod- def update_settings(cls, settings: BaseSettings) -> None:- settings.setdict(cls.custom_settings or {}, priority="spider")++ def update_settings(self, settings: BaseSettings) -> None:++ settings.setdict(self.custom_settings or {}, priority="spider")@classmethod- def handles_request(cls, request):+ def handles_request(cls, request: Request) -> bool:return url_is_from_spider(request.url, cls)@staticmethoddiff --cc tests/test_crawl.pyindex 021849f41,ca9084294..496ab77a5--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@@ -1,20 -1,55 +1,59 @@@import json- import socketimport logging+ import unittest++import warnings+ from ipaddress import IPv4Address+ from socket import gethostbyname+ from urllib.parse import urlparse+ from pytest import markfrom testfixtures import LogCapturefrom twisted.internet import defer+ from twisted.internet.ssl import Certificate+ from twisted.python.failure import Failurefrom twisted.trial.unittest import TestCase- from scrapy.http import Request+ from scrapy import signals-from scrapy.crawler import CrawlerRunner-from scrapy.exceptions import StopDownload+from scrapy.crawler import Crawler, CrawlerRunner++from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload+from scrapy.extensions.throttle import AutoThrottle+ from scrapy.http import Request+ from scrapy.http.response import Response+from scrapy.settings import Settings+ from scrapy.utils.python import to_unicode+from scrapy.utils.spider import DefaultSpider- from tests import mock- from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \- BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider+ from scrapy.utils.test import get_crawler+ from tests import NON_EXISTING_RESOLVABLEfrom tests.mockserver import MockServer+ from tests.spiders import (+ AsyncDefAsyncioGenComplexSpider,+ AsyncDefAsyncioGenExcSpider,+ AsyncDefAsyncioGenLoopSpider,+ AsyncDefAsyncioGenSpider,+ AsyncDefAsyncioReqsReturnSpider,+ AsyncDefAsyncioReturnSingleElementSpider,+ AsyncDefAsyncioReturnSpider,+ AsyncDefAsyncioSpider,+ AsyncDefDeferredDirectSpider,+ AsyncDefDeferredMaybeWrappedSpider,+ AsyncDefDeferredWrappedSpider,+ AsyncDefSpider,+ BrokenStartRequestsSpider,+ BytesReceivedCallbackSpider,+ BytesReceivedErrbackSpider,+ CrawlSpiderWithAsyncCallback,+ CrawlSpiderWithAsyncGeneratorCallback,+ CrawlSpiderWithErrback,+ CrawlSpiderWithParseMethod,+ CrawlSpiderWithProcessRequestCallbackKeywordArguments,+ DelaySpider,+ DuplicateStartRequestsSpider,+ FollowAllSpider,+ HeadersReceivedCallbackSpider,+ HeadersReceivedErrbackSpider,+ SimpleSpider,+ SingleRequestSpider,+ )class CrawlTestCase(TestCase):@@@ -71,9 -123,8 +127,10 @@@self.assertTrue(crawler.spider.t2 == 0)self.assertTrue(crawler.spider.t2_err > 0)self.assertTrue(crawler.spider.t2_err > crawler.spider.t1)++# server hangs after receiving response headers- crawler = runner.create_crawler(DelaySpider)- yield runner.crawl(crawler, n=0.5, b=1)++ crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})+ yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver)self.assertTrue(crawler.spider.t1 > 0)self.assertTrue(crawler.spider.t2 == 0)self.assertTrue(crawler.spider.t2_err > 0)@@@ -136,15 -195,18 +201,19 @@@@defer.inlineCallbacksdef test_start_requests_dupes(self):settings = {"CONCURRENT_REQUESTS": 1}- runner = CrawlerRunner(settings)- crawler = runner.create_crawler(DuplicateStartRequestsSpider)- yield runner.crawl(crawler, dont_filter=True, distinct_urls=2,- dupe_factor=3)+ crawler = get_crawler(DuplicateStartRequestsSpider, settings)+ yield crawler.crawl(+ dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver+ )self.assertEqual(crawler.spider.visited, 6)- crawler = runner.create_crawler(DuplicateStartRequestsSpider)- yield runner.crawl(crawler, dont_filter=False, distinct_urls=3,- dupe_factor=4)++ crawler = get_crawler(DuplicateStartRequestsSpider, settings)+ yield crawler.crawl(+ dont_filter=False,+ distinct_urls=3,+ dupe_factor=4,+ mockserver=self.mockserver,+ )self.assertEqual(crawler.spider.visited, 3)@defer.inlineCallbacks@@@ -276,29 -407,296 +414,328 @@@ with multiples lineself._assert_retried(log)self.assertIn("Got response 200", str(log))+ @defer.inlineCallbacks- def test_populate_spider_settings(self):- spider_settings = {'TEST1': 'spider', 'TEST2': 'spider',- 'AUTOTHROTTLE_ENABLED': True}- project_settings = {'TEST1': 'project', 'TEST3': 'project'}++ def test_populate_spidercls_settings(self):++ spider_settings = {++ "TEST1": "spider",++ "TEST2": "spider",++ "AUTOTHROTTLE_ENABLED": True,++ }++ project_settings = {"TEST1": "project", "TEST3": "project"}++ class CustomSettingsSpider(DefaultSpider):+ custom_settings = spider_settings++ def parse(self, response):+ return++ settings = Settings()- settings.setdict(project_settings, priority='project')- crawler = Crawler(CustomSettingsSpider, settings)++ settings.setdict(project_settings, priority="project")++ with warnings.catch_warnings():++ warnings.simplefilter("ignore", ScrapyDeprecationWarning)++ crawler = Crawler(CustomSettingsSpider, settings)+ yield crawler.crawl()+- self.assertEqual(crawler.settings.get('TEST1'), 'spider')- self.assertEqual(crawler.settings.get('TEST2'), 'spider')- self.assertEqual(crawler.settings.get('TEST3'), 'project')++ self.assertEqual(crawler.settings.get("TEST1"), "spider")++ self.assertEqual(crawler.settings.get("TEST2"), "spider")++ self.assertEqual(crawler.settings.get("TEST3"), "project")++ enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]+ self.assertIn(AutoThrottle, enabled_exts)++ self.assertFalse(settings.frozen)+ self.assertTrue(crawler.settings.frozen)++++ class CrawlSpiderTestCase(TestCase):+ def setUp(self):+ self.mockserver = MockServer()+ self.mockserver.__enter__()++ def tearDown(self):+ self.mockserver.__exit__(None, None, None)++ @defer.inlineCallbacks+ def _run_spider(self, spider_cls):+ items = []++ def _on_item_scraped(item):+ items.append(item)++ crawler = get_crawler(spider_cls)+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)+ with LogCapture() as log:+ yield crawler.crawl(+ self.mockserver.url("/status?n=200"), mockserver=self.mockserver+ )+ return log, items, crawler.stats++ @defer.inlineCallbacks+ def test_crawlspider_with_parse(self):+ crawler = get_crawler(CrawlSpiderWithParseMethod)+ with LogCapture() as log:+ yield crawler.crawl(mockserver=self.mockserver)++ self.assertIn("[parse] status 200 (foo: None)", str(log))+ self.assertIn("[parse] status 201 (foo: None)", str(log))+ self.assertIn("[parse] status 202 (foo: bar)", str(log))++ @defer.inlineCallbacks+ def test_crawlspider_with_async_callback(self):+ crawler = get_crawler(CrawlSpiderWithAsyncCallback)+ with LogCapture() as log:+ yield crawler.crawl(mockserver=self.mockserver)++ self.assertIn("[parse_async] status 200 (foo: None)", str(log))+ self.assertIn("[parse_async] status 201 (foo: None)", str(log))+ self.assertIn("[parse_async] status 202 (foo: bar)", str(log))++ @defer.inlineCallbacks+ def test_crawlspider_with_async_generator_callback(self):+ crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback)+ with LogCapture() as log:+ yield crawler.crawl(mockserver=self.mockserver)++ self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log))+ self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log))+ self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log))++ @defer.inlineCallbacks+ def test_crawlspider_with_errback(self):+ crawler = get_crawler(CrawlSpiderWithErrback)+ with LogCapture() as log:+ yield crawler.crawl(mockserver=self.mockserver)++ self.assertIn("[parse] status 200 (foo: None)", str(log))+ self.assertIn("[parse] status 201 (foo: None)", str(log))+ self.assertIn("[parse] status 202 (foo: bar)", str(log))+ self.assertIn("[errback] status 404", str(log))+ self.assertIn("[errback] status 500", str(log))+ self.assertIn("[errback] status 501", str(log))++ @defer.inlineCallbacks+ def test_crawlspider_process_request_cb_kwargs(self):+ crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments)+ with LogCapture() as log:+ yield crawler.crawl(mockserver=self.mockserver)++ self.assertIn("[parse] status 200 (foo: process_request)", str(log))+ self.assertIn("[parse] status 201 (foo: process_request)", str(log))+ self.assertIn("[parse] status 202 (foo: bar)", str(log))++ @defer.inlineCallbacks+ def test_async_def_parse(self):+ crawler = get_crawler(AsyncDefSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ self.mockserver.url("/status?n=200"), mockserver=self.mockserver+ )+ self.assertIn("Got response 200", str(log))++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncio_parse(self):+ crawler = get_crawler(+ AsyncDefAsyncioSpider,+ {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"+ },+ )+ with LogCapture() as log:+ yield crawler.crawl(+ self.mockserver.url("/status?n=200"), mockserver=self.mockserver+ )+ self.assertIn("Got response 200", str(log))++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncio_parse_items_list(self):+ log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider)+ self.assertIn("Got response 200", str(log))+ self.assertIn({"id": 1}, items)+ self.assertIn({"id": 2}, items)++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncio_parse_items_single_element(self):+ items = []++ def _on_item_scraped(item):+ items.append(item)++ crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider)+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)+ with LogCapture() as log:+ yield crawler.crawl(+ self.mockserver.url("/status?n=200"), mockserver=self.mockserver+ )+ self.assertIn("Got response 200", str(log))+ self.assertIn({"foo": 42}, items)++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncgen_parse(self):+ log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider)+ self.assertIn("Got response 200", str(log))+ itemcount = stats.get_value("item_scraped_count")+ self.assertEqual(itemcount, 1)++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncgen_parse_loop(self):+ log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider)+ self.assertIn("Got response 200", str(log))+ itemcount = stats.get_value("item_scraped_count")+ self.assertEqual(itemcount, 10)+ for i in range(10):+ self.assertIn({"foo": i}, items)++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncgen_parse_exc(self):+ log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider)+ log = str(log)+ self.assertIn("Spider error processing", log)+ self.assertIn("ValueError", log)+ itemcount = stats.get_value("item_scraped_count")+ self.assertEqual(itemcount, 7)+ for i in range(7):+ self.assertIn({"foo": i}, items)++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncgen_parse_complex(self):+ _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider)+ itemcount = stats.get_value("item_scraped_count")+ self.assertEqual(itemcount, 156)+ # some random items+ for i in [1, 4, 21, 22, 207, 311]:+ self.assertIn({"index": i}, items)+ for i in [10, 30, 122]:+ self.assertIn({"index2": i}, items)++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncio_parse_reqs_list(self):+ log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider)+ for req_id in range(3):+ self.assertIn(f"Got response 200, req_id {req_id}", str(log))++ @mark.only_not_asyncio()+ @defer.inlineCallbacks+ def test_async_def_deferred_direct(self):+ _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider)+ self.assertEqual(items, [{"code": 200}])++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_deferred_wrapped(self):+ log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider)+ self.assertEqual(items, [{"code": 200}])++ @defer.inlineCallbacks+ def test_async_def_deferred_maybe_wrapped(self):+ _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider)+ self.assertEqual(items, [{"code": 200}])++ @defer.inlineCallbacks+ def test_response_ssl_certificate_none(self):+ crawler = get_crawler(SingleRequestSpider)+ url = self.mockserver.url("/echo?body=test", is_secure=False)+ yield crawler.crawl(seed=url, mockserver=self.mockserver)+ self.assertIsNone(crawler.spider.meta["responses"][0].certificate)++ @defer.inlineCallbacks+ def test_response_ssl_certificate(self):+ crawler = get_crawler(SingleRequestSpider)+ url = self.mockserver.url("/echo?body=test", is_secure=True)+ yield crawler.crawl(seed=url, mockserver=self.mockserver)+ cert = crawler.spider.meta["responses"][0].certificate+ self.assertIsInstance(cert, Certificate)+ self.assertEqual(cert.getSubject().commonName, b"localhost")+ self.assertEqual(cert.getIssuer().commonName, b"localhost")++ @mark.xfail(reason="Responses with no body return early and contain no certificate")+ @defer.inlineCallbacks+ def test_response_ssl_certificate_empty_response(self):+ crawler = get_crawler(SingleRequestSpider)+ url = self.mockserver.url("/status?n=200", is_secure=True)+ yield crawler.crawl(seed=url, mockserver=self.mockserver)+ cert = crawler.spider.meta["responses"][0].certificate+ self.assertIsInstance(cert, Certificate)+ self.assertEqual(cert.getSubject().commonName, b"localhost")+ self.assertEqual(cert.getIssuer().commonName, b"localhost")++ @defer.inlineCallbacks+ def test_dns_server_ip_address_none(self):+ crawler = get_crawler(SingleRequestSpider)+ url = self.mockserver.url("/status?n=200")+ yield crawler.crawl(seed=url, mockserver=self.mockserver)+ ip_address = crawler.spider.meta["responses"][0].ip_address+ self.assertIsNone(ip_address)++ @defer.inlineCallbacks+ def test_dns_server_ip_address(self):+ crawler = get_crawler(SingleRequestSpider)+ url = self.mockserver.url("/echo?body=test")+ expected_netloc, _ = urlparse(url).netloc.split(":")+ yield crawler.crawl(seed=url, mockserver=self.mockserver)+ ip_address = crawler.spider.meta["responses"][0].ip_address+ self.assertIsInstance(ip_address, IPv4Address)+ self.assertEqual(str(ip_address), gethostbyname(expected_netloc))++ @defer.inlineCallbacks+ def test_bytes_received_stop_download_callback(self):+ crawler = get_crawler(BytesReceivedCallbackSpider)+ yield crawler.crawl(mockserver=self.mockserver)+ self.assertIsNone(crawler.spider.meta.get("failure"))+ self.assertIsInstance(crawler.spider.meta["response"], Response)+ self.assertEqual(+ crawler.spider.meta["response"].body,+ crawler.spider.meta.get("bytes_received"),+ )+ self.assertLess(+ len(crawler.spider.meta["response"].body),+ crawler.spider.full_response_length,+ )++ @defer.inlineCallbacks+ def test_bytes_received_stop_download_errback(self):+ crawler = get_crawler(BytesReceivedErrbackSpider)+ yield crawler.crawl(mockserver=self.mockserver)+ self.assertIsNone(crawler.spider.meta.get("response"))+ self.assertIsInstance(crawler.spider.meta["failure"], Failure)+ self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)+ self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response)+ self.assertEqual(+ crawler.spider.meta["failure"].value.response.body,+ crawler.spider.meta.get("bytes_received"),+ )+ self.assertLess(+ len(crawler.spider.meta["failure"].value.response.body),+ crawler.spider.full_response_length,+ )++ @defer.inlineCallbacks+ def test_headers_received_stop_download_callback(self):+ crawler = get_crawler(HeadersReceivedCallbackSpider)+ yield crawler.crawl(mockserver=self.mockserver)+ self.assertIsNone(crawler.spider.meta.get("failure"))+ self.assertIsInstance(crawler.spider.meta["response"], Response)+ self.assertEqual(+ crawler.spider.meta["response"].headers,+ crawler.spider.meta.get("headers_received"),+ )++ @defer.inlineCallbacks+ def test_headers_received_stop_download_errback(self):+ crawler = get_crawler(HeadersReceivedErrbackSpider)+ yield crawler.crawl(mockserver=self.mockserver)+ self.assertIsNone(crawler.spider.meta.get("response"))+ self.assertIsInstance(crawler.spider.meta["failure"], Failure)+ self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)+ self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response)+ self.assertEqual(+ crawler.spider.meta["failure"].value.response.headers,+ crawler.spider.meta.get("headers_received"),+ )diff --cc tests/test_crawler.pyindex 96bf0c866,4c5c48e6d..1fe865ee4--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@@ -1,12 -1,28 +1,27 @@@+ import logging+ import os+ import platform+ import subprocess+ import sysimport warnings- import unittest+ from pathlib import Path++ from packaging.version import parse as parse_version+ from pytest import mark, raises+ from twisted.internet import defer+ from twisted.trial import unittest+ from w3lib import __version__ as w3lib_versionimport scrapy- from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess+ from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner+ from scrapy.exceptions import ScrapyDeprecationWarning+ from scrapy.extensions import telnet-from scrapy.extensions.throttle import AutoThrottlefrom scrapy.settings import Settings, default_settingsfrom scrapy.spiderloader import SpiderLoader+ from scrapy.utils.log import configure_logging, get_scrapy_root_handlerfrom scrapy.utils.spider import DefaultSpider- from scrapy.utils.misc import load_object+ from scrapy.utils.test import get_crawler+ from tests.mockserver import MockServer, get_mockserver_envclass BaseCrawlerTest(unittest.TestCase):@@@ -17,33 -32,125 +31,94 @@@class CrawlerTestCase(BaseCrawlerTest):- def test_populate_spidercls_settings(self):- spider_settings = {"TEST1": "spider", "TEST2": "spider"}- project_settings = {"TEST1": "project", "TEST3": "project"}-- class CustomSettingsSpider(DefaultSpider):- custom_settings = spider_settings-- settings = Settings()- settings.setdict(project_settings, priority="project")- with warnings.catch_warnings():- warnings.simplefilter("ignore", ScrapyDeprecationWarning)- crawler = Crawler(CustomSettingsSpider, settings)-- self.assertEqual(crawler.settings.get("TEST1"), "spider")- self.assertEqual(crawler.settings.get("TEST2"), "spider")- self.assertEqual(crawler.settings.get("TEST3"), "project")-- self.assertFalse(settings.frozen)- self.assertTrue(crawler.settings.frozen)-+ def test_crawler_accepts_dict(self):+ crawler = get_crawler(DefaultSpider, {"foo": "bar"})+ self.assertEqual(crawler.settings["foo"], "bar")+ self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")- def setUp(self):- self.crawler = Crawler(DefaultSpider, Settings())+ def test_crawler_accepts_None(self):+ with warnings.catch_warnings():+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)+ crawler = Crawler(DefaultSpider)+ self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")- def test_deprecated_attribute_spiders(self):- with warnings.catch_warnings(record=True) as w:- spiders = self.crawler.spiders- self.assertEqual(len(w), 1)- self.assertIn("Crawler.spiders", str(w[0].message))- sl_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS'])- self.assertIsInstance(spiders, sl_cls)+ def test_crawler_rejects_spider_objects(self):+ with raises(ValueError):+ Crawler(DefaultSpider())- self.crawler.spiders- self.assertEqual(len(w), 1, "Warn deprecated access only once")- def test_crawler_accepts_dict(self):- crawler = Crawler(DefaultSpider, {'foo': 'bar'})- self.assertEqual(crawler.settings['foo'], 'bar')- self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')-class SpiderSettingsTestCase(unittest.TestCase):- def test_spider_custom_settings(self):- class MySpider(scrapy.Spider):- name = "spider"- custom_settings = {"AUTOTHROTTLE_ENABLED": True}-- crawler = get_crawler(MySpider)- enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]- self.assertIn(AutoThrottle, enabled_exts)--+ class CrawlerLoggingTestCase(unittest.TestCase):+ def test_no_root_handler_installed(self):+ handler = get_scrapy_root_handler()+ if handler is not None:+ logging.root.removeHandler(handler)- def test_crawler_accepts_None(self):- crawler = Crawler(DefaultSpider)- self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')+ class MySpider(scrapy.Spider):+ name = "spider"++ get_crawler(MySpider)+ assert get_scrapy_root_handler() is None++ def test_spider_custom_settings_log_level(self):+ log_file = Path(self.mktemp())+ log_file.write_text("previous message\n", encoding="utf-8")+ class MySpider(scrapy.Spider):+ name = "spider"+ custom_settings = {+ "LOG_LEVEL": "INFO",+ "LOG_FILE": str(log_file),+ # settings to avoid extra warnings+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",+ "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE,+ }- class SpiderLoaderWithWrongInterface(object):+ configure_logging()+ self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)+ crawler = get_crawler(MySpider)+ self.assertEqual(get_scrapy_root_handler().level, logging.INFO)+ info_count = crawler.stats.get_value("log_count/INFO")+ logging.debug("debug message")+ logging.info("info message")+ logging.warning("warning message")+ logging.error("error message")+ logged = log_file.read_text(encoding="utf-8")++ self.assertIn("previous message", logged)+ self.assertNotIn("debug message", logged)+ self.assertIn("info message", logged)+ self.assertIn("warning message", logged)+ self.assertIn("error message", logged)+ self.assertEqual(crawler.stats.get_value("log_count/ERROR"), 1)+ self.assertEqual(crawler.stats.get_value("log_count/WARNING"), 1)+ self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1)+ self.assertEqual(crawler.stats.get_value("log_count/DEBUG", 0), 0)++ def test_spider_custom_settings_log_append(self):+ log_file = Path(self.mktemp())+ log_file.write_text("previous message\n", encoding="utf-8")++ class MySpider(scrapy.Spider):+ name = "spider"+ custom_settings = {+ "LOG_FILE": str(log_file),+ "LOG_FILE_APPEND": False,+ # disable telnet if not available to avoid an extra warning+ "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE,+ }++ configure_logging()+ get_crawler(MySpider)+ logging.debug("debug message")++ logged = log_file.read_text(encoding="utf-8")++ self.assertNotIn("previous message", logged)+ self.assertIn("debug message", logged)+++ class SpiderLoaderWithWrongInterface:def unneeded_method(self):passdiff --cc tests/test_downloader_handlers.pyindex 7feb6b9c1,8459408ff..57211d97a--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@@ -297,29 -728,29 +728,30 @@@ class Http11MockServerTestCase(unittest@defer.inlineCallbacksdef test_download_gzip_response(self):+ crawler = get_crawler(SingleRequestSpider, self.settings_dict)+ body = b"1" * 100 # PayloadResource requires body length to be 100+ request = Request(+ self.mockserver.url("/payload"),+ method="POST",+ body=body,+ meta={"download_maxsize": 50},+ )+ yield crawler.crawl(seed=request)+ failure = crawler.spider.meta["failure"]+ # download_maxsize < 100, hence the CancelledError+ self.assertIsInstance(failure.value, defer.CancelledError)- if six.PY2 and twisted_version > (12, 3, 0):-- crawler = get_crawler(SingleRequestSpider)- body = '1'*100 # PayloadResource requires body length to be 100- request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50})- yield crawler.crawl(seed=request)- failure = crawler.spider.meta['failure']- # download_maxsize < 100, hence the CancelledError- self.assertIsInstance(failure.value, defer.CancelledError)-- crawler = get_crawler(SingleRequestSpider)- request.headers.setdefault('Accept-Encoding', 'gzip,deflate')- request = request.replace(url='http://localhost:8998/xpayload')- yield crawler.crawl(seed=request)-- # download_maxsize = 50 is enough for the gzipped response- failure = crawler.spider.meta.get('failure')- self.assertTrue(failure == None)- reason = crawler.spider.meta['close_reason']- self.assertTrue(reason, 'finished')- else:- raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0 and python 2.x")+ # See issue https://twistedmatrix.com/trac/ticket/8175+ raise unittest.SkipTest("xpayload fails on PY3")++ crawler = get_crawler(SingleRequestSpider, self.settings_dict)+ request.headers.setdefault(b"Accept-Encoding", b"gzip,deflate")+ request = request.replace(url=self.mockserver.url("/xpayload"))+ yield crawler.crawl(seed=request)+ # download_maxsize = 50 is enough for the gzipped response+ failure = crawler.spider.meta.get("failure")+ self.assertIsNone(failure)+ reason = crawler.spider.meta["close_reason"]+ self.assertTrue(reason, "finished")class UriResource(resource.Resource):diff --cc tests/test_spider.pyindex 63a3d2e61,57c2b79cb..ba61d2402--- a/tests/test_spider.py+++ b/tests/test_spider.py@@@ -93,30 -86,24 +86,30 @@@ class SpiderTest(unittest.TestCase)self.assertTrue(spider.closed_called)def test_update_settings(self):- spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}- project_settings = {'TEST1': 'project', 'TEST3': 'project'}+ spider_settings = {"TEST1": "spider", "TEST2": "spider"}+ project_settings = {"TEST1": "project", "TEST3": "project"}self.spider_class.custom_settings = spider_settings- settings = Settings(project_settings, priority='project')+ settings = Settings(project_settings, priority="project")- spider = self.spider_class('example.com')- self.spider_class.update_settings(settings)++ spider = self.spider_class("example.com")+ spider.update_settings(settings)- self.assertEqual(settings.get('TEST1'), 'spider')- self.assertEqual(settings.get('TEST2'), 'spider')- self.assertEqual(settings.get('TEST3'), 'project')+ self.assertEqual(settings.get("TEST1"), "spider")+ self.assertEqual(settings.get("TEST2"), "spider")+ self.assertEqual(settings.get("TEST3"), "project")- spider_instance_settings = {'TEST1': 'spider_instance'}++ spider_instance_settings = {"TEST1": "spider_instance"}+ spider.custom_settings = spider_instance_settings+ spider.update_settings(settings)- self.assertEqual(settings.get('TEST1'), 'spider_instance')++ self.assertEqual(settings.get("TEST1"), "spider_instance")+def test_logger(self):- spider = self.spider_class('example.com')- with LogCapture() as l:- spider.logger.info('test log msg')- l.check(('example.com', 'INFO', 'test log msg'))+ spider = self.spider_class("example.com")+ with LogCapture() as lc:+ spider.logger.info("test log msg")+ lc.check(("example.com", "INFO", "test log msg"))- record = l.records[0]- self.assertIn('spider', record.__dict__)+ record = lc.records[0]+ self.assertIn("spider", record.__dict__)self.assertIs(record.spider, spider)def test_log(self):