Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into change-init-order

Andrey Rakhmatullin69282c53187e
diff --cc scrapy/crawler.py
index 0754276f3,25823b6ac..ec853e0d8
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@@ -26,49 -69,72 +69,72 @@@ class Crawler
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
- self.spidercls = spidercls
- self.settings = settings.copy()
+ self.spidercls: Type[Spider] = spidercls
+ self.settings: Settings = settings.copy()
- self.spidercls.update_settings(self.settings)
+
+ self.addons: AddonManager = AddonManager(self)
+ self.addons.load_settings(self.settings)
+
+ self.signals: SignalManager = SignalManager(self)
- self.signals = SignalManager(self)
- self.stats = load_object(self.settings['STATS_CLASS'])(self)
+ self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self)
- handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))
+ handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))
logging.root.addHandler(handler)
+
+ d = dict(overridden_settings(self.settings))
+ logger.info(
+ "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
+ )
+
+ if get_scrapy_root_handler() is not None:
+ # scrapy root handler already installed: update it with new settings
+ install_scrapy_root_handler(self.settings)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
- lf_cls = load_object(self.settings['LOG_FORMATTER'])
- self.logformatter = lf_cls.from_crawler(self)
-
- self.crawling = False
- self.spider = None
- self.engine = None
-
- @property
- def spiders(self):
- if not hasattr(self, '_spiders'):
- warnings.warn("Crawler.spiders is deprecated, use "
- "CrawlerRunner.spider_loader or instantiate "
- "scrapy.spiderloader.SpiderLoader with your "
- "settings.",
- category=ScrapyDeprecationWarning, stacklevel=2)
- self._spiders = _get_spider_loader(self.settings.frozencopy())
- return self._spiders
-
- @defer.inlineCallbacks
- def crawl(self, *args, **kwargs):
- assert not self.crawling, "Crawling already taking place"
+ lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
+ self.logformatter: LogFormatter = lf_cls.from_crawler(self)
+
+ self.request_fingerprinter: RequestFingerprinter = create_instance(
+ load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
+ settings=self.settings,
+ crawler=self,
+ )
+
+ reactor_class: str = self.settings["TWISTED_REACTOR"]
+ event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]
+ if init_reactor:
+ # this needs to be done after the spider settings are merged,
+ # but before something imports twisted.internet.reactor
+ if reactor_class:
+ install_reactor(reactor_class, event_loop)
+ else:
+ from twisted.internet import reactor # noqa: F401
+ log_reactor_info()
+ if reactor_class:
+ verify_installed_reactor(reactor_class)
+ if is_asyncio_reactor_installed() and event_loop:
+ verify_installed_asyncio_event_loop(event_loop)
+
- self.extensions: ExtensionManager = ExtensionManager.from_crawler(self)
-
- self.settings.freeze()
+ self.crawling: bool = False
+ self.spider: Optional[Spider] = None
+ self.engine: Optional[ExecutionEngine] = None
+
+ @inlineCallbacks
+ def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
+ if self.crawling:
+ raise RuntimeError("Crawling already taking place")
self.crawling = True
try:
self.spider = self._create_spider(*args, **kwargs)
+ self.spider.update_settings(self.settings)
+ self.settings.freeze()
- self.extensions = ExtensionManager.from_crawler(self)
++ self.extensions: ExtensionManager = ExtensionManager.from_crawler(self)
+
self.engine = self._create_engine()
start_requests = iter(self.spider.start_requests())
yield self.engine.open_spider(self.spider, start_requests)
diff --cc scrapy/spiders/__init__.py
index 4b2771415,e16d71727..590158d1d
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@@ -65,21 -65,30 +65,29 @@@ class Spider(object_ref)
self.settings = crawler.settings
crawler.signals.connect(self.close, signals.spider_closed)
- def start_requests(self):
+ def start_requests(self) -> Iterable[Request]:
+ if not self.start_urls and hasattr(self, "start_url"):
+ raise AttributeError(
+ "Crawling could not start: 'start_urls' not found "
+ "or empty (but found 'start_url' attribute instead, "
+ "did you miss an 's'?)"
+ )
for url in self.start_urls:
- yield self.make_requests_from_url(url)
+ yield Request(url, dont_filter=True)
- def make_requests_from_url(self, url):
- return Request(url, dont_filter=True)
+ def _parse(self, response: Response, **kwargs: Any) -> Any:
+ return self.parse(response, **kwargs)
- def parse(self, response):
- raise NotImplementedError
+ def parse(self, response: Response, **kwargs: Any) -> Any:
+ raise NotImplementedError(
+ f"{self.__class__.__name__}.parse callback is not defined"
+ )
- def update_settings(self, settings):
- settings.setdict(self.custom_settings or {}, priority='spider')
- @classmethod
- def update_settings(cls, settings: BaseSettings) -> None:
- settings.setdict(cls.custom_settings or {}, priority="spider")
++ def update_settings(self, settings: BaseSettings) -> None:
++ settings.setdict(self.custom_settings or {}, priority="spider")
@classmethod
- def handles_request(cls, request):
+ def handles_request(cls, request: Request) -> bool:
return url_is_from_spider(request.url, cls)
@staticmethod
diff --cc tests/test_crawl.py
index 021849f41,ca9084294..496ab77a5
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@@ -1,20 -1,55 +1,59 @@@
import json
- import socket
import logging
+ import unittest
++import warnings
+ from ipaddress import IPv4Address
+ from socket import gethostbyname
+ from urllib.parse import urlparse
+ from pytest import mark
from testfixtures import LogCapture
from twisted.internet import defer
+ from twisted.internet.ssl import Certificate
+ from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
- from scrapy.http import Request
+ from scrapy import signals
-from scrapy.crawler import CrawlerRunner
-from scrapy.exceptions import StopDownload
+from scrapy.crawler import Crawler, CrawlerRunner
++from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
+from scrapy.extensions.throttle import AutoThrottle
+ from scrapy.http import Request
+ from scrapy.http.response import Response
+from scrapy.settings import Settings
+ from scrapy.utils.python import to_unicode
+from scrapy.utils.spider import DefaultSpider
- from tests import mock
- from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
- BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
+ from scrapy.utils.test import get_crawler
+ from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver import MockServer
+ from tests.spiders import (
+ AsyncDefAsyncioGenComplexSpider,
+ AsyncDefAsyncioGenExcSpider,
+ AsyncDefAsyncioGenLoopSpider,
+ AsyncDefAsyncioGenSpider,
+ AsyncDefAsyncioReqsReturnSpider,
+ AsyncDefAsyncioReturnSingleElementSpider,
+ AsyncDefAsyncioReturnSpider,
+ AsyncDefAsyncioSpider,
+ AsyncDefDeferredDirectSpider,
+ AsyncDefDeferredMaybeWrappedSpider,
+ AsyncDefDeferredWrappedSpider,
+ AsyncDefSpider,
+ BrokenStartRequestsSpider,
+ BytesReceivedCallbackSpider,
+ BytesReceivedErrbackSpider,
+ CrawlSpiderWithAsyncCallback,
+ CrawlSpiderWithAsyncGeneratorCallback,
+ CrawlSpiderWithErrback,
+ CrawlSpiderWithParseMethod,
+ CrawlSpiderWithProcessRequestCallbackKeywordArguments,
+ DelaySpider,
+ DuplicateStartRequestsSpider,
+ FollowAllSpider,
+ HeadersReceivedCallbackSpider,
+ HeadersReceivedErrbackSpider,
+ SimpleSpider,
+ SingleRequestSpider,
+ )
class CrawlTestCase(TestCase):
@@@ -71,9 -123,8 +127,10 @@@
self.assertTrue(crawler.spider.t2 == 0)
self.assertTrue(crawler.spider.t2_err > 0)
self.assertTrue(crawler.spider.t2_err > crawler.spider.t1)
++
# server hangs after receiving response headers
- crawler = runner.create_crawler(DelaySpider)
- yield runner.crawl(crawler, n=0.5, b=1)
++ crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
+ yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver)
self.assertTrue(crawler.spider.t1 > 0)
self.assertTrue(crawler.spider.t2 == 0)
self.assertTrue(crawler.spider.t2_err > 0)
@@@ -136,15 -195,18 +201,19 @@@
@defer.inlineCallbacks
def test_start_requests_dupes(self):
settings = {"CONCURRENT_REQUESTS": 1}
- runner = CrawlerRunner(settings)
- crawler = runner.create_crawler(DuplicateStartRequestsSpider)
- yield runner.crawl(crawler, dont_filter=True, distinct_urls=2,
- dupe_factor=3)
+ crawler = get_crawler(DuplicateStartRequestsSpider, settings)
+ yield crawler.crawl(
+ dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver
+ )
self.assertEqual(crawler.spider.visited, 6)
- crawler = runner.create_crawler(DuplicateStartRequestsSpider)
- yield runner.crawl(crawler, dont_filter=False, distinct_urls=3,
- dupe_factor=4)
++ crawler = get_crawler(DuplicateStartRequestsSpider, settings)
+ yield crawler.crawl(
+ dont_filter=False,
+ distinct_urls=3,
+ dupe_factor=4,
+ mockserver=self.mockserver,
+ )
self.assertEqual(crawler.spider.visited, 3)
@defer.inlineCallbacks
@@@ -276,29 -407,296 +414,328 @@@ with multiples line
self._assert_retried(log)
self.assertIn("Got response 200", str(log))
+ @defer.inlineCallbacks
- def test_populate_spider_settings(self):
- spider_settings = {'TEST1': 'spider', 'TEST2': 'spider',
- 'AUTOTHROTTLE_ENABLED': True}
- project_settings = {'TEST1': 'project', 'TEST3': 'project'}
++ def test_populate_spidercls_settings(self):
++ spider_settings = {
++ "TEST1": "spider",
++ "TEST2": "spider",
++ "AUTOTHROTTLE_ENABLED": True,
++ }
++ project_settings = {"TEST1": "project", "TEST3": "project"}
+
+ class CustomSettingsSpider(DefaultSpider):
+ custom_settings = spider_settings
+
+ def parse(self, response):
+ return
+
+ settings = Settings()
- settings.setdict(project_settings, priority='project')
- crawler = Crawler(CustomSettingsSpider, settings)
++ settings.setdict(project_settings, priority="project")
++ with warnings.catch_warnings():
++ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
++ crawler = Crawler(CustomSettingsSpider, settings)
+ yield crawler.crawl()
+
- self.assertEqual(crawler.settings.get('TEST1'), 'spider')
- self.assertEqual(crawler.settings.get('TEST2'), 'spider')
- self.assertEqual(crawler.settings.get('TEST3'), 'project')
++ self.assertEqual(crawler.settings.get("TEST1"), "spider")
++ self.assertEqual(crawler.settings.get("TEST2"), "spider")
++ self.assertEqual(crawler.settings.get("TEST3"), "project")
+
+ enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
+ self.assertIn(AutoThrottle, enabled_exts)
+
+ self.assertFalse(settings.frozen)
+ self.assertTrue(crawler.settings.frozen)
++
+
+ class CrawlSpiderTestCase(TestCase):
+ def setUp(self):
+ self.mockserver = MockServer()
+ self.mockserver.__enter__()
+
+ def tearDown(self):
+ self.mockserver.__exit__(None, None, None)
+
+ @defer.inlineCallbacks
+ def _run_spider(self, spider_cls):
+ items = []
+
+ def _on_item_scraped(item):
+ items.append(item)
+
+ crawler = get_crawler(spider_cls)
+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+ )
+ return log, items, crawler.stats
+
+ @defer.inlineCallbacks
+ def test_crawlspider_with_parse(self):
+ crawler = get_crawler(CrawlSpiderWithParseMethod)
+ with LogCapture() as log:
+ yield crawler.crawl(mockserver=self.mockserver)
+
+ self.assertIn("[parse] status 200 (foo: None)", str(log))
+ self.assertIn("[parse] status 201 (foo: None)", str(log))
+ self.assertIn("[parse] status 202 (foo: bar)", str(log))
+
+ @defer.inlineCallbacks
+ def test_crawlspider_with_async_callback(self):
+ crawler = get_crawler(CrawlSpiderWithAsyncCallback)
+ with LogCapture() as log:
+ yield crawler.crawl(mockserver=self.mockserver)
+
+ self.assertIn("[parse_async] status 200 (foo: None)", str(log))
+ self.assertIn("[parse_async] status 201 (foo: None)", str(log))
+ self.assertIn("[parse_async] status 202 (foo: bar)", str(log))
+
+ @defer.inlineCallbacks
+ def test_crawlspider_with_async_generator_callback(self):
+ crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback)
+ with LogCapture() as log:
+ yield crawler.crawl(mockserver=self.mockserver)
+
+ self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log))
+ self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log))
+ self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log))
+
+ @defer.inlineCallbacks
+ def test_crawlspider_with_errback(self):
+ crawler = get_crawler(CrawlSpiderWithErrback)
+ with LogCapture() as log:
+ yield crawler.crawl(mockserver=self.mockserver)
+
+ self.assertIn("[parse] status 200 (foo: None)", str(log))
+ self.assertIn("[parse] status 201 (foo: None)", str(log))
+ self.assertIn("[parse] status 202 (foo: bar)", str(log))
+ self.assertIn("[errback] status 404", str(log))
+ self.assertIn("[errback] status 500", str(log))
+ self.assertIn("[errback] status 501", str(log))
+
+ @defer.inlineCallbacks
+ def test_crawlspider_process_request_cb_kwargs(self):
+ crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments)
+ with LogCapture() as log:
+ yield crawler.crawl(mockserver=self.mockserver)
+
+ self.assertIn("[parse] status 200 (foo: process_request)", str(log))
+ self.assertIn("[parse] status 201 (foo: process_request)", str(log))
+ self.assertIn("[parse] status 202 (foo: bar)", str(log))
+
+ @defer.inlineCallbacks
+ def test_async_def_parse(self):
+ crawler = get_crawler(AsyncDefSpider)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+ )
+ self.assertIn("Got response 200", str(log))
+
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncio_parse(self):
+ crawler = get_crawler(
+ AsyncDefAsyncioSpider,
+ {
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+ },
+ )
+ with LogCapture() as log:
+ yield crawler.crawl(
+ self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+ )
+ self.assertIn("Got response 200", str(log))
+
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncio_parse_items_list(self):
+ log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider)
+ self.assertIn("Got response 200", str(log))
+ self.assertIn({"id": 1}, items)
+ self.assertIn({"id": 2}, items)
+
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncio_parse_items_single_element(self):
+ items = []
+
+ def _on_item_scraped(item):
+ items.append(item)
+
+ crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider)
+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+ )
+ self.assertIn("Got response 200", str(log))
+ self.assertIn({"foo": 42}, items)
+
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncgen_parse(self):
+ log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider)
+ self.assertIn("Got response 200", str(log))
+ itemcount = stats.get_value("item_scraped_count")
+ self.assertEqual(itemcount, 1)
+
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncgen_parse_loop(self):
+ log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider)
+ self.assertIn("Got response 200", str(log))
+ itemcount = stats.get_value("item_scraped_count")
+ self.assertEqual(itemcount, 10)
+ for i in range(10):
+ self.assertIn({"foo": i}, items)
+
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncgen_parse_exc(self):
+ log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider)
+ log = str(log)
+ self.assertIn("Spider error processing", log)
+ self.assertIn("ValueError", log)
+ itemcount = stats.get_value("item_scraped_count")
+ self.assertEqual(itemcount, 7)
+ for i in range(7):
+ self.assertIn({"foo": i}, items)
+
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncgen_parse_complex(self):
+ _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider)
+ itemcount = stats.get_value("item_scraped_count")
+ self.assertEqual(itemcount, 156)
+ # some random items
+ for i in [1, 4, 21, 22, 207, 311]:
+ self.assertIn({"index": i}, items)
+ for i in [10, 30, 122]:
+ self.assertIn({"index2": i}, items)
+
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncio_parse_reqs_list(self):
+ log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider)
+ for req_id in range(3):
+ self.assertIn(f"Got response 200, req_id {req_id}", str(log))
+
+ @mark.only_not_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_deferred_direct(self):
+ _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider)
+ self.assertEqual(items, [{"code": 200}])
+
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_deferred_wrapped(self):
+ log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider)
+ self.assertEqual(items, [{"code": 200}])
+
+ @defer.inlineCallbacks
+ def test_async_def_deferred_maybe_wrapped(self):
+ _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider)
+ self.assertEqual(items, [{"code": 200}])
+
+ @defer.inlineCallbacks
+ def test_response_ssl_certificate_none(self):
+ crawler = get_crawler(SingleRequestSpider)
+ url = self.mockserver.url("/echo?body=test", is_secure=False)
+ yield crawler.crawl(seed=url, mockserver=self.mockserver)
+ self.assertIsNone(crawler.spider.meta["responses"][0].certificate)
+
+ @defer.inlineCallbacks
+ def test_response_ssl_certificate(self):
+ crawler = get_crawler(SingleRequestSpider)
+ url = self.mockserver.url("/echo?body=test", is_secure=True)
+ yield crawler.crawl(seed=url, mockserver=self.mockserver)
+ cert = crawler.spider.meta["responses"][0].certificate
+ self.assertIsInstance(cert, Certificate)
+ self.assertEqual(cert.getSubject().commonName, b"localhost")
+ self.assertEqual(cert.getIssuer().commonName, b"localhost")
+
+ @mark.xfail(reason="Responses with no body return early and contain no certificate")
+ @defer.inlineCallbacks
+ def test_response_ssl_certificate_empty_response(self):
+ crawler = get_crawler(SingleRequestSpider)
+ url = self.mockserver.url("/status?n=200", is_secure=True)
+ yield crawler.crawl(seed=url, mockserver=self.mockserver)
+ cert = crawler.spider.meta["responses"][0].certificate
+ self.assertIsInstance(cert, Certificate)
+ self.assertEqual(cert.getSubject().commonName, b"localhost")
+ self.assertEqual(cert.getIssuer().commonName, b"localhost")
+
+ @defer.inlineCallbacks
+ def test_dns_server_ip_address_none(self):
+ crawler = get_crawler(SingleRequestSpider)
+ url = self.mockserver.url("/status?n=200")
+ yield crawler.crawl(seed=url, mockserver=self.mockserver)
+ ip_address = crawler.spider.meta["responses"][0].ip_address
+ self.assertIsNone(ip_address)
+
+ @defer.inlineCallbacks
+ def test_dns_server_ip_address(self):
+ crawler = get_crawler(SingleRequestSpider)
+ url = self.mockserver.url("/echo?body=test")
+ expected_netloc, _ = urlparse(url).netloc.split(":")
+ yield crawler.crawl(seed=url, mockserver=self.mockserver)
+ ip_address = crawler.spider.meta["responses"][0].ip_address
+ self.assertIsInstance(ip_address, IPv4Address)
+ self.assertEqual(str(ip_address), gethostbyname(expected_netloc))
+
+ @defer.inlineCallbacks
+ def test_bytes_received_stop_download_callback(self):
+ crawler = get_crawler(BytesReceivedCallbackSpider)
+ yield crawler.crawl(mockserver=self.mockserver)
+ self.assertIsNone(crawler.spider.meta.get("failure"))
+ self.assertIsInstance(crawler.spider.meta["response"], Response)
+ self.assertEqual(
+ crawler.spider.meta["response"].body,
+ crawler.spider.meta.get("bytes_received"),
+ )
+ self.assertLess(
+ len(crawler.spider.meta["response"].body),
+ crawler.spider.full_response_length,
+ )
+
+ @defer.inlineCallbacks
+ def test_bytes_received_stop_download_errback(self):
+ crawler = get_crawler(BytesReceivedErrbackSpider)
+ yield crawler.crawl(mockserver=self.mockserver)
+ self.assertIsNone(crawler.spider.meta.get("response"))
+ self.assertIsInstance(crawler.spider.meta["failure"], Failure)
+ self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)
+ self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response)
+ self.assertEqual(
+ crawler.spider.meta["failure"].value.response.body,
+ crawler.spider.meta.get("bytes_received"),
+ )
+ self.assertLess(
+ len(crawler.spider.meta["failure"].value.response.body),
+ crawler.spider.full_response_length,
+ )
+
+ @defer.inlineCallbacks
+ def test_headers_received_stop_download_callback(self):
+ crawler = get_crawler(HeadersReceivedCallbackSpider)
+ yield crawler.crawl(mockserver=self.mockserver)
+ self.assertIsNone(crawler.spider.meta.get("failure"))
+ self.assertIsInstance(crawler.spider.meta["response"], Response)
+ self.assertEqual(
+ crawler.spider.meta["response"].headers,
+ crawler.spider.meta.get("headers_received"),
+ )
+
+ @defer.inlineCallbacks
+ def test_headers_received_stop_download_errback(self):
+ crawler = get_crawler(HeadersReceivedErrbackSpider)
+ yield crawler.crawl(mockserver=self.mockserver)
+ self.assertIsNone(crawler.spider.meta.get("response"))
+ self.assertIsInstance(crawler.spider.meta["failure"], Failure)
+ self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)
+ self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response)
+ self.assertEqual(
+ crawler.spider.meta["failure"].value.response.headers,
+ crawler.spider.meta.get("headers_received"),
+ )
diff --cc tests/test_crawler.py
index 96bf0c866,4c5c48e6d..1fe865ee4
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@@ -1,12 -1,28 +1,27 @@@
+ import logging
+ import os
+ import platform
+ import subprocess
+ import sys
import warnings
- import unittest
+ from pathlib import Path
+
+ from packaging.version import parse as parse_version
+ from pytest import mark, raises
+ from twisted.internet import defer
+ from twisted.trial import unittest
+ from w3lib import __version__ as w3lib_version
import scrapy
- from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
+ from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner
+ from scrapy.exceptions import ScrapyDeprecationWarning
+ from scrapy.extensions import telnet
-from scrapy.extensions.throttle import AutoThrottle
from scrapy.settings import Settings, default_settings
from scrapy.spiderloader import SpiderLoader
+ from scrapy.utils.log import configure_logging, get_scrapy_root_handler
from scrapy.utils.spider import DefaultSpider
- from scrapy.utils.misc import load_object
+ from scrapy.utils.test import get_crawler
+ from tests.mockserver import MockServer, get_mockserver_env
class BaseCrawlerTest(unittest.TestCase):
@@@ -17,33 -32,125 +31,94 @@@
class CrawlerTestCase(BaseCrawlerTest):
- def test_populate_spidercls_settings(self):
- spider_settings = {"TEST1": "spider", "TEST2": "spider"}
- project_settings = {"TEST1": "project", "TEST3": "project"}
-
- class CustomSettingsSpider(DefaultSpider):
- custom_settings = spider_settings
-
- settings = Settings()
- settings.setdict(project_settings, priority="project")
- with warnings.catch_warnings():
- warnings.simplefilter("ignore", ScrapyDeprecationWarning)
- crawler = Crawler(CustomSettingsSpider, settings)
-
- self.assertEqual(crawler.settings.get("TEST1"), "spider")
- self.assertEqual(crawler.settings.get("TEST2"), "spider")
- self.assertEqual(crawler.settings.get("TEST3"), "project")
-
- self.assertFalse(settings.frozen)
- self.assertTrue(crawler.settings.frozen)
-
+ def test_crawler_accepts_dict(self):
+ crawler = get_crawler(DefaultSpider, {"foo": "bar"})
+ self.assertEqual(crawler.settings["foo"], "bar")
+ self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
- def setUp(self):
- self.crawler = Crawler(DefaultSpider, Settings())
+ def test_crawler_accepts_None(self):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+ crawler = Crawler(DefaultSpider)
+ self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
- def test_deprecated_attribute_spiders(self):
- with warnings.catch_warnings(record=True) as w:
- spiders = self.crawler.spiders
- self.assertEqual(len(w), 1)
- self.assertIn("Crawler.spiders", str(w[0].message))
- sl_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS'])
- self.assertIsInstance(spiders, sl_cls)
+ def test_crawler_rejects_spider_objects(self):
+ with raises(ValueError):
+ Crawler(DefaultSpider())
- self.crawler.spiders
- self.assertEqual(len(w), 1, "Warn deprecated access only once")
- def test_crawler_accepts_dict(self):
- crawler = Crawler(DefaultSpider, {'foo': 'bar'})
- self.assertEqual(crawler.settings['foo'], 'bar')
- self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
-class SpiderSettingsTestCase(unittest.TestCase):
- def test_spider_custom_settings(self):
- class MySpider(scrapy.Spider):
- name = "spider"
- custom_settings = {"AUTOTHROTTLE_ENABLED": True}
-
- crawler = get_crawler(MySpider)
- enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
- self.assertIn(AutoThrottle, enabled_exts)
-
-
+ class CrawlerLoggingTestCase(unittest.TestCase):
+ def test_no_root_handler_installed(self):
+ handler = get_scrapy_root_handler()
+ if handler is not None:
+ logging.root.removeHandler(handler)
- def test_crawler_accepts_None(self):
- crawler = Crawler(DefaultSpider)
- self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
+ class MySpider(scrapy.Spider):
+ name = "spider"
+
+ get_crawler(MySpider)
+ assert get_scrapy_root_handler() is None
+
+ def test_spider_custom_settings_log_level(self):
+ log_file = Path(self.mktemp())
+ log_file.write_text("previous message\n", encoding="utf-8")
+ class MySpider(scrapy.Spider):
+ name = "spider"
+ custom_settings = {
+ "LOG_LEVEL": "INFO",
+ "LOG_FILE": str(log_file),
+ # settings to avoid extra warnings
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+ "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE,
+ }
- class SpiderLoaderWithWrongInterface(object):
+ configure_logging()
+ self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
+ crawler = get_crawler(MySpider)
+ self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
+ info_count = crawler.stats.get_value("log_count/INFO")
+ logging.debug("debug message")
+ logging.info("info message")
+ logging.warning("warning message")
+ logging.error("error message")
+ logged = log_file.read_text(encoding="utf-8")
+
+ self.assertIn("previous message", logged)
+ self.assertNotIn("debug message", logged)
+ self.assertIn("info message", logged)
+ self.assertIn("warning message", logged)
+ self.assertIn("error message", logged)
+ self.assertEqual(crawler.stats.get_value("log_count/ERROR"), 1)
+ self.assertEqual(crawler.stats.get_value("log_count/WARNING"), 1)
+ self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1)
+ self.assertEqual(crawler.stats.get_value("log_count/DEBUG", 0), 0)
+
+ def test_spider_custom_settings_log_append(self):
+ log_file = Path(self.mktemp())
+ log_file.write_text("previous message\n", encoding="utf-8")
+
+ class MySpider(scrapy.Spider):
+ name = "spider"
+ custom_settings = {
+ "LOG_FILE": str(log_file),
+ "LOG_FILE_APPEND": False,
+ # disable telnet if not available to avoid an extra warning
+ "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE,
+ }
+
+ configure_logging()
+ get_crawler(MySpider)
+ logging.debug("debug message")
+
+ logged = log_file.read_text(encoding="utf-8")
+
+ self.assertNotIn("previous message", logged)
+ self.assertIn("debug message", logged)
+
+
+ class SpiderLoaderWithWrongInterface:
def unneeded_method(self):
pass
diff --cc tests/test_downloader_handlers.py
index 7feb6b9c1,8459408ff..57211d97a
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@@ -297,29 -728,29 +728,30 @@@ class Http11MockServerTestCase(unittest
@defer.inlineCallbacks
def test_download_gzip_response(self):
+ crawler = get_crawler(SingleRequestSpider, self.settings_dict)
+ body = b"1" * 100 # PayloadResource requires body length to be 100
+ request = Request(
+ self.mockserver.url("/payload"),
+ method="POST",
+ body=body,
+ meta={"download_maxsize": 50},
+ )
+ yield crawler.crawl(seed=request)
+ failure = crawler.spider.meta["failure"]
+ # download_maxsize < 100, hence the CancelledError
+ self.assertIsInstance(failure.value, defer.CancelledError)
- if six.PY2 and twisted_version > (12, 3, 0):
-
- crawler = get_crawler(SingleRequestSpider)
- body = '1'*100 # PayloadResource requires body length to be 100
- request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50})
- yield crawler.crawl(seed=request)
- failure = crawler.spider.meta['failure']
- # download_maxsize < 100, hence the CancelledError
- self.assertIsInstance(failure.value, defer.CancelledError)
-
- crawler = get_crawler(SingleRequestSpider)
- request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
- request = request.replace(url='http://localhost:8998/xpayload')
- yield crawler.crawl(seed=request)
-
- # download_maxsize = 50 is enough for the gzipped response
- failure = crawler.spider.meta.get('failure')
- self.assertTrue(failure == None)
- reason = crawler.spider.meta['close_reason']
- self.assertTrue(reason, 'finished')
- else:
- raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0 and python 2.x")
+ # See issue https://twistedmatrix.com/trac/ticket/8175
+ raise unittest.SkipTest("xpayload fails on PY3")
++ crawler = get_crawler(SingleRequestSpider, self.settings_dict)
+ request.headers.setdefault(b"Accept-Encoding", b"gzip,deflate")
+ request = request.replace(url=self.mockserver.url("/xpayload"))
+ yield crawler.crawl(seed=request)
+ # download_maxsize = 50 is enough for the gzipped response
+ failure = crawler.spider.meta.get("failure")
+ self.assertIsNone(failure)
+ reason = crawler.spider.meta["close_reason"]
+ self.assertTrue(reason, "finished")
class UriResource(resource.Resource):
diff --cc tests/test_spider.py
index 63a3d2e61,57c2b79cb..ba61d2402
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@@ -93,30 -86,24 +86,30 @@@ class SpiderTest(unittest.TestCase)
self.assertTrue(spider.closed_called)
def test_update_settings(self):
- spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
- project_settings = {'TEST1': 'project', 'TEST3': 'project'}
+ spider_settings = {"TEST1": "spider", "TEST2": "spider"}
+ project_settings = {"TEST1": "project", "TEST3": "project"}
self.spider_class.custom_settings = spider_settings
- settings = Settings(project_settings, priority='project')
+ settings = Settings(project_settings, priority="project")
- spider = self.spider_class('example.com')
- self.spider_class.update_settings(settings)
++ spider = self.spider_class("example.com")
+ spider.update_settings(settings)
- self.assertEqual(settings.get('TEST1'), 'spider')
- self.assertEqual(settings.get('TEST2'), 'spider')
- self.assertEqual(settings.get('TEST3'), 'project')
+ self.assertEqual(settings.get("TEST1"), "spider")
+ self.assertEqual(settings.get("TEST2"), "spider")
+ self.assertEqual(settings.get("TEST3"), "project")
- spider_instance_settings = {'TEST1': 'spider_instance'}
++ spider_instance_settings = {"TEST1": "spider_instance"}
+ spider.custom_settings = spider_instance_settings
+ spider.update_settings(settings)
- self.assertEqual(settings.get('TEST1'), 'spider_instance')
++ self.assertEqual(settings.get("TEST1"), "spider_instance")
+
def test_logger(self):
- spider = self.spider_class('example.com')
- with LogCapture() as l:
- spider.logger.info('test log msg')
- l.check(('example.com', 'INFO', 'test log msg'))
+ spider = self.spider_class("example.com")
+ with LogCapture() as lc:
+ spider.logger.info("test log msg")
+ lc.check(("example.com", "INFO", "test log msg"))
- record = l.records[0]
- self.assertIn('spider', record.__dict__)
+ record = lc.records[0]
+ self.assertIn("spider", record.__dict__)
self.assertIs(record.spider, spider)
def test_log(self):