Scrapy, a fast high-level web crawling & scraping framework for Python.
Ignore SyntaxError as well when SPIDER_LOADER_WARN_ONLY is set to True (#6484)
mmoriniere46cddc6ecfbe
diff --git a/docs/news.rst b/docs/news.rstindex 758b22d80..58b51c9ea 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -8,6 +8,12 @@ Release notesScrapy VERSION (YYYY-MM-DD)---------------------------+New features+~~~~~~~~~~~~++- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``,+ ``SpiderLoader`` does not raise :exc:`SyntaxError` but emits a warning instead.+Deprecations~~~~~~~~~~~~diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 904bd7ecc..02fca7ff4 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1580,7 +1580,7 @@ SPIDER_LOADER_WARN_ONLYDefault: ``False``By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,-it will fail loudly if there is any ``ImportError`` exception.+it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception.But you can choose to silence this exception and turn it into a simplewarning by setting ``SPIDER_LOADER_WARN_ONLY = True``.diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.pyindex b8fe65668..f5fd899b2 100644--- a/scrapy/spiderloader.py+++ b/scrapy/spiderloader.py@@ -64,7 +64,7 @@ class SpiderLoader:try:for module in walk_modules(name):self._load_spiders(module)- except ImportError:+ except (ImportError, SyntaxError):if self.warn_only:warnings.warn(f"\n{traceback.format_exc()}Could not load spiders "diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.pyindex f950739f2..32699d837 100644--- a/tests/test_spiderloader/__init__.py+++ b/tests/test_spiderloader/__init__.py@@ -4,6 +4,7 @@ import tempfileimport warningsfrom pathlib import Pathfrom tempfile import mkdtemp+from unittest import mockfrom twisted.trial import unittestfrom zope.interface.verify import verifyObject@@ -136,6 +137,33 @@ class SpiderLoaderTest(unittest.TestCase):spiders = spider_loader.list()self.assertEqual(spiders, [])+ def test_syntax_error_exception(self):+ module = "tests.test_spiderloader.test_spiders.spider1"+ with mock.patch.object(SpiderLoader, "_load_spiders") as m:+ m.side_effect = SyntaxError+ settings = Settings({"SPIDER_MODULES": [module]})+ self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings)++ def test_syntax_error_warning(self):+ with warnings.catch_warnings(record=True) as w, mock.patch.object(+ SpiderLoader, "_load_spiders"+ ) as m:+ m.side_effect = SyntaxError+ module = "tests.test_spiderloader.test_spiders.spider1"+ settings = Settings(+ {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}+ )+ spider_loader = SpiderLoader.from_settings(settings)+ if str(w[0].message).startswith("_SixMetaPathImporter"):+ # needed on 3.10 because of https://github.com/benjaminp/six/issues/349,+ # at least until all six versions we can import (including botocore.vendored.six)+ # are updated to 1.16.0++ w.pop(0)+ self.assertIn("Could not load spiders from module", str(w[0].message))++ spiders = spider_loader.list()+ self.assertEqual(spiders, [])+class DuplicateSpiderNameLoaderTest(unittest.TestCase):def setUp(self):