Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Ignore SyntaxError as well when SPIDER_LOADER_WARN_ONLY is set to True (#6484)

mmoriniere46cddc6ecfbe
diff --git a/docs/news.rst b/docs/news.rst
index 758b22d80..58b51c9ea 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -8,6 +8,12 @@ Release notes
Scrapy VERSION (YYYY-MM-DD)
---------------------------
+New features
+~~~~~~~~~~~~
+
+- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``,
+ ``SpiderLoader`` does not raise :exc:`SyntaxError` but emits a warning instead.
+
Deprecations
~~~~~~~~~~~~
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 904bd7ecc..02fca7ff4 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1580,7 +1580,7 @@ SPIDER_LOADER_WARN_ONLY
Default: ``False``
By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
-it will fail loudly if there is any ``ImportError`` exception.
+it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception.
But you can choose to silence this exception and turn it into a simple
warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index b8fe65668..f5fd899b2 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -64,7 +64,7 @@ class SpiderLoader:
try:
for module in walk_modules(name):
self._load_spiders(module)
- except ImportError:
+ except (ImportError, SyntaxError):
if self.warn_only:
warnings.warn(
f"\n{traceback.format_exc()}Could not load spiders "
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index f950739f2..32699d837 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -4,6 +4,7 @@ import tempfile
import warnings
from pathlib import Path
from tempfile import mkdtemp
+from unittest import mock
from twisted.trial import unittest
from zope.interface.verify import verifyObject
@@ -136,6 +137,33 @@ class SpiderLoaderTest(unittest.TestCase):
spiders = spider_loader.list()
self.assertEqual(spiders, [])
+ def test_syntax_error_exception(self):
+ module = "tests.test_spiderloader.test_spiders.spider1"
+ with mock.patch.object(SpiderLoader, "_load_spiders") as m:
+ m.side_effect = SyntaxError
+ settings = Settings({"SPIDER_MODULES": [module]})
+ self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings)
+
+ def test_syntax_error_warning(self):
+ with warnings.catch_warnings(record=True) as w, mock.patch.object(
+ SpiderLoader, "_load_spiders"
+ ) as m:
+ m.side_effect = SyntaxError
+ module = "tests.test_spiderloader.test_spiders.spider1"
+ settings = Settings(
+ {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
+ )
+ spider_loader = SpiderLoader.from_settings(settings)
+ if str(w[0].message).startswith("_SixMetaPathImporter"):
+ # needed on 3.10 because of https://github.com/benjaminp/six/issues/349,
+ # at least until all six versions we can import (including botocore.vendored.six)
+ # are updated to 1.16.0+
+ w.pop(0)
+ self.assertIn("Could not load spiders from module", str(w[0].message))
+
+ spiders = spider_loader.list()
+ self.assertEqual(spiders, [])
+
class DuplicateSpiderNameLoaderTest(unittest.TestCase):
def setUp(self):