Scrapy, a fast high-level web crawling & scraping framework for Python.
Syntax Error Fixed (#6738)
Mehraz Hossain Rumman3ca882fba867
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex ca0af569f..a59a61050 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -2047,6 +2047,21 @@ also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddlewareif :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` andthere is no overriding User-Agent header specified for the request.+.. setting:: WARN_ON_GENERATOR_RETURN_VALUE++WARN_ON_GENERATOR_RETURN_VALUE+------------------------------++Default: ``True``++When enabled, Scrapy will warn if generator-based callback methods (like+``parse``) contain return statements with non-``None`` values. This helps detect+potential mistakes in spider development.++Disable this setting to prevent syntax errors that may occur when dynamically+modifying generator function source code during runtime, skip AST parsing of+callback functions, or improve performance in auto-reloading development+environments.Settings documented elsewhere:------------------------------diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 645e50301..680fded7a 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -351,3 +351,5 @@ SPIDER_CONTRACTS_BASE = {"scrapy.contracts.default.ReturnsContract": 2,"scrapy.contracts.default.ScrapesContract": 3,}++WARN_ON_GENERATOR_RETURN_VALUE = Truediff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex d319e7950..b7b436260 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -286,6 +286,8 @@ def warn_on_generator_with_return_value(Logs a warning if a callable is a generator function and includesa 'return' statement with a value different than None"""+ if not spider.settings.getbool("WARN_ON_GENERATOR_RETURN_VALUE"):+ returntry:if is_generator_with_return_value(callable):warnings.warn(diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.pyindex 81a83c3d7..ad31e5185 100644--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py@@ -2,6 +2,8 @@ import warningsfrom functools import partialfrom unittest import mock+import pytest+from scrapy.utils.misc import (is_generator_with_return_value,warn_on_generator_with_return_value,@@ -40,7 +42,24 @@ def generator_that_returns_stuff():class TestUtilsMisc:- def test_generators_return_something(self):+ @pytest.fixture+ def mock_spider(self):+ class MockSettings:+ def __init__(self, settings_dict=None):+ self.settings_dict = settings_dict or {+ "WARN_ON_GENERATOR_RETURN_VALUE": True+ }++ def getbool(self, name, default=False):+ return self.settings_dict.get(name, default)++ class MockSpider:+ def __init__(self):+ self.settings = MockSettings()++ return MockSpider()++ def test_generators_return_something(self, mock_spider):def f1():yield 1return 2@@ -75,30 +94,30 @@ https://example.orgassert is_generator_with_return_value(i1)with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, top_level_return_something)+ warn_on_generator_with_return_value(mock_spider, top_level_return_something)assert len(w) == 1assert (- 'The "NoneType.top_level_return_something" method is a generator'+ 'The "MockSpider.top_level_return_something" method is a generator'in str(w[0].message))with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, f1)+ warn_on_generator_with_return_value(mock_spider, f1)assert len(w) == 1- assert 'The "NoneType.f1" method is a generator' in str(w[0].message)+ assert 'The "MockSpider.f1" method is a generator' in str(w[0].message)with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, g1)+ warn_on_generator_with_return_value(mock_spider, g1)assert len(w) == 1- assert 'The "NoneType.g1" method is a generator' in str(w[0].message)+ assert 'The "MockSpider.g1" method is a generator' in str(w[0].message)with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, h1)+ warn_on_generator_with_return_value(mock_spider, h1)assert len(w) == 1- assert 'The "NoneType.h1" method is a generator' in str(w[0].message)+ assert 'The "MockSpider.h1" method is a generator' in str(w[0].message)with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, i1)+ warn_on_generator_with_return_value(mock_spider, i1)assert len(w) == 1- assert 'The "NoneType.i1" method is a generator' in str(w[0].message)+ assert 'The "MockSpider.i1" method is a generator' in str(w[0].message)- def test_generators_return_none(self):+ def test_generators_return_none(self, mock_spider):def f2():yield 1@@ -142,31 +161,31 @@ https://example.orgassert not is_generator_with_return_value(l2)with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, top_level_return_none)+ warn_on_generator_with_return_value(mock_spider, top_level_return_none)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, f2)+ warn_on_generator_with_return_value(mock_spider, f2)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, g2)+ warn_on_generator_with_return_value(mock_spider, g2)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, h2)+ warn_on_generator_with_return_value(mock_spider, h2)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, i2)+ warn_on_generator_with_return_value(mock_spider, i2)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, j2)+ warn_on_generator_with_return_value(mock_spider, j2)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, k2)+ warn_on_generator_with_return_value(mock_spider, k2)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, l2)+ warn_on_generator_with_return_value(mock_spider, l2)assert len(w) == 0- def test_generators_return_none_with_decorator(self):+ def test_generators_return_none_with_decorator(self, mock_spider):def decorator(func):def inner_func():func()@@ -223,36 +242,36 @@ https://example.orgassert not is_generator_with_return_value(l3)with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, top_level_return_none)+ warn_on_generator_with_return_value(mock_spider, top_level_return_none)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, f3)+ warn_on_generator_with_return_value(mock_spider, f3)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, g3)+ warn_on_generator_with_return_value(mock_spider, g3)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, h3)+ warn_on_generator_with_return_value(mock_spider, h3)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, i3)+ warn_on_generator_with_return_value(mock_spider, i3)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, j3)+ warn_on_generator_with_return_value(mock_spider, j3)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, k3)+ warn_on_generator_with_return_value(mock_spider, k3)assert len(w) == 0with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, l3)+ warn_on_generator_with_return_value(mock_spider, l3)assert len(w) == 0@mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error)- def test_indentation_error(self):+ def test_indentation_error(self, mock_spider):with warnings.catch_warnings(record=True) as w:- warn_on_generator_with_return_value(None, top_level_return_none)+ warn_on_generator_with_return_value(mock_spider, top_level_return_none)assert len(w) == 1assert "Unable to determine" in str(w[0].message)@@ -262,3 +281,32 @@ https://example.orgpartial_cb = partial(cb, arg1=42)assert not is_generator_with_return_value(partial_cb)++ def test_warn_on_generator_with_return_value_settings_disabled(self):+ class MockSettings:+ def __init__(self, settings_dict=None):+ self.settings_dict = settings_dict or {}++ def getbool(self, name, default=False):+ return self.settings_dict.get(name, default)++ class MockSpider:+ def __init__(self):+ self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False})++ spider = MockSpider()++ def gen_with_return():+ yield 1+ return "value"++ with warnings.catch_warnings(record=True) as w:+ warn_on_generator_with_return_value(spider, gen_with_return)+ assert len(w) == 0++ spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True++ with warnings.catch_warnings(record=True) as w:+ warn_on_generator_with_return_value(spider, gen_with_return)+ assert len(w) == 1+ assert "is a generator" in str(w[0].message)