Scrapy, a fast high-level web crawling & scraping framework for Python.
Smarter generator check for combined return/yield statements (#4721)
Grisha Temchenkod68aab992e43
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex d6966be8e..bd400bd30 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -5,6 +5,7 @@ import osimport reimport hashlibimport warnings+from collections import dequefrom contextlib import contextmanagerfrom importlib import import_modulefrom pkgutil import iter_modules@@ -184,6 +185,22 @@ def set_environ(**kwargs):os.environ[k] = v+def walk_callable(node):+ """Similar to ``ast.walk``, but walks only function body and skips nested+ functions defined within the node.+ """+ todo = deque([node])+ walked_func_def = False+ while todo:+ node = todo.popleft()+ if isinstance(node, ast.FunctionDef):+ if walked_func_def:+ continue+ walked_func_def = True+ todo.extend(ast.iter_child_nodes(node))+ yield node++_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)@@ -201,7 +218,7 @@ def is_generator_with_return_value(callable):if inspect.isgeneratorfunction(callable):tree = ast.parse(dedent(inspect.getsource(callable)))- for node in ast.walk(tree):+ for node in walk_callable(tree):if isinstance(node, ast.Return) and not returns_none(node):_generator_callbacks_cache[callable] = Truereturn _generator_callbacks_cache[callable]diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.pyindex bdbec1beb..2be38620c 100644--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py@@ -29,9 +29,28 @@ class UtilsMiscPy3TestCase(unittest.TestCase):yield 1yield from g()+ def m():+ yield 1++ def helper():+ return 0++ yield helper()++ def n():+ yield 1++ def helper():+ return 0++ yield helper()+ return 2+assert is_generator_with_return_value(f)assert is_generator_with_return_value(g)assert not is_generator_with_return_value(h)assert not is_generator_with_return_value(i)assert not is_generator_with_return_value(j)assert not is_generator_with_return_value(k) # not recursive+ assert not is_generator_with_return_value(m)+ assert is_generator_with_return_value(n)