Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve get_func_args (#5885)

Serhii Ac2a31974ffc0
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index fc50e0f12..818fa5d6b 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -174,33 +174,33 @@ def binary_is_text(data):
def get_func_args(func, stripself=False):
- """Return the argument name list of a callable"""
- if inspect.isfunction(func):
- spec = inspect.getfullargspec(func)
- func_args = spec.args + spec.kwonlyargs
- elif inspect.isclass(func):
- return get_func_args(func.__init__, True)
- elif inspect.ismethod(func):
- return get_func_args(func.__func__, True)
- elif inspect.ismethoddescriptor(func):
- return []
- elif isinstance(func, partial):
- return [
- x
- for x in get_func_args(func.func)[len(func.args) :]
- if not (func.keywords and x in func.keywords)
- ]
- elif hasattr(func, "__call__"):
- if inspect.isroutine(func):
- return []
- if getattr(func, "__name__", None) == "__call__":
- return []
- return get_func_args(func.__call__, True)
+ """Return the argument name list of a callable object"""
+ if not callable(func):
+ raise TypeError(f"func must be callable, got '{type(func).__name__}'")
+
+ args = []
+ try:
+ sig = inspect.signature(func)
+ except ValueError:
+ return args
+
+ if isinstance(func, partial):
+ partial_args = func.args
+ partial_kw = func.keywords
+
+ for name, param in sig.parameters.items():
+ if param.name in partial_args:
+ continue
+ if partial_kw and param.name in partial_kw:
+ continue
+ args.append(name)
else:
- raise TypeError(f"{type(func)} is not callable")
- if stripself:
- func_args.pop(0)
- return func_args
+ for name in sig.parameters.keys():
+ args.append(name)
+
+ if stripself and args and args[0] == "self":
+ args = args[1:]
+ return args
def get_spec(func):
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 57f40c2e5..80d2e8da1 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -235,20 +235,16 @@ class UtilsPythonTestCase(unittest.TestCase):
self.assertEqual(get_func_args(partial_f3), ["c"])
self.assertEqual(get_func_args(cal), ["a", "b", "c"])
self.assertEqual(get_func_args(object), [])
+ self.assertEqual(get_func_args(str.split, stripself=True), ["sep", "maxsplit"])
+ self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"])
if platform.python_implementation() == "CPython":
- # TODO: how do we fix this to return the actual argument names?
- self.assertEqual(get_func_args(str.split), [])
- self.assertEqual(get_func_args(" ".join), [])
+ # doesn't work on CPython: https://bugs.python.org/issue42785
self.assertEqual(get_func_args(operator.itemgetter(2)), [])
elif platform.python_implementation() == "PyPy":
- self.assertEqual(
- get_func_args(str.split, stripself=True), ["sep", "maxsplit"]
- )
self.assertEqual(
get_func_args(operator.itemgetter(2), stripself=True), ["obj"]
)
- self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"])
def test_without_none_values(self):
self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])