Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add Perflint rules to ruff.

Andrey Rakhmatullin1e4c81e9dce7
diff --git a/pyproject.toml b/pyproject.toml
index 973d43162..b63a3631b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -240,6 +240,8 @@ extend-select = [
"ISC",
# flake8-logging
"LOG",
+ # Perflint
+ "PERF",
# pygrep-hooks
"PGH",
# flake8-pie
@@ -314,6 +316,8 @@ ignore = [
"D402",
# First word of the first line should be properly capitalized
"D403",
+ # `try`-`except` within a loop incurs performance overhead
+ "PERF203",
# Use of `assert` detected; needed for mypy
"S101",
# FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 905959c25..237bd8331 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -7,10 +7,7 @@ _T = TypeVar("_T")
async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]:
- results = []
- async for x in result:
- results.append(x)
- return results
+ return [x async for x in result]
async def as_async_generator(
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 511511301..e954b625c 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -235,8 +235,7 @@ def get_func_args(func: Callable[..., Any], stripself: bool = False) -> list[str
continue
args.append(name)
else:
- for name in sig.parameters.keys():
- args.append(name)
+ args = list(sig.parameters)
if stripself and args and args[0] == "self":
args = args[1:]
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 031d6180d..2debbe0d7 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1757,13 +1757,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
crawler = get_crawler(spider_cls, settings)
yield crawler.crawl()
- for file_path, feed_options in FEEDS.items():
+ for file_path in FEEDS:
content[str(file_path)] = (
Path(file_path).read_bytes() if Path(file_path).exists() else None
)
finally:
- for file_path in FEEDS.keys():
+ for file_path in FEEDS:
if not Path(file_path).exists():
continue
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index af3b7543d..f2a57bd88 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -289,10 +289,7 @@ class ProcessSpiderOutputNonIterableMiddleware:
class ProcessSpiderOutputCoroutineMiddleware:
async def process_spider_output(self, response, result, spider):
- results = []
- for r in result:
- results.append(r)
- return results
+ return result
class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index 9ae66c57c..8adeea5c0 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -8,9 +8,7 @@ class AsyncgenUtilsTest(unittest.TestCase):
@deferred_f_from_coro_f
async def test_as_async_generator(self):
ag = as_async_generator(range(42))
- results = []
- async for i in ag:
- results.append(i)
+ results = [i async for i in ag]
self.assertEqual(results, list(range(42)))
@deferred_f_from_coro_f
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index ec377bb19..4c81e3a2f 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -26,15 +26,14 @@ class XmliterBaseTestCase:
"""
response = XmlResponse(url="http://example.com", body=body)
- attrs = []
- for x in self.xmliter(response, "product"):
- attrs.append(
- (
- x.attrib["id"],
- x.xpath("name/text()").getall(),
- x.xpath("./type/text()").getall(),
- )
+ attrs = [
+ (
+ x.attrib["id"],
+ x.xpath("name/text()").getall(),
+ x.xpath("./type/text()").getall(),
)
+ for x in self.xmliter(response, "product")
+ ]
self.assertEqual(
attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])]
@@ -99,15 +98,14 @@ class XmliterBaseTestCase:
# Unicode body needs encoding information
XmlResponse(url="http://example.com", body=body, encoding="utf-8"),
):
- attrs = []
- for x in self.xmliter(r, "þingflokkur"):
- attrs.append(
- (
- x.attrib["id"],
- x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(),
- x.xpath("./tímabil/fyrstaþing/text()").getall(),
- )
+ attrs = [
+ (
+ x.attrib["id"],
+ x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(),
+ x.xpath("./tímabil/fyrstaþing/text()").getall(),
)
+ for x in self.xmliter(r, "þingflokkur")
+ ]
self.assertEqual(
attrs,
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index f80f2517a..83004cec4 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -58,13 +58,6 @@ class MutableAsyncChainTest(unittest.TestCase):
for i in range(5, 7):
yield i
- @staticmethod
- async def collect_asyncgen_exc(asyncgen):
- results = []
- async for x in asyncgen:
- results.append(x)
- return results
-
@deferred_f_from_coro_f
async def test_mutableasyncchain(self):
m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))