Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

flake8-bugbear

Andrey Rakhmatullin3d8dbd564840
diff --git a/.flake8 b/.flake8
index 222ba7179..57117d2cf 100644
--- a/.flake8
+++ b/.flake8
@@ -4,6 +4,21 @@ max-line-length = 119
ignore =
# black disagrees with flake8 about these
E203, E501, E701, E704, W503
+ # Assigning to `os.environ` doesn't clear the environment.
+ B003
+ # Do not use mutable data structures for argument defaults.
+ B006
+ # Loop control variable not used within the loop body.
+ B007
+ # Do not perform function calls in argument defaults.
+ B008
+ # return/continue/break inside finally blocks cause exceptions to be
+ # silenced.
+ B012
+ # Star-arg unpacking after a keyword argument is strongly discouraged
+ B026
+ # No explicit stacklevel argument found.
+ B028
# docstring does contain unindexed parameters
P102
# other string does contain unindexed parameters
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6b60eff68..f70effc5d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -9,6 +9,7 @@ repos:
hooks:
- id: flake8
additional_dependencies:
+ - flake8-bugbear
- flake8-comprehensions
- flake8-debugger
- flake8-docstrings
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 3e327105e..09e95cf5d 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -234,7 +234,7 @@ class MediaPipeline(ABC):
# Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
context = getattr(result.value, "__context__", None)
if isinstance(context, StopIteration):
- setattr(result.value, "__context__", None)
+ result.value.__context__ = None
info.downloading.remove(fp)
info.downloaded[fp] = result # cache result
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index ddb68c86b..877eb4388 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -407,7 +407,7 @@ def maybeDeferred_coro(
"""Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
try:
result = f(*args, **kw)
- except: # noqa: E722
+ except: # noqa: E722,B001
return defer.fail(failure.Failure(captureVars=Deferred.debug))
if isinstance(result, Deferred):
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 059d8e04d..f56950fdd 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -269,7 +269,7 @@ def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]:
if inspect.isfunction(func) or inspect.ismethod(func):
spec = inspect.getfullargspec(func)
- elif hasattr(func, "__call__"):
+ elif hasattr(func, "__call__"): # noqa: B004
spec = inspect.getfullargspec(func.__call__)
else:
raise TypeError(f"{type(func)} is not callable")
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index 89cfbd2ec..bb6d807ee 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -100,7 +100,10 @@ def send_catch_log_deferred(
d.addErrback(logerror, receiver)
# TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html
d.addBoth(
- lambda result: (receiver, result) # pylint: disable=cell-var-from-loop
+ lambda result: (
+ receiver, # pylint: disable=cell-var-from-loop # noqa: B023
+ result,
+ )
)
dfds.append(d)
d = DeferredList(dfds)
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 25ded143c..4835e936b 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -20,7 +20,7 @@ class CmdlineTest(unittest.TestCase):
self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"
def _execute(self, *new_args, **kwargs):
- encoding = getattr(sys.stdout, "encoding") or "utf-8"
+ encoding = sys.stdout.encoding or "utf-8"
args = (sys.executable, "-m", "scrapy.cmdline") + new_args
proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs)
comm = proc.communicate()[0].strip()
diff --git a/tests/test_command_version.py b/tests/test_command_version.py
index a52d0d13c..18c1c531c 100644
--- a/tests/test_command_version.py
+++ b/tests/test_command_version.py
@@ -12,7 +12,7 @@ class VersionTest(ProcessTest, unittest.TestCase):
@defer.inlineCallbacks
def test_output(self):
- encoding = getattr(sys.stdout, "encoding") or "utf-8"
+ encoding = sys.stdout.encoding or "utf-8"
_, out, _ = yield self.execute([])
self.assertEqual(
out.strip().decode(encoding),
@@ -21,7 +21,7 @@ class VersionTest(ProcessTest, unittest.TestCase):
@defer.inlineCallbacks
def test_verbose_output(self):
- encoding = getattr(sys.stdout, "encoding") or "utf-8"
+ encoding = sys.stdout.encoding or "utf-8"
_, out, _ = yield self.execute(["-v"])
headers = [
line.partition(":")[0].strip()
diff --git a/tests/test_commands.py b/tests/test_commands.py
index d829b1701..a23b7f4a9 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -101,7 +101,7 @@ class ProjectTest(unittest.TestCase):
def kill_proc():
p.kill()
p.communicate()
- assert False, "Command took too much time to complete"
+ raise AssertionError("Command took too much time to complete")
timer = Timer(15, kill_proc)
try:
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index d3fd63847..884491d01 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -892,7 +892,7 @@ class S3TestCase(unittest.TestCase):
except Exception as e:
self.assertIsInstance(e, (TypeError, NotConfigured))
else:
- assert False
+ raise AssertionError()
def test_request_signing1(self):
# gets an object from the johnsmith bucket.
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 33544e8db..86526420f 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -459,7 +459,7 @@ class EngineTest(unittest.TestCase):
def kill_proc():
p.kill()
p.communicate()
- assert False, "Command took too much time to complete"
+ raise AssertionError("Command took too much time to complete")
timer = Timer(15, kill_proc)
try:
diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py
index 7312eb036..d3f416347 100644
--- a/tests/test_request_dict.py
+++ b/tests/test_request_dict.py
@@ -147,7 +147,7 @@ class RequestSerializationTest(unittest.TestCase):
spider = MySpider()
r = Request("http://www.example.com", callback=spider.parse)
- setattr(spider, "parse", None)
+ spider.parse = None
self.assertRaises(ValueError, r.to_dict, spider=spider)
def test_callback_not_available(self):