Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

bandit: allow-list false positives

Adrián Chaves2bfd9a2257c7
diff --git a/.bandit.yml b/.bandit.yml
index 6e8331c0f..4fcd75c57 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -1,19 +1,10 @@
skips:
-- B101 # assert_used
-- B105 # hardcoded_password_string
-- B301 # pickle
-- B307 # eval
-- B311 # random
+- B101 # assert_used, needed for mypy
- B320 # xml_bad_etree
- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180
- B324 # hashlib "Use of weak SHA1 hash for security"
- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
-- B403 # import_pickle
-- B404 # import_subprocess
-- B406 # import_xml_sax
- B410 # import_lxml
- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
- B503 # ssl_with_bad_defaults
-- B603 # subprocess_without_shell_equals_true
-- B605 # start_process_with_a_shell
exclude_dirs: ['tests']
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index e1ccdc451..aaf5a439f 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,4 +1,4 @@
-import subprocess
+import subprocess # nosec
import sys
import time
from urllib.parse import urlencode
@@ -29,7 +29,9 @@ class _BenchServer:
from scrapy.utils.test import get_testenv
pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
- self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv())
+ self.proc = subprocess.Popen(
+ pargs, stdout=subprocess.PIPE, env=get_testenv()
+ ) # nosec
self.proc.stdout.readline()
def __exit__(self, exc_type, exc_value, traceback):
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index 03a8ed5c7..e85d2c9ec 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -37,4 +37,4 @@ class Command(ScrapyCommand):
sfile = sys.modules[spidercls.__module__].__file__
sfile = sfile.replace(".pyc", ".py")
- self.exitcode = os.system(f'{editor} "{sfile}"')
+ self.exitcode = os.system(f'{editor} "{sfile}"') # nosec
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 68cbe8ff6..567ebcdc0 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -113,7 +113,7 @@ class Command(ScrapyCommand):
if template_file:
self._genspider(module, name, url, opts.template, template_file)
if opts.edit:
- self.exitcode = os.system(f'scrapy edit "{name}"')
+ self.exitcode = os.system(f'scrapy edit "{name}"') # nosec
def _genspider(self, module, name, url, template_name, template_file):
"""Generate the spider module, based on the given template"""
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index c84525160..666282856 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -40,7 +40,7 @@ class Slot:
def download_delay(self) -> float:
if self.randomize_delay:
- return random.uniform(0.5 * self.delay, 1.5 * self.delay)
+ return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosec
return self.delay
def close(self) -> None:
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index f85f1dad8..79fd4e56f 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -5,10 +5,10 @@ Item Exporters are used to export/serialize items into different formats.
import csv
import io
import marshal
-import pickle
+import pickle # nosec
import pprint
from collections.abc import Mapping
-from xml.sax.saxutils import XMLGenerator
+from xml.sax.saxutils import XMLGenerator # nosec
from itemadapter import ItemAdapter, is_item
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 7e4f047a8..335728502 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,6 +1,6 @@
import gzip
import logging
-import pickle
+import pickle # nosec
from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from pathlib import Path
@@ -274,7 +274,7 @@ class DbmCacheStorage:
if 0 < self.expiration_secs < time() - float(ts):
return # expired
- return pickle.loads(db[f"{key}_data"])
+ return pickle.loads(db[f"{key}_data"]) # nosec
class FilesystemCacheStorage:
@@ -352,7 +352,7 @@ class FilesystemCacheStorage:
if 0 < self.expiration_secs < time() - mtime:
return # expired
with self._open(metapath, "rb") as f:
- return pickle.load(f)
+ return pickle.load(f) # nosec
def parse_cachecontrol(header):
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 929a3be70..43359401b 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -1,4 +1,4 @@
-import pickle
+import pickle # nosec
from pathlib import Path
from scrapy import signals
@@ -31,7 +31,7 @@ class SpiderState:
def spider_opened(self, spider):
if self.jobdir and Path(self.statefn).exists():
with Path(self.statefn).open("rb") as f:
- spider.state = pickle.load(f)
+ spider.state = pickle.load(f) # nosec
else:
spider.state = {}
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 49ab1b5ef..2b3d95a0e 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -177,7 +177,7 @@ FILES_STORE_S3_ACL = "private"
FILES_STORE_GCS_ACL = ""
FTP_USER = "anonymous"
-FTP_PASSWORD = "guest"
+FTP_PASSWORD = "guest" # nosec
FTP_PASSIVE_MODE = True
GCS_PROJECT_ID = None
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 05909977a..63ea33892 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -50,7 +50,7 @@ class Shell:
else:
self.populate_vars()
if self.code:
- print(eval(self.code, globals(), self.vars))
+ print(eval(self.code, globals(), self.vars)) # nosec
else:
"""
Detect interactive shell setting in scrapy.cfg
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index f665ad88c..e20f60f06 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -3,7 +3,7 @@ Scheduler queues
"""
import marshal
-import pickle
+import pickle # nosec
from os import PathLike
from pathlib import Path
from typing import Union
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 38884a9f0..f6f704d4b 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -14,7 +14,7 @@ class Root(Resource):
def render(self, request):
total = _getarg(request, b"total", 100, int)
show = _getarg(request, b"show", 10, int)
- nlist = [random.randint(1, total) for _ in range(show)]
+ nlist = [random.randint(1, total) for _ in range(show)] # nosec
request.write(b"<html><head></head><body>")
args = request.args.copy()
for nl in nlist:
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index a5f2a8c6e..0b2722663 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -30,7 +30,7 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]:
checks: List[Tuple[str, Any]] = []
for test in tests:
try:
- checks += [(test, eval(test))]
+ checks += [(test, eval(test))] # nosec
except Exception as e:
checks += [(test, f"{type(e).__name__} (exception)")]