Scrapy, a fast high-level web crawling & scraping framework for Python.
bandit: allow-list false positives
Adrián Chaves2bfd9a2257c7
diff --git a/.bandit.yml b/.bandit.ymlindex 6e8331c0f..4fcd75c57 100644--- a/.bandit.yml+++ b/.bandit.yml@@ -1,19 +1,10 @@skips:-- B101 # assert_used-- B105 # hardcoded_password_string-- B301 # pickle-- B307 # eval-- B311 # random+- B101 # assert_used, needed for mypy- B320 # xml_bad_etree- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180- B324 # hashlib "Use of weak SHA1 hash for security"- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180-- B403 # import_pickle-- B404 # import_subprocess-- B406 # import_xml_sax- B410 # import_lxml- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082- B503 # ssl_with_bad_defaults-- B603 # subprocess_without_shell_equals_true-- B605 # start_process_with_a_shellexclude_dirs: ['tests']diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.pyindex e1ccdc451..aaf5a439f 100644--- a/scrapy/commands/bench.py+++ b/scrapy/commands/bench.py@@ -1,4 +1,4 @@-import subprocess+import subprocess # nosecimport sysimport timefrom urllib.parse import urlencode@@ -29,7 +29,9 @@ class _BenchServer:from scrapy.utils.test import get_testenvpargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]- self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv())+ self.proc = subprocess.Popen(+ pargs, stdout=subprocess.PIPE, env=get_testenv()+ ) # nosecself.proc.stdout.readline()def __exit__(self, exc_type, exc_value, traceback):diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.pyindex 03a8ed5c7..e85d2c9ec 100644--- a/scrapy/commands/edit.py+++ b/scrapy/commands/edit.py@@ -37,4 +37,4 @@ class Command(ScrapyCommand):sfile = sys.modules[spidercls.__module__].__file__sfile = sfile.replace(".pyc", ".py")- self.exitcode = os.system(f'{editor} "{sfile}"')+ self.exitcode = os.system(f'{editor} "{sfile}"') # nosecdiff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex 68cbe8ff6..567ebcdc0 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -113,7 +113,7 @@ class Command(ScrapyCommand):if template_file:self._genspider(module, name, url, opts.template, template_file)if opts.edit:- self.exitcode = os.system(f'scrapy edit "{name}"')+ self.exitcode = os.system(f'scrapy edit "{name}"') # nosecdef _genspider(self, module, name, url, template_name, template_file):"""Generate the spider module, based on the given template"""diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex c84525160..666282856 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -40,7 +40,7 @@ class Slot:def download_delay(self) -> float:if self.randomize_delay:- return random.uniform(0.5 * self.delay, 1.5 * self.delay)+ return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosecreturn self.delaydef close(self) -> None:diff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex f85f1dad8..79fd4e56f 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -5,10 +5,10 @@ Item Exporters are used to export/serialize items into different formats.import csvimport ioimport marshal-import pickle+import pickle # nosecimport pprintfrom collections.abc import Mapping-from xml.sax.saxutils import XMLGenerator+from xml.sax.saxutils import XMLGenerator # nosecfrom itemadapter import ItemAdapter, is_itemdiff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex 7e4f047a8..335728502 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -1,6 +1,6 @@import gzipimport logging-import pickle+import pickle # nosecfrom email.utils import mktime_tz, parsedate_tzfrom importlib import import_modulefrom pathlib import Path@@ -274,7 +274,7 @@ class DbmCacheStorage:if 0 < self.expiration_secs < time() - float(ts):return # expired- return pickle.loads(db[f"{key}_data"])+ return pickle.loads(db[f"{key}_data"]) # nosecclass FilesystemCacheStorage:@@ -352,7 +352,7 @@ class FilesystemCacheStorage:if 0 < self.expiration_secs < time() - mtime:return # expiredwith self._open(metapath, "rb") as f:- return pickle.load(f)+ return pickle.load(f) # nosecdef parse_cachecontrol(header):diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.pyindex 929a3be70..43359401b 100644--- a/scrapy/extensions/spiderstate.py+++ b/scrapy/extensions/spiderstate.py@@ -1,4 +1,4 @@-import pickle+import pickle # nosecfrom pathlib import Pathfrom scrapy import signals@@ -31,7 +31,7 @@ class SpiderState:def spider_opened(self, spider):if self.jobdir and Path(self.statefn).exists():with Path(self.statefn).open("rb") as f:- spider.state = pickle.load(f)+ spider.state = pickle.load(f) # nosecelse:spider.state = {}diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 49ab1b5ef..2b3d95a0e 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -177,7 +177,7 @@ FILES_STORE_S3_ACL = "private"FILES_STORE_GCS_ACL = ""FTP_USER = "anonymous"-FTP_PASSWORD = "guest"+FTP_PASSWORD = "guest" # nosecFTP_PASSIVE_MODE = TrueGCS_PROJECT_ID = Nonediff --git a/scrapy/shell.py b/scrapy/shell.pyindex 05909977a..63ea33892 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -50,7 +50,7 @@ class Shell:else:self.populate_vars()if self.code:- print(eval(self.code, globals(), self.vars))+ print(eval(self.code, globals(), self.vars)) # nosecelse:"""Detect interactive shell setting in scrapy.cfgdiff --git a/scrapy/squeues.py b/scrapy/squeues.pyindex f665ad88c..e20f60f06 100644--- a/scrapy/squeues.py+++ b/scrapy/squeues.py@@ -3,7 +3,7 @@ Scheduler queues"""import marshal-import pickle+import pickle # nosecfrom os import PathLikefrom pathlib import Pathfrom typing import Uniondiff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.pyindex 38884a9f0..f6f704d4b 100644--- a/scrapy/utils/benchserver.py+++ b/scrapy/utils/benchserver.py@@ -14,7 +14,7 @@ class Root(Resource):def render(self, request):total = _getarg(request, b"total", 100, int)show = _getarg(request, b"show", 10, int)- nlist = [random.randint(1, total) for _ in range(show)]+ nlist = [random.randint(1, total) for _ in range(show)] # nosecrequest.write(b"<html><head></head><body>")args = request.args.copy()for nl in nlist:diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.pyindex a5f2a8c6e..0b2722663 100644--- a/scrapy/utils/engine.py+++ b/scrapy/utils/engine.py@@ -30,7 +30,7 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]:checks: List[Tuple[str, Any]] = []for test in tests:try:- checks += [(test, eval(test))]+ checks += [(test, eval(test))] # nosecexcept Exception as e:checks += [(test, f"{type(e).__name__} (exception)")]