Scrapy, a fast high-level web crawling & scraping framework for Python.
Ruff: migrate pyupgrade and bandit, enable some other rules (#6577)
Andrey Rakhmatullincde0845ab2ac
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 49db3f610..b273e269b 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -3,12 +3,6 @@ repos:rev: v0.8.1hooks:- id: ruff-- repo: https://github.com/PyCQA/bandit- rev: 1.7.9- hooks:- - id: bandit- args: ["-c", "pyproject.toml"]- additional_dependencies: ["bandit[toml]"]- repo: https://github.com/psf/black.gitrev: 24.4.2hooks:@@ -23,8 +17,3 @@ repos:- id: blacken-docsadditional_dependencies:- black==24.4.2-- repo: https://github.com/asottile/pyupgrade- rev: v3.18.0- hooks:- - id: pyupgrade- args: [--py39-plus]diff --git a/pyproject.toml b/pyproject.tomlindex 1378bab50..977792178 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -92,16 +92,6 @@ follow_imports = "skip"module = "scrapy.settings.default_settings"ignore_errors = true-[tool.bandit]-skips = [- "B101", # assert_used, needed for mypy- "B321", # ftplib, https://github.com/scrapy/scrapy/issues/4180- "B402", # import_ftplib, https://github.com/scrapy/scrapy/issues/4180- "B411", # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082- "B503", # ssl_with_bad_defaults-]-exclude_dirs = ["tests"]-[tool.bumpversion]current_version = "2.12.0"commit = true@@ -242,10 +232,30 @@ extend-select = ["C4",# pydocstyle"D",+ # flake8-future-annotations+ "FA",+ # refurb+ "FURB",+ # flake8-implicit-str-concat+ "ISC",+ # flake8-logging+ "LOG",+ # pygrep-hooks+ "PGH",+ # flake8-quotes+ "Q",+ # flake8-bandit+ "S",+ # flake8-slots+ "SLOT",# flake8-debugger"T10",# flake8-type-checking"TC",+ # pyupgrade+ "UP",+ # flake8-2020+ "YTT",]ignore = [# Assigning to `os.environ` doesn't clear the environment.@@ -296,6 +306,12 @@ ignore = ["D402",# First word of the first line should be properly capitalized"D403",+ # Use of `assert` detected; needed for mypy+ "S101",+ # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180+ "S321",+ # Argument default set to insecure SSL protocol+ "S503",][tool.ruff.lint.per-file-ignores]@@ -307,6 +323,9 @@ ignore = ["scrapy/selector/__init__.py" = ["F401"]"scrapy/spiders/__init__.py" = ["E402", "F401"]+# Skip bandit in tests+"tests/**" = ["S"]+# Issues pending a review:"docs/conf.py" = ["E402"]"scrapy/utils/url.py" = ["F403", "F405"]diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.pyindex b96c63eb7..714bc38da 100644--- a/scrapy/commands/bench.py+++ b/scrapy/commands/bench.py@@ -1,6 +1,6 @@from __future__ import annotations-import subprocess # nosec+import subprocessimport sysimport timefrom typing import TYPE_CHECKING, Any@@ -40,9 +40,9 @@ class _BenchServer:from scrapy.utils.test import get_testenvpargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]- self.proc = subprocess.Popen(+ self.proc = subprocess.Popen( # noqa: S603pargs, stdout=subprocess.PIPE, env=get_testenv()- ) # nosec+ )assert self.proc.stdoutself.proc.stdout.readline()diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.pyindex 438375e02..0e046cece 100644--- a/scrapy/commands/edit.py+++ b/scrapy/commands/edit.py@@ -41,4 +41,4 @@ class Command(ScrapyCommand):sfile = sys.modules[spidercls.__module__].__file__assert sfilesfile = sfile.replace(".pyc", ".py")- self.exitcode = os.system(f'{editor} "{sfile}"') # nosec+ self.exitcode = os.system(f'{editor} "{sfile}"') # noqa: S605diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex 2e70b2865..38f917c7e 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -118,7 +118,7 @@ class Command(ScrapyCommand):if template_file:self._genspider(module, name, url, opts.template, template_file)if opts.edit:- self.exitcode = os.system(f'scrapy edit "{name}"') # nosec+ self.exitcode = os.system(f'scrapy edit "{name}"') # noqa: S605def _generate_template_variables(self,diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 434b316e9..78dc16df6 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -52,7 +52,7 @@ class Slot:def download_delay(self) -> float:if self.randomize_delay:- return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosec+ return random.uniform(0.5 * self.delay, 1.5 * self.delay) # noqa: S311return self.delaydef close(self) -> None:diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex 9fab172a8..723fe5e93 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -115,7 +115,7 @@ def get_retry_request(return new_requeststats.inc_value(f"{stats_base_key}/max_reached")logger.error(- "Gave up retrying %(request)s (failed %(retry_times)d times): " "%(reason)s",+ "Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s",{"request": request, "retry_times": retry_times, "reason": reason},extra={"spider": spider},)diff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex 9380b7e78..b6997ef67 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -6,13 +6,13 @@ from __future__ import annotationsimport csvimport marshal-import pickle # nosec+import pickleimport pprintfrom collections.abc import Callable, Iterable, Mappingfrom io import BytesIO, TextIOWrapperfrom typing import TYPE_CHECKING, Any-from xml.sax.saxutils import XMLGenerator # nosec-from xml.sax.xmlreader import AttributesImpl # nosec+from xml.sax.saxutils import XMLGenerator+from xml.sax.xmlreader import AttributesImplfrom itemadapter import ItemAdapter, is_itemdiff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex f6415ad8e..edea7cc39 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -681,7 +681,7 @@ class FeedExporter:return Trueexcept NotConfigured as e:logger.error(- "Disabled feed storage scheme: %(scheme)s. " "Reason: %(reason)s",+ "Disabled feed storage scheme: %(scheme)s. Reason: %(reason)s",{"scheme": scheme, "reason": str(e)},)else:diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex 0edcce888..965d6434b 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport gzipimport logging-import pickle # nosec+import picklefrom email.utils import mktime_tz, parsedate_tzfrom importlib import import_modulefrom pathlib import Path@@ -309,7 +309,7 @@ class DbmCacheStorage:if 0 < self.expiration_secs < time() - float(ts):return None # expired- return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec+ return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # noqa: S301class FilesystemCacheStorage:@@ -392,7 +392,7 @@ class FilesystemCacheStorage:if 0 < self.expiration_secs < time() - mtime:return None # expiredwith self._open(metapath, "rb") as f:- return cast(dict[str, Any], pickle.load(f)) # nosec+ return cast(dict[str, Any], pickle.load(f)) # noqa: S301def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]:diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.pyindex 642919be9..7b8756572 100644--- a/scrapy/extensions/spiderstate.py+++ b/scrapy/extensions/spiderstate.py@@ -1,6 +1,6 @@from __future__ import annotations-import pickle # nosec+import picklefrom pathlib import Pathfrom typing import TYPE_CHECKING@@ -41,7 +41,7 @@ class SpiderState:def spider_opened(self, spider: Spider) -> None:if self.jobdir and Path(self.statefn).exists():with Path(self.statefn).open("rb") as f:- spider.state = pickle.load(f) # type: ignore[attr-defined] # nosec+ spider.state = pickle.load(f) # type: ignore[attr-defined] # noqa: S301else:spider.state = {} # type: ignore[attr-defined]diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.pyindex b3c3d7c7a..de3b24de0 100644--- a/scrapy/http/request/form.py+++ b/scrapy/http/request/form.py@@ -11,11 +11,13 @@ from collections.abc import Iterablefrom typing import TYPE_CHECKING, Any, Optional, Union, castfrom urllib.parse import urlencode, urljoin, urlsplit, urlunsplit-from lxml.html import FormElement # nosec-from lxml.html import InputElement # nosec-from lxml.html import MultipleSelectOptions # nosec-from lxml.html import SelectElement # nosec-from lxml.html import TextareaElement # nosec+from lxml.html import (+ FormElement,+ InputElement,+ MultipleSelectOptions,+ SelectElement,+ TextareaElement,+)from w3lib.html import strip_html5_whitespacefrom scrapy.http.request import Requestdiff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex d50388548..387805f57 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -107,7 +107,7 @@ class Response(object_ref):self._url: str = urlelse:raise TypeError(- f"{type(self).__name__} url must be str, " f"got {type(url).__name__}"+ f"{type(self).__name__} url must be str, got {type(url).__name__}")@propertydiff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex c713f6188..f954b5e9e 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -308,7 +308,7 @@ def _url_from_selector(sel: parsel.Selector) -> str:raise _InvalidSelector(f"Unsupported selector: {sel}")if sel.root.tag not in ("a", "link"):raise _InvalidSelector(- "Only <a> and <link> elements are supported; " f"got <{sel.root.tag}>"+ f"Only <a> and <link> elements are supported; got <{sel.root.tag}>")href = sel.root.get("href")if href is None:diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 192f937ce..bd96ccf19 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -12,7 +12,7 @@ from functools import partialfrom typing import TYPE_CHECKING, Any, Union, castfrom urllib.parse import urljoin, urlparse-from lxml import etree # nosec+from lxml import etreefrom parsel.csstranslator import HTMLTranslatorfrom w3lib.html import strip_html5_whitespacefrom w3lib.url import canonicalize_url, safe_url_string@@ -26,7 +26,7 @@ from scrapy.utils.url import url_has_any_extension, url_is_from_any_domainif TYPE_CHECKING:- from lxml.html import HtmlElement # nosec+ from lxml.html import HtmlElementfrom scrapy import Selectorfrom scrapy.http import TextResponsediff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex bebf6039b..16bd45c00 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -66,7 +66,7 @@ def _md5sum(file: IO[bytes]) -> str:>>> _md5sum(BytesIO(b'file content to hash'))'784406af91dd5a54fbb9c84c2236595a'"""- m = hashlib.md5() # nosec+ m = hashlib.md5() # noqa: S324while True:d = file.read(8096)if not d:@@ -399,7 +399,7 @@ class FTPFilesStore:ftp.set_pasv(False)file_path = f"{self.basedir}/{path}"last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())- m = hashlib.md5() # nosec+ m = hashlib.md5() # noqa: S324ftp.retrbinary(f"RETR {file_path}", m.update)return {"last_modified": last_modified, "checksum": m.hexdigest()}# The file doesn't exist@@ -734,7 +734,7 @@ class FilesPipeline(MediaPipeline):*,item: Any = None,) -> str:- media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec+ media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324media_ext = Path(request.url).suffix# Handles empty and wild extensions by trying to guess the# mime type then extension or default to empty string otherwisediff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex e86e7c493..29dc13f0a 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -258,7 +258,7 @@ class ImagesPipeline(FilesPipeline):*,item: Any = None,) -> str:- image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec+ image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324return f"full/{image_guid}.jpg"def thumb_path(@@ -270,5 +270,5 @@ class ImagesPipeline(FilesPipeline):*,item: Any = None,) -> str:- thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec+ thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324return f"thumbs/{thumb_id}/{thumb_guid}.jpg"diff --git a/scrapy/pqueues.py b/scrapy/pqueues.pyindex 4dea5afea..5b2f81335 100644--- a/scrapy/pqueues.py+++ b/scrapy/pqueues.py@@ -33,7 +33,7 @@ def _path_safe(text: str) -> str:pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text])# as we replace some letters we can get collision for different slots# add we add unique part- unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # nosec+ unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # noqa: S324return "-".join([pathable_slot, unique_slot])diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 7ba0128a5..89ab21fbe 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -178,7 +178,7 @@ FILES_STORE_S3_ACL = "private"FILES_STORE_GCS_ACL = ""FTP_USER = "anonymous"-FTP_PASSWORD = "guest" # nosec+FTP_PASSWORD = "guest" # noqa: S105FTP_PASSIVE_MODE = TrueGCS_PROJECT_ID = Nonediff --git a/scrapy/shell.py b/scrapy/shell.pyindex 31349c4ff..5d0ab1e4d 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -70,7 +70,7 @@ class Shell:else:self.populate_vars()if self.code:- print(eval(self.code, globals(), self.vars)) # nosec+ print(eval(self.code, globals(), self.vars)) # noqa: S307else:"""Detect interactive shell setting in scrapy.cfgdiff --git a/scrapy/squeues.py b/scrapy/squeues.pyindex 7732187fd..80bb37e93 100644--- a/scrapy/squeues.py+++ b/scrapy/squeues.py@@ -5,7 +5,7 @@ Scheduler queuesfrom __future__ import annotationsimport marshal-import pickle # nosec+import picklefrom pathlib import Pathfrom typing import TYPE_CHECKING, Anydiff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.pyindex 550516141..923ec005e 100644--- a/scrapy/utils/benchserver.py+++ b/scrapy/utils/benchserver.py@@ -15,7 +15,7 @@ class Root(Resource):def render(self, request: Request) -> bytes:total = _getarg(request, b"total", 100, int)show = _getarg(request, b"show", 10, int)- nlist = [random.randint(1, total) for _ in range(show)] # nosec+ nlist = [random.randint(1, total) for _ in range(show)] # noqa: S311request.write(b"<html><head></head><body>")assert request.args is not Noneargs = request.args.copy()diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.pyindex 1430ed8d6..1948009e8 100644--- a/scrapy/utils/engine.py+++ b/scrapy/utils/engine.py@@ -32,7 +32,7 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]:checks: list[tuple[str, Any]] = []for test in tests:try:- checks += [(test, eval(test))] # nosec+ checks += [(test, eval(test))] # noqa: S307except Exception as e:checks += [(test, f"{type(e).__name__} (exception)")]diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.pyindex ba58d939c..e8ed7b60a 100644--- a/scrapy/utils/iterators.py+++ b/scrapy/utils/iterators.py@@ -7,7 +7,7 @@ from io import StringIOfrom typing import TYPE_CHECKING, Any, Literal, cast, overloadfrom warnings import warn-from lxml import etree # nosec+from lxml import etreefrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Response, TextResponse@@ -41,10 +41,10 @@ def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]:nodename_patt = re.escape(nodename)- DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S)- HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S)- END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S)- NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S)+ DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.DOTALL)+ HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.DOTALL)+ END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.DOTALL)+ NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.DOTALL)text = _body_or_str(obj)document_header_match = re.search(DOCUMENT_HEADER_RE, text)@@ -58,7 +58,9 @@ def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]:for tagname in reversed(re.findall(END_TAG_RE, header_end)):assert header_end_idxtag = re.search(- rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S+ rf"<\s*{tagname}.*?xmlns[:=][^>]*>",+ text[: header_end_idx[1]],+ re.DOTALL,)if tag:for x in re.findall(NAMESPACE_RE, tag.group()):diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex eefadd07d..5ce4863f6 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -116,7 +116,7 @@ def md5sum(file: IO[bytes]) -> str:ScrapyDeprecationWarning,stacklevel=2,)- m = hashlib.md5() # nosec+ m = hashlib.md5() # noqa: S324while True:d = file.read(8096)if not d:diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex b9babb08f..511511301 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -136,7 +136,7 @@ def to_bytes(return textif not isinstance(text, str):raise TypeError(- "to_bytes must receive a str or bytes " f"object, got {type(text).__name__}"+ f"to_bytes must receive a str or bytes object, got {type(text).__name__}")if encoding is None:encoding = "utf-8"diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 20e3151da..ad811e804 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -94,7 +94,9 @@ def fingerprint("headers": headers,}fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)- cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() # nosec+ cache[cache_key] = hashlib.sha1( # noqa: S324+ fingerprint_json.encode()+ ).digest()return cache[cache_key]diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex 7c8ca51f2..a7ad4544d 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -104,7 +104,7 @@ def open_in_browser(elif isinstance(response, TextResponse):ext = ".txt"else:- raise TypeError("Unsupported response type: " f"{response.__class__.__name__}")+ raise TypeError(f"Unsupported response type: {response.__class__.__name__}")fd, fname = tempfile.mkstemp(ext)os.write(fd, body)os.close(fd)diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.pyindex c572580ae..b60fe929e 100644--- a/scrapy/utils/sitemap.py+++ b/scrapy/utils/sitemap.py@@ -10,7 +10,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Anyfrom urllib.parse import urljoin-import lxml.etree # nosec+import lxml.etreeif TYPE_CHECKING:from collections.abc import Iterable, Iterator@@ -24,7 +24,7 @@ class Sitemap:xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True, resolve_entities=False)- self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # nosec+ self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320rt = self._root.tagself.type = self._root.tag.split("}", 1)[1] if "}" in rt else rtdiff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex a5cc22c1c..2539f30c7 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -89,7 +89,7 @@ def escape_ajax(url: str) -> str:def add_http_if_no_scheme(url: str) -> str:"""Add http as the default scheme if it is missing from the url."""- match = re.match(r"^\w+://", url, flags=re.I)+ match = re.match(r"^\w+://", url, flags=re.IGNORECASE)if not match:parts = urlparse(url)scheme = "http:" if parts.netloc else "http://"diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.pyindex 4e9e29286..996a5cdb3 100644--- a/scrapy/utils/versions.py+++ b/scrapy/utils/versions.py@@ -3,7 +3,7 @@ import sysimport cryptographyimport cssselect-import lxml.etree # nosec+import lxml.etreeimport parselimport twistedimport w3libdiff --git a/tests/test_commands.py b/tests/test_commands.pyindex e7df7b6e8..32b69de8a 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -238,7 +238,7 @@ class StartprojectTemplatesTest(ProjectTest):args = ["--set", f"TEMPLATES_DIR={self.tmpl}"]p, out, err = self.proc("startproject", self.project_name, *args)self.assertIn(- f"New Scrapy project '{self.project_name}', " "using template directory",+ f"New Scrapy project '{self.project_name}', using template directory",out,)self.assertIn(self.tmpl_proj, out)diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 2ebc0b5e4..8d645eada 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -67,8 +67,8 @@ class TestSpider(Spider):allowed_domains = ["scrapytest.org", "localhost"]itemurl_re = re.compile(r"item\d+.html")- name_re = re.compile(r"<h1>(.*?)</h1>", re.M)- price_re = re.compile(r">Price: \$(.*?)<", re.M)+ name_re = re.compile(r"<h1>(.*?)</h1>", re.MULTILINE)+ price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE)item_cls: type = TestItemdiff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 96a2c42b7..2be5e09bc 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -631,7 +631,7 @@ class TestGCSFilesStore(unittest.TestCase):"""assert_gcs_environ()try:- import google.cloud.storage # noqa+ import google.cloud.storage # noqa: F401except ModuleNotFoundError:raise unittest.SkipTest("google-cloud-storage is not installed")with mock.patch("google.cloud.storage") as _:diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex 541979dcc..e127cc2e3 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -66,14 +66,14 @@ class BaseRobotParserTest:self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second"))def test_length_based_precedence(self):- robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"+ robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)self.assertTrue(rp.allowed("https://www.site.local/page", "*"))def test_order_based_precedence(self):- robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"+ robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.pyindex 35d1508c6..ef07d625f 100644--- a/tests/test_utils_trackref.py+++ b/tests/test_utils_trackref.py@@ -21,9 +21,9 @@ class TrackrefTestCase(unittest.TestCase):trackref.live_refs.clear()def test_format_live_refs(self):- o1 = Foo() # NOQA- o2 = Bar() # NOQA- o3 = Foo() # NOQA+ o1 = Foo() # noqa: F841+ o2 = Bar() # noqa: F841+ o3 = Foo() # noqa: F841self.assertEqual(trackref.format_live_refs(),"""\@@ -50,7 +50,7 @@ Bar 1 oldest: 0s ago@mock.patch("sys.stdout", new_callable=StringIO)def test_print_live_refs_with_objects(self, stdout):- o1 = Foo() # NOQA+ o1 = Foo() # noqa: F841trackref.print_live_refs()self.assertEqual(stdout.getvalue(),@@ -61,11 +61,11 @@ Foo 1 oldest: 0s ago\n\n""",)def test_get_oldest(self):- o1 = Foo() # NOQA+ o1 = Foo() # noqa: F841o1_time = time()- o2 = Bar() # NOQA+ o2 = Bar() # noqa: F841o3_time = time()if o3_time <= o1_time:@@ -74,15 +74,15 @@ Foo 1 oldest: 0s ago\n\n""",if o3_time <= o1_time:raise SkipTest("time.time is not precise enough")- o3 = Foo() # NOQA+ o3 = Foo() # noqa: F841self.assertIs(trackref.get_oldest("Foo"), o1)self.assertIs(trackref.get_oldest("Bar"), o2)self.assertIsNone(trackref.get_oldest("XXX"))def test_iter_all(self):- o1 = Foo() # NOQA- o2 = Bar() # NOQA- o3 = Foo() # NOQA+ o1 = Foo() # noqa: F841+ o2 = Bar() # noqa: F841+ o3 = Foo() # noqa: F841self.assertEqual(set(trackref.iter_all("Foo")),{o1, o3},diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex 1cad68b9c..0a594aa7c 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -161,7 +161,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):# test minimal sent headersfactory = client.ScrapyHTTPClientFactory(Request("http://foo/bar"))- self._test(factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"\r\n")+ self._test(factory, b"GET /bar HTTP/1.0\r\nHost: foo\r\n\r\n")# test a simple POST with body and content-typefactory = client.ScrapyHTTPClientFactory(@@ -191,7 +191,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):self._test(factory,- b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Content-Length: 0\r\n" b"\r\n",+ b"POST /bar HTTP/1.0\r\nHost: foo\r\nContent-Length: 0\r\n\r\n",)# test with single and multivalued headers