Scrapy, a fast high-level web crawling & scraping framework for Python.
Refactoring of test_utils_*. (#6905)
Andrey Rakhmatullind70f8a3f1425
diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.pyindex 988ad50b9..1edef0b4a 100644--- a/tests/test_command_startproject.py+++ b/tests/test_command_startproject.py@@ -106,8 +106,6 @@ def get_permissions_dict(class TestStartprojectTemplates(TestProjectBase):- maxDiff = None-def setup_method(self):super().setup_method()self.tmpl = str(Path(self.temp_path, "templates"))diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex 9d2e5f997..1714bd4db 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -150,8 +150,6 @@ class KeywordArgumentsSpider(MockServerSpider):class TestCallbackKeywordArguments(TestCase):- maxDiff = None-@classmethoddef setUpClass(cls):cls.mockserver = MockServer()diff --git a/tests/test_toplevel.py b/tests/test_toplevel.pyindex a4f31096e..66a6f5318 100644--- a/tests/test_toplevel.py+++ b/tests/test_toplevel.py@@ -1,31 +1,35 @@import scrapy-class TestToplevel:- def test_version(self):- assert isinstance(scrapy.__version__, str)+def test_version():+ assert isinstance(scrapy.__version__, str)- def test_version_info(self):- assert isinstance(scrapy.version_info, tuple)- def test_request_shortcut(self):- from scrapy.http import FormRequest, Request+def test_version_info():+ assert isinstance(scrapy.version_info, tuple)- assert scrapy.Request is Request- assert scrapy.FormRequest is FormRequest- def test_spider_shortcut(self):- from scrapy.spiders import Spider+def test_request_shortcut():+ from scrapy.http import FormRequest, Request- assert scrapy.Spider is Spider+ assert scrapy.Request is Request+ assert scrapy.FormRequest is FormRequest- def test_selector_shortcut(self):- from scrapy.selector import Selector- assert scrapy.Selector is Selector+def test_spider_shortcut():+ from scrapy.spiders import Spider- def test_item_shortcut(self):- from scrapy.item import Field, Item+ assert scrapy.Spider is Spider- assert scrapy.Item is Item- assert scrapy.Field is Field++def test_selector_shortcut():+ from scrapy.selector import Selector++ assert scrapy.Selector is Selector+++def test_item_shortcut():+ from scrapy.item import Field, Item++ assert scrapy.Item is Item+ assert scrapy.Field is Fielddiff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.pydeleted file mode 100644index 0e1e89e81..000000000--- a/tests/test_urlparse_monkeypatches.py+++ /dev/null@@ -1,10 +0,0 @@-from urllib.parse import urlparse---class TestUrlparse:- def test_s3_url(self):- p = urlparse("s3://bucket/key/name?param=value")- assert p.scheme == "s3"- assert p.hostname == "bucket"- assert p.path == "/key/name"- assert p.query == "param=value"diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex 26f158380..ed7dda18d 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -47,12 +47,11 @@ class TestBuildComponentList:assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"]-class TestUtilsConf:- def test_arglist_to_dict(self):- assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == {- "arg1": "val1",- "arg2": "val2",- }+def test_arglist_to_dict():+ assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == {+ "arg1": "val1",+ "arg2": "val2",+ }class TestFeedExportConfig:diff --git a/tests/test_utils_console.py b/tests/test_utils_console.pyindex 6598bdce7..dc1d96f66 100644--- a/tests/test_utils_console.py+++ b/tests/test_utils_console.py@@ -18,23 +18,24 @@ except ImportError:ipy = False-class TestUtilsConsole:- def test_get_shell_embed_func(self):- shell = get_shell_embed_func(["invalid"])- assert shell is None-- shell = get_shell_embed_func(["invalid", "python"])- assert callable(shell)- assert shell.__name__ == "_embed_standard_shell"-- @pytest.mark.skipif(not bpy, reason="bpython not available in testenv")- def test_get_shell_embed_func2(self):- shell = get_shell_embed_func(["bpython"])- assert callable(shell)- assert shell.__name__ == "_embed_bpython_shell"-- @pytest.mark.skipif(not ipy, reason="IPython not available in testenv")- def test_get_shell_embed_func3(self):- # default shell should be 'ipython'- shell = get_shell_embed_func()- assert shell.__name__ == "_embed_ipython_shell"+def test_get_shell_embed_func():+ shell = get_shell_embed_func(["invalid"])+ assert shell is None++ shell = get_shell_embed_func(["invalid", "python"])+ assert callable(shell)+ assert shell.__name__ == "_embed_standard_shell"++[email protected](not bpy, reason="bpython not available in testenv")+def test_get_shell_embed_func_bpython():+ shell = get_shell_embed_func(["bpython"])+ assert callable(shell)+ assert shell.__name__ == "_embed_bpython_shell"++[email protected](not ipy, reason="IPython not available in testenv")+def test_get_shell_embed_func_ipython():+ # default shell should be 'ipython'+ shell = get_shell_embed_func()+ assert shell.__name__ == "_embed_ipython_shell"diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.pyindex e8dd88049..02362693a 100644--- a/tests/test_utils_curl.py+++ b/tests/test_utils_curl.py@@ -1,4 +1,5 @@import warnings+from typing import Anyimport pytestfrom w3lib.http import basic_auth_header@@ -8,9 +9,8 @@ from scrapy.utils.curl import curl_to_request_kwargsclass TestCurlToRequestKwargs:- maxDiff = 5000-- def _test_command(self, curl_command, expected_result):+ @staticmethod+ def _test_command(curl_command: str, expected_result: dict[str, Any]) -> None:result = curl_to_request_kwargs(curl_command)assert result == expected_resulttry:diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.pyindex 75b6b0e99..352e49165 100644--- a/tests/test_utils_datatypes.py+++ b/tests/test_utils_datatypes.py@@ -1,5 +1,6 @@import copyimport warnings+from abc import ABC, abstractmethodfrom collections.abc import Iterator, Mapping, MutableMappingimport pytest@@ -16,7 +17,12 @@ from scrapy.utils.datatypes import (from scrapy.utils.python import garbage_collect-class CaseInsensitiveDictBase:+class TestCaseInsensitiveDictBase(ABC):+ @property+ @abstractmethod+ def dict_class(self) -> type[MutableMapping]:+ raise NotImplementedError+def test_init_dict(self):seq = {"red": 1, "black": 3}d = self.dict_class(seq)@@ -199,7 +205,7 @@ class CaseInsensitiveDictBase:assert h1.get("header1") == h3.get("HEADER1")-class TestCaseInsensitiveDict(CaseInsensitiveDictBase):+class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):dict_class = CaseInsensitiveDictdef test_repr(self):@@ -216,7 +222,7 @@ class TestCaseInsensitiveDict(CaseInsensitiveDictBase):@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")-class TestCaselessDict(CaseInsensitiveDictBase):+class TestCaselessDict(TestCaseInsensitiveDictBase):dict_class = CaselessDictdef test_deprecation_message(self):diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.pyindex 662de0dc3..a88b5e008 100644--- a/tests/test_utils_deprecate.py+++ b/tests/test_utils_deprecate.py@@ -1,6 +1,7 @@import inspectimport warningsfrom unittest import mock+from warnings import WarningMessageimport pytest@@ -21,7 +22,9 @@ class NewName(SomeBaseClass):class TestWarnWhenSubclassed:- def _mywarnings(self, w, category=MyWarning):+ def _mywarnings(+ self, w: list[WarningMessage], category: type[Warning] = MyWarning+ ) -> list[WarningMessage]:return [x for x in w if x.category is MyWarning]def test_no_warning_on_definition(self):diff --git a/tests/test_utils_display.py b/tests/test_utils_display.pyindex cea564653..20251ca59 100644--- a/tests/test_utils_display.py+++ b/tests/test_utils_display.py@@ -3,88 +3,92 @@ from unittest import mockfrom scrapy.utils.display import pformat, pprint--class TestDisplay:- object = {"a": 1}- colorized_strings = {+value = {"a": 1}+colorized_strings = {+ ((- (- "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"- "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}"- )- + suffix- )- for suffix in (- # https://github.com/pygments/pygments/issues/2313- "\n", # pygments ≤ 2.13- "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14+ "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"+ "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}")- }- plain_string = "{'a': 1}"-- @mock.patch("sys.platform", "linux")- @mock.patch("sys.stdout.isatty")- def test_pformat(self, isatty):- isatty.return_value = True- assert pformat(self.object) in self.colorized_strings-- @mock.patch("sys.stdout.isatty")- def test_pformat_dont_colorize(self, isatty):- isatty.return_value = True- assert pformat(self.object, colorize=False) == self.plain_string-- def test_pformat_not_tty(self):- assert pformat(self.object) == self.plain_string-- @mock.patch("sys.platform", "win32")- @mock.patch("platform.version")- @mock.patch("sys.stdout.isatty")- def test_pformat_old_windows(self, isatty, version):- isatty.return_value = True- version.return_value = "10.0.14392"- assert pformat(self.object) in self.colorized_strings-- @mock.patch("sys.platform", "win32")- @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")- @mock.patch("platform.version")- @mock.patch("sys.stdout.isatty")- def test_pformat_windows_no_terminal_processing(- self, isatty, version, terminal_processing- ):- isatty.return_value = True- version.return_value = "10.0.14393"- terminal_processing.return_value = False- assert pformat(self.object) == self.plain_string-- @mock.patch("sys.platform", "win32")- @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")- @mock.patch("platform.version")- @mock.patch("sys.stdout.isatty")- def test_pformat_windows(self, isatty, version, terminal_processing):- isatty.return_value = True- version.return_value = "10.0.14393"- terminal_processing.return_value = True- assert pformat(self.object) in self.colorized_strings-- @mock.patch("sys.platform", "linux")- @mock.patch("sys.stdout.isatty")- def test_pformat_no_pygments(self, isatty):- isatty.return_value = True-- import builtins-- real_import = builtins.__import__-- def mock_import(name, globals, locals, fromlist, level):- if "pygments" in name:- raise ImportError- return real_import(name, globals, locals, fromlist, level)-- builtins.__import__ = mock_import- assert pformat(self.object) == self.plain_string- builtins.__import__ = real_import-- def test_pprint(self):- with mock.patch("sys.stdout", new=StringIO()) as mock_out:- pprint(self.object)- assert mock_out.getvalue() == "{'a': 1}\n"+ + suffix+ )+ for suffix in (+ # https://github.com/pygments/pygments/issues/2313+ "\n", # pygments ≤ 2.13+ "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14+ )+}+plain_string = "{'a': 1}"++[email protected]("sys.platform", "linux")[email protected]("sys.stdout.isatty")+def test_pformat(isatty):+ isatty.return_value = True+ assert pformat(value) in colorized_strings++[email protected]("sys.stdout.isatty")+def test_pformat_dont_colorize(isatty):+ isatty.return_value = True+ assert pformat(value, colorize=False) == plain_string+++def test_pformat_not_tty():+ assert pformat(value) == plain_string++[email protected]("sys.platform", "win32")[email protected]("platform.version")[email protected]("sys.stdout.isatty")+def test_pformat_old_windows(isatty, version):+ isatty.return_value = True+ version.return_value = "10.0.14392"+ assert pformat(value) in colorized_strings++[email protected]("sys.platform", "win32")[email protected]("scrapy.utils.display._enable_windows_terminal_processing")[email protected]("platform.version")[email protected]("sys.stdout.isatty")+def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing):+ isatty.return_value = True+ version.return_value = "10.0.14393"+ terminal_processing.return_value = False+ assert pformat(value) == plain_string++[email protected]("sys.platform", "win32")[email protected]("scrapy.utils.display._enable_windows_terminal_processing")[email protected]("platform.version")[email protected]("sys.stdout.isatty")+def test_pformat_windows(isatty, version, terminal_processing):+ isatty.return_value = True+ version.return_value = "10.0.14393"+ terminal_processing.return_value = True+ assert pformat(value) in colorized_strings++[email protected]("sys.platform", "linux")[email protected]("sys.stdout.isatty")+def test_pformat_no_pygments(isatty):+ isatty.return_value = True++ import builtins++ real_import = builtins.__import__++ def mock_import(name, globals, locals, fromlist, level):+ if "pygments" in name:+ raise ImportError+ return real_import(name, globals, locals, fromlist, level)++ builtins.__import__ = mock_import+ assert pformat(value) == plain_string+ builtins.__import__ = real_import+++def test_pprint():+ with mock.patch("sys.stdout", new=StringIO()) as mock_out:+ pprint(value)+ assert mock_out.getvalue() == "{'a': 1}\n"diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.pyindex c43ed152b..06fdf9cba 100644--- a/tests/test_utils_gz.py+++ b/tests/test_utils_gz.py@@ -11,47 +11,51 @@ from tests import tests_datadirSAMPLEDIR = Path(tests_datadir, "compressed")-class TestGunzip:- def test_gunzip_basic(self):- r1 = Response(- "http://www.example.com",- body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(),- )- assert gzip_magic_number(r1)-- r2 = Response("http://www.example.com", body=gunzip(r1.body))- assert not gzip_magic_number(r2)- assert len(r2.body) == 9950-- def test_gunzip_truncated(self):- text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes())- assert text.endswith(b"</html")-- def test_gunzip_no_gzip_file_raises(self):- with pytest.raises(BadGzipFile):- gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes())-- def test_gunzip_truncated_short(self):- r1 = Response(- "http://www.example.com",- body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(),- )- assert gzip_magic_number(r1)-- r2 = Response("http://www.example.com", body=gunzip(r1.body))- assert r2.body.endswith(b"</html>")- assert not gzip_magic_number(r2)-- def test_is_gzipped_empty(self):- r1 = Response("http://www.example.com")- assert not gzip_magic_number(r1)-- def test_gunzip_illegal_eof(self):- text = html_to_unicode(- "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes())- )[1]- expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text(- encoding="utf-8"- )- assert len(text) == len(expected_text)- assert text == expected_text+def test_gunzip_basic():+ r1 = Response(+ "http://www.example.com",+ body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(),+ )+ assert gzip_magic_number(r1)++ r2 = Response("http://www.example.com", body=gunzip(r1.body))+ assert not gzip_magic_number(r2)+ assert len(r2.body) == 9950+++def test_gunzip_truncated():+ text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes())+ assert text.endswith(b"</html")+++def test_gunzip_no_gzip_file_raises():+ with pytest.raises(BadGzipFile):+ gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes())+++def test_gunzip_truncated_short():+ r1 = Response(+ "http://www.example.com",+ body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(),+ )+ assert gzip_magic_number(r1)++ r2 = Response("http://www.example.com", body=gunzip(r1.body))+ assert r2.body.endswith(b"</html>")+ assert not gzip_magic_number(r2)+++def test_is_gzipped_empty():+ r1 = Response("http://www.example.com")+ assert not gzip_magic_number(r1)+++def test_gunzip_illegal_eof():+ text = html_to_unicode(+ "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes())+ )[1]+ expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text(+ encoding="utf-8"+ )+ assert len(text) == len(expected_text)+ assert text == expected_textdiff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.pyindex 0c05ef7d6..9bd86f7fb 100644--- a/tests/test_utils_httpobj.py+++ b/tests/test_utils_httpobj.py@@ -4,18 +4,17 @@ from scrapy.http import Requestfrom scrapy.utils.httpobj import urlparse_cached-class TestHttpobjUtils:- def test_urlparse_cached(self):- url = "http://www.example.com/index.html"- request1 = Request(url)- request2 = Request(url)- req1a = urlparse_cached(request1)- req1b = urlparse_cached(request1)- req2 = urlparse_cached(request2)- urlp = urlparse(url)+def test_urlparse_cached():+ url = "http://www.example.com/index.html"+ request1 = Request(url)+ request2 = Request(url)+ req1a = urlparse_cached(request1)+ req1b = urlparse_cached(request1)+ req2 = urlparse_cached(request2)+ urlp = urlparse(url)- assert req1a == req2- assert req1a == urlp- assert req1a is req1b- assert req1a is not req2- assert req1a is not req2+ assert req1a == req2+ assert req1a == urlp+ assert req1a is req1b+ assert req1a is not req2+ assert req1a is not req2diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.pyindex fa0d37866..ac32fff2c 100644--- a/tests/test_utils_iterators.py+++ b/tests/test_utils_iterators.py@@ -1,3 +1,8 @@+from __future__ import annotations++from abc import ABC, abstractmethod+from typing import TYPE_CHECKING, Any+import pytestfrom scrapy.exceptions import ScrapyDeprecationWarning@@ -5,9 +10,19 @@ from scrapy.http import Response, TextResponse, XmlResponsefrom scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxmlfrom tests import get_testdata+if TYPE_CHECKING:+ from collections.abc import Iterator++ from scrapy import Selector+++class TestXmliterBase(ABC):+ @abstractmethod+ def xmliter(+ self, obj: Response | str | bytes, nodename: str, *args: Any+ ) -> Iterator[Selector]:+ raise NotImplementedError-class XmliterBase:- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_xmliter(self):body = b"""<?xml version="1.0" encoding="UTF-8"?>@@ -39,7 +54,6 @@ class XmliterBase:("002", ["Name 2"], ["Type 2"]),]- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_xmliter_unusual_node(self):body = b"""<?xml version="1.0" encoding="UTF-8"?><root>@@ -53,7 +67,6 @@ class XmliterBase:]assert nodenames == [["matchme..."]]- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_xmliter_unicode(self):# example taken from https://github.com/scrapy/scrapy/issues/1665body = """<?xml version="1.0" encoding="UTF-8"?>@@ -113,7 +126,6 @@ class XmliterBase:("27", ["A"], ["27"]),]- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_xmliter_text(self):body = ('<?xml version="1.0" encoding="UTF-8"?>'@@ -125,7 +137,6 @@ class XmliterBase:["two"],]- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_xmliter_namespaces(self):body = b"""<?xml version="1.0" encoding="UTF-8"?>@@ -163,7 +174,6 @@ class XmliterBase:assert node.xpath("id/text()").getall() == []assert node.xpath("price/text()").getall() == []- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_xmliter_namespaced_nodename(self):body = b"""<?xml version="1.0" encoding="UTF-8"?>@@ -191,7 +201,6 @@ class XmliterBase:"http://www.mydummycompany.com/images/item1.jpg"]- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_xmliter_namespaced_nodename_missing(self):body = b"""<?xml version="1.0" encoding="UTF-8"?>@@ -216,7 +225,6 @@ class XmliterBase:with pytest.raises(StopIteration):next(my_iter)- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_xmliter_exception(self):body = ('<?xml version="1.0" encoding="UTF-8"?>'@@ -229,13 +237,11 @@ class XmliterBase:with pytest.raises(StopIteration):next(iter)- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_xmliter_objtype_exception(self):i = self.xmliter(42, "product")with pytest.raises(TypeError):next(i)- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_xmliter_encoding(self):body = (b'<?xml version="1.0" encoding="ISO-8859-9"?>\n'@@ -250,8 +256,12 @@ class XmliterBase:)-class TestXmliter(XmliterBase):- xmliter = staticmethod(xmliter)[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")+class TestXmliter(TestXmliterBase):+ def xmliter(+ self, obj: Response | str | bytes, nodename: str, *args: Any+ ) -> Iterator[Selector]:+ return xmliter(obj, nodename)def test_deprecation(self):body = b"""@@ -267,8 +277,11 @@ class TestXmliter(XmliterBase):next(self.xmliter(body, "product"))-class TestLxmlXmliter(XmliterBase):- xmliter = staticmethod(xmliter_lxml)+class TestLxmlXmliter(TestXmliterBase):+ def xmliter(+ self, obj: Response | str | bytes, nodename: str, *args: Any+ ) -> Iterator[Selector]:+ return xmliter_lxml(obj, nodename, *args)def test_xmliter_iterate_namespace(self):body = b"""@@ -493,23 +506,32 @@ class TestUtilsCsv:]-class TestHelper:+class TestBodyOrStr:bbody = b"utf8-body"ubody = bbody.decode("utf8")- txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8")- response = Response(url="http://example.org/", body=bbody)-- def test_body_or_str(self):- for obj in (self.bbody, self.ubody, self.txtresponse, self.response):- r1 = _body_or_str(obj)- self._assert_type_and_value(r1, self.ubody, obj)- r2 = _body_or_str(obj, unicode=True)- self._assert_type_and_value(r2, self.ubody, obj)- r3 = _body_or_str(obj, unicode=False)- self._assert_type_and_value(r3, self.bbody, obj)- assert type(r1) is type(r2)- assert type(r1) is not type(r3)-- def _assert_type_and_value(self, a, b, obj):++ @pytest.mark.parametrize(+ "obj",+ [+ bbody,+ ubody,+ TextResponse(url="http://example.org/", body=bbody, encoding="utf-8"),+ Response(url="http://example.org/", body=bbody),+ ],+ )+ def test_body_or_str(self, obj: Response | str | bytes) -> None:+ r1 = _body_or_str(obj)+ self._assert_type_and_value(r1, self.ubody, obj)+ r2 = _body_or_str(obj, unicode=True)+ self._assert_type_and_value(r2, self.ubody, obj)+ r3 = _body_or_str(obj, unicode=False)+ self._assert_type_and_value(r3, self.bbody, obj)+ assert type(r1) is type(r2)+ assert type(r1) is not type(r3)++ @staticmethod+ def _assert_type_and_value(+ a: str | bytes, b: str | bytes, obj: Response | str | bytes+ ) -> None:assert type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}"assert a == bdiff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex 56375606c..f40e424ff 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -22,7 +22,9 @@ from scrapy.utils.test import get_crawlerfrom tests.spiders import LogSpiderif TYPE_CHECKING:- from collections.abc import Mapping, MutableMapping+ from collections.abc import Generator, Mapping, MutableMapping++ from scrapy.crawler import Crawlerclass TestFailureToExcInfo:@@ -70,33 +72,41 @@ class TestTopLevelFormatter:class TestLogCounterHandler:- def setup_method(self):+ @pytest.fixture+ def crawler(self) -> Crawler:settings = {"LOG_LEVEL": "WARNING"}- self.logger = logging.getLogger("test")- self.logger.setLevel(logging.NOTSET)- self.logger.propagate = False- self.crawler = get_crawler(settings_dict=settings)- self.handler = LogCounterHandler(self.crawler)- self.logger.addHandler(self.handler)+ return get_crawler(settings_dict=settings)++ @pytest.fixture+ def logger(self, crawler: Crawler) -> Generator[logging.Logger]:+ logger = logging.getLogger("test")+ logger.setLevel(logging.NOTSET)+ logger.propagate = False+ handler = LogCounterHandler(crawler)+ logger.addHandler(handler)++ yield logger- def teardown_method(self):- self.logger.propagate = True- self.logger.removeHandler(self.handler)+ logger.propagate = True+ logger.removeHandler(handler)- def test_init(self):- assert self.crawler.stats.get_value("log_count/DEBUG") is None- assert self.crawler.stats.get_value("log_count/INFO") is None- assert self.crawler.stats.get_value("log_count/WARNING") is None- assert self.crawler.stats.get_value("log_count/ERROR") is None- assert self.crawler.stats.get_value("log_count/CRITICAL") is None+ def test_init(self, crawler: Crawler, logger: logging.Logger) -> None:+ assert crawler.stats+ assert crawler.stats.get_value("log_count/DEBUG") is None+ assert crawler.stats.get_value("log_count/INFO") is None+ assert crawler.stats.get_value("log_count/WARNING") is None+ assert crawler.stats.get_value("log_count/ERROR") is None+ assert crawler.stats.get_value("log_count/CRITICAL") is None- def test_accepted_level(self):- self.logger.error("test log msg")- assert self.crawler.stats.get_value("log_count/ERROR") == 1+ def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None:+ logger.error("test log msg")+ assert crawler.stats+ assert crawler.stats.get_value("log_count/ERROR") == 1- def test_filtered_out_level(self):- self.logger.debug("test log msg")- assert self.crawler.stats.get_value("log_count/INFO") is None+ def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None:+ logger.debug("test log msg")+ assert crawler.stats+ assert crawler.stats.get_value("log_count/INFO") is Noneclass TestStreamLogger:@@ -135,7 +145,7 @@ class TestStreamLogger:)def test_spider_logger_adapter_process(base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict-):+) -> None:logger = logging.getLogger("test")spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra)@@ -149,59 +159,75 @@ def test_spider_logger_adapter_process(class TestLogging:- def setup_method(self):- self.log_stream = StringIO()- handler = logging.StreamHandler(self.log_stream)+ @pytest.fixture+ def log_stream(self) -> StringIO:+ return StringIO()++ @pytest.fixture+ def spider(self) -> LogSpider:+ return LogSpider()++ @pytest.fixture(autouse=True)+ def logger(self, log_stream: StringIO) -> Generator[logging.Logger]:+ handler = logging.StreamHandler(log_stream)logger = logging.getLogger("log_spider")logger.addHandler(handler)logger.setLevel(logging.DEBUG)- self.handler = handler- self.logger = logger- self.spider = LogSpider()- def teardown_method(self):- self.logger.removeHandler(self.handler)+ yield logger++ logger.removeHandler(handler)- def test_debug_logging(self):+ def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:log_message = "Foo message"- self.spider.log_debug(log_message)- log_contents = self.log_stream.getvalue()+ spider.log_debug(log_message)+ log_contents = log_stream.getvalue()assert log_contents == f"{log_message}\n"- def test_info_logging(self):+ def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None:log_message = "Bar message"- self.spider.log_info(log_message)- log_contents = self.log_stream.getvalue()+ spider.log_info(log_message)+ log_contents = log_stream.getvalue()assert log_contents == f"{log_message}\n"- def test_warning_logging(self):+ def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None:log_message = "Baz message"- self.spider.log_warning(log_message)- log_contents = self.log_stream.getvalue()+ spider.log_warning(log_message)+ log_contents = log_stream.getvalue()assert log_contents == f"{log_message}\n"- def test_error_logging(self):+ def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None:log_message = "Foo bar message"- self.spider.log_error(log_message)- log_contents = self.log_stream.getvalue()+ spider.log_error(log_message)+ log_contents = log_stream.getvalue()assert log_contents == f"{log_message}\n"- def test_critical_logging(self):+ def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None:log_message = "Foo bar baz message"- self.spider.log_critical(log_message)- log_contents = self.log_stream.getvalue()+ spider.log_critical(log_message)+ log_contents = log_stream.getvalue()assert log_contents == f"{log_message}\n"class TestLoggingWithExtra:- def setup_method(self):- self.log_stream = StringIO()- handler = logging.StreamHandler(self.log_stream)+ regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$")++ @pytest.fixture+ def log_stream(self) -> StringIO:+ return StringIO()++ @pytest.fixture+ def spider(self) -> LogSpider:+ return LogSpider()++ @pytest.fixture(autouse=True)+ def logger(self, log_stream: StringIO) -> Generator[logging.Logger]:+ handler = logging.StreamHandler(log_stream)formatter = logging.Formatter('{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}')@@ -209,80 +235,79 @@ class TestLoggingWithExtra:logger = logging.getLogger("log_spider")logger.addHandler(handler)logger.setLevel(logging.DEBUG)- self.handler = handler- self.logger = logger- self.spider = LogSpider()- self.regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$")- def teardown_method(self):- self.logger.removeHandler(self.handler)+ yield logger++ logger.removeHandler(handler)- def test_debug_logging(self):+ def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:log_message = "Foo message"extra = {"important_info": "foo"}- self.spider.log_debug(log_message, extra)- log_contents = self.log_stream.getvalue()- log_contents = json.loads(log_contents)+ spider.log_debug(log_message, extra)+ log_contents_str = log_stream.getvalue()+ log_contents = json.loads(log_contents_str)assert log_contents["levelname"] == "DEBUG"assert log_contents["message"] == log_messageassert self.regex_pattern.match(log_contents["spider"])assert log_contents["important_info"] == extra["important_info"]- def test_info_logging(self):+ def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None:log_message = "Bar message"extra = {"important_info": "bar"}- self.spider.log_info(log_message, extra)- log_contents = self.log_stream.getvalue()- log_contents = json.loads(log_contents)+ spider.log_info(log_message, extra)+ log_contents_str = log_stream.getvalue()+ log_contents = json.loads(log_contents_str)assert log_contents["levelname"] == "INFO"assert log_contents["message"] == log_messageassert self.regex_pattern.match(log_contents["spider"])assert log_contents["important_info"] == extra["important_info"]- def test_warning_logging(self):+ def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None:log_message = "Baz message"extra = {"important_info": "baz"}- self.spider.log_warning(log_message, extra)- log_contents = self.log_stream.getvalue()- log_contents = json.loads(log_contents)+ spider.log_warning(log_message, extra)+ log_contents_str = log_stream.getvalue()+ log_contents = json.loads(log_contents_str)assert log_contents["levelname"] == "WARNING"assert log_contents["message"] == log_messageassert self.regex_pattern.match(log_contents["spider"])assert log_contents["important_info"] == extra["important_info"]- def test_error_logging(self):+ def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None:log_message = "Foo bar message"extra = {"important_info": "foo bar"}- self.spider.log_error(log_message, extra)- log_contents = self.log_stream.getvalue()- log_contents = json.loads(log_contents)+ spider.log_error(log_message, extra)+ log_contents_str = log_stream.getvalue()+ log_contents = json.loads(log_contents_str)assert log_contents["levelname"] == "ERROR"assert log_contents["message"] == log_messageassert self.regex_pattern.match(log_contents["spider"])assert log_contents["important_info"] == extra["important_info"]- def test_critical_logging(self):+ def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None:log_message = "Foo bar baz message"extra = {"important_info": "foo bar baz"}- self.spider.log_critical(log_message, extra)- log_contents = self.log_stream.getvalue()- log_contents = json.loads(log_contents)+ spider.log_critical(log_message, extra)+ log_contents_str = log_stream.getvalue()+ log_contents = json.loads(log_contents_str)assert log_contents["levelname"] == "CRITICAL"assert log_contents["message"] == log_messageassert self.regex_pattern.match(log_contents["spider"])assert log_contents["important_info"] == extra["important_info"]- def test_overwrite_spider_extra(self):+ def test_overwrite_spider_extra(+ self, log_stream: StringIO, spider: LogSpider+ ) -> None:log_message = "Foo message"extra = {"important_info": "foo", "spider": "shouldn't change"}- self.spider.log_error(log_message, extra)- log_contents = self.log_stream.getvalue()- log_contents = json.loads(log_contents)+ spider.log_error(log_message, extra)+ log_contents_str = log_stream.getvalue()+ log_contents = json.loads(log_contents_str)assert log_contents["levelname"] == "ERROR"assert log_contents["message"] == log_messagediff --git a/tests/test_utils_project.py b/tests/test_utils_project.pyindex aa250be69..20a3d940c 100644--- a/tests/test_utils_project.py+++ b/tests/test_utils_project.py@@ -1,18 +1,17 @@-import contextlibimport os-import shutil-import tempfileimport warningsfrom pathlib import Path+import pytest+from scrapy.utils.misc import set_environfrom scrapy.utils.project import data_path, get_project_settings-def inside_a_project():+def proj_path(tmp_path):prev_dir = Path.cwd()- project_dir = tempfile.mkdtemp()+ project_dir = tmp_pathtry:os.chdir(project_dir)@@ -21,21 +20,19 @@ def inside_a_project():yield project_dirfinally:os.chdir(prev_dir)- shutil.rmtree(project_dir)-class TestProjectUtils:- def test_data_path_outside_project(self):- assert str(Path(".scrapy", "somepath")) == data_path("somepath")- abspath = str(Path(os.path.sep, "absolute", "path"))- assert abspath == data_path(abspath)+def test_data_path_outside_project():+ assert str(Path(".scrapy", "somepath")) == data_path("somepath")+ abspath = str(Path(os.path.sep, "absolute", "path"))+ assert abspath == data_path(abspath)+- def test_data_path_inside_project(self):- with inside_a_project() as proj_path:- expected = Path(proj_path, ".scrapy", "somepath")- assert expected.resolve() == Path(data_path("somepath")).resolve()- abspath = str(Path(os.path.sep, "absolute", "path").resolve())- assert abspath == data_path(abspath)+def test_data_path_inside_project(proj_path: Path) -> None:+ expected = proj_path / ".scrapy" / "somepath"+ assert expected.resolve() == Path(data_path("somepath")).resolve()+ abspath = str(Path(os.path.sep, "absolute", "path").resolve())+ assert abspath == data_path(abspath)class TestGetProjectSettings:diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex 291646ad7..c933e0ac9 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -1,7 +1,10 @@+from __future__ import annotations+import functoolsimport operatorimport platformimport sys+from typing import TYPE_CHECKING, TypeVarimport pytestfrom twisted.trial import unittest@@ -20,16 +23,22 @@ from scrapy.utils.python import (without_none_values,)+if TYPE_CHECKING:+ from collections.abc import Iterable, Mapping+++_KT = TypeVar("_KT")+_VT = TypeVar("_VT")+-class TestMutableChain:- def test_mutablechain(self):- m = MutableChain(range(2), [2, 3], (4, 5))- m.extend(range(6, 7))- m.extend([7, 8])- m.extend([9, 10], (11, 12))- assert next(m) == 0- assert m.__next__() == 1- assert list(m) == list(range(2, 13))+def test_mutablechain():+ m = MutableChain(range(2), [2, 3], (4, 5))+ m.extend(range(6, 7))+ m.extend([7, 8])+ m.extend([9, 10], (11, 12))+ assert next(m) == 0+ assert m.__next__() == 1+ assert list(m) == list(range(2, 13))class TestMutableAsyncChain(unittest.TestCase):@@ -112,144 +121,150 @@ class TestToBytes:assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b"-class TestMemoizedMethod:- def test_memoizemethod_noargs(self):- class A:- @memoizemethod_noargs- def cached(self):- return object()+def test_memoizemethod_noargs():+ class A:+ @memoizemethod_noargs+ def cached(self):+ return object()- def noncached(self):- return object()+ def noncached(self):+ return object()- a = A()- one = a.cached()- two = a.cached()- three = a.noncached()- assert one is two- assert one is not three+ a = A()+ one = a.cached()+ two = a.cached()+ three = a.noncached()+ assert one is two+ assert one is not three-class TestBinaryIsText:- def test_binaryistext(self):- assert binary_is_text(b"hello")+ ("value", "expected"),+ [+ (b"hello", True),+ ("hello".encode("utf-16"), True),+ (b"<div>Price \xa3</div>", True),+ (b"\x02\xa3", False),+ ],+)+def test_binaryistext(value: bytes, expected: bool) -> None:+ assert binary_is_text(value) is expected- def test_utf_16_strings_contain_null_bytes(self):- assert binary_is_text("hello".encode("utf-16"))- def test_one_with_encoding(self):- assert binary_is_text(b"<div>Price \xa3</div>")[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")+def test_equal_attributes():+ class Obj:+ pass- def test_real_binary_bytes(self):- assert not binary_is_text(b"\x02\xa3")+ a = Obj()+ b = Obj()+ # no attributes given return False+ assert not equal_attributes(a, b, [])+ # nonexistent attributes+ assert not equal_attributes(a, b, ["x", "y"])+ a.x = 1+ b.x = 1+ # equal attribute+ assert equal_attributes(a, b, ["x"])-class TestUtilsPython:- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")- def test_equal_attributes(self):- class Obj:- pass+ b.y = 2+ # obj1 has no attribute y+ assert not equal_attributes(a, b, ["x", "y"])- a = Obj()- b = Obj()- # no attributes given return False- assert not equal_attributes(a, b, [])- # nonexistent attributes- assert not equal_attributes(a, b, ["x", "y"])+ a.y = 2+ # equal attributes+ assert equal_attributes(a, b, ["x", "y"])- a.x = 1- b.x = 1- # equal attribute- assert equal_attributes(a, b, ["x"])+ a.y = 1+ # different attributes+ assert not equal_attributes(a, b, ["x", "y"])- b.y = 2- # obj1 has no attribute y- assert not equal_attributes(a, b, ["x", "y"])+ # test callable+ a.meta = {}+ b.meta = {}+ assert equal_attributes(a, b, ["meta"])- a.y = 2- # equal attributes- assert equal_attributes(a, b, ["x", "y"])+ # compare ['meta']['a']+ a.meta["z"] = 1+ b.meta["z"] = 1- a.y = 1- # different attributes- assert not equal_attributes(a, b, ["x", "y"])+ get_z = operator.itemgetter("z")+ get_meta = operator.attrgetter("meta")- # test callable- a.meta = {}- b.meta = {}- assert equal_attributes(a, b, ["meta"])+ def compare_z(obj):+ return get_z(get_meta(obj))- # compare ['meta']['a']- a.meta["z"] = 1- b.meta["z"] = 1+ assert equal_attributes(a, b, [compare_z, "x"])+ # fail z equality+ a.meta["z"] = 2+ assert not equal_attributes(a, b, [compare_z, "x"])- get_z = operator.itemgetter("z")- get_meta = operator.attrgetter("meta")- def compare_z(obj):- return get_z(get_meta(obj))+def test_get_func_args():+ def f1(a, b, c):+ pass- assert equal_attributes(a, b, [compare_z, "x"])- # fail z equality- a.meta["z"] = 2- assert not equal_attributes(a, b, [compare_z, "x"])+ def f2(a, b=None, c=None):+ pass- def test_get_func_args(self):- def f1(a, b, c):+ def f3(a, b=None, *, c=None):+ pass++ class A:+ def __init__(self, a, b, c):pass- def f2(a, b=None, c=None):+ def method(self, a, b, c):pass- def f3(a, b=None, *, c=None):+ class Callable:+ def __call__(self, a, b, c):pass- class A:- def __init__(self, a, b, c):- pass-- def method(self, a, b, c):- pass-- class Callable:- def __call__(self, a, b, c):- pass-- a = A(1, 2, 3)- cal = Callable()- partial_f1 = functools.partial(f1, None)- partial_f2 = functools.partial(f1, b=None)- partial_f3 = functools.partial(partial_f2, None)-- assert get_func_args(f1) == ["a", "b", "c"]- assert get_func_args(f2) == ["a", "b", "c"]- assert get_func_args(f3) == ["a", "b", "c"]- assert get_func_args(A) == ["a", "b", "c"]- assert get_func_args(a.method) == ["a", "b", "c"]- assert get_func_args(partial_f1) == ["b", "c"]- assert get_func_args(partial_f2) == ["a", "c"]- assert get_func_args(partial_f3) == ["c"]- assert get_func_args(cal) == ["a", "b", "c"]- assert get_func_args(object) == []- assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]- assert get_func_args(" ".join, stripself=True) == ["iterable"]-- if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy":- # the correct and correctly extracted signature- assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"]- elif platform.python_implementation() == "CPython":- # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature- # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951)- assert get_func_args(operator.itemgetter(2), stripself=True) in [- [],- ["args", "kwargs"],- ]-- def test_without_none_values(self):- assert without_none_values([1, None, 3, 4]) == [1, 3, 4]- assert without_none_values((1, None, 3, 4)) == (1, 3, 4)- assert without_none_values({"one": 1, "none": None, "three": 3, "four": 4}) == {- "one": 1,- "three": 3,- "four": 4,- }+ a = A(1, 2, 3)+ cal = Callable()+ partial_f1 = functools.partial(f1, None)+ partial_f2 = functools.partial(f1, b=None)+ partial_f3 = functools.partial(partial_f2, None)++ assert get_func_args(f1) == ["a", "b", "c"]+ assert get_func_args(f2) == ["a", "b", "c"]+ assert get_func_args(f3) == ["a", "b", "c"]+ assert get_func_args(A) == ["a", "b", "c"]+ assert get_func_args(a.method) == ["a", "b", "c"]+ assert get_func_args(partial_f1) == ["b", "c"]+ assert get_func_args(partial_f2) == ["a", "c"]+ assert get_func_args(partial_f3) == ["c"]+ assert get_func_args(cal) == ["a", "b", "c"]+ assert get_func_args(object) == []+ assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]+ assert get_func_args(" ".join, stripself=True) == ["iterable"]++ if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy":+ # the correct and correctly extracted signature+ assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"]+ elif platform.python_implementation() == "CPython":+ # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature+ # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951)+ assert get_func_args(operator.itemgetter(2), stripself=True) in [+ [],+ ["args", "kwargs"],+ ]+++ ("value", "expected"),+ [+ ([1, None, 3, 4], [1, 3, 4]),+ ((1, None, 3, 4), (1, 3, 4)),+ (+ {"one": 1, "none": None, "three": 3, "four": 4},+ {"one": 1, "three": 3, "four": 4},+ ),+ ],+)+def test_without_none_values(+ value: Mapping[_KT, _VT] | Iterable[_KT], expected: dict[_KT, _VT] | Iterable[_KT]+) -> None:+ assert without_none_values(value) == expecteddiff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 5b8509753..9c4cb7159 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -20,45 +20,52 @@ from scrapy.utils.request import (from scrapy.utils.test import get_crawler-class TestUtilsRequest:- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")- def test_request_authenticate(self):- r = Request("http://www.example.com")- request_authenticate(r, "someuser", "somepass")- assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M="-- def test_request_httprepr(self):- r1 = Request("http://www.example.com")- assert (- request_httprepr(r1) == b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"- )[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")+def test_request_authenticate():+ r = Request("http://www.example.com")+ request_authenticate(r, "someuser", "somepass")+ assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M="- r1 = Request("http://www.example.com/some/page.html?arg=1")- assert (- request_httprepr(r1)- == b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n"- )- r1 = Request(- "http://www.example.com",- method="POST",- headers={"Content-type": b"text/html"},- body=b"Some body",- )- assert (- request_httprepr(r1)- == b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body"- )+ ("r", "expected"),+ [+ (+ Request("http://www.example.com"),+ b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n",+ ),+ (+ Request("http://www.example.com/some/page.html?arg=1"),+ b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n",+ ),+ (+ Request(+ "http://www.example.com",+ method="POST",+ headers={"Content-type": b"text/html"},+ body=b"Some body",+ ),+ b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body",+ ),+ ],+)+def test_request_httprepr(r: Request, expected: bytes) -> None:+ assert request_httprepr(r) == expected- def test_request_httprepr_for_non_http_request(self):- # the representation is not important but it must not fail.- request_httprepr(Request("file:///tmp/foo.txt"))- request_httprepr(Request("ftp://localhost/tmp/foo.txt"))+ "r",+ [+ Request("file:///tmp/foo.txt"),+ Request("ftp://localhost/tmp/foo.txt"),+ ],+)+def test_request_httprepr_for_non_http_request(r: Request) -> None:+ # the representation is not important but it must not fail.+ request_httprepr(r)-class TestFingerprint:- maxDiff = None+class TestFingerprint:function: staticmethod = staticmethod(fingerprint)cache: (WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]@@ -229,35 +236,6 @@ class TestFingerprint:assert actual == expected-REQUEST_OBJECTS_TO_TEST = (- Request("http://www.example.com/"),- Request("http://www.example.com/query?id=111&cat=222"),- Request("http://www.example.com/query?cat=222&id=111"),- Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"),- Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"),- Request("http://www.example.com/members/offers.html"),- Request(- "http://www.example.com/members/offers.html",- headers={"SESSIONID": b"somehash"},- ),- Request(- "http://www.example.com/",- headers={"Accept-Language": b"en"},- ),- Request(- "http://www.example.com/",- headers={- "Accept-Language": b"en",- "SESSIONID": b"somehash",- },- ),- Request("http://www.example.com/test.html"),- Request("http://www.example.com/test.html#fragment"),- Request("http://www.example.com", method="POST"),- Request("http://www.example.com", method="POST", body=b"request body"),-)--class TestRequestFingerprinter:def test_default_implementation(self):crawler = get_crawler()diff --git a/tests/test_utils_response.py b/tests/test_utils_response.pyindex 80f2f25d5..179ca49e4 100644--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@ -4,7 +4,7 @@ from urllib.parse import urlparseimport pytest-from scrapy.http import HtmlResponse, Response, TextResponse+from scrapy.http import HtmlResponse, Responsefrom scrapy.utils.python import to_bytesfrom scrapy.utils.response import (_remove_html_comments,@@ -15,229 +15,203 @@ from scrapy.utils.response import ()-class TestResponseUtils:- dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response")-- def test_open_in_browser(self):- url = "http:///www.example.com/some/page.html"- body = b"<html> <head> <title>test page</title> </head> <body>test body</body> </html>"-- def browser_open(burl):- path = urlparse(burl).path- if not path or not Path(path).exists():- path = burl.replace("file://", "")- bbody = Path(path).read_bytes()- assert b'<base href="' + to_bytes(url) + b'">' in bbody- return True-- response = HtmlResponse(url, body=body)- assert open_in_browser(response, _openfunc=browser_open), "Browser not called"-- resp = Response(url, body=body)- with pytest.raises(TypeError):- open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg-- def test_get_meta_refresh(self):- r1 = HtmlResponse(- "http://www.example.com",- body=b"""- <html>- <head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>- <body>blahablsdfsal&</body>- </html>""",- )- r2 = HtmlResponse(- "http://www.example.com",- body=b"""- <html>- <head><title>Dummy</title><noScript>- <meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>- </noSCRIPT>- <body>blahablsdfsal&</body>- </html>""",- )- r3 = HtmlResponse(- "http://www.example.com",- body=b"""- <noscript><meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage</noscript>- <script type="text/javascript">- if(!checkCookies()){- document.write('<meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage">');- }- </script>- """,- )- assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage")- assert get_meta_refresh(r2) == (None, None)- assert get_meta_refresh(r3) == (None, None)-- def test_get_base_url(self):- resp = HtmlResponse(- "http://www.example.com",- body=b"""- <html>- <head><base href="http://www.example.com/img/" target="_blank"></head>- <body>blahablsdfsal&</body>- </html>""",- )- assert get_base_url(resp) == "http://www.example.com/img/"-- resp2 = HtmlResponse(- "http://www.example.com",- body=b"""- <html><body>blahablsdfsal&</body></html>""",- )- assert get_base_url(resp2) == "http://www.example.com"-- def test_response_status_message(self):- assert response_status_message(200) == "200 OK"- assert response_status_message(404) == "404 Not Found"- assert response_status_message(573) == "573 Unknown Status"-- def test_inject_base_url(self):- url = "http://www.example.com"-- def check_base_url(burl):- path = urlparse(burl).path- if not path or not Path(path).exists():- path = burl.replace("file://", "")- bbody = Path(path).read_bytes()- assert bbody.count(b'<base href="' + to_bytes(url) + b'">') == 1- return True-- r1 = HtmlResponse(- url,- body=b"""- <html>- <head><title>Dummy</title></head>- <body><p>Hello world.</p></body>- </html>""",- )- r2 = HtmlResponse(- url,- body=b"""- <html>- <head id="foo"><title>Dummy</title></head>- <body>Hello world.</body>- </html>""",- )- r3 = HtmlResponse(- url,- body=b"""- <html>- <head><title>Dummy</title></head>- <body>- <header>Hello header</header>- <p>Hello world.</p>- </body>- </html>""",- )- r4 = HtmlResponse(- url,- body=b"""- <html>- <!-- <head>Dummy comment</head> -->- <head><title>Dummy</title></head>- <body><p>Hello world.</p></body>- </html>""",- )- r5 = HtmlResponse(- url,- body=b"""- <html>- <!--[if IE]>- <head><title>IE head</title></head>- <![endif]-->- <!--[if !IE]>-->- <head><title>Standard head</title></head>- <!--<![endif]-->- <body><p>Hello world.</p></body>- </html>""",- )-- assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"- assert open_in_browser(r2, _openfunc=check_base_url), (- "Inject base url with argumented head"- )- assert open_in_browser(r3, _openfunc=check_base_url), (- "Inject unique base url with misleading tag"- )- assert open_in_browser(r4, _openfunc=check_base_url), (- "Inject unique base url with misleading comment"- )- assert open_in_browser(r5, _openfunc=check_base_url), (- "Inject unique base url with conditional comment"- )-- def test_open_in_browser_redos_comment(self):- MAX_CPU_TIME = 0.02-- # Exploit input from- # https://makenowjust-labs.github.io/recheck/playground/- # for /<!--.*?-->/ (old pattern to remove comments).- body = b"-><!--\x00" * 25_000 + b"->\n<!---->"-- response = HtmlResponse("https://example.com", body=body)-- start_time = process_time()-- open_in_browser(response, lambda url: True)-- end_time = process_time()- assert end_time - start_time < MAX_CPU_TIME-- def test_open_in_browser_redos_head(self):- MAX_CPU_TIME = 0.02-- # Exploit input from- # https://makenowjust-labs.github.io/recheck/playground/- # for /(<head(?:>|\s.*?>))/ (old pattern to find the head element).- body = b"<head\t" * 8_000-- response = HtmlResponse("https://example.com", body=body)-- start_time = process_time()-- open_in_browser(response, lambda url: True)-- end_time = process_time()- assert end_time - start_time < MAX_CPU_TIME+def test_open_in_browser():+ url = "http:///www.example.com/some/page.html"+ body = (+ b"<html> <head> <title>test page</title> </head> <body>test body</body> </html>"+ )++ def browser_open(burl: str) -> bool:+ path = urlparse(burl).path+ if not path or not Path(path).exists():+ path = burl.replace("file://", "")+ bbody = Path(path).read_bytes()+ assert b'<base href="' + to_bytes(url) + b'">' in bbody+ return True++ response = HtmlResponse(url, body=body)+ assert open_in_browser(response, _openfunc=browser_open), "Browser not called"++ resp = Response(url, body=body)+ with pytest.raises(TypeError):+ open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg+++def test_get_meta_refresh():+ r1 = HtmlResponse(+ "http://www.example.com",+ body=b"""+ <html>+ <head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>+ <body>blahablsdfsal&</body>+ </html>""",+ )+ r2 = HtmlResponse(+ "http://www.example.com",+ body=b"""+ <html>+ <head><title>Dummy</title><noScript>+ <meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>+ </noSCRIPT>+ <body>blahablsdfsal&</body>+ </html>""",+ )+ r3 = HtmlResponse(+ "http://www.example.com",+ body=b"""+<noscript><meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage</noscript>+<script type="text/javascript">+if(!checkCookies()){+ document.write('<meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage">');+}+</script>+ """,+ )+ assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage")+ assert get_meta_refresh(r2) == (None, None)+ assert get_meta_refresh(r3) == (None, None)+++def test_get_base_url():+ resp = HtmlResponse(+ "http://www.example.com",+ body=b"""+ <html>+ <head><base href="http://www.example.com/img/" target="_blank"></head>+ <body>blahablsdfsal&</body>+ </html>""",+ )+ assert get_base_url(resp) == "http://www.example.com/img/"++ resp2 = HtmlResponse(+ "http://www.example.com",+ body=b"""+ <html><body>blahablsdfsal&</body></html>""",+ )+ assert get_base_url(resp2) == "http://www.example.com"+++def test_response_status_message():+ assert response_status_message(200) == "200 OK"+ assert response_status_message(404) == "404 Not Found"+ assert response_status_message(573) == "573 Unknown Status"+++def test_inject_base_url():+ url = "http://www.example.com"++ def check_base_url(burl):+ path = urlparse(burl).path+ if not path or not Path(path).exists():+ path = burl.replace("file://", "")+ bbody = Path(path).read_bytes()+ assert bbody.count(b'<base href="' + to_bytes(url) + b'">') == 1+ return True++ r1 = HtmlResponse(+ url,+ body=b"""+ <html>+ <head><title>Dummy</title></head>+ <body><p>Hello world.</p></body>+ </html>""",+ )+ r2 = HtmlResponse(+ url,+ body=b"""+ <html>+ <head id="foo"><title>Dummy</title></head>+ <body>Hello world.</body>+ </html>""",+ )+ r3 = HtmlResponse(+ url,+ body=b"""+ <html>+ <head><title>Dummy</title></head>+ <body>+ <header>Hello header</header>+ <p>Hello world.</p>+ </body>+ </html>""",+ )+ r4 = HtmlResponse(+ url,+ body=b"""+ <html>+ <!-- <head>Dummy comment</head> -->+ <head><title>Dummy</title></head>+ <body><p>Hello world.</p></body>+ </html>""",+ )+ r5 = HtmlResponse(+ url,+ body=b"""+ <html>+ <!--[if IE]>+ <head><title>IE head</title></head>+ <![endif]-->+ <!--[if !IE]>-->+ <head><title>Standard head</title></head>+ <!--<![endif]-->+ <body><p>Hello world.</p></body>+ </html>""",+ )++ assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"+ assert open_in_browser(r2, _openfunc=check_base_url), (+ "Inject base url with argumented head"+ )+ assert open_in_browser(r3, _openfunc=check_base_url), (+ "Inject unique base url with misleading tag"+ )+ assert open_in_browser(r4, _openfunc=check_base_url), (+ "Inject unique base url with misleading comment"+ )+ assert open_in_browser(r5, _openfunc=check_base_url), (+ "Inject unique base url with conditional comment"+ )+++def test_open_in_browser_redos_comment():+ MAX_CPU_TIME = 0.02++ # Exploit input from+ # https://makenowjust-labs.github.io/recheck/playground/+ # for /<!--.*?-->/ (old pattern to remove comments).+ body = b"-><!--\x00" * 25_000 + b"->\n<!---->"+ response = HtmlResponse("https://example.com", body=body)+ start_time = process_time()+ open_in_browser(response, lambda url: True)+ end_time = process_time()+ assert end_time - start_time < MAX_CPU_TIME+++def test_open_in_browser_redos_head():+ MAX_CPU_TIME = 0.02++ # Exploit input from+ # https://makenowjust-labs.github.io/recheck/playground/+ # for /(<head(?:>|\s.*?>))/ (old pattern to find the head element).+ body = b"<head\t" * 8_000+ response = HtmlResponse("https://example.com", body=body)+ start_time = process_time()+ open_in_browser(response, lambda url: True)+ end_time = process_time()+ assert end_time - start_time < MAX_CPU_TIME@pytest.mark.parametrize(("input_body", "output_body"),[- (- b"a<!--",- b"a",- ),- (- b"a<!---->b",- b"ab",- ),- (- b"a<!--b-->c",- b"ac",- ),- (- b"a<!--b-->c<!--",- b"ac",- ),- (- b"a<!--b-->c<!--d",- b"ac",- ),- (- b"a<!--b-->c<!---->d",- b"acd",- ),- (- b"a<!--b--><!--c-->d",- b"ad",- ),+ (b"a<!--", b"a"),+ (b"a<!---->b", b"ab"),+ (b"a<!--b-->c", b"ac"),+ (b"a<!--b-->c<!--", b"ac"),+ (b"a<!--b-->c<!--d", b"ac"),+ (b"a<!--b-->c<!---->d", b"acd"),+ (b"a<!--b--><!--c-->d", b"ad"),],)def test_remove_html_comments(input_body, output_body):- assert _remove_html_comments(input_body) == output_body, (- f"{_remove_html_comments(input_body)=} == {output_body=}"- )+ assert _remove_html_comments(input_body) == output_bodydiff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.pyindex 2ee3850b0..dcaac7358 100644--- a/tests/test_utils_serialize.py+++ b/tests/test_utils_serialize.py@@ -4,6 +4,7 @@ import jsonfrom decimal import Decimalimport attr+import pytestfrom twisted.internet import deferfrom scrapy.http import Request, Response@@ -11,10 +12,11 @@ from scrapy.utils.serialize import ScrapyJSONEncoderclass TestJsonEncoder:- def setup_method(self):- self.encoder = ScrapyJSONEncoder(sort_keys=True)+ @pytest.fixture+ def encoder(self) -> ScrapyJSONEncoder:+ return ScrapyJSONEncoder(sort_keys=True)- def test_encode_decode(self):+ def test_encode_decode(self, encoder: ScrapyJSONEncoder) -> None:dt = datetime.datetime(2010, 1, 2, 10, 11, 12)dts = "2010-01-02 10:11:12"d = datetime.date(2010, 1, 2)@@ -38,24 +40,24 @@ class TestJsonEncoder:(s, ss),(dt_set, dt_sets),]:- assert self.encoder.encode(input) == json.dumps(output, sort_keys=True)+ assert encoder.encode(input) == json.dumps(output, sort_keys=True)- def test_encode_deferred(self):- assert "Deferred" in self.encoder.encode(defer.Deferred())+ def test_encode_deferred(self, encoder: ScrapyJSONEncoder) -> None:+ assert "Deferred" in encoder.encode(defer.Deferred())- def test_encode_request(self):+ def test_encode_request(self, encoder: ScrapyJSONEncoder) -> None:r = Request("http://www.example.com/lala")- rs = self.encoder.encode(r)+ rs = encoder.encode(r)assert r.method in rsassert r.url in rs- def test_encode_response(self):+ def test_encode_response(self, encoder: ScrapyJSONEncoder) -> None:r = Response("http://www.example.com/lala")- rs = self.encoder.encode(r)+ rs = encoder.encode(r)assert r.url in rsassert str(r.status) in rs- def test_encode_dataclass_item(self) -> None:+ def test_encode_dataclass_item(self, encoder: ScrapyJSONEncoder) -> None:@dataclasses.dataclassclass TestDataClass:name: str@@ -63,10 +65,10 @@ class TestJsonEncoder:price: intitem = TestDataClass(name="Product", url="http://product.org", price=1)- encoded = self.encoder.encode(item)+ encoded = encoder.encode(item)assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}'- def test_encode_attrs_item(self):+ def test_encode_attrs_item(self, encoder: ScrapyJSONEncoder) -> None:@attr.sclass AttrsItem:name = attr.ib(type=str)@@ -74,5 +76,5 @@ class TestJsonEncoder:price = attr.ib(type=int)item = AttrsItem(name="Product", url="http://product.org", price=1)- encoded = self.encoder.encode(item)+ encoded = encoder.encode(item)assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}'diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.pyindex 36d612009..464a31777 100644--- a/tests/test_utils_sitemap.py+++ b/tests/test_utils_sitemap.py@@ -1,158 +1,162 @@from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots-class TestSitemap:- def test_sitemap(self):- s = Sitemap(- b"""<?xml version="1.0" encoding="UTF-8"?>+def test_sitemap():+ s = Sitemap(+ b"""<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.google.com/schemas/sitemap/0.84">- <url>- <loc>http://www.example.com/</loc>- <lastmod>2009-08-16</lastmod>- <changefreq>daily</changefreq>- <priority>1</priority>- </url>- <url>- <loc>http://www.example.com/Special-Offers.html</loc>- <lastmod>2009-08-16</lastmod>- <changefreq>weekly</changefreq>- <priority>0.8</priority>- </url>+<url>+<loc>http://www.example.com/</loc>+<lastmod>2009-08-16</lastmod>+<changefreq>daily</changefreq>+<priority>1</priority>+</url>+<url>+<loc>http://www.example.com/Special-Offers.html</loc>+<lastmod>2009-08-16</lastmod>+<changefreq>weekly</changefreq>+<priority>0.8</priority>+</url></urlset>"""- )- assert s.type == "urlset"- assert list(s) == [- {- "priority": "1",- "loc": "http://www.example.com/",- "lastmod": "2009-08-16",- "changefreq": "daily",- },- {- "priority": "0.8",- "loc": "http://www.example.com/Special-Offers.html",- "lastmod": "2009-08-16",- "changefreq": "weekly",- },- ]+ )+ assert s.type == "urlset"+ assert list(s) == [+ {+ "priority": "1",+ "loc": "http://www.example.com/",+ "lastmod": "2009-08-16",+ "changefreq": "daily",+ },+ {+ "priority": "0.8",+ "loc": "http://www.example.com/Special-Offers.html",+ "lastmod": "2009-08-16",+ "changefreq": "weekly",+ },+ ]- def test_sitemap_index(self):- s = Sitemap(- b"""<?xml version="1.0" encoding="UTF-8"?>++def test_sitemap_index():+ s = Sitemap(+ b"""<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">- <sitemap>- <loc>http://www.example.com/sitemap1.xml.gz</loc>- <lastmod>2004-10-01T18:23:17+00:00</lastmod>- </sitemap>- <sitemap>- <loc>http://www.example.com/sitemap2.xml.gz</loc>- <lastmod>2005-01-01</lastmod>- </sitemap>+<sitemap>+ <loc>http://www.example.com/sitemap1.xml.gz</loc>+ <lastmod>2004-10-01T18:23:17+00:00</lastmod>+</sitemap>+<sitemap>+ <loc>http://www.example.com/sitemap2.xml.gz</loc>+ <lastmod>2005-01-01</lastmod>+</sitemap></sitemapindex>"""- )- assert s.type == "sitemapindex"- assert list(s) == [- {- "loc": "http://www.example.com/sitemap1.xml.gz",- "lastmod": "2004-10-01T18:23:17+00:00",- },- {- "loc": "http://www.example.com/sitemap2.xml.gz",- "lastmod": "2005-01-01",- },- ]+ )+ assert s.type == "sitemapindex"+ assert list(s) == [+ {+ "loc": "http://www.example.com/sitemap1.xml.gz",+ "lastmod": "2004-10-01T18:23:17+00:00",+ },+ {+ "loc": "http://www.example.com/sitemap2.xml.gz",+ "lastmod": "2005-01-01",+ },+ ]- def test_sitemap_strip(self):- """Assert we can deal with trailing spaces inside <loc> tags - we've- seen those- """- s = Sitemap(- b"""<?xml version="1.0" encoding="UTF-8"?>++def test_sitemap_strip():+ """Assert we can deal with trailing spaces inside <loc> tags - we've+ seen those+ """+ s = Sitemap(+ b"""<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.google.com/schemas/sitemap/0.84">- <url>- <loc> http://www.example.com/</loc>- <lastmod>2009-08-16</lastmod>- <changefreq>daily</changefreq>- <priority>1</priority>- </url>- <url>- <loc> http://www.example.com/2</loc>- <lastmod />- </url>+<url>+<loc> http://www.example.com/</loc>+<lastmod>2009-08-16</lastmod>+<changefreq>daily</changefreq>+<priority>1</priority>+</url>+<url>+<loc> http://www.example.com/2</loc>+<lastmod />+</url></urlset>"""- )- assert list(s) == [- {- "priority": "1",- "loc": "http://www.example.com/",- "lastmod": "2009-08-16",- "changefreq": "daily",- },- {"loc": "http://www.example.com/2", "lastmod": ""},- ]+ )+ assert list(s) == [+ {+ "priority": "1",+ "loc": "http://www.example.com/",+ "lastmod": "2009-08-16",+ "changefreq": "daily",+ },+ {"loc": "http://www.example.com/2", "lastmod": ""},+ ]+- def test_sitemap_wrong_ns(self):- """We have seen sitemaps with wrongs ns. Presumably, Google still works- with these, though is not 100% confirmed"""- s = Sitemap(- b"""<?xml version="1.0" encoding="UTF-8"?>+def test_sitemap_wrong_ns():+ """We have seen sitemaps with wrongs ns. Presumably, Google still works+ with these, though is not 100% confirmed"""+ s = Sitemap(+ b"""<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.google.com/schemas/sitemap/0.84">- <url xmlns="">- <loc> http://www.example.com/</loc>- <lastmod>2009-08-16</lastmod>- <changefreq>daily</changefreq>- <priority>1</priority>- </url>- <url xmlns="">- <loc> http://www.example.com/2</loc>- <lastmod />- </url>+<url xmlns="">+<loc> http://www.example.com/</loc>+<lastmod>2009-08-16</lastmod>+<changefreq>daily</changefreq>+<priority>1</priority>+</url>+<url xmlns="">+<loc> http://www.example.com/2</loc>+<lastmod />+</url></urlset>"""- )- assert list(s) == [- {- "priority": "1",- "loc": "http://www.example.com/",- "lastmod": "2009-08-16",- "changefreq": "daily",- },- {"loc": "http://www.example.com/2", "lastmod": ""},- ]+ )+ assert list(s) == [+ {+ "priority": "1",+ "loc": "http://www.example.com/",+ "lastmod": "2009-08-16",+ "changefreq": "daily",+ },+ {"loc": "http://www.example.com/2", "lastmod": ""},+ ]- def test_sitemap_wrong_ns2(self):- """We have seen sitemaps with wrongs ns. Presumably, Google still works- with these, though is not 100% confirmed"""- s = Sitemap(- b"""<?xml version="1.0" encoding="UTF-8"?>++def test_sitemap_wrong_ns2():+ """We have seen sitemaps with wrongs ns. Presumably, Google still works+ with these, though is not 100% confirmed"""+ s = Sitemap(+ b"""<?xml version="1.0" encoding="UTF-8"?><urlset>- <url xmlns="">- <loc> http://www.example.com/</loc>- <lastmod>2009-08-16</lastmod>- <changefreq>daily</changefreq>- <priority>1</priority>- </url>- <url xmlns="">- <loc> http://www.example.com/2</loc>- <lastmod />- </url>+<url xmlns="">+<loc> http://www.example.com/</loc>+<lastmod>2009-08-16</lastmod>+<changefreq>daily</changefreq>+<priority>1</priority>+</url>+<url xmlns="">+<loc> http://www.example.com/2</loc>+<lastmod />+</url></urlset>"""- )- assert s.type == "urlset"- assert list(s) == [- {- "priority": "1",- "loc": "http://www.example.com/",- "lastmod": "2009-08-16",- "changefreq": "daily",- },- {"loc": "http://www.example.com/2", "lastmod": ""},- ]+ )+ assert s.type == "urlset"+ assert list(s) == [+ {+ "priority": "1",+ "loc": "http://www.example.com/",+ "lastmod": "2009-08-16",+ "changefreq": "daily",+ },+ {"loc": "http://www.example.com/2", "lastmod": ""},+ ]+- def test_sitemap_urls_from_robots(self):- robots = """User-agent: *+def test_sitemap_urls_from_robots():+ robots = """User-agent: *Disallow: /aff/Disallow: /wl/@@ -170,19 +174,18 @@ Sitemap: /sitemap-relative-url.xmlDisallow: /forum/search/Disallow: /forum/active/"""- assert list(- sitemap_urls_from_robots(robots, base_url="http://example.com")- ) == [- "http://example.com/sitemap.xml",- "http://example.com/sitemap-product-index.xml",- "http://example.com/sitemap-uppercase.xml",- "http://example.com/sitemap-relative-url.xml",- ]+ assert list(sitemap_urls_from_robots(robots, base_url="http://example.com")) == [+ "http://example.com/sitemap.xml",+ "http://example.com/sitemap-product-index.xml",+ "http://example.com/sitemap-uppercase.xml",+ "http://example.com/sitemap-relative-url.xml",+ ]- def test_sitemap_blanklines(self):- """Assert we can deal with starting blank lines before <xml> tag"""- s = Sitemap(- b"""++def test_sitemap_blanklines():+ """Assert we can deal with starting blank lines before <xml> tag"""+ s = Sitemap(+ b"""<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">@@ -205,69 +208,69 @@ Disallow: /forum/active/<!-- end cache --></sitemapindex>"""- )- assert list(s) == [- {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"},- {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"},- {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"},- ]+ )+ assert list(s) == [+ {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"},+ {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"},+ {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"},+ ]- def test_comment(self):- s = Sitemap(- b"""<?xml version="1.0" encoding="UTF-8"?>- <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"- xmlns:xhtml="http://www.w3.org/1999/xhtml">- <url>- <loc>http://www.example.com/</loc>- <!-- this is a comment on which the parser might raise an exception if implemented incorrectly -->- </url>- </urlset>"""- )- assert list(s) == [{"loc": "http://www.example.com/"}]+def test_comment():+ s = Sitemap(+ b"""<?xml version="1.0" encoding="UTF-8"?>+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"+ xmlns:xhtml="http://www.w3.org/1999/xhtml">+ <url>+ <loc>http://www.example.com/</loc>+ <!-- this is a comment on which the parser might raise an exception if implemented incorrectly -->+ </url>+</urlset>"""+ )+ assert list(s) == [{"loc": "http://www.example.com/"}]- def test_alternate(self):- s = Sitemap(- b"""<?xml version="1.0" encoding="UTF-8"?>- <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"- xmlns:xhtml="http://www.w3.org/1999/xhtml">- <url>- <loc>http://www.example.com/english/</loc>- <xhtml:link rel="alternate" hreflang="de"- href="http://www.example.com/deutsch/"/>- <xhtml:link rel="alternate" hreflang="de-ch"- href="http://www.example.com/schweiz-deutsch/"/>- <xhtml:link rel="alternate" hreflang="en"- href="http://www.example.com/english/"/>- <xhtml:link rel="alternate" hreflang="en"/><!-- wrong tag without href -->- </url>- </urlset>"""- )- assert list(s) == [- {- "loc": "http://www.example.com/english/",- "alternate": [- "http://www.example.com/deutsch/",- "http://www.example.com/schweiz-deutsch/",- "http://www.example.com/english/",- ],- }- ]+def test_alternate():+ s = Sitemap(+ b"""<?xml version="1.0" encoding="UTF-8"?>+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"+ xmlns:xhtml="http://www.w3.org/1999/xhtml">+ <url>+ <loc>http://www.example.com/english/</loc>+ <xhtml:link rel="alternate" hreflang="de"+ href="http://www.example.com/deutsch/"/>+ <xhtml:link rel="alternate" hreflang="de-ch"+ href="http://www.example.com/schweiz-deutsch/"/>+ <xhtml:link rel="alternate" hreflang="en"+ href="http://www.example.com/english/"/>+ <xhtml:link rel="alternate" hreflang="en"/><!-- wrong tag without href -->+ </url>+</urlset>"""+ )+ assert list(s) == [+ {+ "loc": "http://www.example.com/english/",+ "alternate": [+ "http://www.example.com/deutsch/",+ "http://www.example.com/schweiz-deutsch/",+ "http://www.example.com/english/",+ ],+ }+ ]- def test_xml_entity_expansion(self):- s = Sitemap(- b"""<?xml version="1.0" encoding="utf-8"?>- <!DOCTYPE foo [- <!ELEMENT foo ANY >- <!ENTITY xxe SYSTEM "file:///etc/passwd" >- ]>- <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">- <url>- <loc>http://127.0.0.1:8000/&xxe;</loc>- </url>- </urlset>- """- )- assert list(s) == [{"loc": "http://127.0.0.1:8000/"}]+def test_xml_entity_expansion():+ s = Sitemap(+ b"""<?xml version="1.0" encoding="utf-8"?>+ <!DOCTYPE foo [+ <!ELEMENT foo ANY >+ <!ENTITY xxe SYSTEM "file:///etc/passwd" >+ ]>+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url>+ <loc>http://127.0.0.1:8000/&xxe;</loc>+ </url>+ </urlset>+ """+ )+ assert list(s) == [{"loc": "http://127.0.0.1:8000/"}]diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.pyindex 43e603f6c..8600faa8d 100644--- a/tests/test_utils_spider.py+++ b/tests/test_utils_spider.py@@ -12,19 +12,19 @@ class MySpider2(Spider):name = "myspider2"-class TestUtilsSpiders:- def test_iterate_spider_output(self):- i = Item()- r = Request("http://scrapytest.org")- o = object()-- assert list(iterate_spider_output(i)) == [i]- assert list(iterate_spider_output(r)) == [r]- assert list(iterate_spider_output(o)) == [o]- assert list(iterate_spider_output([r, i, o])) == [r, i, o]-- def test_iter_spider_classes(self):- import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self-- it = iter_spider_classes(tests.test_utils_spider)- assert set(it) == {MySpider1, MySpider2}+def test_iterate_spider_output():+ i = Item()+ r = Request("http://scrapytest.org")+ o = object()++ assert list(iterate_spider_output(i)) == [i]+ assert list(iterate_spider_output(r)) == [r]+ assert list(iterate_spider_output(o)) == [o]+ assert list(iterate_spider_output([r, i, o])) == [r, i, o]+++def test_iter_spider_classes():+ import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self++ it = iter_spider_classes(tests.test_utils_spider)+ assert set(it) == {MySpider1, MySpider2}diff --git a/tests/test_utils_template.py b/tests/test_utils_template.pyindex 41d9b8933..4515ce36e 100644--- a/tests/test_utils_template.py+++ b/tests/test_utils_template.py@@ -1,22 +1,21 @@from scrapy.utils.template import render_templatefile-class TestUtilsRenderTemplateFile:- def test_simple_render(self, tmp_path):- context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}- template = "from ${project_name}.spiders.${name} import ${classname}"- rendered = "from proj.spiders.spi import TheSpider"+def test_simple_render(tmp_path):+ context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}+ template = "from ${project_name}.spiders.${name} import ${classname}"+ rendered = "from proj.spiders.spi import TheSpider"- template_path = tmp_path / "templ.py.tmpl"- render_path = tmp_path / "templ.py"+ template_path = tmp_path / "templ.py.tmpl"+ render_path = tmp_path / "templ.py"- template_path.write_text(template, encoding="utf8")- assert template_path.is_file() # Failure of test itself+ template_path.write_text(template, encoding="utf8")+ assert template_path.is_file() # Failure of test itself- render_templatefile(template_path, **context)+ render_templatefile(template_path, **context)- assert not template_path.exists()- assert render_path.read_text(encoding="utf8") == rendered+ assert not template_path.exists()+ assert render_path.read_text(encoding="utf8") == rendered- render_path.unlink()- assert not render_path.exists() # Failure of test itself+ render_path.unlink()+ assert not render_path.exists() # Failure of test itselfdiff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.pyindex e3d6675bf..3967c3365 100644--- a/tests/test_utils_trackref.py+++ b/tests/test_utils_trackref.py@@ -15,71 +15,76 @@ class Bar(trackref.object_ref):pass-class TestTrackref:- def setup_method(self):- trackref.live_refs.clear()-- def test_format_live_refs(self):- o1 = Foo() # noqa: F841- o2 = Bar() # noqa: F841- o3 = Foo() # noqa: F841- assert (- trackref.format_live_refs()- == """\[email protected](autouse=True)+def clear_refs() -> None:+ trackref.live_refs.clear()+++def test_format_live_refs():+ o1 = Foo() # noqa: F841+ o2 = Bar() # noqa: F841+ o3 = Foo() # noqa: F841+ assert (+ trackref.format_live_refs()+ == """\Live ReferencesBar 1 oldest: 0s agoFoo 2 oldest: 0s ago"""- )+ )- assert (- trackref.format_live_refs(ignore=Foo)- == """\+ assert (+ trackref.format_live_refs(ignore=Foo)+ == """\Live ReferencesBar 1 oldest: 0s ago"""- )-- @mock.patch("sys.stdout", new_callable=StringIO)- def test_print_live_refs_empty(self, stdout):- trackref.print_live_refs()- assert stdout.getvalue() == "Live References\n\n\n"-- @mock.patch("sys.stdout", new_callable=StringIO)- def test_print_live_refs_with_objects(self, stdout):- o1 = Foo() # noqa: F841- trackref.print_live_refs()- assert (- stdout.getvalue()- == """\+ )++[email protected]("sys.stdout", new_callable=StringIO)+def test_print_live_refs_empty(stdout):+ trackref.print_live_refs()+ assert stdout.getvalue() == "Live References\n\n\n"++[email protected]("sys.stdout", new_callable=StringIO)+def test_print_live_refs_with_objects(stdout):+ o1 = Foo() # noqa: F841+ trackref.print_live_refs()+ assert (+ stdout.getvalue()+ == """\Live ReferencesFoo 1 oldest: 0s ago\n\n"""- )+ )- def test_get_oldest(self):- o1 = Foo()- o1_time = time()+def test_get_oldest():+ o1 = Foo()- o2 = Bar()+ o1_time = time()+ o2 = Bar()++ o3_time = time()+ if o3_time <= o1_time:+ sleep(0.01)o3_time = time()- if o3_time <= o1_time:- sleep(0.01)- o3_time = time()- if o3_time <= o1_time:- pytest.skip("time.time is not precise enough")-- o3 = Foo() # noqa: F841- assert trackref.get_oldest("Foo") is o1- assert trackref.get_oldest("Bar") is o2- assert trackref.get_oldest("XXX") is None-- def test_iter_all(self):- o1 = Foo()- o2 = Bar() # noqa: F841- o3 = Foo()- assert set(trackref.iter_all("Foo")) == {o1, o3}+ if o3_time <= o1_time:+ pytest.skip("time.time is not precise enough")++ o3 = Foo() # noqa: F841+ assert trackref.get_oldest("Foo") is o1+ assert trackref.get_oldest("Bar") is o2+ assert trackref.get_oldest("XXX") is None+++def test_iter_all():+ o1 = Foo()+ o2 = Bar() # noqa: F841+ o3 = Foo()+ assert set(trackref.iter_all("Foo")) == {o1, o3}diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex c85dcc55e..daf09f945 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -4,7 +4,6 @@ import pytestfrom scrapy.linkextractors import IGNORED_EXTENSIONSfrom scrapy.spiders import Spider-from scrapy.utils.misc import arg_to_iterfrom scrapy.utils.url import ( # type: ignore[attr-defined]_is_filesystem_path,_public_w3lib_objects,@@ -17,261 +16,156 @@ from scrapy.utils.url import ( # type: ignore[attr-defined])-class TestUrlUtils:- def test_url_is_from_any_domain(self):- url = "http://www.wheele-bin-art.co.uk/get/product/123"- assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])- assert not url_is_from_any_domain(url, ["art.co.uk"])+def test_url_is_from_any_domain():+ url = "http://www.wheele-bin-art.co.uk/get/product/123"+ assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])+ assert not url_is_from_any_domain(url, ["art.co.uk"])- url = "http://wheele-bin-art.co.uk/get/product/123"- assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])- assert not url_is_from_any_domain(url, ["art.co.uk"])+ url = "http://wheele-bin-art.co.uk/get/product/123"+ assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])+ assert not url_is_from_any_domain(url, ["art.co.uk"])- url = "http://www.Wheele-Bin-Art.co.uk/get/product/123"- assert url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"])- assert url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"])+ url = "http://www.Wheele-Bin-Art.co.uk/get/product/123"+ assert url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"])+ assert url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"])- url = "http://192.169.0.15:8080/mypage.html"- assert url_is_from_any_domain(url, ["192.169.0.15:8080"])- assert not url_is_from_any_domain(url, ["192.169.0.15"])+ url = "http://192.169.0.15:8080/mypage.html"+ assert url_is_from_any_domain(url, ["192.169.0.15:8080"])+ assert not url_is_from_any_domain(url, ["192.169.0.15"])- url = (- "javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20"- "javascript:%20document.orderform_2581_1190810811.submit%28%29"- )- assert not url_is_from_any_domain(url, ["testdomain.com"])- assert not url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"])-- def test_url_is_from_spider(self):- spider = Spider(name="example.com")- assert url_is_from_spider("http://www.example.com/some/page.html", spider)- assert url_is_from_spider("http://sub.example.com/some/page.html", spider)- assert not url_is_from_spider("http://www.example.org/some/page.html", spider)- assert not url_is_from_spider("http://www.example.net/some/page.html", spider)-- def test_url_is_from_spider_class_attributes(self):- class MySpider(Spider):- name = "example.com"-- assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)- assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)- assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)- assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)-- def test_url_is_from_spider_with_allowed_domains(self):- spider = Spider(- name="example.com", allowed_domains=["example.org", "example.net"]- )- assert url_is_from_spider("http://www.example.com/some/page.html", spider)- assert url_is_from_spider("http://sub.example.com/some/page.html", spider)- assert url_is_from_spider("http://example.com/some/page.html", spider)- assert url_is_from_spider("http://www.example.org/some/page.html", spider)- assert url_is_from_spider("http://www.example.net/some/page.html", spider)- assert not url_is_from_spider("http://www.example.us/some/page.html", spider)-- spider = Spider(- name="example.com", allowed_domains={"example.com", "example.net"}- )- assert url_is_from_spider("http://www.example.com/some/page.html", spider)-- spider = Spider(- name="example.com", allowed_domains=("example.com", "example.net")- )- assert url_is_from_spider("http://www.example.com/some/page.html", spider)-- def test_url_is_from_spider_with_allowed_domains_class_attributes(self):- class MySpider(Spider):- name = "example.com"- allowed_domains = ("example.org", "example.net")-- assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)- assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)- assert url_is_from_spider("http://example.com/some/page.html", MySpider)- assert url_is_from_spider("http://www.example.org/some/page.html", MySpider)- assert url_is_from_spider("http://www.example.net/some/page.html", MySpider)- assert not url_is_from_spider("http://www.example.us/some/page.html", MySpider)-- def test_url_has_any_extension(self):- deny_extensions = {"." + e for e in arg_to_iter(IGNORED_EXTENSIONS)}- assert url_has_any_extension(- "http://www.example.com/archive.tar.gz", deny_extensions- )- assert url_has_any_extension("http://www.example.com/page.doc", deny_extensions)- assert url_has_any_extension("http://www.example.com/page.pdf", deny_extensions)- assert not url_has_any_extension(- "http://www.example.com/page.htm", deny_extensions- )- assert not url_has_any_extension("http://www.example.com/", deny_extensions)- assert not url_has_any_extension(- "http://www.example.com/page.doc.html", deny_extensions- )---class TestAddHttpIfNoScheme:- def test_add_scheme(self):- assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"-- def test_without_subdomain(self):- assert add_http_if_no_scheme("example.com") == "http://example.com"-- def test_path(self):- assert (- add_http_if_no_scheme("www.example.com/some/page.html")- == "http://www.example.com/some/page.html"- )-- def test_port(self):- assert (- add_http_if_no_scheme("www.example.com:80") == "http://www.example.com:80"- )+ url = (+ "javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20"+ "javascript:%20document.orderform_2581_1190810811.submit%28%29"+ )+ assert not url_is_from_any_domain(url, ["testdomain.com"])+ assert not url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"])- def test_fragment(self):- assert (- add_http_if_no_scheme("www.example.com/some/page#frag")- == "http://www.example.com/some/page#frag"- )- def test_query(self):- assert (- add_http_if_no_scheme("www.example.com/do?a=1&b=2&c=3")- == "http://www.example.com/do?a=1&b=2&c=3"- )+def test_url_is_from_spider():+ class MySpider(Spider):+ name = "example.com"- def test_username_password(self):- assert (- add_http_if_no_scheme("username:[email protected]")- == "http://username:[email protected]"- )-- def test_complete_url(self):- assert (- add_http_if_no_scheme(- "username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag"- )- == "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag"- )-- def test_preserve_http(self):- assert (- add_http_if_no_scheme("http://www.example.com") == "http://www.example.com"- )+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)+ assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)+ assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)+ assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)- def test_preserve_http_without_subdomain(self):- assert add_http_if_no_scheme("http://example.com") == "http://example.com"-- def test_preserve_http_path(self):- assert (- add_http_if_no_scheme("http://www.example.com/some/page.html")- == "http://www.example.com/some/page.html"- )-- def test_preserve_http_port(self):- assert (- add_http_if_no_scheme("http://www.example.com:80")- == "http://www.example.com:80"- )-- def test_preserve_http_fragment(self):- assert (- add_http_if_no_scheme("http://www.example.com/some/page#frag")- == "http://www.example.com/some/page#frag"- )-- def test_preserve_http_query(self):- assert (- add_http_if_no_scheme("http://www.example.com/do?a=1&b=2&c=3")- == "http://www.example.com/do?a=1&b=2&c=3"- )-- def test_preserve_http_username_password(self):- assert (- add_http_if_no_scheme("http://username:[email protected]")- == "http://username:[email protected]"- )-- def test_preserve_http_complete_url(self):- assert (- add_http_if_no_scheme(- "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag"- )- == "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag"- )-- def test_protocol_relative(self):- assert add_http_if_no_scheme("//www.example.com") == "http://www.example.com"-- def test_protocol_relative_without_subdomain(self):- assert add_http_if_no_scheme("//example.com") == "http://example.com"-- def test_protocol_relative_path(self):- assert (- add_http_if_no_scheme("//www.example.com/some/page.html")- == "http://www.example.com/some/page.html"- )-- def test_protocol_relative_port(self):- assert (- add_http_if_no_scheme("//www.example.com:80") == "http://www.example.com:80"- )-- def test_protocol_relative_fragment(self):- assert (- add_http_if_no_scheme("//www.example.com/some/page#frag")- == "http://www.example.com/some/page#frag"- )- def test_protocol_relative_query(self):- assert (- add_http_if_no_scheme("//www.example.com/do?a=1&b=2&c=3")- == "http://www.example.com/do?a=1&b=2&c=3"- )+def test_url_is_from_spider_class_attributes():+ class MySpider(Spider):+ name = "example.com"- def test_protocol_relative_username_password(self):- assert (- add_http_if_no_scheme("//username:[email protected]")- == "http://username:[email protected]"- )+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)+ assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)+ assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)+ assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)- def test_protocol_relative_complete_url(self):- assert (- add_http_if_no_scheme(- "//username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag"- )- == "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag"- )- def test_preserve_https(self):- assert (- add_http_if_no_scheme("https://www.example.com")- == "https://www.example.com"- )+def test_url_is_from_spider_with_allowed_domains():+ class MySpider(Spider):+ name = "example.com"+ allowed_domains = ["example.org", "example.net"]- def test_preserve_ftp(self):- assert add_http_if_no_scheme("ftp://www.example.com") == "ftp://www.example.com"+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)+ assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)+ assert url_is_from_spider("http://example.com/some/page.html", MySpider)+ assert url_is_from_spider("http://www.example.org/some/page.html", MySpider)+ assert url_is_from_spider("http://www.example.net/some/page.html", MySpider)+ assert not url_is_from_spider("http://www.example.us/some/page.html", MySpider)+ class MySpider2(Spider):+ name = "example.com"+ allowed_domains = {"example.com", "example.net"}-class TestGuessScheme:- pass+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider2)+ class MySpider3(Spider):+ name = "example.com"+ allowed_domains = ("example.com", "example.net")-def create_guess_scheme_t(args):- def do_expected(self):- url = guess_scheme(args[0])- assert url.startswith(args[1]), (- f"Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`"- )+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider3)- return do_expected+ ("url", "expected"),+ [+ ("http://www.example.com/archive.tar.gz", True),+ ("http://www.example.com/page.doc", True),+ ("http://www.example.com/page.pdf", True),+ ("http://www.example.com/page.htm", False),+ ("http://www.example.com/", False),+ ("http://www.example.com/page.doc.html", False),+ ],+)+def test_url_has_any_extension(url: str, expected: bool) -> None:+ deny_extensions = {"." + e for e in IGNORED_EXTENSIONS}+ assert url_has_any_extension(url, deny_extensions) is expected-def create_skipped_scheme_t(args):- def do_expected(self):- pytest.skip(args[2])- return do_expected+ ("url", "expected"),+ [+ ("www.example.com", "http://www.example.com"),+ ("example.com", "http://example.com"),+ ("www.example.com/some/page.html", "http://www.example.com/some/page.html"),+ ("www.example.com:80", "http://www.example.com:80"),+ ("www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),+ ("www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),+ (+ "username:[email protected]",+ "http://username:[email protected]",+ ),+ (+ "username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",+ "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",+ ),+ ("http://www.example.com", "http://www.example.com"),+ ("http://example.com", "http://example.com"),+ (+ "http://www.example.com/some/page.html",+ "http://www.example.com/some/page.html",+ ),+ ("http://www.example.com:80", "http://www.example.com:80"),+ (+ "http://www.example.com/some/page#frag",+ "http://www.example.com/some/page#frag",+ ),+ (+ "http://www.example.com/do?a=1&b=2&c=3",+ "http://www.example.com/do?a=1&b=2&c=3",+ ),+ (+ "http://username:[email protected]",+ "http://username:[email protected]",+ ),+ (+ "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",+ "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",+ ),+ ("//www.example.com", "http://www.example.com"),+ ("//example.com", "http://example.com"),+ ("//www.example.com/some/page.html", "http://www.example.com/some/page.html"),+ ("//www.example.com:80", "http://www.example.com:80"),+ ("//www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),+ ("//www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),+ (+ "//username:[email protected]",+ "http://username:[email protected]",+ ),+ (+ "//username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",+ "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",+ ),+ ("https://www.example.com", "https://www.example.com"),+ ("ftp://www.example.com", "ftp://www.example.com"),+ ],+)+def test_add_http_if_no_scheme(url: str, expected: str) -> None:+ assert add_http_if_no_scheme(url) == expected-for k, args in enumerate(+ ("url", "expected"),[("/index", "file://"),("/index.html", "file://"),@@ -295,14 +189,13 @@ for k, args in enumerate(("/", "http://"),(".../test", "http://"),],- start=1,-):- t_method = create_guess_scheme_t(args)- t_method.__name__ = f"test_uri_{k:03}"- setattr(TestGuessScheme, t_method.__name__, t_method)--# TODO: the following tests do not pass with current implementation-for k, skip_args in enumerate(+)+def test_guess_scheme(url: str, expected: str):+ assert guess_scheme(url).startswith(expected)+++ ("url", "expected", "reason"),[(r"C:\absolute\path\to\a\file.html",@@ -310,25 +203,21 @@ for k, skip_args in enumerate("Windows filepath are not supported for scrapy shell",),],- start=1,-):- t_method = create_skipped_scheme_t(skip_args)- t_method.__name__ = f"test_uri_skipped_{k:03}"- setattr(TestGuessScheme, t_method.__name__, t_method)+)+def test_guess_scheme_skipped(url: str, expected: str, reason: str):+ pytest.skip(reason)class TestStripUrl:- def test_noop(self):- assert (- strip_url("http://www.example.com/index.html")- == "http://www.example.com/index.html"- )-- def test_noop_query_string(self):- assert (- strip_url("http://www.example.com/index.html?somekey=somevalue")- == "http://www.example.com/index.html?somekey=somevalue"- )+ @pytest.mark.parametrize(+ "url",+ [+ "http://www.example.com/index.html",+ "http://www.example.com/index.html?somekey=somevalue",+ ],+ )+ def test_noop(self, url: str) -> None:+ assert strip_url(url) == urldef test_fragments(self):assert (@@ -339,16 +228,20 @@ class TestStripUrl:== "http://www.example.com/index.html?somekey=somevalue#section")- def test_path(self):- for input_url, origin, output_url in [+ @pytest.mark.parametrize(+ ("url", "origin", "expected"),+ [("http://www.example.com/", False, "http://www.example.com/"),("http://www.example.com", False, "http://www.example.com"),("http://www.example.com", True, "http://www.example.com/"),- ]:- assert strip_url(input_url, origin_only=origin) == output_url+ ],+ )+ def test_path(self, url: str, origin: bool, expected: str) -> None:+ assert strip_url(url, origin_only=origin) == expected- def test_credentials(self):- for i, o in [+ @pytest.mark.parametrize(+ ("url", "expected"),+ [("http://[email protected]/index.html?somekey=somevalue#section","http://www.example.com/index.html?somekey=somevalue",@@ -361,34 +254,29 @@ class TestStripUrl:"ftp://username:[email protected]/index.html?somekey=somevalue#section","ftp://www.example.com/index.html?somekey=somevalue",),- ]:- assert strip_url(i, strip_credentials=True) == o-- def test_credentials_encoded_delims(self):- for i, o in [- # user: "username@"- # password: none+ # user: "username@", password: none("http://username%[email protected]/index.html?somekey=somevalue#section","http://www.example.com/index.html?somekey=somevalue",),- # user: "username:pass"- # password: ""+ # user: "username:pass", password: ""("https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section","https://www.example.com/index.html?somekey=somevalue",),- # user: "me"- # password: "[email protected]"+ # user: "me", password: "[email protected]"("ftp://me:user%[email protected]/index.html?somekey=somevalue#section","ftp://www.example.com/index.html?somekey=somevalue",),- ]:- assert strip_url(i, strip_credentials=True) == o+ ],+ )+ def test_credentials(self, url: str, expected: str) -> None:+ assert strip_url(url, strip_credentials=True) == expected- def test_default_ports_creds_off(self):- for i, o in [+ @pytest.mark.parametrize(+ ("url", "expected"),+ [("http://username:[email protected]:80/index.html?somekey=somevalue#section","http://www.example.com/index.html?somekey=somevalue",@@ -421,11 +309,14 @@ class TestStripUrl:"ftp://username:[email protected]:221/file.txt","ftp://www.example.com:221/file.txt",),- ]:- assert strip_url(i) == o+ ],+ )+ def test_default_ports_creds_off(self, url: str, expected: str) -> None:+ assert strip_url(url) == expected- def test_default_ports(self):- for i, o in [+ @pytest.mark.parametrize(+ ("url", "expected"),+ [("http://username:[email protected]:80/index.html","http://username:[email protected]/index.html",@@ -458,11 +349,16 @@ class TestStripUrl:"ftp://username:[email protected]:221/file.txt","ftp://username:[email protected]:221/file.txt",),- ]:- assert strip_url(i, strip_default_port=True, strip_credentials=False) == o+ ],+ )+ def test_default_ports(self, url: str, expected: str) -> None:+ assert (+ strip_url(url, strip_default_port=True, strip_credentials=False) == expected+ )- def test_default_ports_keep(self):- for i, o in [+ @pytest.mark.parametrize(+ ("url", "expected"),+ [("http://username:[email protected]:80/index.html?somekey=somevalue&someotherkey=sov#section","http://username:[email protected]:80/index.html?somekey=somevalue&someotherkey=sov",@@ -495,11 +391,17 @@ class TestStripUrl:"ftp://username:[email protected]:221/file.txt","ftp://username:[email protected]:221/file.txt",),- ]:- assert strip_url(i, strip_default_port=False, strip_credentials=False) == o+ ],+ )+ def test_default_ports_keep(self, url: str, expected: str) -> None:+ assert (+ strip_url(url, strip_default_port=False, strip_credentials=False)+ == expected+ )- def test_origin_only(self):- for i, o in [+ @pytest.mark.parametrize(+ ("url", "expected"),+ [("http://username:[email protected]/index.html","http://www.example.com/",@@ -516,29 +418,33 @@ class TestStripUrl:"https://username:[email protected]:443/index.html","https://www.example.com/",),- ]:- assert strip_url(i, origin_only=True) == o---class TestIsPath:- def test_path(self):- for input_value, output_value in (- # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell- # Unix-like OS, Microsoft Windows / cmd.exe- ("/home/user/docs/Letter.txt", True),- ("./inthisdir", True),- ("../../greatgrandparent", True),- ("~/.rcinfo", True),- (r"C:\user\docs\Letter.txt", True),- ("/user/docs/Letter.txt", True),- (r"C:\Letter.txt", True),- (r"\\Server01\user\docs\Letter.txt", True),- (r"\\?\UNC\Server01\user\docs\Letter.txt", True),- (r"\\?\C:\user\docs\Letter.txt", True),- (r"C:\user\docs\somefile.ext:alternate_stream_name", True),- (r"https://example.com", False),- ):- assert _is_filesystem_path(input_value) == output_value, input_value+ ],+ )+ def test_origin_only(self, url: str, expected: str) -> None:+ assert strip_url(url, origin_only=True) == expected+++ ("path", "expected"),+ [+ # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell+ # Unix-like OS, Microsoft Windows / cmd.exe+ ("/home/user/docs/Letter.txt", True),+ ("./inthisdir", True),+ ("../../greatgrandparent", True),+ ("~/.rcinfo", True),+ (r"C:\user\docs\Letter.txt", True),+ ("/user/docs/Letter.txt", True),+ (r"C:\Letter.txt", True),+ (r"\\Server01\user\docs\Letter.txt", True),+ (r"\\?\UNC\Server01\user\docs\Letter.txt", True),+ (r"\\?\C:\user\docs\Letter.txt", True),+ (r"C:\user\docs\somefile.ext:alternate_stream_name", True),+ (r"https://example.com", False),+ ],+)+def test__is_filesystem_path(path: str, expected: bool) -> None:+ assert _is_filesystem_path(path) == expected@pytest.mark.parametrize(@@ -550,7 +456,7 @@ class TestIsPath:*_public_w3lib_objects,],)-def test_deprecated_imports_from_w3lib(obj_name):+def test_deprecated_imports_from_w3lib(obj_name: str) -> None:with warnings.catch_warnings(record=True) as warns:obj_type = "attribute" if obj_name == "_safe_chars" else "function"message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."@@ -559,4 +465,5 @@ def test_deprecated_imports_from_w3lib(obj_name):getattr(import_module("scrapy.utils.url"), obj_name)+ assert isinstance(warns[0].message, Warning)assert message in warns[0].message.args