Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Refactoring of test_utils_*. (#6905)

Andrey Rakhmatullind70f8a3f1425
diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py
index 988ad50b9..1edef0b4a 100644
--- a/tests/test_command_startproject.py
+++ b/tests/test_command_startproject.py
@@ -106,8 +106,6 @@ def get_permissions_dict(
class TestStartprojectTemplates(TestProjectBase):
- maxDiff = None
-
def setup_method(self):
super().setup_method()
self.tmpl = str(Path(self.temp_path, "templates"))
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 9d2e5f997..1714bd4db 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -150,8 +150,6 @@ class KeywordArgumentsSpider(MockServerSpider):
class TestCallbackKeywordArguments(TestCase):
- maxDiff = None
-
@classmethod
def setUpClass(cls):
cls.mockserver = MockServer()
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index a4f31096e..66a6f5318 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -1,31 +1,35 @@
import scrapy
-class TestToplevel:
- def test_version(self):
- assert isinstance(scrapy.__version__, str)
+def test_version():
+ assert isinstance(scrapy.__version__, str)
- def test_version_info(self):
- assert isinstance(scrapy.version_info, tuple)
- def test_request_shortcut(self):
- from scrapy.http import FormRequest, Request
+def test_version_info():
+ assert isinstance(scrapy.version_info, tuple)
- assert scrapy.Request is Request
- assert scrapy.FormRequest is FormRequest
- def test_spider_shortcut(self):
- from scrapy.spiders import Spider
+def test_request_shortcut():
+ from scrapy.http import FormRequest, Request
- assert scrapy.Spider is Spider
+ assert scrapy.Request is Request
+ assert scrapy.FormRequest is FormRequest
- def test_selector_shortcut(self):
- from scrapy.selector import Selector
- assert scrapy.Selector is Selector
+def test_spider_shortcut():
+ from scrapy.spiders import Spider
- def test_item_shortcut(self):
- from scrapy.item import Field, Item
+ assert scrapy.Spider is Spider
- assert scrapy.Item is Item
- assert scrapy.Field is Field
+
+def test_selector_shortcut():
+ from scrapy.selector import Selector
+
+ assert scrapy.Selector is Selector
+
+
+def test_item_shortcut():
+ from scrapy.item import Field, Item
+
+ assert scrapy.Item is Item
+ assert scrapy.Field is Field
diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py
deleted file mode 100644
index 0e1e89e81..000000000
--- a/tests/test_urlparse_monkeypatches.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from urllib.parse import urlparse
-
-
-class TestUrlparse:
- def test_s3_url(self):
- p = urlparse("s3://bucket/key/name?param=value")
- assert p.scheme == "s3"
- assert p.hostname == "bucket"
- assert p.path == "/key/name"
- assert p.query == "param=value"
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 26f158380..ed7dda18d 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -47,12 +47,11 @@ class TestBuildComponentList:
assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"]
-class TestUtilsConf:
- def test_arglist_to_dict(self):
- assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == {
- "arg1": "val1",
- "arg2": "val2",
- }
+def test_arglist_to_dict():
+ assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == {
+ "arg1": "val1",
+ "arg2": "val2",
+ }
class TestFeedExportConfig:
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
index 6598bdce7..dc1d96f66 100644
--- a/tests/test_utils_console.py
+++ b/tests/test_utils_console.py
@@ -18,23 +18,24 @@ except ImportError:
ipy = False
-class TestUtilsConsole:
- def test_get_shell_embed_func(self):
- shell = get_shell_embed_func(["invalid"])
- assert shell is None
-
- shell = get_shell_embed_func(["invalid", "python"])
- assert callable(shell)
- assert shell.__name__ == "_embed_standard_shell"
-
- @pytest.mark.skipif(not bpy, reason="bpython not available in testenv")
- def test_get_shell_embed_func2(self):
- shell = get_shell_embed_func(["bpython"])
- assert callable(shell)
- assert shell.__name__ == "_embed_bpython_shell"
-
- @pytest.mark.skipif(not ipy, reason="IPython not available in testenv")
- def test_get_shell_embed_func3(self):
- # default shell should be 'ipython'
- shell = get_shell_embed_func()
- assert shell.__name__ == "_embed_ipython_shell"
+def test_get_shell_embed_func():
+ shell = get_shell_embed_func(["invalid"])
+ assert shell is None
+
+ shell = get_shell_embed_func(["invalid", "python"])
+ assert callable(shell)
+ assert shell.__name__ == "_embed_standard_shell"
+
+
[email protected](not bpy, reason="bpython not available in testenv")
+def test_get_shell_embed_func_bpython():
+ shell = get_shell_embed_func(["bpython"])
+ assert callable(shell)
+ assert shell.__name__ == "_embed_bpython_shell"
+
+
[email protected](not ipy, reason="IPython not available in testenv")
+def test_get_shell_embed_func_ipython():
+ # default shell should be 'ipython'
+ shell = get_shell_embed_func()
+ assert shell.__name__ == "_embed_ipython_shell"
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index e8dd88049..02362693a 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -1,4 +1,5 @@
import warnings
+from typing import Any
import pytest
from w3lib.http import basic_auth_header
@@ -8,9 +9,8 @@ from scrapy.utils.curl import curl_to_request_kwargs
class TestCurlToRequestKwargs:
- maxDiff = 5000
-
- def _test_command(self, curl_command, expected_result):
+ @staticmethod
+ def _test_command(curl_command: str, expected_result: dict[str, Any]) -> None:
result = curl_to_request_kwargs(curl_command)
assert result == expected_result
try:
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 75b6b0e99..352e49165 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,5 +1,6 @@
import copy
import warnings
+from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping, MutableMapping
import pytest
@@ -16,7 +17,12 @@ from scrapy.utils.datatypes import (
from scrapy.utils.python import garbage_collect
-class CaseInsensitiveDictBase:
+class TestCaseInsensitiveDictBase(ABC):
+ @property
+ @abstractmethod
+ def dict_class(self) -> type[MutableMapping]:
+ raise NotImplementedError
+
def test_init_dict(self):
seq = {"red": 1, "black": 3}
d = self.dict_class(seq)
@@ -199,7 +205,7 @@ class CaseInsensitiveDictBase:
assert h1.get("header1") == h3.get("HEADER1")
-class TestCaseInsensitiveDict(CaseInsensitiveDictBase):
+class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
dict_class = CaseInsensitiveDict
def test_repr(self):
@@ -216,7 +222,7 @@ class TestCaseInsensitiveDict(CaseInsensitiveDictBase):
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestCaselessDict(CaseInsensitiveDictBase):
+class TestCaselessDict(TestCaseInsensitiveDictBase):
dict_class = CaselessDict
def test_deprecation_message(self):
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 662de0dc3..a88b5e008 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -1,6 +1,7 @@
import inspect
import warnings
from unittest import mock
+from warnings import WarningMessage
import pytest
@@ -21,7 +22,9 @@ class NewName(SomeBaseClass):
class TestWarnWhenSubclassed:
- def _mywarnings(self, w, category=MyWarning):
+ def _mywarnings(
+ self, w: list[WarningMessage], category: type[Warning] = MyWarning
+ ) -> list[WarningMessage]:
return [x for x in w if x.category is MyWarning]
def test_no_warning_on_definition(self):
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
index cea564653..20251ca59 100644
--- a/tests/test_utils_display.py
+++ b/tests/test_utils_display.py
@@ -3,88 +3,92 @@ from unittest import mock
from scrapy.utils.display import pformat, pprint
-
-class TestDisplay:
- object = {"a": 1}
- colorized_strings = {
+value = {"a": 1}
+colorized_strings = {
+ (
(
- (
- "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
- "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}"
- )
- + suffix
- )
- for suffix in (
- # https://github.com/pygments/pygments/issues/2313
- "\n", # pygments ≤ 2.13
- "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14
+ "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
+ "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}"
)
- }
- plain_string = "{'a': 1}"
-
- @mock.patch("sys.platform", "linux")
- @mock.patch("sys.stdout.isatty")
- def test_pformat(self, isatty):
- isatty.return_value = True
- assert pformat(self.object) in self.colorized_strings
-
- @mock.patch("sys.stdout.isatty")
- def test_pformat_dont_colorize(self, isatty):
- isatty.return_value = True
- assert pformat(self.object, colorize=False) == self.plain_string
-
- def test_pformat_not_tty(self):
- assert pformat(self.object) == self.plain_string
-
- @mock.patch("sys.platform", "win32")
- @mock.patch("platform.version")
- @mock.patch("sys.stdout.isatty")
- def test_pformat_old_windows(self, isatty, version):
- isatty.return_value = True
- version.return_value = "10.0.14392"
- assert pformat(self.object) in self.colorized_strings
-
- @mock.patch("sys.platform", "win32")
- @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
- @mock.patch("platform.version")
- @mock.patch("sys.stdout.isatty")
- def test_pformat_windows_no_terminal_processing(
- self, isatty, version, terminal_processing
- ):
- isatty.return_value = True
- version.return_value = "10.0.14393"
- terminal_processing.return_value = False
- assert pformat(self.object) == self.plain_string
-
- @mock.patch("sys.platform", "win32")
- @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
- @mock.patch("platform.version")
- @mock.patch("sys.stdout.isatty")
- def test_pformat_windows(self, isatty, version, terminal_processing):
- isatty.return_value = True
- version.return_value = "10.0.14393"
- terminal_processing.return_value = True
- assert pformat(self.object) in self.colorized_strings
-
- @mock.patch("sys.platform", "linux")
- @mock.patch("sys.stdout.isatty")
- def test_pformat_no_pygments(self, isatty):
- isatty.return_value = True
-
- import builtins
-
- real_import = builtins.__import__
-
- def mock_import(name, globals, locals, fromlist, level):
- if "pygments" in name:
- raise ImportError
- return real_import(name, globals, locals, fromlist, level)
-
- builtins.__import__ = mock_import
- assert pformat(self.object) == self.plain_string
- builtins.__import__ = real_import
-
- def test_pprint(self):
- with mock.patch("sys.stdout", new=StringIO()) as mock_out:
- pprint(self.object)
- assert mock_out.getvalue() == "{'a': 1}\n"
+ + suffix
+ )
+ for suffix in (
+ # https://github.com/pygments/pygments/issues/2313
+ "\n", # pygments ≤ 2.13
+ "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14
+ )
+}
+plain_string = "{'a': 1}"
+
+
[email protected]("sys.platform", "linux")
[email protected]("sys.stdout.isatty")
+def test_pformat(isatty):
+ isatty.return_value = True
+ assert pformat(value) in colorized_strings
+
+
[email protected]("sys.stdout.isatty")
+def test_pformat_dont_colorize(isatty):
+ isatty.return_value = True
+ assert pformat(value, colorize=False) == plain_string
+
+
+def test_pformat_not_tty():
+ assert pformat(value) == plain_string
+
+
[email protected]("sys.platform", "win32")
[email protected]("platform.version")
[email protected]("sys.stdout.isatty")
+def test_pformat_old_windows(isatty, version):
+ isatty.return_value = True
+ version.return_value = "10.0.14392"
+ assert pformat(value) in colorized_strings
+
+
[email protected]("sys.platform", "win32")
[email protected]("scrapy.utils.display._enable_windows_terminal_processing")
[email protected]("platform.version")
[email protected]("sys.stdout.isatty")
+def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing):
+ isatty.return_value = True
+ version.return_value = "10.0.14393"
+ terminal_processing.return_value = False
+ assert pformat(value) == plain_string
+
+
[email protected]("sys.platform", "win32")
[email protected]("scrapy.utils.display._enable_windows_terminal_processing")
[email protected]("platform.version")
[email protected]("sys.stdout.isatty")
+def test_pformat_windows(isatty, version, terminal_processing):
+ isatty.return_value = True
+ version.return_value = "10.0.14393"
+ terminal_processing.return_value = True
+ assert pformat(value) in colorized_strings
+
+
[email protected]("sys.platform", "linux")
[email protected]("sys.stdout.isatty")
+def test_pformat_no_pygments(isatty):
+ isatty.return_value = True
+
+ import builtins
+
+ real_import = builtins.__import__
+
+ def mock_import(name, globals, locals, fromlist, level):
+ if "pygments" in name:
+ raise ImportError
+ return real_import(name, globals, locals, fromlist, level)
+
+ builtins.__import__ = mock_import
+ assert pformat(value) == plain_string
+ builtins.__import__ = real_import
+
+
+def test_pprint():
+ with mock.patch("sys.stdout", new=StringIO()) as mock_out:
+ pprint(value)
+ assert mock_out.getvalue() == "{'a': 1}\n"
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index c43ed152b..06fdf9cba 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -11,47 +11,51 @@ from tests import tests_datadir
SAMPLEDIR = Path(tests_datadir, "compressed")
-class TestGunzip:
- def test_gunzip_basic(self):
- r1 = Response(
- "http://www.example.com",
- body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(),
- )
- assert gzip_magic_number(r1)
-
- r2 = Response("http://www.example.com", body=gunzip(r1.body))
- assert not gzip_magic_number(r2)
- assert len(r2.body) == 9950
-
- def test_gunzip_truncated(self):
- text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes())
- assert text.endswith(b"</html")
-
- def test_gunzip_no_gzip_file_raises(self):
- with pytest.raises(BadGzipFile):
- gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes())
-
- def test_gunzip_truncated_short(self):
- r1 = Response(
- "http://www.example.com",
- body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(),
- )
- assert gzip_magic_number(r1)
-
- r2 = Response("http://www.example.com", body=gunzip(r1.body))
- assert r2.body.endswith(b"</html>")
- assert not gzip_magic_number(r2)
-
- def test_is_gzipped_empty(self):
- r1 = Response("http://www.example.com")
- assert not gzip_magic_number(r1)
-
- def test_gunzip_illegal_eof(self):
- text = html_to_unicode(
- "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes())
- )[1]
- expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text(
- encoding="utf-8"
- )
- assert len(text) == len(expected_text)
- assert text == expected_text
+def test_gunzip_basic():
+ r1 = Response(
+ "http://www.example.com",
+ body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(),
+ )
+ assert gzip_magic_number(r1)
+
+ r2 = Response("http://www.example.com", body=gunzip(r1.body))
+ assert not gzip_magic_number(r2)
+ assert len(r2.body) == 9950
+
+
+def test_gunzip_truncated():
+ text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes())
+ assert text.endswith(b"</html")
+
+
+def test_gunzip_no_gzip_file_raises():
+ with pytest.raises(BadGzipFile):
+ gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes())
+
+
+def test_gunzip_truncated_short():
+ r1 = Response(
+ "http://www.example.com",
+ body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(),
+ )
+ assert gzip_magic_number(r1)
+
+ r2 = Response("http://www.example.com", body=gunzip(r1.body))
+ assert r2.body.endswith(b"</html>")
+ assert not gzip_magic_number(r2)
+
+
+def test_is_gzipped_empty():
+ r1 = Response("http://www.example.com")
+ assert not gzip_magic_number(r1)
+
+
+def test_gunzip_illegal_eof():
+ text = html_to_unicode(
+ "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes())
+ )[1]
+ expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text(
+ encoding="utf-8"
+ )
+ assert len(text) == len(expected_text)
+ assert text == expected_text
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 0c05ef7d6..9bd86f7fb 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -4,18 +4,17 @@ from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
-class TestHttpobjUtils:
- def test_urlparse_cached(self):
- url = "http://www.example.com/index.html"
- request1 = Request(url)
- request2 = Request(url)
- req1a = urlparse_cached(request1)
- req1b = urlparse_cached(request1)
- req2 = urlparse_cached(request2)
- urlp = urlparse(url)
+def test_urlparse_cached():
+ url = "http://www.example.com/index.html"
+ request1 = Request(url)
+ request2 = Request(url)
+ req1a = urlparse_cached(request1)
+ req1b = urlparse_cached(request1)
+ req2 = urlparse_cached(request2)
+ urlp = urlparse(url)
- assert req1a == req2
- assert req1a == urlp
- assert req1a is req1b
- assert req1a is not req2
- assert req1a is not req2
+ assert req1a == req2
+ assert req1a == urlp
+ assert req1a is req1b
+ assert req1a is not req2
+ assert req1a is not req2
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index fa0d37866..ac32fff2c 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,3 +1,8 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -5,9 +10,19 @@ from scrapy.http import Response, TextResponse, XmlResponse
from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml
from tests import get_testdata
+if TYPE_CHECKING:
+ from collections.abc import Iterator
+
+ from scrapy import Selector
+
+
+class TestXmliterBase(ABC):
+ @abstractmethod
+ def xmliter(
+ self, obj: Response | str | bytes, nodename: str, *args: Any
+ ) -> Iterator[Selector]:
+ raise NotImplementedError
-class XmliterBase:
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter(self):
body = b"""
<?xml version="1.0" encoding="UTF-8"?>
@@ -39,7 +54,6 @@ class XmliterBase:
("002", ["Name 2"], ["Type 2"]),
]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_unusual_node(self):
body = b"""<?xml version="1.0" encoding="UTF-8"?>
<root>
@@ -53,7 +67,6 @@ class XmliterBase:
]
assert nodenames == [["matchme..."]]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_unicode(self):
# example taken from https://github.com/scrapy/scrapy/issues/1665
body = """<?xml version="1.0" encoding="UTF-8"?>
@@ -113,7 +126,6 @@ class XmliterBase:
("27", ["A"], ["27"]),
]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_text(self):
body = (
'<?xml version="1.0" encoding="UTF-8"?>'
@@ -125,7 +137,6 @@ class XmliterBase:
["two"],
]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_namespaces(self):
body = b"""
<?xml version="1.0" encoding="UTF-8"?>
@@ -163,7 +174,6 @@ class XmliterBase:
assert node.xpath("id/text()").getall() == []
assert node.xpath("price/text()").getall() == []
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_namespaced_nodename(self):
body = b"""
<?xml version="1.0" encoding="UTF-8"?>
@@ -191,7 +201,6 @@ class XmliterBase:
"http://www.mydummycompany.com/images/item1.jpg"
]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_namespaced_nodename_missing(self):
body = b"""
<?xml version="1.0" encoding="UTF-8"?>
@@ -216,7 +225,6 @@ class XmliterBase:
with pytest.raises(StopIteration):
next(my_iter)
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_exception(self):
body = (
'<?xml version="1.0" encoding="UTF-8"?>'
@@ -229,13 +237,11 @@ class XmliterBase:
with pytest.raises(StopIteration):
next(iter)
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_objtype_exception(self):
i = self.xmliter(42, "product")
with pytest.raises(TypeError):
next(i)
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_encoding(self):
body = (
b'<?xml version="1.0" encoding="ISO-8859-9"?>\n'
@@ -250,8 +256,12 @@ class XmliterBase:
)
-class TestXmliter(XmliterBase):
- xmliter = staticmethod(xmliter)
[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
+class TestXmliter(TestXmliterBase):
+ def xmliter(
+ self, obj: Response | str | bytes, nodename: str, *args: Any
+ ) -> Iterator[Selector]:
+ return xmliter(obj, nodename)
def test_deprecation(self):
body = b"""
@@ -267,8 +277,11 @@ class TestXmliter(XmliterBase):
next(self.xmliter(body, "product"))
-class TestLxmlXmliter(XmliterBase):
- xmliter = staticmethod(xmliter_lxml)
+class TestLxmlXmliter(TestXmliterBase):
+ def xmliter(
+ self, obj: Response | str | bytes, nodename: str, *args: Any
+ ) -> Iterator[Selector]:
+ return xmliter_lxml(obj, nodename, *args)
def test_xmliter_iterate_namespace(self):
body = b"""
@@ -493,23 +506,32 @@ class TestUtilsCsv:
]
-class TestHelper:
+class TestBodyOrStr:
bbody = b"utf8-body"
ubody = bbody.decode("utf8")
- txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8")
- response = Response(url="http://example.org/", body=bbody)
-
- def test_body_or_str(self):
- for obj in (self.bbody, self.ubody, self.txtresponse, self.response):
- r1 = _body_or_str(obj)
- self._assert_type_and_value(r1, self.ubody, obj)
- r2 = _body_or_str(obj, unicode=True)
- self._assert_type_and_value(r2, self.ubody, obj)
- r3 = _body_or_str(obj, unicode=False)
- self._assert_type_and_value(r3, self.bbody, obj)
- assert type(r1) is type(r2)
- assert type(r1) is not type(r3)
-
- def _assert_type_and_value(self, a, b, obj):
+
+ @pytest.mark.parametrize(
+ "obj",
+ [
+ bbody,
+ ubody,
+ TextResponse(url="http://example.org/", body=bbody, encoding="utf-8"),
+ Response(url="http://example.org/", body=bbody),
+ ],
+ )
+ def test_body_or_str(self, obj: Response | str | bytes) -> None:
+ r1 = _body_or_str(obj)
+ self._assert_type_and_value(r1, self.ubody, obj)
+ r2 = _body_or_str(obj, unicode=True)
+ self._assert_type_and_value(r2, self.ubody, obj)
+ r3 = _body_or_str(obj, unicode=False)
+ self._assert_type_and_value(r3, self.bbody, obj)
+ assert type(r1) is type(r2)
+ assert type(r1) is not type(r3)
+
+ @staticmethod
+ def _assert_type_and_value(
+ a: str | bytes, b: str | bytes, obj: Response | str | bytes
+ ) -> None:
assert type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}"
assert a == b
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 56375606c..f40e424ff 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -22,7 +22,9 @@ from scrapy.utils.test import get_crawler
from tests.spiders import LogSpider
if TYPE_CHECKING:
- from collections.abc import Mapping, MutableMapping
+ from collections.abc import Generator, Mapping, MutableMapping
+
+ from scrapy.crawler import Crawler
class TestFailureToExcInfo:
@@ -70,33 +72,41 @@ class TestTopLevelFormatter:
class TestLogCounterHandler:
- def setup_method(self):
+ @pytest.fixture
+ def crawler(self) -> Crawler:
settings = {"LOG_LEVEL": "WARNING"}
- self.logger = logging.getLogger("test")
- self.logger.setLevel(logging.NOTSET)
- self.logger.propagate = False
- self.crawler = get_crawler(settings_dict=settings)
- self.handler = LogCounterHandler(self.crawler)
- self.logger.addHandler(self.handler)
+ return get_crawler(settings_dict=settings)
+
+ @pytest.fixture
+ def logger(self, crawler: Crawler) -> Generator[logging.Logger]:
+ logger = logging.getLogger("test")
+ logger.setLevel(logging.NOTSET)
+ logger.propagate = False
+ handler = LogCounterHandler(crawler)
+ logger.addHandler(handler)
+
+ yield logger
- def teardown_method(self):
- self.logger.propagate = True
- self.logger.removeHandler(self.handler)
+ logger.propagate = True
+ logger.removeHandler(handler)
- def test_init(self):
- assert self.crawler.stats.get_value("log_count/DEBUG") is None
- assert self.crawler.stats.get_value("log_count/INFO") is None
- assert self.crawler.stats.get_value("log_count/WARNING") is None
- assert self.crawler.stats.get_value("log_count/ERROR") is None
- assert self.crawler.stats.get_value("log_count/CRITICAL") is None
+ def test_init(self, crawler: Crawler, logger: logging.Logger) -> None:
+ assert crawler.stats
+ assert crawler.stats.get_value("log_count/DEBUG") is None
+ assert crawler.stats.get_value("log_count/INFO") is None
+ assert crawler.stats.get_value("log_count/WARNING") is None
+ assert crawler.stats.get_value("log_count/ERROR") is None
+ assert crawler.stats.get_value("log_count/CRITICAL") is None
- def test_accepted_level(self):
- self.logger.error("test log msg")
- assert self.crawler.stats.get_value("log_count/ERROR") == 1
+ def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None:
+ logger.error("test log msg")
+ assert crawler.stats
+ assert crawler.stats.get_value("log_count/ERROR") == 1
- def test_filtered_out_level(self):
- self.logger.debug("test log msg")
- assert self.crawler.stats.get_value("log_count/INFO") is None
+ def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None:
+ logger.debug("test log msg")
+ assert crawler.stats
+ assert crawler.stats.get_value("log_count/INFO") is None
class TestStreamLogger:
@@ -135,7 +145,7 @@ class TestStreamLogger:
)
def test_spider_logger_adapter_process(
base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict
-):
+) -> None:
logger = logging.getLogger("test")
spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra)
@@ -149,59 +159,75 @@ def test_spider_logger_adapter_process(
class TestLogging:
- def setup_method(self):
- self.log_stream = StringIO()
- handler = logging.StreamHandler(self.log_stream)
+ @pytest.fixture
+ def log_stream(self) -> StringIO:
+ return StringIO()
+
+ @pytest.fixture
+ def spider(self) -> LogSpider:
+ return LogSpider()
+
+ @pytest.fixture(autouse=True)
+ def logger(self, log_stream: StringIO) -> Generator[logging.Logger]:
+ handler = logging.StreamHandler(log_stream)
logger = logging.getLogger("log_spider")
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
- self.handler = handler
- self.logger = logger
- self.spider = LogSpider()
- def teardown_method(self):
- self.logger.removeHandler(self.handler)
+ yield logger
+
+ logger.removeHandler(handler)
- def test_debug_logging(self):
+ def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo message"
- self.spider.log_debug(log_message)
- log_contents = self.log_stream.getvalue()
+ spider.log_debug(log_message)
+ log_contents = log_stream.getvalue()
assert log_contents == f"{log_message}\n"
- def test_info_logging(self):
+ def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Bar message"
- self.spider.log_info(log_message)
- log_contents = self.log_stream.getvalue()
+ spider.log_info(log_message)
+ log_contents = log_stream.getvalue()
assert log_contents == f"{log_message}\n"
- def test_warning_logging(self):
+ def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Baz message"
- self.spider.log_warning(log_message)
- log_contents = self.log_stream.getvalue()
+ spider.log_warning(log_message)
+ log_contents = log_stream.getvalue()
assert log_contents == f"{log_message}\n"
- def test_error_logging(self):
+ def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo bar message"
- self.spider.log_error(log_message)
- log_contents = self.log_stream.getvalue()
+ spider.log_error(log_message)
+ log_contents = log_stream.getvalue()
assert log_contents == f"{log_message}\n"
- def test_critical_logging(self):
+ def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo bar baz message"
- self.spider.log_critical(log_message)
- log_contents = self.log_stream.getvalue()
+ spider.log_critical(log_message)
+ log_contents = log_stream.getvalue()
assert log_contents == f"{log_message}\n"
class TestLoggingWithExtra:
- def setup_method(self):
- self.log_stream = StringIO()
- handler = logging.StreamHandler(self.log_stream)
+ regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$")
+
+ @pytest.fixture
+ def log_stream(self) -> StringIO:
+ return StringIO()
+
+ @pytest.fixture
+ def spider(self) -> LogSpider:
+ return LogSpider()
+
+ @pytest.fixture(autouse=True)
+ def logger(self, log_stream: StringIO) -> Generator[logging.Logger]:
+ handler = logging.StreamHandler(log_stream)
formatter = logging.Formatter(
'{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}'
)
@@ -209,80 +235,79 @@ class TestLoggingWithExtra:
logger = logging.getLogger("log_spider")
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
- self.handler = handler
- self.logger = logger
- self.spider = LogSpider()
- self.regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$")
- def teardown_method(self):
- self.logger.removeHandler(self.handler)
+ yield logger
+
+ logger.removeHandler(handler)
- def test_debug_logging(self):
+ def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo message"
extra = {"important_info": "foo"}
- self.spider.log_debug(log_message, extra)
- log_contents = self.log_stream.getvalue()
- log_contents = json.loads(log_contents)
+ spider.log_debug(log_message, extra)
+ log_contents_str = log_stream.getvalue()
+ log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "DEBUG"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
- def test_info_logging(self):
+ def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Bar message"
extra = {"important_info": "bar"}
- self.spider.log_info(log_message, extra)
- log_contents = self.log_stream.getvalue()
- log_contents = json.loads(log_contents)
+ spider.log_info(log_message, extra)
+ log_contents_str = log_stream.getvalue()
+ log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "INFO"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
- def test_warning_logging(self):
+ def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Baz message"
extra = {"important_info": "baz"}
- self.spider.log_warning(log_message, extra)
- log_contents = self.log_stream.getvalue()
- log_contents = json.loads(log_contents)
+ spider.log_warning(log_message, extra)
+ log_contents_str = log_stream.getvalue()
+ log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "WARNING"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
- def test_error_logging(self):
+ def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo bar message"
extra = {"important_info": "foo bar"}
- self.spider.log_error(log_message, extra)
- log_contents = self.log_stream.getvalue()
- log_contents = json.loads(log_contents)
+ spider.log_error(log_message, extra)
+ log_contents_str = log_stream.getvalue()
+ log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "ERROR"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
- def test_critical_logging(self):
+ def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo bar baz message"
extra = {"important_info": "foo bar baz"}
- self.spider.log_critical(log_message, extra)
- log_contents = self.log_stream.getvalue()
- log_contents = json.loads(log_contents)
+ spider.log_critical(log_message, extra)
+ log_contents_str = log_stream.getvalue()
+ log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "CRITICAL"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
- def test_overwrite_spider_extra(self):
+ def test_overwrite_spider_extra(
+ self, log_stream: StringIO, spider: LogSpider
+ ) -> None:
log_message = "Foo message"
extra = {"important_info": "foo", "spider": "shouldn't change"}
- self.spider.log_error(log_message, extra)
- log_contents = self.log_stream.getvalue()
- log_contents = json.loads(log_contents)
+ spider.log_error(log_message, extra)
+ log_contents_str = log_stream.getvalue()
+ log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "ERROR"
assert log_contents["message"] == log_message
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index aa250be69..20a3d940c 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -1,18 +1,17 @@
-import contextlib
import os
-import shutil
-import tempfile
import warnings
from pathlib import Path
+import pytest
+
from scrapy.utils.misc import set_environ
from scrapy.utils.project import data_path, get_project_settings
-def inside_a_project():
+def proj_path(tmp_path):
prev_dir = Path.cwd()
- project_dir = tempfile.mkdtemp()
+ project_dir = tmp_path
try:
os.chdir(project_dir)
@@ -21,21 +20,19 @@ def inside_a_project():
yield project_dir
finally:
os.chdir(prev_dir)
- shutil.rmtree(project_dir)
-class TestProjectUtils:
- def test_data_path_outside_project(self):
- assert str(Path(".scrapy", "somepath")) == data_path("somepath")
- abspath = str(Path(os.path.sep, "absolute", "path"))
- assert abspath == data_path(abspath)
+def test_data_path_outside_project():
+ assert str(Path(".scrapy", "somepath")) == data_path("somepath")
+ abspath = str(Path(os.path.sep, "absolute", "path"))
+ assert abspath == data_path(abspath)
+
- def test_data_path_inside_project(self):
- with inside_a_project() as proj_path:
- expected = Path(proj_path, ".scrapy", "somepath")
- assert expected.resolve() == Path(data_path("somepath")).resolve()
- abspath = str(Path(os.path.sep, "absolute", "path").resolve())
- assert abspath == data_path(abspath)
+def test_data_path_inside_project(proj_path: Path) -> None:
+ expected = proj_path / ".scrapy" / "somepath"
+ assert expected.resolve() == Path(data_path("somepath")).resolve()
+ abspath = str(Path(os.path.sep, "absolute", "path").resolve())
+ assert abspath == data_path(abspath)
class TestGetProjectSettings:
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 291646ad7..c933e0ac9 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -1,7 +1,10 @@
+from __future__ import annotations
+
import functools
import operator
import platform
import sys
+from typing import TYPE_CHECKING, TypeVar
import pytest
from twisted.trial import unittest
@@ -20,16 +23,22 @@ from scrapy.utils.python import (
without_none_values,
)
+if TYPE_CHECKING:
+ from collections.abc import Iterable, Mapping
+
+
+_KT = TypeVar("_KT")
+_VT = TypeVar("_VT")
+
-class TestMutableChain:
- def test_mutablechain(self):
- m = MutableChain(range(2), [2, 3], (4, 5))
- m.extend(range(6, 7))
- m.extend([7, 8])
- m.extend([9, 10], (11, 12))
- assert next(m) == 0
- assert m.__next__() == 1
- assert list(m) == list(range(2, 13))
+def test_mutablechain():
+ m = MutableChain(range(2), [2, 3], (4, 5))
+ m.extend(range(6, 7))
+ m.extend([7, 8])
+ m.extend([9, 10], (11, 12))
+ assert next(m) == 0
+ assert m.__next__() == 1
+ assert list(m) == list(range(2, 13))
class TestMutableAsyncChain(unittest.TestCase):
@@ -112,144 +121,150 @@ class TestToBytes:
assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b"
-class TestMemoizedMethod:
- def test_memoizemethod_noargs(self):
- class A:
- @memoizemethod_noargs
- def cached(self):
- return object()
+def test_memoizemethod_noargs():
+ class A:
+ @memoizemethod_noargs
+ def cached(self):
+ return object()
- def noncached(self):
- return object()
+ def noncached(self):
+ return object()
- a = A()
- one = a.cached()
- two = a.cached()
- three = a.noncached()
- assert one is two
- assert one is not three
+ a = A()
+ one = a.cached()
+ two = a.cached()
+ three = a.noncached()
+ assert one is two
+ assert one is not three
-class TestBinaryIsText:
- def test_binaryistext(self):
- assert binary_is_text(b"hello")
+ ("value", "expected"),
+ [
+ (b"hello", True),
+ ("hello".encode("utf-16"), True),
+ (b"<div>Price \xa3</div>", True),
+ (b"\x02\xa3", False),
+ ],
+)
+def test_binaryistext(value: bytes, expected: bool) -> None:
+ assert binary_is_text(value) is expected
- def test_utf_16_strings_contain_null_bytes(self):
- assert binary_is_text("hello".encode("utf-16"))
- def test_one_with_encoding(self):
- assert binary_is_text(b"<div>Price \xa3</div>")
[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
+def test_equal_attributes():
+ class Obj:
+ pass
- def test_real_binary_bytes(self):
- assert not binary_is_text(b"\x02\xa3")
+ a = Obj()
+ b = Obj()
+ # no attributes given return False
+ assert not equal_attributes(a, b, [])
+ # nonexistent attributes
+ assert not equal_attributes(a, b, ["x", "y"])
+ a.x = 1
+ b.x = 1
+ # equal attribute
+ assert equal_attributes(a, b, ["x"])
-class TestUtilsPython:
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_equal_attributes(self):
- class Obj:
- pass
+ b.y = 2
+ # obj1 has no attribute y
+ assert not equal_attributes(a, b, ["x", "y"])
- a = Obj()
- b = Obj()
- # no attributes given return False
- assert not equal_attributes(a, b, [])
- # nonexistent attributes
- assert not equal_attributes(a, b, ["x", "y"])
+ a.y = 2
+ # equal attributes
+ assert equal_attributes(a, b, ["x", "y"])
- a.x = 1
- b.x = 1
- # equal attribute
- assert equal_attributes(a, b, ["x"])
+ a.y = 1
+ # different attributes
+ assert not equal_attributes(a, b, ["x", "y"])
- b.y = 2
- # obj1 has no attribute y
- assert not equal_attributes(a, b, ["x", "y"])
+ # test callable
+ a.meta = {}
+ b.meta = {}
+ assert equal_attributes(a, b, ["meta"])
- a.y = 2
- # equal attributes
- assert equal_attributes(a, b, ["x", "y"])
+ # compare ['meta']['a']
+ a.meta["z"] = 1
+ b.meta["z"] = 1
- a.y = 1
- # different attributes
- assert not equal_attributes(a, b, ["x", "y"])
+ get_z = operator.itemgetter("z")
+ get_meta = operator.attrgetter("meta")
- # test callable
- a.meta = {}
- b.meta = {}
- assert equal_attributes(a, b, ["meta"])
+ def compare_z(obj):
+ return get_z(get_meta(obj))
- # compare ['meta']['a']
- a.meta["z"] = 1
- b.meta["z"] = 1
+ assert equal_attributes(a, b, [compare_z, "x"])
+ # fail z equality
+ a.meta["z"] = 2
+ assert not equal_attributes(a, b, [compare_z, "x"])
- get_z = operator.itemgetter("z")
- get_meta = operator.attrgetter("meta")
- def compare_z(obj):
- return get_z(get_meta(obj))
+def test_get_func_args():
+ def f1(a, b, c):
+ pass
- assert equal_attributes(a, b, [compare_z, "x"])
- # fail z equality
- a.meta["z"] = 2
- assert not equal_attributes(a, b, [compare_z, "x"])
+ def f2(a, b=None, c=None):
+ pass
- def test_get_func_args(self):
- def f1(a, b, c):
+ def f3(a, b=None, *, c=None):
+ pass
+
+ class A:
+ def __init__(self, a, b, c):
pass
- def f2(a, b=None, c=None):
+ def method(self, a, b, c):
pass
- def f3(a, b=None, *, c=None):
+ class Callable:
+ def __call__(self, a, b, c):
pass
- class A:
- def __init__(self, a, b, c):
- pass
-
- def method(self, a, b, c):
- pass
-
- class Callable:
- def __call__(self, a, b, c):
- pass
-
- a = A(1, 2, 3)
- cal = Callable()
- partial_f1 = functools.partial(f1, None)
- partial_f2 = functools.partial(f1, b=None)
- partial_f3 = functools.partial(partial_f2, None)
-
- assert get_func_args(f1) == ["a", "b", "c"]
- assert get_func_args(f2) == ["a", "b", "c"]
- assert get_func_args(f3) == ["a", "b", "c"]
- assert get_func_args(A) == ["a", "b", "c"]
- assert get_func_args(a.method) == ["a", "b", "c"]
- assert get_func_args(partial_f1) == ["b", "c"]
- assert get_func_args(partial_f2) == ["a", "c"]
- assert get_func_args(partial_f3) == ["c"]
- assert get_func_args(cal) == ["a", "b", "c"]
- assert get_func_args(object) == []
- assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]
- assert get_func_args(" ".join, stripself=True) == ["iterable"]
-
- if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy":
- # the correct and correctly extracted signature
- assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"]
- elif platform.python_implementation() == "CPython":
- # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature
- # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951)
- assert get_func_args(operator.itemgetter(2), stripself=True) in [
- [],
- ["args", "kwargs"],
- ]
-
- def test_without_none_values(self):
- assert without_none_values([1, None, 3, 4]) == [1, 3, 4]
- assert without_none_values((1, None, 3, 4)) == (1, 3, 4)
- assert without_none_values({"one": 1, "none": None, "three": 3, "four": 4}) == {
- "one": 1,
- "three": 3,
- "four": 4,
- }
+ a = A(1, 2, 3)
+ cal = Callable()
+ partial_f1 = functools.partial(f1, None)
+ partial_f2 = functools.partial(f1, b=None)
+ partial_f3 = functools.partial(partial_f2, None)
+
+ assert get_func_args(f1) == ["a", "b", "c"]
+ assert get_func_args(f2) == ["a", "b", "c"]
+ assert get_func_args(f3) == ["a", "b", "c"]
+ assert get_func_args(A) == ["a", "b", "c"]
+ assert get_func_args(a.method) == ["a", "b", "c"]
+ assert get_func_args(partial_f1) == ["b", "c"]
+ assert get_func_args(partial_f2) == ["a", "c"]
+ assert get_func_args(partial_f3) == ["c"]
+ assert get_func_args(cal) == ["a", "b", "c"]
+ assert get_func_args(object) == []
+ assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]
+ assert get_func_args(" ".join, stripself=True) == ["iterable"]
+
+ if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy":
+ # the correct and correctly extracted signature
+ assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"]
+ elif platform.python_implementation() == "CPython":
+ # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature
+ # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951)
+ assert get_func_args(operator.itemgetter(2), stripself=True) in [
+ [],
+ ["args", "kwargs"],
+ ]
+
+
+ ("value", "expected"),
+ [
+ ([1, None, 3, 4], [1, 3, 4]),
+ ((1, None, 3, 4), (1, 3, 4)),
+ (
+ {"one": 1, "none": None, "three": 3, "four": 4},
+ {"one": 1, "three": 3, "four": 4},
+ ),
+ ],
+)
+def test_without_none_values(
+ value: Mapping[_KT, _VT] | Iterable[_KT], expected: dict[_KT, _VT] | Iterable[_KT]
+) -> None:
+ assert without_none_values(value) == expected
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 5b8509753..9c4cb7159 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -20,45 +20,52 @@ from scrapy.utils.request import (
from scrapy.utils.test import get_crawler
-class TestUtilsRequest:
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_request_authenticate(self):
- r = Request("http://www.example.com")
- request_authenticate(r, "someuser", "somepass")
- assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M="
-
- def test_request_httprepr(self):
- r1 = Request("http://www.example.com")
- assert (
- request_httprepr(r1) == b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
- )
[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
+def test_request_authenticate():
+ r = Request("http://www.example.com")
+ request_authenticate(r, "someuser", "somepass")
+ assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M="
- r1 = Request("http://www.example.com/some/page.html?arg=1")
- assert (
- request_httprepr(r1)
- == b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
- )
- r1 = Request(
- "http://www.example.com",
- method="POST",
- headers={"Content-type": b"text/html"},
- body=b"Some body",
- )
- assert (
- request_httprepr(r1)
- == b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body"
- )
+ ("r", "expected"),
+ [
+ (
+ Request("http://www.example.com"),
+ b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
+ ),
+ (
+ Request("http://www.example.com/some/page.html?arg=1"),
+ b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
+ ),
+ (
+ Request(
+ "http://www.example.com",
+ method="POST",
+ headers={"Content-type": b"text/html"},
+ body=b"Some body",
+ ),
+ b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body",
+ ),
+ ],
+)
+def test_request_httprepr(r: Request, expected: bytes) -> None:
+ assert request_httprepr(r) == expected
- def test_request_httprepr_for_non_http_request(self):
- # the representation is not important but it must not fail.
- request_httprepr(Request("file:///tmp/foo.txt"))
- request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
+ "r",
+ [
+ Request("file:///tmp/foo.txt"),
+ Request("ftp://localhost/tmp/foo.txt"),
+ ],
+)
+def test_request_httprepr_for_non_http_request(r: Request) -> None:
+ # the representation is not important but it must not fail.
+ request_httprepr(r)
-class TestFingerprint:
- maxDiff = None
+class TestFingerprint:
function: staticmethod = staticmethod(fingerprint)
cache: (
WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]
@@ -229,35 +236,6 @@ class TestFingerprint:
assert actual == expected
-REQUEST_OBJECTS_TO_TEST = (
- Request("http://www.example.com/"),
- Request("http://www.example.com/query?id=111&cat=222"),
- Request("http://www.example.com/query?cat=222&id=111"),
- Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"),
- Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"),
- Request("http://www.example.com/members/offers.html"),
- Request(
- "http://www.example.com/members/offers.html",
- headers={"SESSIONID": b"somehash"},
- ),
- Request(
- "http://www.example.com/",
- headers={"Accept-Language": b"en"},
- ),
- Request(
- "http://www.example.com/",
- headers={
- "Accept-Language": b"en",
- "SESSIONID": b"somehash",
- },
- ),
- Request("http://www.example.com/test.html"),
- Request("http://www.example.com/test.html#fragment"),
- Request("http://www.example.com", method="POST"),
- Request("http://www.example.com", method="POST", body=b"request body"),
-)
-
-
class TestRequestFingerprinter:
def test_default_implementation(self):
crawler = get_crawler()
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 80f2f25d5..179ca49e4 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -4,7 +4,7 @@ from urllib.parse import urlparse
import pytest
-from scrapy.http import HtmlResponse, Response, TextResponse
+from scrapy.http import HtmlResponse, Response
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (
_remove_html_comments,
@@ -15,229 +15,203 @@ from scrapy.utils.response import (
)
-class TestResponseUtils:
- dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response")
-
- def test_open_in_browser(self):
- url = "http:///www.example.com/some/page.html"
- body = b"<html> <head> <title>test page</title> </head> <body>test body</body> </html>"
-
- def browser_open(burl):
- path = urlparse(burl).path
- if not path or not Path(path).exists():
- path = burl.replace("file://", "")
- bbody = Path(path).read_bytes()
- assert b'<base href="' + to_bytes(url) + b'">' in bbody
- return True
-
- response = HtmlResponse(url, body=body)
- assert open_in_browser(response, _openfunc=browser_open), "Browser not called"
-
- resp = Response(url, body=body)
- with pytest.raises(TypeError):
- open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg
-
- def test_get_meta_refresh(self):
- r1 = HtmlResponse(
- "http://www.example.com",
- body=b"""
- <html>
- <head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
- <body>blahablsdfsal&amp;</body>
- </html>""",
- )
- r2 = HtmlResponse(
- "http://www.example.com",
- body=b"""
- <html>
- <head><title>Dummy</title><noScript>
- <meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
- </noSCRIPT>
- <body>blahablsdfsal&amp;</body>
- </html>""",
- )
- r3 = HtmlResponse(
- "http://www.example.com",
- body=b"""
- <noscript><meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage</noscript>
- <script type="text/javascript">
- if(!checkCookies()){
- document.write('<meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage">');
- }
- </script>
- """,
- )
- assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage")
- assert get_meta_refresh(r2) == (None, None)
- assert get_meta_refresh(r3) == (None, None)
-
- def test_get_base_url(self):
- resp = HtmlResponse(
- "http://www.example.com",
- body=b"""
- <html>
- <head><base href="http://www.example.com/img/" target="_blank"></head>
- <body>blahablsdfsal&amp;</body>
- </html>""",
- )
- assert get_base_url(resp) == "http://www.example.com/img/"
-
- resp2 = HtmlResponse(
- "http://www.example.com",
- body=b"""
- <html><body>blahablsdfsal&amp;</body></html>""",
- )
- assert get_base_url(resp2) == "http://www.example.com"
-
- def test_response_status_message(self):
- assert response_status_message(200) == "200 OK"
- assert response_status_message(404) == "404 Not Found"
- assert response_status_message(573) == "573 Unknown Status"
-
- def test_inject_base_url(self):
- url = "http://www.example.com"
-
- def check_base_url(burl):
- path = urlparse(burl).path
- if not path or not Path(path).exists():
- path = burl.replace("file://", "")
- bbody = Path(path).read_bytes()
- assert bbody.count(b'<base href="' + to_bytes(url) + b'">') == 1
- return True
-
- r1 = HtmlResponse(
- url,
- body=b"""
- <html>
- <head><title>Dummy</title></head>
- <body><p>Hello world.</p></body>
- </html>""",
- )
- r2 = HtmlResponse(
- url,
- body=b"""
- <html>
- <head id="foo"><title>Dummy</title></head>
- <body>Hello world.</body>
- </html>""",
- )
- r3 = HtmlResponse(
- url,
- body=b"""
- <html>
- <head><title>Dummy</title></head>
- <body>
- <header>Hello header</header>
- <p>Hello world.</p>
- </body>
- </html>""",
- )
- r4 = HtmlResponse(
- url,
- body=b"""
- <html>
- <!-- <head>Dummy comment</head> -->
- <head><title>Dummy</title></head>
- <body><p>Hello world.</p></body>
- </html>""",
- )
- r5 = HtmlResponse(
- url,
- body=b"""
- <html>
- <!--[if IE]>
- <head><title>IE head</title></head>
- <![endif]-->
- <!--[if !IE]>-->
- <head><title>Standard head</title></head>
- <!--<![endif]-->
- <body><p>Hello world.</p></body>
- </html>""",
- )
-
- assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
- assert open_in_browser(r2, _openfunc=check_base_url), (
- "Inject base url with argumented head"
- )
- assert open_in_browser(r3, _openfunc=check_base_url), (
- "Inject unique base url with misleading tag"
- )
- assert open_in_browser(r4, _openfunc=check_base_url), (
- "Inject unique base url with misleading comment"
- )
- assert open_in_browser(r5, _openfunc=check_base_url), (
- "Inject unique base url with conditional comment"
- )
-
- def test_open_in_browser_redos_comment(self):
- MAX_CPU_TIME = 0.02
-
- # Exploit input from
- # https://makenowjust-labs.github.io/recheck/playground/
- # for /<!--.*?-->/ (old pattern to remove comments).
- body = b"-><!--\x00" * 25_000 + b"->\n<!---->"
-
- response = HtmlResponse("https://example.com", body=body)
-
- start_time = process_time()
-
- open_in_browser(response, lambda url: True)
-
- end_time = process_time()
- assert end_time - start_time < MAX_CPU_TIME
-
- def test_open_in_browser_redos_head(self):
- MAX_CPU_TIME = 0.02
-
- # Exploit input from
- # https://makenowjust-labs.github.io/recheck/playground/
- # for /(<head(?:>|\s.*?>))/ (old pattern to find the head element).
- body = b"<head\t" * 8_000
-
- response = HtmlResponse("https://example.com", body=body)
-
- start_time = process_time()
-
- open_in_browser(response, lambda url: True)
-
- end_time = process_time()
- assert end_time - start_time < MAX_CPU_TIME
+def test_open_in_browser():
+ url = "http:///www.example.com/some/page.html"
+ body = (
+ b"<html> <head> <title>test page</title> </head> <body>test body</body> </html>"
+ )
+
+ def browser_open(burl: str) -> bool:
+ path = urlparse(burl).path
+ if not path or not Path(path).exists():
+ path = burl.replace("file://", "")
+ bbody = Path(path).read_bytes()
+ assert b'<base href="' + to_bytes(url) + b'">' in bbody
+ return True
+
+ response = HtmlResponse(url, body=body)
+ assert open_in_browser(response, _openfunc=browser_open), "Browser not called"
+
+ resp = Response(url, body=body)
+ with pytest.raises(TypeError):
+ open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg
+
+
+def test_get_meta_refresh():
+ r1 = HtmlResponse(
+ "http://www.example.com",
+ body=b"""
+ <html>
+ <head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
+ <body>blahablsdfsal&amp;</body>
+ </html>""",
+ )
+ r2 = HtmlResponse(
+ "http://www.example.com",
+ body=b"""
+ <html>
+ <head><title>Dummy</title><noScript>
+ <meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
+ </noSCRIPT>
+ <body>blahablsdfsal&amp;</body>
+ </html>""",
+ )
+ r3 = HtmlResponse(
+ "http://www.example.com",
+ body=b"""
+<noscript><meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage</noscript>
+<script type="text/javascript">
+if(!checkCookies()){
+ document.write('<meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage">');
+}
+</script>
+ """,
+ )
+ assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage")
+ assert get_meta_refresh(r2) == (None, None)
+ assert get_meta_refresh(r3) == (None, None)
+
+
+def test_get_base_url():
+ resp = HtmlResponse(
+ "http://www.example.com",
+ body=b"""
+ <html>
+ <head><base href="http://www.example.com/img/" target="_blank"></head>
+ <body>blahablsdfsal&amp;</body>
+ </html>""",
+ )
+ assert get_base_url(resp) == "http://www.example.com/img/"
+
+ resp2 = HtmlResponse(
+ "http://www.example.com",
+ body=b"""
+ <html><body>blahablsdfsal&amp;</body></html>""",
+ )
+ assert get_base_url(resp2) == "http://www.example.com"
+
+
+def test_response_status_message():
+ assert response_status_message(200) == "200 OK"
+ assert response_status_message(404) == "404 Not Found"
+ assert response_status_message(573) == "573 Unknown Status"
+
+
+def test_inject_base_url():
+ url = "http://www.example.com"
+
+ def check_base_url(burl):
+ path = urlparse(burl).path
+ if not path or not Path(path).exists():
+ path = burl.replace("file://", "")
+ bbody = Path(path).read_bytes()
+ assert bbody.count(b'<base href="' + to_bytes(url) + b'">') == 1
+ return True
+
+ r1 = HtmlResponse(
+ url,
+ body=b"""
+ <html>
+ <head><title>Dummy</title></head>
+ <body><p>Hello world.</p></body>
+ </html>""",
+ )
+ r2 = HtmlResponse(
+ url,
+ body=b"""
+ <html>
+ <head id="foo"><title>Dummy</title></head>
+ <body>Hello world.</body>
+ </html>""",
+ )
+ r3 = HtmlResponse(
+ url,
+ body=b"""
+ <html>
+ <head><title>Dummy</title></head>
+ <body>
+ <header>Hello header</header>
+ <p>Hello world.</p>
+ </body>
+ </html>""",
+ )
+ r4 = HtmlResponse(
+ url,
+ body=b"""
+ <html>
+ <!-- <head>Dummy comment</head> -->
+ <head><title>Dummy</title></head>
+ <body><p>Hello world.</p></body>
+ </html>""",
+ )
+ r5 = HtmlResponse(
+ url,
+ body=b"""
+ <html>
+ <!--[if IE]>
+ <head><title>IE head</title></head>
+ <![endif]-->
+ <!--[if !IE]>-->
+ <head><title>Standard head</title></head>
+ <!--<![endif]-->
+ <body><p>Hello world.</p></body>
+ </html>""",
+ )
+
+ assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
+ assert open_in_browser(r2, _openfunc=check_base_url), (
+ "Inject base url with argumented head"
+ )
+ assert open_in_browser(r3, _openfunc=check_base_url), (
+ "Inject unique base url with misleading tag"
+ )
+ assert open_in_browser(r4, _openfunc=check_base_url), (
+ "Inject unique base url with misleading comment"
+ )
+ assert open_in_browser(r5, _openfunc=check_base_url), (
+ "Inject unique base url with conditional comment"
+ )
+
+
+def test_open_in_browser_redos_comment():
+ MAX_CPU_TIME = 0.02
+
+ # Exploit input from
+ # https://makenowjust-labs.github.io/recheck/playground/
+ # for /<!--.*?-->/ (old pattern to remove comments).
+ body = b"-><!--\x00" * 25_000 + b"->\n<!---->"
+ response = HtmlResponse("https://example.com", body=body)
+ start_time = process_time()
+ open_in_browser(response, lambda url: True)
+ end_time = process_time()
+ assert end_time - start_time < MAX_CPU_TIME
+
+
+def test_open_in_browser_redos_head():
+ MAX_CPU_TIME = 0.02
+
+ # Exploit input from
+ # https://makenowjust-labs.github.io/recheck/playground/
+ # for /(<head(?:>|\s.*?>))/ (old pattern to find the head element).
+ body = b"<head\t" * 8_000
+ response = HtmlResponse("https://example.com", body=body)
+ start_time = process_time()
+ open_in_browser(response, lambda url: True)
+ end_time = process_time()
+ assert end_time - start_time < MAX_CPU_TIME
@pytest.mark.parametrize(
("input_body", "output_body"),
[
- (
- b"a<!--",
- b"a",
- ),
- (
- b"a<!---->b",
- b"ab",
- ),
- (
- b"a<!--b-->c",
- b"ac",
- ),
- (
- b"a<!--b-->c<!--",
- b"ac",
- ),
- (
- b"a<!--b-->c<!--d",
- b"ac",
- ),
- (
- b"a<!--b-->c<!---->d",
- b"acd",
- ),
- (
- b"a<!--b--><!--c-->d",
- b"ad",
- ),
+ (b"a<!--", b"a"),
+ (b"a<!---->b", b"ab"),
+ (b"a<!--b-->c", b"ac"),
+ (b"a<!--b-->c<!--", b"ac"),
+ (b"a<!--b-->c<!--d", b"ac"),
+ (b"a<!--b-->c<!---->d", b"acd"),
+ (b"a<!--b--><!--c-->d", b"ad"),
],
)
def test_remove_html_comments(input_body, output_body):
- assert _remove_html_comments(input_body) == output_body, (
- f"{_remove_html_comments(input_body)=} == {output_body=}"
- )
+ assert _remove_html_comments(input_body) == output_body
diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py
index 2ee3850b0..dcaac7358 100644
--- a/tests/test_utils_serialize.py
+++ b/tests/test_utils_serialize.py
@@ -4,6 +4,7 @@ import json
from decimal import Decimal
import attr
+import pytest
from twisted.internet import defer
from scrapy.http import Request, Response
@@ -11,10 +12,11 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
class TestJsonEncoder:
- def setup_method(self):
- self.encoder = ScrapyJSONEncoder(sort_keys=True)
+ @pytest.fixture
+ def encoder(self) -> ScrapyJSONEncoder:
+ return ScrapyJSONEncoder(sort_keys=True)
- def test_encode_decode(self):
+ def test_encode_decode(self, encoder: ScrapyJSONEncoder) -> None:
dt = datetime.datetime(2010, 1, 2, 10, 11, 12)
dts = "2010-01-02 10:11:12"
d = datetime.date(2010, 1, 2)
@@ -38,24 +40,24 @@ class TestJsonEncoder:
(s, ss),
(dt_set, dt_sets),
]:
- assert self.encoder.encode(input) == json.dumps(output, sort_keys=True)
+ assert encoder.encode(input) == json.dumps(output, sort_keys=True)
- def test_encode_deferred(self):
- assert "Deferred" in self.encoder.encode(defer.Deferred())
+ def test_encode_deferred(self, encoder: ScrapyJSONEncoder) -> None:
+ assert "Deferred" in encoder.encode(defer.Deferred())
- def test_encode_request(self):
+ def test_encode_request(self, encoder: ScrapyJSONEncoder) -> None:
r = Request("http://www.example.com/lala")
- rs = self.encoder.encode(r)
+ rs = encoder.encode(r)
assert r.method in rs
assert r.url in rs
- def test_encode_response(self):
+ def test_encode_response(self, encoder: ScrapyJSONEncoder) -> None:
r = Response("http://www.example.com/lala")
- rs = self.encoder.encode(r)
+ rs = encoder.encode(r)
assert r.url in rs
assert str(r.status) in rs
- def test_encode_dataclass_item(self) -> None:
+ def test_encode_dataclass_item(self, encoder: ScrapyJSONEncoder) -> None:
@dataclasses.dataclass
class TestDataClass:
name: str
@@ -63,10 +65,10 @@ class TestJsonEncoder:
price: int
item = TestDataClass(name="Product", url="http://product.org", price=1)
- encoded = self.encoder.encode(item)
+ encoded = encoder.encode(item)
assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}'
- def test_encode_attrs_item(self):
+ def test_encode_attrs_item(self, encoder: ScrapyJSONEncoder) -> None:
@attr.s
class AttrsItem:
name = attr.ib(type=str)
@@ -74,5 +76,5 @@ class TestJsonEncoder:
price = attr.ib(type=int)
item = AttrsItem(name="Product", url="http://product.org", price=1)
- encoded = self.encoder.encode(item)
+ encoded = encoder.encode(item)
assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}'
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index 36d612009..464a31777 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -1,158 +1,162 @@
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
-class TestSitemap:
- def test_sitemap(self):
- s = Sitemap(
- b"""<?xml version="1.0" encoding="UTF-8"?>
+def test_sitemap():
+ s = Sitemap(
+ b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
- <url>
- <loc>http://www.example.com/</loc>
- <lastmod>2009-08-16</lastmod>
- <changefreq>daily</changefreq>
- <priority>1</priority>
- </url>
- <url>
- <loc>http://www.example.com/Special-Offers.html</loc>
- <lastmod>2009-08-16</lastmod>
- <changefreq>weekly</changefreq>
- <priority>0.8</priority>
- </url>
+<url>
+<loc>http://www.example.com/</loc>
+<lastmod>2009-08-16</lastmod>
+<changefreq>daily</changefreq>
+<priority>1</priority>
+</url>
+<url>
+<loc>http://www.example.com/Special-Offers.html</loc>
+<lastmod>2009-08-16</lastmod>
+<changefreq>weekly</changefreq>
+<priority>0.8</priority>
+</url>
</urlset>"""
- )
- assert s.type == "urlset"
- assert list(s) == [
- {
- "priority": "1",
- "loc": "http://www.example.com/",
- "lastmod": "2009-08-16",
- "changefreq": "daily",
- },
- {
- "priority": "0.8",
- "loc": "http://www.example.com/Special-Offers.html",
- "lastmod": "2009-08-16",
- "changefreq": "weekly",
- },
- ]
+ )
+ assert s.type == "urlset"
+ assert list(s) == [
+ {
+ "priority": "1",
+ "loc": "http://www.example.com/",
+ "lastmod": "2009-08-16",
+ "changefreq": "daily",
+ },
+ {
+ "priority": "0.8",
+ "loc": "http://www.example.com/Special-Offers.html",
+ "lastmod": "2009-08-16",
+ "changefreq": "weekly",
+ },
+ ]
- def test_sitemap_index(self):
- s = Sitemap(
- b"""<?xml version="1.0" encoding="UTF-8"?>
+
+def test_sitemap_index():
+ s = Sitemap(
+ b"""<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
- <sitemap>
- <loc>http://www.example.com/sitemap1.xml.gz</loc>
- <lastmod>2004-10-01T18:23:17+00:00</lastmod>
- </sitemap>
- <sitemap>
- <loc>http://www.example.com/sitemap2.xml.gz</loc>
- <lastmod>2005-01-01</lastmod>
- </sitemap>
+<sitemap>
+ <loc>http://www.example.com/sitemap1.xml.gz</loc>
+ <lastmod>2004-10-01T18:23:17+00:00</lastmod>
+</sitemap>
+<sitemap>
+ <loc>http://www.example.com/sitemap2.xml.gz</loc>
+ <lastmod>2005-01-01</lastmod>
+</sitemap>
</sitemapindex>"""
- )
- assert s.type == "sitemapindex"
- assert list(s) == [
- {
- "loc": "http://www.example.com/sitemap1.xml.gz",
- "lastmod": "2004-10-01T18:23:17+00:00",
- },
- {
- "loc": "http://www.example.com/sitemap2.xml.gz",
- "lastmod": "2005-01-01",
- },
- ]
+ )
+ assert s.type == "sitemapindex"
+ assert list(s) == [
+ {
+ "loc": "http://www.example.com/sitemap1.xml.gz",
+ "lastmod": "2004-10-01T18:23:17+00:00",
+ },
+ {
+ "loc": "http://www.example.com/sitemap2.xml.gz",
+ "lastmod": "2005-01-01",
+ },
+ ]
- def test_sitemap_strip(self):
- """Assert we can deal with trailing spaces inside <loc> tags - we've
- seen those
- """
- s = Sitemap(
- b"""<?xml version="1.0" encoding="UTF-8"?>
+
+def test_sitemap_strip():
+ """Assert we can deal with trailing spaces inside <loc> tags - we've
+ seen those
+ """
+ s = Sitemap(
+ b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
- <url>
- <loc> http://www.example.com/</loc>
- <lastmod>2009-08-16</lastmod>
- <changefreq>daily</changefreq>
- <priority>1</priority>
- </url>
- <url>
- <loc> http://www.example.com/2</loc>
- <lastmod />
- </url>
+<url>
+<loc> http://www.example.com/</loc>
+<lastmod>2009-08-16</lastmod>
+<changefreq>daily</changefreq>
+<priority>1</priority>
+</url>
+<url>
+<loc> http://www.example.com/2</loc>
+<lastmod />
+</url>
</urlset>
"""
- )
- assert list(s) == [
- {
- "priority": "1",
- "loc": "http://www.example.com/",
- "lastmod": "2009-08-16",
- "changefreq": "daily",
- },
- {"loc": "http://www.example.com/2", "lastmod": ""},
- ]
+ )
+ assert list(s) == [
+ {
+ "priority": "1",
+ "loc": "http://www.example.com/",
+ "lastmod": "2009-08-16",
+ "changefreq": "daily",
+ },
+ {"loc": "http://www.example.com/2", "lastmod": ""},
+ ]
+
- def test_sitemap_wrong_ns(self):
- """We have seen sitemaps with wrongs ns. Presumably, Google still works
- with these, though is not 100% confirmed"""
- s = Sitemap(
- b"""<?xml version="1.0" encoding="UTF-8"?>
+def test_sitemap_wrong_ns():
+ """We have seen sitemaps with wrongs ns. Presumably, Google still works
+ with these, though is not 100% confirmed"""
+ s = Sitemap(
+ b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
- <url xmlns="">
- <loc> http://www.example.com/</loc>
- <lastmod>2009-08-16</lastmod>
- <changefreq>daily</changefreq>
- <priority>1</priority>
- </url>
- <url xmlns="">
- <loc> http://www.example.com/2</loc>
- <lastmod />
- </url>
+<url xmlns="">
+<loc> http://www.example.com/</loc>
+<lastmod>2009-08-16</lastmod>
+<changefreq>daily</changefreq>
+<priority>1</priority>
+</url>
+<url xmlns="">
+<loc> http://www.example.com/2</loc>
+<lastmod />
+</url>
</urlset>
"""
- )
- assert list(s) == [
- {
- "priority": "1",
- "loc": "http://www.example.com/",
- "lastmod": "2009-08-16",
- "changefreq": "daily",
- },
- {"loc": "http://www.example.com/2", "lastmod": ""},
- ]
+ )
+ assert list(s) == [
+ {
+ "priority": "1",
+ "loc": "http://www.example.com/",
+ "lastmod": "2009-08-16",
+ "changefreq": "daily",
+ },
+ {"loc": "http://www.example.com/2", "lastmod": ""},
+ ]
- def test_sitemap_wrong_ns2(self):
- """We have seen sitemaps with wrongs ns. Presumably, Google still works
- with these, though is not 100% confirmed"""
- s = Sitemap(
- b"""<?xml version="1.0" encoding="UTF-8"?>
+
+def test_sitemap_wrong_ns2():
+ """We have seen sitemaps with wrongs ns. Presumably, Google still works
+ with these, though is not 100% confirmed"""
+ s = Sitemap(
+ b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset>
- <url xmlns="">
- <loc> http://www.example.com/</loc>
- <lastmod>2009-08-16</lastmod>
- <changefreq>daily</changefreq>
- <priority>1</priority>
- </url>
- <url xmlns="">
- <loc> http://www.example.com/2</loc>
- <lastmod />
- </url>
+<url xmlns="">
+<loc> http://www.example.com/</loc>
+<lastmod>2009-08-16</lastmod>
+<changefreq>daily</changefreq>
+<priority>1</priority>
+</url>
+<url xmlns="">
+<loc> http://www.example.com/2</loc>
+<lastmod />
+</url>
</urlset>
"""
- )
- assert s.type == "urlset"
- assert list(s) == [
- {
- "priority": "1",
- "loc": "http://www.example.com/",
- "lastmod": "2009-08-16",
- "changefreq": "daily",
- },
- {"loc": "http://www.example.com/2", "lastmod": ""},
- ]
+ )
+ assert s.type == "urlset"
+ assert list(s) == [
+ {
+ "priority": "1",
+ "loc": "http://www.example.com/",
+ "lastmod": "2009-08-16",
+ "changefreq": "daily",
+ },
+ {"loc": "http://www.example.com/2", "lastmod": ""},
+ ]
+
- def test_sitemap_urls_from_robots(self):
- robots = """User-agent: *
+def test_sitemap_urls_from_robots():
+ robots = """User-agent: *
Disallow: /aff/
Disallow: /wl/
@@ -170,19 +174,18 @@ Sitemap: /sitemap-relative-url.xml
Disallow: /forum/search/
Disallow: /forum/active/
"""
- assert list(
- sitemap_urls_from_robots(robots, base_url="http://example.com")
- ) == [
- "http://example.com/sitemap.xml",
- "http://example.com/sitemap-product-index.xml",
- "http://example.com/sitemap-uppercase.xml",
- "http://example.com/sitemap-relative-url.xml",
- ]
+ assert list(sitemap_urls_from_robots(robots, base_url="http://example.com")) == [
+ "http://example.com/sitemap.xml",
+ "http://example.com/sitemap-product-index.xml",
+ "http://example.com/sitemap-uppercase.xml",
+ "http://example.com/sitemap-relative-url.xml",
+ ]
- def test_sitemap_blanklines(self):
- """Assert we can deal with starting blank lines before <xml> tag"""
- s = Sitemap(
- b"""
+
+def test_sitemap_blanklines():
+ """Assert we can deal with starting blank lines before <xml> tag"""
+ s = Sitemap(
+ b"""
<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
@@ -205,69 +208,69 @@ Disallow: /forum/active/
<!-- end cache -->
</sitemapindex>
"""
- )
- assert list(s) == [
- {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"},
- {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"},
- {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"},
- ]
+ )
+ assert list(s) == [
+ {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"},
+ {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"},
+ {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"},
+ ]
- def test_comment(self):
- s = Sitemap(
- b"""<?xml version="1.0" encoding="UTF-8"?>
- <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
- xmlns:xhtml="http://www.w3.org/1999/xhtml">
- <url>
- <loc>http://www.example.com/</loc>
- <!-- this is a comment on which the parser might raise an exception if implemented incorrectly -->
- </url>
- </urlset>"""
- )
- assert list(s) == [{"loc": "http://www.example.com/"}]
+def test_comment():
+ s = Sitemap(
+ b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+ xmlns:xhtml="http://www.w3.org/1999/xhtml">
+ <url>
+ <loc>http://www.example.com/</loc>
+ <!-- this is a comment on which the parser might raise an exception if implemented incorrectly -->
+ </url>
+</urlset>"""
+ )
+ assert list(s) == [{"loc": "http://www.example.com/"}]
- def test_alternate(self):
- s = Sitemap(
- b"""<?xml version="1.0" encoding="UTF-8"?>
- <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
- xmlns:xhtml="http://www.w3.org/1999/xhtml">
- <url>
- <loc>http://www.example.com/english/</loc>
- <xhtml:link rel="alternate" hreflang="de"
- href="http://www.example.com/deutsch/"/>
- <xhtml:link rel="alternate" hreflang="de-ch"
- href="http://www.example.com/schweiz-deutsch/"/>
- <xhtml:link rel="alternate" hreflang="en"
- href="http://www.example.com/english/"/>
- <xhtml:link rel="alternate" hreflang="en"/><!-- wrong tag without href -->
- </url>
- </urlset>"""
- )
- assert list(s) == [
- {
- "loc": "http://www.example.com/english/",
- "alternate": [
- "http://www.example.com/deutsch/",
- "http://www.example.com/schweiz-deutsch/",
- "http://www.example.com/english/",
- ],
- }
- ]
+def test_alternate():
+ s = Sitemap(
+ b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+ xmlns:xhtml="http://www.w3.org/1999/xhtml">
+ <url>
+ <loc>http://www.example.com/english/</loc>
+ <xhtml:link rel="alternate" hreflang="de"
+ href="http://www.example.com/deutsch/"/>
+ <xhtml:link rel="alternate" hreflang="de-ch"
+ href="http://www.example.com/schweiz-deutsch/"/>
+ <xhtml:link rel="alternate" hreflang="en"
+ href="http://www.example.com/english/"/>
+ <xhtml:link rel="alternate" hreflang="en"/><!-- wrong tag without href -->
+ </url>
+</urlset>"""
+ )
+ assert list(s) == [
+ {
+ "loc": "http://www.example.com/english/",
+ "alternate": [
+ "http://www.example.com/deutsch/",
+ "http://www.example.com/schweiz-deutsch/",
+ "http://www.example.com/english/",
+ ],
+ }
+ ]
- def test_xml_entity_expansion(self):
- s = Sitemap(
- b"""<?xml version="1.0" encoding="utf-8"?>
- <!DOCTYPE foo [
- <!ELEMENT foo ANY >
- <!ENTITY xxe SYSTEM "file:///etc/passwd" >
- ]>
- <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
- <url>
- <loc>http://127.0.0.1:8000/&xxe;</loc>
- </url>
- </urlset>
- """
- )
- assert list(s) == [{"loc": "http://127.0.0.1:8000/"}]
+def test_xml_entity_expansion():
+ s = Sitemap(
+ b"""<?xml version="1.0" encoding="utf-8"?>
+ <!DOCTYPE foo [
+ <!ELEMENT foo ANY >
+ <!ENTITY xxe SYSTEM "file:///etc/passwd" >
+ ]>
+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url>
+ <loc>http://127.0.0.1:8000/&xxe;</loc>
+ </url>
+ </urlset>
+ """
+ )
+ assert list(s) == [{"loc": "http://127.0.0.1:8000/"}]
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index 43e603f6c..8600faa8d 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -12,19 +12,19 @@ class MySpider2(Spider):
name = "myspider2"
-class TestUtilsSpiders:
- def test_iterate_spider_output(self):
- i = Item()
- r = Request("http://scrapytest.org")
- o = object()
-
- assert list(iterate_spider_output(i)) == [i]
- assert list(iterate_spider_output(r)) == [r]
- assert list(iterate_spider_output(o)) == [o]
- assert list(iterate_spider_output([r, i, o])) == [r, i, o]
-
- def test_iter_spider_classes(self):
- import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self
-
- it = iter_spider_classes(tests.test_utils_spider)
- assert set(it) == {MySpider1, MySpider2}
+def test_iterate_spider_output():
+ i = Item()
+ r = Request("http://scrapytest.org")
+ o = object()
+
+ assert list(iterate_spider_output(i)) == [i]
+ assert list(iterate_spider_output(r)) == [r]
+ assert list(iterate_spider_output(o)) == [o]
+ assert list(iterate_spider_output([r, i, o])) == [r, i, o]
+
+
+def test_iter_spider_classes():
+ import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self
+
+ it = iter_spider_classes(tests.test_utils_spider)
+ assert set(it) == {MySpider1, MySpider2}
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 41d9b8933..4515ce36e 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -1,22 +1,21 @@
from scrapy.utils.template import render_templatefile
-class TestUtilsRenderTemplateFile:
- def test_simple_render(self, tmp_path):
- context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
- template = "from ${project_name}.spiders.${name} import ${classname}"
- rendered = "from proj.spiders.spi import TheSpider"
+def test_simple_render(tmp_path):
+ context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
+ template = "from ${project_name}.spiders.${name} import ${classname}"
+ rendered = "from proj.spiders.spi import TheSpider"
- template_path = tmp_path / "templ.py.tmpl"
- render_path = tmp_path / "templ.py"
+ template_path = tmp_path / "templ.py.tmpl"
+ render_path = tmp_path / "templ.py"
- template_path.write_text(template, encoding="utf8")
- assert template_path.is_file() # Failure of test itself
+ template_path.write_text(template, encoding="utf8")
+ assert template_path.is_file() # Failure of test itself
- render_templatefile(template_path, **context)
+ render_templatefile(template_path, **context)
- assert not template_path.exists()
- assert render_path.read_text(encoding="utf8") == rendered
+ assert not template_path.exists()
+ assert render_path.read_text(encoding="utf8") == rendered
- render_path.unlink()
- assert not render_path.exists() # Failure of test itself
+ render_path.unlink()
+ assert not render_path.exists() # Failure of test itself
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index e3d6675bf..3967c3365 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -15,71 +15,76 @@ class Bar(trackref.object_ref):
pass
-class TestTrackref:
- def setup_method(self):
- trackref.live_refs.clear()
-
- def test_format_live_refs(self):
- o1 = Foo() # noqa: F841
- o2 = Bar() # noqa: F841
- o3 = Foo() # noqa: F841
- assert (
- trackref.format_live_refs()
- == """\
[email protected](autouse=True)
+def clear_refs() -> None:
+ trackref.live_refs.clear()
+
+
+def test_format_live_refs():
+ o1 = Foo() # noqa: F841
+ o2 = Bar() # noqa: F841
+ o3 = Foo() # noqa: F841
+ assert (
+ trackref.format_live_refs()
+ == """\
Live References
Bar 1 oldest: 0s ago
Foo 2 oldest: 0s ago
"""
- )
+ )
- assert (
- trackref.format_live_refs(ignore=Foo)
- == """\
+ assert (
+ trackref.format_live_refs(ignore=Foo)
+ == """\
Live References
Bar 1 oldest: 0s ago
"""
- )
-
- @mock.patch("sys.stdout", new_callable=StringIO)
- def test_print_live_refs_empty(self, stdout):
- trackref.print_live_refs()
- assert stdout.getvalue() == "Live References\n\n\n"
-
- @mock.patch("sys.stdout", new_callable=StringIO)
- def test_print_live_refs_with_objects(self, stdout):
- o1 = Foo() # noqa: F841
- trackref.print_live_refs()
- assert (
- stdout.getvalue()
- == """\
+ )
+
+
[email protected]("sys.stdout", new_callable=StringIO)
+def test_print_live_refs_empty(stdout):
+ trackref.print_live_refs()
+ assert stdout.getvalue() == "Live References\n\n\n"
+
+
[email protected]("sys.stdout", new_callable=StringIO)
+def test_print_live_refs_with_objects(stdout):
+ o1 = Foo() # noqa: F841
+ trackref.print_live_refs()
+ assert (
+ stdout.getvalue()
+ == """\
Live References
Foo 1 oldest: 0s ago\n\n"""
- )
+ )
- def test_get_oldest(self):
- o1 = Foo()
- o1_time = time()
+def test_get_oldest():
+ o1 = Foo()
- o2 = Bar()
+ o1_time = time()
+ o2 = Bar()
+
+ o3_time = time()
+ if o3_time <= o1_time:
+ sleep(0.01)
o3_time = time()
- if o3_time <= o1_time:
- sleep(0.01)
- o3_time = time()
- if o3_time <= o1_time:
- pytest.skip("time.time is not precise enough")
-
- o3 = Foo() # noqa: F841
- assert trackref.get_oldest("Foo") is o1
- assert trackref.get_oldest("Bar") is o2
- assert trackref.get_oldest("XXX") is None
-
- def test_iter_all(self):
- o1 = Foo()
- o2 = Bar() # noqa: F841
- o3 = Foo()
- assert set(trackref.iter_all("Foo")) == {o1, o3}
+ if o3_time <= o1_time:
+ pytest.skip("time.time is not precise enough")
+
+ o3 = Foo() # noqa: F841
+ assert trackref.get_oldest("Foo") is o1
+ assert trackref.get_oldest("Bar") is o2
+ assert trackref.get_oldest("XXX") is None
+
+
+def test_iter_all():
+ o1 = Foo()
+ o2 = Bar() # noqa: F841
+ o3 = Foo()
+ assert set(trackref.iter_all("Foo")) == {o1, o3}
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index c85dcc55e..daf09f945 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -4,7 +4,6 @@ import pytest
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.spiders import Spider
-from scrapy.utils.misc import arg_to_iter
from scrapy.utils.url import ( # type: ignore[attr-defined]
_is_filesystem_path,
_public_w3lib_objects,
@@ -17,261 +16,156 @@ from scrapy.utils.url import ( # type: ignore[attr-defined]
)
-class TestUrlUtils:
- def test_url_is_from_any_domain(self):
- url = "http://www.wheele-bin-art.co.uk/get/product/123"
- assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
- assert not url_is_from_any_domain(url, ["art.co.uk"])
+def test_url_is_from_any_domain():
+ url = "http://www.wheele-bin-art.co.uk/get/product/123"
+ assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
+ assert not url_is_from_any_domain(url, ["art.co.uk"])
- url = "http://wheele-bin-art.co.uk/get/product/123"
- assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
- assert not url_is_from_any_domain(url, ["art.co.uk"])
+ url = "http://wheele-bin-art.co.uk/get/product/123"
+ assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
+ assert not url_is_from_any_domain(url, ["art.co.uk"])
- url = "http://www.Wheele-Bin-Art.co.uk/get/product/123"
- assert url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"])
- assert url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"])
+ url = "http://www.Wheele-Bin-Art.co.uk/get/product/123"
+ assert url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"])
+ assert url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"])
- url = "http://192.169.0.15:8080/mypage.html"
- assert url_is_from_any_domain(url, ["192.169.0.15:8080"])
- assert not url_is_from_any_domain(url, ["192.169.0.15"])
+ url = "http://192.169.0.15:8080/mypage.html"
+ assert url_is_from_any_domain(url, ["192.169.0.15:8080"])
+ assert not url_is_from_any_domain(url, ["192.169.0.15"])
- url = (
- "javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20"
- "javascript:%20document.orderform_2581_1190810811.submit%28%29"
- )
- assert not url_is_from_any_domain(url, ["testdomain.com"])
- assert not url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"])
-
- def test_url_is_from_spider(self):
- spider = Spider(name="example.com")
- assert url_is_from_spider("http://www.example.com/some/page.html", spider)
- assert url_is_from_spider("http://sub.example.com/some/page.html", spider)
- assert not url_is_from_spider("http://www.example.org/some/page.html", spider)
- assert not url_is_from_spider("http://www.example.net/some/page.html", spider)
-
- def test_url_is_from_spider_class_attributes(self):
- class MySpider(Spider):
- name = "example.com"
-
- assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
- assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
- assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
- assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
-
- def test_url_is_from_spider_with_allowed_domains(self):
- spider = Spider(
- name="example.com", allowed_domains=["example.org", "example.net"]
- )
- assert url_is_from_spider("http://www.example.com/some/page.html", spider)
- assert url_is_from_spider("http://sub.example.com/some/page.html", spider)
- assert url_is_from_spider("http://example.com/some/page.html", spider)
- assert url_is_from_spider("http://www.example.org/some/page.html", spider)
- assert url_is_from_spider("http://www.example.net/some/page.html", spider)
- assert not url_is_from_spider("http://www.example.us/some/page.html", spider)
-
- spider = Spider(
- name="example.com", allowed_domains={"example.com", "example.net"}
- )
- assert url_is_from_spider("http://www.example.com/some/page.html", spider)
-
- spider = Spider(
- name="example.com", allowed_domains=("example.com", "example.net")
- )
- assert url_is_from_spider("http://www.example.com/some/page.html", spider)
-
- def test_url_is_from_spider_with_allowed_domains_class_attributes(self):
- class MySpider(Spider):
- name = "example.com"
- allowed_domains = ("example.org", "example.net")
-
- assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
- assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
- assert url_is_from_spider("http://example.com/some/page.html", MySpider)
- assert url_is_from_spider("http://www.example.org/some/page.html", MySpider)
- assert url_is_from_spider("http://www.example.net/some/page.html", MySpider)
- assert not url_is_from_spider("http://www.example.us/some/page.html", MySpider)
-
- def test_url_has_any_extension(self):
- deny_extensions = {"." + e for e in arg_to_iter(IGNORED_EXTENSIONS)}
- assert url_has_any_extension(
- "http://www.example.com/archive.tar.gz", deny_extensions
- )
- assert url_has_any_extension("http://www.example.com/page.doc", deny_extensions)
- assert url_has_any_extension("http://www.example.com/page.pdf", deny_extensions)
- assert not url_has_any_extension(
- "http://www.example.com/page.htm", deny_extensions
- )
- assert not url_has_any_extension("http://www.example.com/", deny_extensions)
- assert not url_has_any_extension(
- "http://www.example.com/page.doc.html", deny_extensions
- )
-
-
-class TestAddHttpIfNoScheme:
- def test_add_scheme(self):
- assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"
-
- def test_without_subdomain(self):
- assert add_http_if_no_scheme("example.com") == "http://example.com"
-
- def test_path(self):
- assert (
- add_http_if_no_scheme("www.example.com/some/page.html")
- == "http://www.example.com/some/page.html"
- )
-
- def test_port(self):
- assert (
- add_http_if_no_scheme("www.example.com:80") == "http://www.example.com:80"
- )
+ url = (
+ "javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20"
+ "javascript:%20document.orderform_2581_1190810811.submit%28%29"
+ )
+ assert not url_is_from_any_domain(url, ["testdomain.com"])
+ assert not url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"])
- def test_fragment(self):
- assert (
- add_http_if_no_scheme("www.example.com/some/page#frag")
- == "http://www.example.com/some/page#frag"
- )
- def test_query(self):
- assert (
- add_http_if_no_scheme("www.example.com/do?a=1&b=2&c=3")
- == "http://www.example.com/do?a=1&b=2&c=3"
- )
+def test_url_is_from_spider():
+ class MySpider(Spider):
+ name = "example.com"
- def test_username_password(self):
- assert (
- add_http_if_no_scheme("username:[email protected]")
- == "http://username:[email protected]"
- )
-
- def test_complete_url(self):
- assert (
- add_http_if_no_scheme(
- "username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag"
- )
- == "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag"
- )
-
- def test_preserve_http(self):
- assert (
- add_http_if_no_scheme("http://www.example.com") == "http://www.example.com"
- )
+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
+ assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
+ assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
+ assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
- def test_preserve_http_without_subdomain(self):
- assert add_http_if_no_scheme("http://example.com") == "http://example.com"
-
- def test_preserve_http_path(self):
- assert (
- add_http_if_no_scheme("http://www.example.com/some/page.html")
- == "http://www.example.com/some/page.html"
- )
-
- def test_preserve_http_port(self):
- assert (
- add_http_if_no_scheme("http://www.example.com:80")
- == "http://www.example.com:80"
- )
-
- def test_preserve_http_fragment(self):
- assert (
- add_http_if_no_scheme("http://www.example.com/some/page#frag")
- == "http://www.example.com/some/page#frag"
- )
-
- def test_preserve_http_query(self):
- assert (
- add_http_if_no_scheme("http://www.example.com/do?a=1&b=2&c=3")
- == "http://www.example.com/do?a=1&b=2&c=3"
- )
-
- def test_preserve_http_username_password(self):
- assert (
- add_http_if_no_scheme("http://username:[email protected]")
- == "http://username:[email protected]"
- )
-
- def test_preserve_http_complete_url(self):
- assert (
- add_http_if_no_scheme(
- "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag"
- )
- == "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag"
- )
-
- def test_protocol_relative(self):
- assert add_http_if_no_scheme("//www.example.com") == "http://www.example.com"
-
- def test_protocol_relative_without_subdomain(self):
- assert add_http_if_no_scheme("//example.com") == "http://example.com"
-
- def test_protocol_relative_path(self):
- assert (
- add_http_if_no_scheme("//www.example.com/some/page.html")
- == "http://www.example.com/some/page.html"
- )
-
- def test_protocol_relative_port(self):
- assert (
- add_http_if_no_scheme("//www.example.com:80") == "http://www.example.com:80"
- )
-
- def test_protocol_relative_fragment(self):
- assert (
- add_http_if_no_scheme("//www.example.com/some/page#frag")
- == "http://www.example.com/some/page#frag"
- )
- def test_protocol_relative_query(self):
- assert (
- add_http_if_no_scheme("//www.example.com/do?a=1&b=2&c=3")
- == "http://www.example.com/do?a=1&b=2&c=3"
- )
+def test_url_is_from_spider_class_attributes():
+ class MySpider(Spider):
+ name = "example.com"
- def test_protocol_relative_username_password(self):
- assert (
- add_http_if_no_scheme("//username:[email protected]")
- == "http://username:[email protected]"
- )
+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
+ assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
+ assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
+ assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
- def test_protocol_relative_complete_url(self):
- assert (
- add_http_if_no_scheme(
- "//username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag"
- )
- == "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag"
- )
- def test_preserve_https(self):
- assert (
- add_http_if_no_scheme("https://www.example.com")
- == "https://www.example.com"
- )
+def test_url_is_from_spider_with_allowed_domains():
+ class MySpider(Spider):
+ name = "example.com"
+ allowed_domains = ["example.org", "example.net"]
- def test_preserve_ftp(self):
- assert add_http_if_no_scheme("ftp://www.example.com") == "ftp://www.example.com"
+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
+ assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
+ assert url_is_from_spider("http://example.com/some/page.html", MySpider)
+ assert url_is_from_spider("http://www.example.org/some/page.html", MySpider)
+ assert url_is_from_spider("http://www.example.net/some/page.html", MySpider)
+ assert not url_is_from_spider("http://www.example.us/some/page.html", MySpider)
+ class MySpider2(Spider):
+ name = "example.com"
+ allowed_domains = {"example.com", "example.net"}
-class TestGuessScheme:
- pass
+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider2)
+ class MySpider3(Spider):
+ name = "example.com"
+ allowed_domains = ("example.com", "example.net")
-def create_guess_scheme_t(args):
- def do_expected(self):
- url = guess_scheme(args[0])
- assert url.startswith(args[1]), (
- f"Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`"
- )
+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider3)
- return do_expected
+ ("url", "expected"),
+ [
+ ("http://www.example.com/archive.tar.gz", True),
+ ("http://www.example.com/page.doc", True),
+ ("http://www.example.com/page.pdf", True),
+ ("http://www.example.com/page.htm", False),
+ ("http://www.example.com/", False),
+ ("http://www.example.com/page.doc.html", False),
+ ],
+)
+def test_url_has_any_extension(url: str, expected: bool) -> None:
+ deny_extensions = {"." + e for e in IGNORED_EXTENSIONS}
+ assert url_has_any_extension(url, deny_extensions) is expected
-def create_skipped_scheme_t(args):
- def do_expected(self):
- pytest.skip(args[2])
- return do_expected
+ ("url", "expected"),
+ [
+ ("www.example.com", "http://www.example.com"),
+ ("example.com", "http://example.com"),
+ ("www.example.com/some/page.html", "http://www.example.com/some/page.html"),
+ ("www.example.com:80", "http://www.example.com:80"),
+ ("www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),
+ ("www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),
+ (
+ "username:[email protected]",
+ "http://username:[email protected]",
+ ),
+ (
+ "username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",
+ "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",
+ ),
+ ("http://www.example.com", "http://www.example.com"),
+ ("http://example.com", "http://example.com"),
+ (
+ "http://www.example.com/some/page.html",
+ "http://www.example.com/some/page.html",
+ ),
+ ("http://www.example.com:80", "http://www.example.com:80"),
+ (
+ "http://www.example.com/some/page#frag",
+ "http://www.example.com/some/page#frag",
+ ),
+ (
+ "http://www.example.com/do?a=1&b=2&c=3",
+ "http://www.example.com/do?a=1&b=2&c=3",
+ ),
+ (
+ "http://username:[email protected]",
+ "http://username:[email protected]",
+ ),
+ (
+ "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",
+ "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",
+ ),
+ ("//www.example.com", "http://www.example.com"),
+ ("//example.com", "http://example.com"),
+ ("//www.example.com/some/page.html", "http://www.example.com/some/page.html"),
+ ("//www.example.com:80", "http://www.example.com:80"),
+ ("//www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),
+ ("//www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),
+ (
+ "//username:[email protected]",
+ "http://username:[email protected]",
+ ),
+ (
+ "//username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",
+ "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",
+ ),
+ ("https://www.example.com", "https://www.example.com"),
+ ("ftp://www.example.com", "ftp://www.example.com"),
+ ],
+)
+def test_add_http_if_no_scheme(url: str, expected: str) -> None:
+ assert add_http_if_no_scheme(url) == expected
-for k, args in enumerate(
+ ("url", "expected"),
[
("/index", "file://"),
("/index.html", "file://"),
@@ -295,14 +189,13 @@ for k, args in enumerate(
("/", "http://"),
(".../test", "http://"),
],
- start=1,
-):
- t_method = create_guess_scheme_t(args)
- t_method.__name__ = f"test_uri_{k:03}"
- setattr(TestGuessScheme, t_method.__name__, t_method)
-
-# TODO: the following tests do not pass with current implementation
-for k, skip_args in enumerate(
+)
+def test_guess_scheme(url: str, expected: str):
+ assert guess_scheme(url).startswith(expected)
+
+
+ ("url", "expected", "reason"),
[
(
r"C:\absolute\path\to\a\file.html",
@@ -310,25 +203,21 @@ for k, skip_args in enumerate(
"Windows filepath are not supported for scrapy shell",
),
],
- start=1,
-):
- t_method = create_skipped_scheme_t(skip_args)
- t_method.__name__ = f"test_uri_skipped_{k:03}"
- setattr(TestGuessScheme, t_method.__name__, t_method)
+)
+def test_guess_scheme_skipped(url: str, expected: str, reason: str):
+ pytest.skip(reason)
class TestStripUrl:
- def test_noop(self):
- assert (
- strip_url("http://www.example.com/index.html")
- == "http://www.example.com/index.html"
- )
-
- def test_noop_query_string(self):
- assert (
- strip_url("http://www.example.com/index.html?somekey=somevalue")
- == "http://www.example.com/index.html?somekey=somevalue"
- )
+ @pytest.mark.parametrize(
+ "url",
+ [
+ "http://www.example.com/index.html",
+ "http://www.example.com/index.html?somekey=somevalue",
+ ],
+ )
+ def test_noop(self, url: str) -> None:
+ assert strip_url(url) == url
def test_fragments(self):
assert (
@@ -339,16 +228,20 @@ class TestStripUrl:
== "http://www.example.com/index.html?somekey=somevalue#section"
)
- def test_path(self):
- for input_url, origin, output_url in [
+ @pytest.mark.parametrize(
+ ("url", "origin", "expected"),
+ [
("http://www.example.com/", False, "http://www.example.com/"),
("http://www.example.com", False, "http://www.example.com"),
("http://www.example.com", True, "http://www.example.com/"),
- ]:
- assert strip_url(input_url, origin_only=origin) == output_url
+ ],
+ )
+ def test_path(self, url: str, origin: bool, expected: str) -> None:
+ assert strip_url(url, origin_only=origin) == expected
- def test_credentials(self):
- for i, o in [
+ @pytest.mark.parametrize(
+ ("url", "expected"),
+ [
(
"http://[email protected]/index.html?somekey=somevalue#section",
"http://www.example.com/index.html?somekey=somevalue",
@@ -361,34 +254,29 @@ class TestStripUrl:
"ftp://username:[email protected]/index.html?somekey=somevalue#section",
"ftp://www.example.com/index.html?somekey=somevalue",
),
- ]:
- assert strip_url(i, strip_credentials=True) == o
-
- def test_credentials_encoded_delims(self):
- for i, o in [
- # user: "username@"
- # password: none
+ # user: "username@", password: none
(
"http://username%[email protected]/index.html?somekey=somevalue#section",
"http://www.example.com/index.html?somekey=somevalue",
),
- # user: "username:pass"
- # password: ""
+ # user: "username:pass", password: ""
(
"https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section",
"https://www.example.com/index.html?somekey=somevalue",
),
- # user: "me"
- # password: "[email protected]"
+ # user: "me", password: "[email protected]"
(
"ftp://me:user%[email protected]/index.html?somekey=somevalue#section",
"ftp://www.example.com/index.html?somekey=somevalue",
),
- ]:
- assert strip_url(i, strip_credentials=True) == o
+ ],
+ )
+ def test_credentials(self, url: str, expected: str) -> None:
+ assert strip_url(url, strip_credentials=True) == expected
- def test_default_ports_creds_off(self):
- for i, o in [
+ @pytest.mark.parametrize(
+ ("url", "expected"),
+ [
(
"http://username:[email protected]:80/index.html?somekey=somevalue#section",
"http://www.example.com/index.html?somekey=somevalue",
@@ -421,11 +309,14 @@ class TestStripUrl:
"ftp://username:[email protected]:221/file.txt",
"ftp://www.example.com:221/file.txt",
),
- ]:
- assert strip_url(i) == o
+ ],
+ )
+ def test_default_ports_creds_off(self, url: str, expected: str) -> None:
+ assert strip_url(url) == expected
- def test_default_ports(self):
- for i, o in [
+ @pytest.mark.parametrize(
+ ("url", "expected"),
+ [
(
"http://username:[email protected]:80/index.html",
"http://username:[email protected]/index.html",
@@ -458,11 +349,16 @@ class TestStripUrl:
"ftp://username:[email protected]:221/file.txt",
"ftp://username:[email protected]:221/file.txt",
),
- ]:
- assert strip_url(i, strip_default_port=True, strip_credentials=False) == o
+ ],
+ )
+ def test_default_ports(self, url: str, expected: str) -> None:
+ assert (
+ strip_url(url, strip_default_port=True, strip_credentials=False) == expected
+ )
- def test_default_ports_keep(self):
- for i, o in [
+ @pytest.mark.parametrize(
+ ("url", "expected"),
+ [
(
"http://username:[email protected]:80/index.html?somekey=somevalue&someotherkey=sov#section",
"http://username:[email protected]:80/index.html?somekey=somevalue&someotherkey=sov",
@@ -495,11 +391,17 @@ class TestStripUrl:
"ftp://username:[email protected]:221/file.txt",
"ftp://username:[email protected]:221/file.txt",
),
- ]:
- assert strip_url(i, strip_default_port=False, strip_credentials=False) == o
+ ],
+ )
+ def test_default_ports_keep(self, url: str, expected: str) -> None:
+ assert (
+ strip_url(url, strip_default_port=False, strip_credentials=False)
+ == expected
+ )
- def test_origin_only(self):
- for i, o in [
+ @pytest.mark.parametrize(
+ ("url", "expected"),
+ [
(
"http://username:[email protected]/index.html",
"http://www.example.com/",
@@ -516,29 +418,33 @@ class TestStripUrl:
"https://username:[email protected]:443/index.html",
"https://www.example.com/",
),
- ]:
- assert strip_url(i, origin_only=True) == o
-
-
-class TestIsPath:
- def test_path(self):
- for input_value, output_value in (
- # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
- # Unix-like OS, Microsoft Windows / cmd.exe
- ("/home/user/docs/Letter.txt", True),
- ("./inthisdir", True),
- ("../../greatgrandparent", True),
- ("~/.rcinfo", True),
- (r"C:\user\docs\Letter.txt", True),
- ("/user/docs/Letter.txt", True),
- (r"C:\Letter.txt", True),
- (r"\\Server01\user\docs\Letter.txt", True),
- (r"\\?\UNC\Server01\user\docs\Letter.txt", True),
- (r"\\?\C:\user\docs\Letter.txt", True),
- (r"C:\user\docs\somefile.ext:alternate_stream_name", True),
- (r"https://example.com", False),
- ):
- assert _is_filesystem_path(input_value) == output_value, input_value
+ ],
+ )
+ def test_origin_only(self, url: str, expected: str) -> None:
+ assert strip_url(url, origin_only=True) == expected
+
+
+ ("path", "expected"),
+ [
+ # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
+ # Unix-like OS, Microsoft Windows / cmd.exe
+ ("/home/user/docs/Letter.txt", True),
+ ("./inthisdir", True),
+ ("../../greatgrandparent", True),
+ ("~/.rcinfo", True),
+ (r"C:\user\docs\Letter.txt", True),
+ ("/user/docs/Letter.txt", True),
+ (r"C:\Letter.txt", True),
+ (r"\\Server01\user\docs\Letter.txt", True),
+ (r"\\?\UNC\Server01\user\docs\Letter.txt", True),
+ (r"\\?\C:\user\docs\Letter.txt", True),
+ (r"C:\user\docs\somefile.ext:alternate_stream_name", True),
+ (r"https://example.com", False),
+ ],
+)
+def test__is_filesystem_path(path: str, expected: bool) -> None:
+ assert _is_filesystem_path(path) == expected
@pytest.mark.parametrize(
@@ -550,7 +456,7 @@ class TestIsPath:
*_public_w3lib_objects,
],
)
-def test_deprecated_imports_from_w3lib(obj_name):
+def test_deprecated_imports_from_w3lib(obj_name: str) -> None:
with warnings.catch_warnings(record=True) as warns:
obj_type = "attribute" if obj_name == "_safe_chars" else "function"
message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."
@@ -559,4 +465,5 @@ def test_deprecated_imports_from_w3lib(obj_name):
getattr(import_module("scrapy.utils.url"), obj_name)
+ assert isinstance(warns[0].message, Warning)
assert message in warns[0].message.args