Scrapy, a fast high-level web crawling & scraping framework for Python.
Complete test coverage for linkextractors (#7639)
Adrian699c93f6b262
diff --git a/pyproject.toml b/pyproject.tomlindex 2d857dc6d..6b0c561f8 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -230,6 +230,7 @@ disable = ["undefined-variable","unused-argument","unused-variable",+ "use-implicit-booleaness-not-comparison","useless-import-alias", # used as a hint to mypy"useless-return", # https://github.com/pylint-dev/pylint/issues/6530"wrong-import-position",diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex 15d358d2a..063b92dac 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -9,7 +9,7 @@ from w3lib import __version__ as w3lib_versionfrom scrapy.http import HtmlResponse, XmlResponsefrom scrapy.link import Link-from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor, LxmlParserLinkExtractorfrom tests import get_testdata@@ -837,3 +837,36 @@ class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):def test_link_allowed_is_false_with_missing_url_prefix(self):bad_link = Link("should_have_prefix.example")assert not LxmlLinkExtractor()._link_allowed(bad_link)+++class TestLxmlParserLinkExtractor:+ def test_extract_links(self):+ html = b'<a href="http://example.com/page.html">Link</a>'+ response = HtmlResponse("http://example.com/", body=html)+ lx = LxmlParserLinkExtractor()+ assert lx.extract_links(response) == [+ Link(url="http://example.com/page.html", text="Link", nofollow=False),+ ]++ def test_strip_false(self):+ # With strip=False, trailing whitespace on a relative href survives urljoin+ # and is visible to process_value (safe_url_string cleans it up afterward).+ # Here process_value rejects URLs that still carry trailing whitespace,+ # demonstrating the difference from strip=True.+ def reject_trailing_whitespace(url):+ return None if url != url.rstrip() else url++ html = b'<a href="page.html ">Link</a>'+ response = HtmlResponse("http://example.com/", body=html)++ lx_strip = LxmlParserLinkExtractor(+ strip=True, process=reject_trailing_whitespace+ )+ assert lx_strip.extract_links(response) == [+ Link(url="http://example.com/page.html", text="Link", nofollow=False),+ ]++ lx_no_strip = LxmlParserLinkExtractor(+ strip=False, process=reject_trailing_whitespace+ )+ assert lx_no_strip.extract_links(response) == []diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex 0d96e1d88..ee9d2ed51 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -102,9 +102,7 @@ class KeywordArgumentsSpider(MockServerSpider):self.checks.append(kwargs["callback"] == "some_callback")self.crawler.stats.inc_value("boolean_checks", 3)elif response.url.endswith("/general_without"):- self.checks.append(- kwargs == {} # pylint: disable=use-implicit-booleaness-not-comparison- )+ self.checks.append(kwargs == {})self.crawler.stats.inc_value("boolean_checks")def parse_no_kwargs(self, response):diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex 7b3c52d65..9599ed6c9 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -1,4 +1,4 @@-# pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison+# pylint: disable=unsubscriptable-object,unsupported-membership-test# (too many false positives)import loggingdiff --git a/tests/test_spider.py b/tests/test_spider.pyindex 23efed77a..526cc8f23 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -22,7 +22,7 @@ class TestSpider:def test_base_spider(self):spider = self.spider_class("example.com")assert spider.name == "example.com"- assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison+ assert spider.start_urls == []def test_spider_args(self):"""``__init__`` method arguments are assigned to spider attributes"""diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex e8fe45749..2e8047e2d 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -175,7 +175,7 @@ def test_get_func_args():assert get_func_args(partial_f2) == ["a", "c"]assert get_func_args(partial_f3) == ["c"]assert get_func_args(cal) == ["a", "b", "c"]- assert get_func_args(object) == [] # pylint: disable=use-implicit-booleaness-not-comparison+ assert get_func_args(object) == []assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]assert get_func_args(" ".join, stripself=True) == ["iterable"]