Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix rel_has_nofollow to be case-insensitive per HTML spec (#7632)
JSap0914b7824db573ca
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 0b67eaa34..47568e656 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -162,7 +162,7 @@ def md5sum(file: IO[bytes]) -> str:def rel_has_nofollow(rel: str | None) -> bool:"""Return True if link rel attribute has nofollow type"""- return rel is not None and "nofollow" in rel.replace(",", " ").split()+ return rel is not None and "nofollow" in rel.lower().replace(",", " ").split()class SupportsFromCrawler(Protocol[_T_co, _P]):diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.pyindex a995e38e6..c4c861404 100644--- a/tests/test_utils_misc/__init__.py+++ b/tests/test_utils_misc/__init__.py@@ -160,3 +160,8 @@ class TestUtilsMisc:assert rel_has_nofollow("nofollowfoo") is Falseassert rel_has_nofollow("foonofollow") is Falseassert rel_has_nofollow("ugc, , nofollow") is True+ # rel attribute values are ASCII case-insensitive per the HTML spec+ assert rel_has_nofollow("NoFollow") is True+ assert rel_has_nofollow("NOFOLLOW") is True+ assert rel_has_nofollow("UGC NoFollow") is True+ assert rel_has_nofollow("ugc,NoFollow") is True