Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix rel_has_nofollow to be case-insensitive per HTML spec (#7632)

JSap0914b7824db573ca
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 0b67eaa34..47568e656 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -162,7 +162,7 @@ def md5sum(file: IO[bytes]) -> str:
def rel_has_nofollow(rel: str | None) -> bool:
"""Return True if link rel attribute has nofollow type"""
- return rel is not None and "nofollow" in rel.replace(",", " ").split()
+ return rel is not None and "nofollow" in rel.lower().replace(",", " ").split()
class SupportsFromCrawler(Protocol[_T_co, _P]):
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index a995e38e6..c4c861404 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -160,3 +160,8 @@ class TestUtilsMisc:
assert rel_has_nofollow("nofollowfoo") is False
assert rel_has_nofollow("foonofollow") is False
assert rel_has_nofollow("ugc, , nofollow") is True
+ # rel attribute values are ASCII case-insensitive per the HTML spec
+ assert rel_has_nofollow("NoFollow") is True
+ assert rel_has_nofollow("NOFOLLOW") is True
+ assert rel_has_nofollow("UGC NoFollow") is True
+ assert rel_has_nofollow("ugc,NoFollow") is True