Scrapy, a fast high-level web crawling & scraping framework for Python.
Warn and ignore allowed_domains defined as a property (#8094)
Adriandcaa6ced5882
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex f67853ece..21ef4aac7 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -6,6 +6,7 @@ library.from __future__ import annotationsimport re+import warningsfrom typing import TYPE_CHECKING, TypeAliasfrom urllib.parse import ParseResult, urlparse, urlunparse@@ -29,7 +30,19 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:def _spider_domains(spider: type[Spider]) -> Iterable[str]:yield spider.name- if allowed_domains := getattr(spider, "allowed_domains", None):+ allowed_domains = getattr(spider, "allowed_domains", None)+ if isinstance(allowed_domains, property):+ warnings.warn(+ f"{spider.__name__}.allowed_domains is a property. Properties "+ "cannot be evaluated on a spider class, only on a spider "+ "instance, so it will be ignored here. This affects matching "+ "URLs to spiders, e.g. in the shell, fetch and parse commands. "+ "Define allowed_domains as a plain class attribute instead.",+ stacklevel=2,+ category=UserWarning,+ )+ return+ if allowed_domains:yield from allowed_domainsdiff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex d9d162c23..403fd6176 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -75,6 +75,18 @@ def test_url_is_from_spider_with_allowed_domains():assert url_is_from_spider("http://www.example.com/some/page.html", MySpider3)+def test_url_is_from_spider_with_allowed_domains_property():+ class MySpider(Spider):+ name = "myspider"++ @property+ def allowed_domains(self):+ return ["example.org"]++ with pytest.warns(UserWarning, match="allowed_domains is a property"):+ assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)++@pytest.mark.parametrize(("url", "expected"),[