Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Warn and ignore allowed_domains defined as a property (#8094)

Adriandcaa6ced5882
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index f67853ece..21ef4aac7 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -6,6 +6,7 @@ library.
from __future__ import annotations
import re
+import warnings
from typing import TYPE_CHECKING, TypeAlias
from urllib.parse import ParseResult, urlparse, urlunparse
@@ -29,7 +30,19 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:
def _spider_domains(spider: type[Spider]) -> Iterable[str]:
yield spider.name
- if allowed_domains := getattr(spider, "allowed_domains", None):
+ allowed_domains = getattr(spider, "allowed_domains", None)
+ if isinstance(allowed_domains, property):
+ warnings.warn(
+ f"{spider.__name__}.allowed_domains is a property. Properties "
+ "cannot be evaluated on a spider class, only on a spider "
+ "instance, so it will be ignored here. This affects matching "
+ "URLs to spiders, e.g. in the shell, fetch and parse commands. "
+ "Define allowed_domains as a plain class attribute instead.",
+ stacklevel=2,
+ category=UserWarning,
+ )
+ return
+ if allowed_domains:
yield from allowed_domains
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index d9d162c23..403fd6176 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -75,6 +75,18 @@ def test_url_is_from_spider_with_allowed_domains():
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider3)
+def test_url_is_from_spider_with_allowed_domains_property():
+ class MySpider(Spider):
+ name = "myspider"
+
+ @property
+ def allowed_domains(self):
+ return ["example.org"]
+
+ with pytest.warns(UserWarning, match="allowed_domains is a property"):
+ assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
+
+
@pytest.mark.parametrize(
("url", "expected"),
[