Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add support for multiple referer policy tokens.

Fabian Schneebauer0d58af86971b
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index a0b6851e5..7706c8c15 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -323,15 +323,17 @@ def _load_policy_class(
try:
return cast(Type[ReferrerPolicy], load_object(policy))
except ValueError:
- try:
- return _policy_classes[policy.lower()]
- except KeyError:
- msg = f"Could not load referrer policy {policy!r}"
- if not warning_only:
- raise RuntimeError(msg)
- else:
- warnings.warn(msg, RuntimeWarning)
- return None
+ tokens = [token.strip() for token in policy.lower().split(",")]
+ for token in tokens[::-1]:
+ if token in _policy_classes:
+ return _policy_classes[token]
+
+ msg = f"Could not load referrer policy {policy!r}"
+ if not warning_only:
+ raise RuntimeError(msg)
+ else:
+ warnings.warn(msg, RuntimeWarning)
+ return None
class RefererMiddleware:
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index afffa87fb..5797edfbd 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -884,6 +884,53 @@ class TestSettingsPolicyByName(TestCase):
with self.assertRaises(RuntimeError):
RefererMiddleware(settings)
+ def test_multiple_policy_tokens(self):
+ # test parsing without space(s) after the comma
+ settings1 = Settings(
+ {
+ "REFERRER_POLICY": ",".join(
+ [
+ "some-custom-unknown-policy",
+ POLICY_SAME_ORIGIN,
+ POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,
+ "another-custom-unknown-policy",
+ ]
+ )
+ }
+ )
+ mw1 = RefererMiddleware(settings1)
+ self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy)
+
+ # test parsing with space(s) after the comma
+ settings2 = Settings(
+ {
+ "REFERRER_POLICY": ", ".join(
+ [
+ POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,
+ "another-custom-unknown-policy",
+ POLICY_UNSAFE_URL,
+ ]
+ )
+ }
+ )
+ mw2 = RefererMiddleware(settings2)
+ self.assertEqual(mw2.default_policy, UnsafeUrlPolicy)
+
+ def test_multiple_policy_tokens_all_invalid(self):
+ settings = Settings(
+ {
+ "REFERRER_POLICY": ",".join(
+ [
+ "some-custom-unknown-policy",
+ "another-custom-unknown-policy",
+ "yet-another-custom-unknown-policy",
+ ]
+ )
+ }
+ )
+ with self.assertRaises(RuntimeError):
+ RefererMiddleware(settings)
+
class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}