Scrapy, a fast high-level web crawling & scraping framework for Python.
Add support for multiple referer policy tokens.
Fabian Schneebauer0d58af86971b
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex a0b6851e5..7706c8c15 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -323,15 +323,17 @@ def _load_policy_class(try:return cast(Type[ReferrerPolicy], load_object(policy))except ValueError:- try:- return _policy_classes[policy.lower()]- except KeyError:- msg = f"Could not load referrer policy {policy!r}"- if not warning_only:- raise RuntimeError(msg)- else:- warnings.warn(msg, RuntimeWarning)- return None+ tokens = [token.strip() for token in policy.lower().split(",")]+ for token in tokens[::-1]:+ if token in _policy_classes:+ return _policy_classes[token]++ msg = f"Could not load referrer policy {policy!r}"+ if not warning_only:+ raise RuntimeError(msg)+ else:+ warnings.warn(msg, RuntimeWarning)+ return Noneclass RefererMiddleware:diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex afffa87fb..5797edfbd 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -884,6 +884,53 @@ class TestSettingsPolicyByName(TestCase):with self.assertRaises(RuntimeError):RefererMiddleware(settings)+ def test_multiple_policy_tokens(self):+ # test parsing without space(s) after the comma+ settings1 = Settings(+ {+ "REFERRER_POLICY": ",".join(+ [+ "some-custom-unknown-policy",+ POLICY_SAME_ORIGIN,+ POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,+ "another-custom-unknown-policy",+ ]+ )+ }+ )+ mw1 = RefererMiddleware(settings1)+ self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy)++ # test parsing with space(s) after the comma+ settings2 = Settings(+ {+ "REFERRER_POLICY": ", ".join(+ [+ POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,+ "another-custom-unknown-policy",+ POLICY_UNSAFE_URL,+ ]+ )+ }+ )+ mw2 = RefererMiddleware(settings2)+ self.assertEqual(mw2.default_policy, UnsafeUrlPolicy)++ def test_multiple_policy_tokens_all_invalid(self):+ settings = Settings(+ {+ "REFERRER_POLICY": ",".join(+ [+ "some-custom-unknown-policy",+ "another-custom-unknown-policy",+ "yet-another-custom-unknown-policy",+ ]+ )+ }+ )+ with self.assertRaises(RuntimeError):+ RefererMiddleware(settings)+class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}