Scrapy, a fast high-level web crawling & scraping framework for Python.
Rename setting to REFERRER_POLICY (with 2 Rs)
Paul Tremberthbc200d1155e0
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rstindex 349dbb3a1..9a0ccd0c1 100644--- a/docs/topics/spider-middleware.rst+++ b/docs/topics/spider-middleware.rst@@ -328,10 +328,10 @@ Default: ``True``Whether to enable referer middleware.-.. setting:: REFERER_POLICY+.. setting:: REFERRER_POLICY-REFERER_POLICY-^^^^^^^^^^^^^^+REFERRER_POLICY+^^^^^^^^^^^^^^^.. versionadded:: 1.4@@ -344,10 +344,10 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``.. note::You can also set the Referrer Policy per request,using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,- with the same acceptable values as for the ``REFERER_POLICY`` setting.+ with the same acceptable values as for the ``REFERRER_POLICY`` setting.-Acceptable values for REFERER_POLICY-************************************+Acceptable values for REFERRER_POLICY+*************************************- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy``subclass — a custom policy or one of the built-in ones (see classes below),diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 15a134dd3..35d9844a7 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -234,7 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default settingREDIRECT_PRIORITY_ADJUST = +2REFERER_ENABLED = True-REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'+REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'RETRY_ENABLED = TrueRETRY_TIMES = 2 # initial response + 2 retries = 3 requestsdiff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex c015e13c8..24c163089 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -266,7 +266,7 @@ class RefererMiddleware(object):def __init__(self, settings=None):self.default_policy = DefaultReferrerPolicyif settings is not None:- policy = settings.get('REFERER_POLICY')+ policy = settings.get('REFERRER_POLICY')if policy is not None:# expect a string for the path to the policy classtry:diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 4779b0ed1..81868efab 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -289,35 +289,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):# --- Tests using settings to set policy using class pathclass TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}class CustomPythonOrgPolicy(ReferrerPolicy):@@ -336,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy):class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware):- settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}+ settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}scenarii = [('https://example.com/', 'https://scrapy.org/', b'https://python.org/'),('http://example.com/', 'http://scrapy.org/', b'http://python.org/'),@@ -385,17 +385,17 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}req_meta = {'referrer_policy': POLICY_UNSAFE_URL}class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}req_meta = {'referrer_policy': POLICY_NO_REFERRER}class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}req_meta = {'referrer_policy': POLICY_UNSAFE_URL}@@ -411,7 +411,7 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),(POLICY_UNSAFE_URL, UnsafeUrlPolicy),]:- settings = Settings({'REFERER_POLICY': s})+ settings = Settings({'REFERRER_POLICY': s})mw = RefererMiddleware(settings)self.assertEquals(mw.default_policy, p)@@ -425,24 +425,24 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),(POLICY_UNSAFE_URL, UnsafeUrlPolicy),]:- settings = Settings({'REFERER_POLICY': s.upper()})+ settings = Settings({'REFERRER_POLICY': s.upper()})mw = RefererMiddleware(settings)self.assertEquals(mw.default_policy, p)def test_invalid_name(self):- settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'})+ settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})with self.assertRaises(NotConfigured):mw = RefererMiddleware(settings)class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}