Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Rename setting to REFERRER_POLICY (with 2 Rs)

Paul Tremberthbc200d1155e0
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 349dbb3a1..9a0ccd0c1 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -328,10 +328,10 @@ Default: ``True``
Whether to enable referer middleware.
-.. setting:: REFERER_POLICY
+.. setting:: REFERRER_POLICY
-REFERER_POLICY
-^^^^^^^^^^^^^^
+REFERRER_POLICY
+^^^^^^^^^^^^^^^
.. versionadded:: 1.4
@@ -344,10 +344,10 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
.. note::
You can also set the Referrer Policy per request,
using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,
- with the same acceptable values as for the ``REFERER_POLICY`` setting.
+ with the same acceptable values as for the ``REFERRER_POLICY`` setting.
-Acceptable values for REFERER_POLICY
-************************************
+Acceptable values for REFERRER_POLICY
+*************************************
- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy``
subclass — a custom policy or one of the built-in ones (see classes below),
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 15a134dd3..35d9844a7 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -234,7 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
REDIRECT_PRIORITY_ADJUST = +2
REFERER_ENABLED = True
-REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
+REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index c015e13c8..24c163089 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -266,7 +266,7 @@ class RefererMiddleware(object):
def __init__(self, settings=None):
self.default_policy = DefaultReferrerPolicy
if settings is not None:
- policy = settings.get('REFERER_POLICY')
+ policy = settings.get('REFERRER_POLICY')
if policy is not None:
# expect a string for the path to the policy class
try:
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 4779b0ed1..81868efab 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -289,35 +289,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):
# --- Tests using settings to set policy using class path
class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}
class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}
class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}
class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}
class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
class CustomPythonOrgPolicy(ReferrerPolicy):
@@ -336,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
+ settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
scenarii = [
('https://example.com/', 'https://scrapy.org/', b'https://python.org/'),
('http://example.com/', 'http://scrapy.org/', b'http://python.org/'),
@@ -385,17 +385,17 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
req_meta = {'referrer_policy': POLICY_NO_REFERRER}
class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
@@ -411,7 +411,7 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
]:
- settings = Settings({'REFERER_POLICY': s})
+ settings = Settings({'REFERRER_POLICY': s})
mw = RefererMiddleware(settings)
self.assertEquals(mw.default_policy, p)
@@ -425,24 +425,24 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
]:
- settings = Settings({'REFERER_POLICY': s.upper()})
+ settings = Settings({'REFERRER_POLICY': s.upper()})
mw = RefererMiddleware(settings)
self.assertEquals(mw.default_policy, p)
def test_invalid_name(self):
- settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'})
+ settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
with self.assertRaises(NotConfigured):
mw = RefererMiddleware(settings)
class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
- settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}