Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix tests

Paul Tremberthb6c761d2b4c9
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 8ee04120a..4f50db689 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -159,7 +159,8 @@ class StrictOriginPolicy(ReferrerPolicy):
name = POLICY_STRICT_ORIGIN
def referrer(self, response, request):
- if ((self.tls_protected(response) and self.potentially_trustworthy(request))
+ if ((self.tls_protected(response) and
+ self.potentially_trustworthy(request))
or not self.tls_protected(response)):
return self.origin_referrer(response)
@@ -208,9 +209,9 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
origin = self.origin(response)
if origin == self.origin(request):
return self.stripped_referrer(response)
- elif ((urlparse_cached(response).scheme in ('https', 'ftps') and
+ elif ((self.tls_protected(response) and
self.potentially_trustworthy(request))
- or urlparse_cached(response).scheme == 'http'):
+ or not self.tls_protected(response)):
return self.origin_referrer(response)
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index df20dfbb9..4779b0ed1 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -232,7 +232,13 @@ class MixinStrictOriginWhenCrossOrigin(object):
# downgrade
('https://example4.com/page.html', 'http://example4.com/not-page.html', None),
('https://example4.com/page.html', 'http://not.example4.com/', None),
- ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', None),
+
+ # non-TLS to non-TLS
+ ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
+
+ # upgrade
+ ('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'),
+ ('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'),
# Different protocols: send origin as referrer
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),