Scrapy, a fast high-level web crawling & scraping framework for Python.
Move URL credentials stripping to a helper function
Paul Tremberth5dd7311cd48e
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex 01f1fdf85..e40e798b8 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -2,14 +2,13 @@RefererMiddleware: populates Request referer field, based on the Response whichoriginated it."""-from six.moves.urllib.parse import ParseResult, urlunparse-from scrapy.http import Request, Responsefrom scrapy.exceptions import NotConfiguredfrom scrapy import signalsfrom scrapy.utils.python import to_native_strfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import load_object+from scrapy.utils.url import strip_url_credentialsLOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)@@ -31,14 +30,10 @@ class ReferrerPolicy(object):raise NotImplementedError()def stripped_referrer(self, req_or_resp):- stripped = self.strip_url(req_or_resp)- if stripped is not None:- return urlunparse(stripped)+ return self.strip_url(req_or_resp)def origin_referrer(self, req_or_resp):- stripped = self.strip_url(req_or_resp, origin_only=True)- if stripped is not None:- return urlunparse(stripped)+ return self.strip_url(req_or_resp, origin_only=True)def strip_url(self, req_or_resp, origin_only=False):"""@@ -56,33 +51,13 @@ class ReferrerPolicy(object):"""if req_or_resp.url is None or not req_or_resp.url:return None- parsed = urlparse_cached(req_or_resp)-- if parsed.scheme in self.NOREFERRER_SCHEMES:- return None-- netloc = parsed.netloc- # strip username and password if present- if parsed.username or parsed.password:- netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '')-- # strip standard protocol numbers- # Note: strictly speaking, standard port numbers should only be- # stripped when comparing origins- if parsed.port:- if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)):- netloc = netloc.replace(':{p.port}'.format(p=parsed), '')-- return ParseResult(parsed.scheme,- netloc,- '/' if origin_only else parsed.path,- '' if origin_only else parsed.params,- '' if origin_only else parsed.query,- '')+ parsed_url = urlparse_cached(req_or_resp)+ if parsed_url.scheme not in self.NOREFERRER_SCHEMES:+ return strip_url_credentials(parsed_url, origin_only=origin_only)def origin(self, req_or_resp):- """Return (scheme, host, path) tuple for a request or response URL."""- return tuple(self.strip_url(req_or_resp, origin_only=True)[:3])+ """Return serialized origin (scheme, host, path) for a request or response URL."""+ return self.strip_url(req_or_resp, origin_only=True)class NoReferrerPolicy(ReferrerPolicy):@@ -178,7 +153,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):if origin == self.origin(request):return self.stripped_referrer(response)else:- return urlunparse(origin + ('', '', ''))+ return originclass UnsafeUrlPolicy(ReferrerPolicy):diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex dc1cce4ac..f3ccfb0e8 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead."""import posixpathimport re-from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse)+from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse)# scrapy.utils.url was moved to w3lib.url and import * ensures this# move doesn't break old code@@ -103,3 +103,35 @@ def guess_scheme(url):return any_to_uri(url)else:return add_http_if_no_scheme(url)+++def strip_url_credentials(url, origin_only=False, keep_fragments=False):++ if url is None:+ return None++ if not isinstance(url, ParseResult):+ parsed_url = urlparse(url)+ else:+ parsed_url = url++ netloc = parsed_url.netloc+ # strip username and password if present+ if parsed_url.username or parsed_url.password:+ netloc = netloc.split('@')[-1]++ # strip standard protocol numbers+ # Note: strictly speaking, standard port numbers should only be+ # stripped when comparing origins+ if parsed_url.port:+ if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443)):+ netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '')++ return urlunparse((+ parsed_url.scheme,+ netloc,+ '/' if origin_only else parsed_url.path,+ '' if origin_only else parsed_url.params,+ '' if origin_only else parsed_url.query,+ '' if not keep_fragments else parsed_url.fragment+ ))diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex f46d1d927..f1a5c3196 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -6,7 +6,8 @@ from six.moves.urllib.parse import urlparsefrom scrapy.spiders import Spiderfrom scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,- add_http_if_no_scheme, guess_scheme, parse_url)+ add_http_if_no_scheme, guess_scheme,+ parse_url, strip_url_credentials)__doctests__ = ['scrapy.utils.url']@@ -241,5 +242,94 @@ for k, args in enumerate ([setattr (GuessSchemeTest, t_method.__name__, t_method)+class StripUrlCredentials(unittest.TestCase):++ def test_noop(self):+ self.assertEqual(strip_url_credentials(+ 'http://www.example.com/index.html'),+ 'http://www.example.com/index.html')++ def test_noop_query_string(self):+ self.assertEqual(strip_url_credentials(+ 'http://www.example.com/index.html?somekey=somevalue'),+ 'http://www.example.com/index.html?somekey=somevalue')++ def test_fragments(self):+ self.assertEqual(strip_url_credentials(+ 'http://www.example.com/index.html?somekey=somevalue#section', keep_fragments=True),+ 'http://www.example.com/index.html?somekey=somevalue#section')++ def test_noop_trailing_path(self):+ self.assertEqual(strip_url_credentials(+ 'http://www.example.com/'),+ 'http://www.example.com/')++ def test_noop_trailing_path2(self):+ self.assertEqual(strip_url_credentials(+ 'http://www.example.com'),+ 'http://www.example.com')++ def test_trailing_path_origin(self):+ self.assertEqual(strip_url_credentials(+ 'http://www.example.com', origin_only=True),+ 'http://www.example.com/')++ def test_username(self):+ # username is stripped (and fragment too)+ self.assertEqual(strip_url_credentials(+ 'http://[email protected]/index.html?somekey=somevalue#section'),+ 'http://www.example.com/index.html?somekey=somevalue')++ def test_username_empty_pass(self):+ # same as above+ self.assertEqual(strip_url_credentials(+ 'https://username:@www.example.com/index.html?somekey=somevalue#section'),+ 'https://www.example.com/index.html?somekey=somevalue')++ def test_username_password(self):+ self.assertEqual(strip_url_credentials(+ 'ftp://username:[email protected]/index.html?somekey=somevalue#section'),+ 'ftp://www.example.com/index.html?somekey=somevalue')++ def test_default_http_port(self):+ self.assertEqual(strip_url_credentials(+ 'http://username:[email protected]:80/index.html'),+ 'http://www.example.com/index.html')++ def test_non_default_http_port(self):+ self.assertEqual(strip_url_credentials(+ 'http://username:[email protected]:8080/index.html'),+ 'http://www.example.com:8080/index.html')++ def test_default_https_port(self):+ self.assertEqual(strip_url_credentials(+ 'https://username:[email protected]:443/index.html'),+ 'https://www.example.com/index.html')++ def test_non_default_https_port(self):+ self.assertEqual(strip_url_credentials(+ 'https://username:[email protected]:442/index.html'),+ 'https://www.example.com:442/index.html')++ def test_origin_only(self):+ self.assertEqual(strip_url_credentials(+ 'http://username:[email protected]/index.html', origin_only=True),+ 'http://www.example.com/')++ def test_default_http_port_origin_only(self):+ self.assertEqual(strip_url_credentials(+ 'http://username:[email protected]:80/index.html', origin_only=True),+ 'http://www.example.com/')++ def test_non_default_http_port_origin_only(self):+ self.assertEqual(strip_url_credentials(+ 'http://username:[email protected]:8008/index.html', origin_only=True),+ 'http://www.example.com:8008/')++ def test_default_https_port_origin_only(self):+ self.assertEqual(strip_url_credentials(+ 'https://username:[email protected]:443/index.html', origin_only=True),+ 'https://www.example.com/')+if __name__ == "__main__":unittest.main()