Scrapy, a fast high-level web crawling & scraping framework for Python.
Rename helper function to strip_url() + add more tests
Paul Tremberth8864d0e8c19d
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex e40e798b8..24e5eac40 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -8,7 +8,7 @@ from scrapy import signalsfrom scrapy.utils.python import to_native_strfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import load_object-from scrapy.utils.url import strip_url_credentials+from scrapy.utils.url import strip_urlLOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)@@ -53,7 +53,11 @@ class ReferrerPolicy(object):return Noneparsed_url = urlparse_cached(req_or_resp)if parsed_url.scheme not in self.NOREFERRER_SCHEMES:- return strip_url_credentials(parsed_url, origin_only=origin_only)+ return strip_url(parsed_url,+ strip_credentials=True,+ strip_fragment=True,+ strip_default_port=True,+ origin_only=origin_only)def origin(self, req_or_resp):"""Return serialized origin (scheme, host, path) for a request or response URL."""diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex f3ccfb0e8..9864f353d 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -105,33 +105,37 @@ def guess_scheme(url):return add_http_if_no_scheme(url)-def strip_url_credentials(url, origin_only=False, keep_fragments=False):+def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):++ """Strip URL string from some of its components:++ - `strip_credentials` removes "user:password@"+ - `strip_default_port` removes ":80" (resp. ":443", ":21")+ from http:// (resp. https://, ftp://) URLs+ - `origin_only` replaces path component with "/", also dropping+ query and fragment components ; it also strips credentials+ - `strip_fragment` drops any #fragment component+ """if url is None:return None-if not isinstance(url, ParseResult):parsed_url = urlparse(url)else:parsed_url = url-netloc = parsed_url.netloc- # strip username and password if present- if parsed_url.username or parsed_url.password:+ if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password):netloc = netloc.split('@')[-1]-- # strip standard protocol numbers- # Note: strictly speaking, standard port numbers should only be- # stripped when comparing origins- if parsed_url.port:- if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443)):+ if strip_default_port and parsed_url.port:+ if (parsed_url.scheme, parsed_url.port) in (('http', 80),+ ('https', 443),+ ('ftp', 21)):netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '')-return urlunparse((parsed_url.scheme,netloc,'/' if origin_only else parsed_url.path,'' if origin_only else parsed_url.params,'' if origin_only else parsed_url.query,- '' if not keep_fragments else parsed_url.fragment+ '' if strip_fragment else parsed_url.fragment))diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex f1a5c3196..1f9845d82 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -7,7 +7,7 @@ from six.moves.urllib.parse import urlparsefrom scrapy.spiders import Spiderfrom scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,add_http_if_no_scheme, guess_scheme,- parse_url, strip_url_credentials)+ parse_url, strip_url)__doctests__ = ['scrapy.utils.url']@@ -242,94 +242,158 @@ for k, args in enumerate ([setattr (GuessSchemeTest, t_method.__name__, t_method)-class StripUrlCredentials(unittest.TestCase):+class StripUrl(unittest.TestCase):def test_noop(self):- self.assertEqual(strip_url_credentials(+ self.assertEqual(strip_url('http://www.example.com/index.html'),'http://www.example.com/index.html')def test_noop_query_string(self):- self.assertEqual(strip_url_credentials(+ self.assertEqual(strip_url('http://www.example.com/index.html?somekey=somevalue'),'http://www.example.com/index.html?somekey=somevalue')def test_fragments(self):- self.assertEqual(strip_url_credentials(- 'http://www.example.com/index.html?somekey=somevalue#section', keep_fragments=True),+ self.assertEqual(strip_url(+ 'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False),'http://www.example.com/index.html?somekey=somevalue#section')- def test_noop_trailing_path(self):- self.assertEqual(strip_url_credentials(- 'http://www.example.com/'),- 'http://www.example.com/')-- def test_noop_trailing_path2(self):- self.assertEqual(strip_url_credentials(- 'http://www.example.com'),- 'http://www.example.com')-- def test_trailing_path_origin(self):- self.assertEqual(strip_url_credentials(- 'http://www.example.com', origin_only=True),- 'http://www.example.com/')-- def test_username(self):- # username is stripped (and fragment too)- self.assertEqual(strip_url_credentials(- 'http://[email protected]/index.html?somekey=somevalue#section'),- 'http://www.example.com/index.html?somekey=somevalue')+ def test_path(self):+ for input_url, origin, output_url in [+ ('http://www.example.com/',+ False,+ 'http://www.example.com/'),- def test_username_empty_pass(self):- # same as above- self.assertEqual(strip_url_credentials(- 'https://username:@www.example.com/index.html?somekey=somevalue#section'),- 'https://www.example.com/index.html?somekey=somevalue')+ ('http://www.example.com',+ False,+ 'http://www.example.com'),- def test_username_password(self):- self.assertEqual(strip_url_credentials(- 'ftp://username:[email protected]/index.html?somekey=somevalue#section'),- 'ftp://www.example.com/index.html?somekey=somevalue')+ ('http://www.example.com',+ True,+ 'http://www.example.com/'),+ ]:+ self.assertEqual(strip_url(input_url, origin_only=origin), output_url)+ self.assertEqual(strip_url(urlparse(input_url), origin_only=origin), output_url)- def test_default_http_port(self):- self.assertEqual(strip_url_credentials(- 'http://username:[email protected]:80/index.html'),- 'http://www.example.com/index.html')+ def test_credentials(self):+ for i, o in [+ ('http://[email protected]/index.html?somekey=somevalue#section',+ 'http://www.example.com/index.html?somekey=somevalue'),++ ('https://username:@www.example.com/index.html?somekey=somevalue#section',+ 'https://www.example.com/index.html?somekey=somevalue'),++ ('ftp://username:[email protected]/index.html?somekey=somevalue#section',+ 'ftp://www.example.com/index.html?somekey=somevalue'),+ ]:+ self.assertEqual(strip_url(i, strip_credentials=True), o)+ self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o)++ def test_default_ports_creds_off(self):+ for i, o in [+ ('http://username:[email protected]:80/index.html?somekey=somevalue#section',+ 'http://www.example.com/index.html?somekey=somevalue'),++ ('http://username:[email protected]:8080/index.html#section',+ 'http://www.example.com:8080/index.html'),++ ('http://username:[email protected]:443/index.html?somekey=somevalue&someotherkey=sov#section',+ 'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'),++ ('https://username:[email protected]:443/index.html',+ 'https://www.example.com/index.html'),++ ('https://username:[email protected]:442/index.html',+ 'https://www.example.com:442/index.html'),++ ('https://username:[email protected]:80/index.html',+ 'https://www.example.com:80/index.html'),++ ('ftp://username:[email protected]:21/file.txt',+ 'ftp://www.example.com/file.txt'),++ ('ftp://username:[email protected]:221/file.txt',+ 'ftp://www.example.com:221/file.txt'),+ ]:+ self.assertEqual(strip_url(i), o)+ self.assertEqual(strip_url(urlparse(i)), o)++ def test_default_ports(self):+ for i, o in [+ ('http://username:[email protected]:80/index.html',+ 'http://username:[email protected]/index.html'),++ ('http://username:[email protected]:8080/index.html',+ 'http://username:[email protected]:8080/index.html'),- def test_non_default_http_port(self):- self.assertEqual(strip_url_credentials(- 'http://username:[email protected]:8080/index.html'),- 'http://www.example.com:8080/index.html')+ ('http://username:[email protected]:443/index.html',+ 'http://username:[email protected]:443/index.html'),- def test_default_https_port(self):- self.assertEqual(strip_url_credentials(- 'https://username:[email protected]:443/index.html'),- 'https://www.example.com/index.html')+ ('https://username:[email protected]:443/index.html',+ 'https://username:[email protected]/index.html'),- def test_non_default_https_port(self):- self.assertEqual(strip_url_credentials(- 'https://username:[email protected]:442/index.html'),- 'https://www.example.com:442/index.html')+ ('https://username:[email protected]:442/index.html',+ 'https://username:[email protected]:442/index.html'),++ ('https://username:[email protected]:80/index.html',+ 'https://username:[email protected]:80/index.html'),++ ('ftp://username:[email protected]:21/file.txt',+ 'ftp://username:[email protected]/file.txt'),++ ('ftp://username:[email protected]:221/file.txt',+ 'ftp://username:[email protected]:221/file.txt'),+ ]:+ self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)+ self.assertEqual(strip_url(urlparse(i), strip_default_port=True, strip_credentials=False), o)++ def test_default_ports_keep(self):+ for i, o in [+ ('http://username:[email protected]:80/index.html?somekey=somevalue&someotherkey=sov#section',+ 'http://username:[email protected]:80/index.html?somekey=somevalue&someotherkey=sov'),++ ('http://username:[email protected]:8080/index.html?somekey=somevalue&someotherkey=sov#section',+ 'http://username:[email protected]:8080/index.html?somekey=somevalue&someotherkey=sov'),++ ('http://username:[email protected]:443/index.html',+ 'http://username:[email protected]:443/index.html'),++ ('https://username:[email protected]:443/index.html',+ 'https://username:[email protected]:443/index.html'),++ ('https://username:[email protected]:442/index.html',+ 'https://username:[email protected]:442/index.html'),++ ('https://username:[email protected]:80/index.html',+ 'https://username:[email protected]:80/index.html'),++ ('ftp://username:[email protected]:21/file.txt',+ 'ftp://username:[email protected]:21/file.txt'),++ ('ftp://username:[email protected]:221/file.txt',+ 'ftp://username:[email protected]:221/file.txt'),+ ]:+ self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)+ self.assertEqual(strip_url(urlparse(i), strip_default_port=False, strip_credentials=False), o)def test_origin_only(self):- self.assertEqual(strip_url_credentials(- 'http://username:[email protected]/index.html', origin_only=True),- 'http://www.example.com/')-- def test_default_http_port_origin_only(self):- self.assertEqual(strip_url_credentials(- 'http://username:[email protected]:80/index.html', origin_only=True),- 'http://www.example.com/')-- def test_non_default_http_port_origin_only(self):- self.assertEqual(strip_url_credentials(- 'http://username:[email protected]:8008/index.html', origin_only=True),- 'http://www.example.com:8008/')-- def test_default_https_port_origin_only(self):- self.assertEqual(strip_url_credentials(- 'https://username:[email protected]:443/index.html', origin_only=True),- 'https://www.example.com/')+ for i, o in [+ ('http://username:[email protected]/index.html',+ 'http://www.example.com/'),++ ('http://username:[email protected]:80/foo/bar?query=value#somefrag',+ 'http://www.example.com/'),++ ('http://username:[email protected]:8008/foo/bar?query=value#somefrag',+ 'http://www.example.com:8008/'),++ ('https://username:[email protected]:443/index.html',+ 'https://www.example.com/'),+ ]:+ self.assertEqual(strip_url(i, origin_only=True), o)+ self.assertEqual(strip_url(urlparse(i), origin_only=True), o)+if __name__ == "__main__":unittest.main()