Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Rename helper function to strip_url() + add more tests

Paul Tremberth8864d0e8c19d
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index e40e798b8..24e5eac40 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -8,7 +8,7 @@ from scrapy import signals
from scrapy.utils.python import to_native_str
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
-from scrapy.utils.url import strip_url_credentials
+from scrapy.utils.url import strip_url
LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
@@ -53,7 +53,11 @@ class ReferrerPolicy(object):
return None
parsed_url = urlparse_cached(req_or_resp)
if parsed_url.scheme not in self.NOREFERRER_SCHEMES:
- return strip_url_credentials(parsed_url, origin_only=origin_only)
+ return strip_url(parsed_url,
+ strip_credentials=True,
+ strip_fragment=True,
+ strip_default_port=True,
+ origin_only=origin_only)
def origin(self, req_or_resp):
"""Return serialized origin (scheme, host, path) for a request or response URL."""
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index f3ccfb0e8..9864f353d 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -105,33 +105,37 @@ def guess_scheme(url):
return add_http_if_no_scheme(url)
-def strip_url_credentials(url, origin_only=False, keep_fragments=False):
+def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):
+
+ """Strip URL string from some of its components:
+
+ - `strip_credentials` removes "user:password@"
+ - `strip_default_port` removes ":80" (resp. ":443", ":21")
+ from http:// (resp. https://, ftp://) URLs
+ - `origin_only` replaces path component with "/", also dropping
+ query and fragment components ; it also strips credentials
+ - `strip_fragment` drops any #fragment component
+ """
if url is None:
return None
-
if not isinstance(url, ParseResult):
parsed_url = urlparse(url)
else:
parsed_url = url
-
netloc = parsed_url.netloc
- # strip username and password if present
- if parsed_url.username or parsed_url.password:
+ if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password):
netloc = netloc.split('@')[-1]
-
- # strip standard protocol numbers
- # Note: strictly speaking, standard port numbers should only be
- # stripped when comparing origins
- if parsed_url.port:
- if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443)):
+ if strip_default_port and parsed_url.port:
+ if (parsed_url.scheme, parsed_url.port) in (('http', 80),
+ ('https', 443),
+ ('ftp', 21)):
netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '')
-
return urlunparse((
parsed_url.scheme,
netloc,
'/' if origin_only else parsed_url.path,
'' if origin_only else parsed_url.params,
'' if origin_only else parsed_url.query,
- '' if not keep_fragments else parsed_url.fragment
+ '' if strip_fragment else parsed_url.fragment
))
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index f1a5c3196..1f9845d82 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -7,7 +7,7 @@ from six.moves.urllib.parse import urlparse
from scrapy.spiders import Spider
from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
add_http_if_no_scheme, guess_scheme,
- parse_url, strip_url_credentials)
+ parse_url, strip_url)
__doctests__ = ['scrapy.utils.url']
@@ -242,94 +242,158 @@ for k, args in enumerate ([
setattr (GuessSchemeTest, t_method.__name__, t_method)
-class StripUrlCredentials(unittest.TestCase):
+class StripUrl(unittest.TestCase):
def test_noop(self):
- self.assertEqual(strip_url_credentials(
+ self.assertEqual(strip_url(
'http://www.example.com/index.html'),
'http://www.example.com/index.html')
def test_noop_query_string(self):
- self.assertEqual(strip_url_credentials(
+ self.assertEqual(strip_url(
'http://www.example.com/index.html?somekey=somevalue'),
'http://www.example.com/index.html?somekey=somevalue')
def test_fragments(self):
- self.assertEqual(strip_url_credentials(
- 'http://www.example.com/index.html?somekey=somevalue#section', keep_fragments=True),
+ self.assertEqual(strip_url(
+ 'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False),
'http://www.example.com/index.html?somekey=somevalue#section')
- def test_noop_trailing_path(self):
- self.assertEqual(strip_url_credentials(
- 'http://www.example.com/'),
- 'http://www.example.com/')
-
- def test_noop_trailing_path2(self):
- self.assertEqual(strip_url_credentials(
- 'http://www.example.com'),
- 'http://www.example.com')
-
- def test_trailing_path_origin(self):
- self.assertEqual(strip_url_credentials(
- 'http://www.example.com', origin_only=True),
- 'http://www.example.com/')
-
- def test_username(self):
- # username is stripped (and fragment too)
- self.assertEqual(strip_url_credentials(
- 'http://[email protected]/index.html?somekey=somevalue#section'),
- 'http://www.example.com/index.html?somekey=somevalue')
+ def test_path(self):
+ for input_url, origin, output_url in [
+ ('http://www.example.com/',
+ False,
+ 'http://www.example.com/'),
- def test_username_empty_pass(self):
- # same as above
- self.assertEqual(strip_url_credentials(
- 'https://username:@www.example.com/index.html?somekey=somevalue#section'),
- 'https://www.example.com/index.html?somekey=somevalue')
+ ('http://www.example.com',
+ False,
+ 'http://www.example.com'),
- def test_username_password(self):
- self.assertEqual(strip_url_credentials(
- 'ftp://username:[email protected]/index.html?somekey=somevalue#section'),
- 'ftp://www.example.com/index.html?somekey=somevalue')
+ ('http://www.example.com',
+ True,
+ 'http://www.example.com/'),
+ ]:
+ self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
+ self.assertEqual(strip_url(urlparse(input_url), origin_only=origin), output_url)
- def test_default_http_port(self):
- self.assertEqual(strip_url_credentials(
- 'http://username:[email protected]:80/index.html'),
- 'http://www.example.com/index.html')
+ def test_credentials(self):
+ for i, o in [
+ ('http://[email protected]/index.html?somekey=somevalue#section',
+ 'http://www.example.com/index.html?somekey=somevalue'),
+
+ ('https://username:@www.example.com/index.html?somekey=somevalue#section',
+ 'https://www.example.com/index.html?somekey=somevalue'),
+
+ ('ftp://username:[email protected]/index.html?somekey=somevalue#section',
+ 'ftp://www.example.com/index.html?somekey=somevalue'),
+ ]:
+ self.assertEqual(strip_url(i, strip_credentials=True), o)
+ self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o)
+
+ def test_default_ports_creds_off(self):
+ for i, o in [
+ ('http://username:[email protected]:80/index.html?somekey=somevalue#section',
+ 'http://www.example.com/index.html?somekey=somevalue'),
+
+ ('http://username:[email protected]:8080/index.html#section',
+ 'http://www.example.com:8080/index.html'),
+
+ ('http://username:[email protected]:443/index.html?somekey=somevalue&someotherkey=sov#section',
+ 'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'),
+
+ ('https://username:[email protected]:443/index.html',
+ 'https://www.example.com/index.html'),
+
+ ('https://username:[email protected]:442/index.html',
+ 'https://www.example.com:442/index.html'),
+
+ ('https://username:[email protected]:80/index.html',
+ 'https://www.example.com:80/index.html'),
+
+ ('ftp://username:[email protected]:21/file.txt',
+ 'ftp://www.example.com/file.txt'),
+
+ ('ftp://username:[email protected]:221/file.txt',
+ 'ftp://www.example.com:221/file.txt'),
+ ]:
+ self.assertEqual(strip_url(i), o)
+ self.assertEqual(strip_url(urlparse(i)), o)
+
+ def test_default_ports(self):
+ for i, o in [
+ ('http://username:[email protected]:80/index.html',
+ 'http://username:[email protected]/index.html'),
+
+ ('http://username:[email protected]:8080/index.html',
+ 'http://username:[email protected]:8080/index.html'),
- def test_non_default_http_port(self):
- self.assertEqual(strip_url_credentials(
- 'http://username:[email protected]:8080/index.html'),
- 'http://www.example.com:8080/index.html')
+ ('http://username:[email protected]:443/index.html',
+ 'http://username:[email protected]:443/index.html'),
- def test_default_https_port(self):
- self.assertEqual(strip_url_credentials(
- 'https://username:[email protected]:443/index.html'),
- 'https://www.example.com/index.html')
+ ('https://username:[email protected]:443/index.html',
+ 'https://username:[email protected]/index.html'),
- def test_non_default_https_port(self):
- self.assertEqual(strip_url_credentials(
- 'https://username:[email protected]:442/index.html'),
- 'https://www.example.com:442/index.html')
+ ('https://username:[email protected]:442/index.html',
+ 'https://username:[email protected]:442/index.html'),
+
+ ('https://username:[email protected]:80/index.html',
+ 'https://username:[email protected]:80/index.html'),
+
+ ('ftp://username:[email protected]:21/file.txt',
+ 'ftp://username:[email protected]/file.txt'),
+
+ ('ftp://username:[email protected]:221/file.txt',
+ 'ftp://username:[email protected]:221/file.txt'),
+ ]:
+ self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
+ self.assertEqual(strip_url(urlparse(i), strip_default_port=True, strip_credentials=False), o)
+
+ def test_default_ports_keep(self):
+ for i, o in [
+ ('http://username:[email protected]:80/index.html?somekey=somevalue&someotherkey=sov#section',
+ 'http://username:[email protected]:80/index.html?somekey=somevalue&someotherkey=sov'),
+
+ ('http://username:[email protected]:8080/index.html?somekey=somevalue&someotherkey=sov#section',
+ 'http://username:[email protected]:8080/index.html?somekey=somevalue&someotherkey=sov'),
+
+ ('http://username:[email protected]:443/index.html',
+ 'http://username:[email protected]:443/index.html'),
+
+ ('https://username:[email protected]:443/index.html',
+ 'https://username:[email protected]:443/index.html'),
+
+ ('https://username:[email protected]:442/index.html',
+ 'https://username:[email protected]:442/index.html'),
+
+ ('https://username:[email protected]:80/index.html',
+ 'https://username:[email protected]:80/index.html'),
+
+ ('ftp://username:[email protected]:21/file.txt',
+ 'ftp://username:[email protected]:21/file.txt'),
+
+ ('ftp://username:[email protected]:221/file.txt',
+ 'ftp://username:[email protected]:221/file.txt'),
+ ]:
+ self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
+ self.assertEqual(strip_url(urlparse(i), strip_default_port=False, strip_credentials=False), o)
def test_origin_only(self):
- self.assertEqual(strip_url_credentials(
- 'http://username:[email protected]/index.html', origin_only=True),
- 'http://www.example.com/')
-
- def test_default_http_port_origin_only(self):
- self.assertEqual(strip_url_credentials(
- 'http://username:[email protected]:80/index.html', origin_only=True),
- 'http://www.example.com/')
-
- def test_non_default_http_port_origin_only(self):
- self.assertEqual(strip_url_credentials(
- 'http://username:[email protected]:8008/index.html', origin_only=True),
- 'http://www.example.com:8008/')
-
- def test_default_https_port_origin_only(self):
- self.assertEqual(strip_url_credentials(
- 'https://username:[email protected]:443/index.html', origin_only=True),
- 'https://www.example.com/')
+ for i, o in [
+ ('http://username:[email protected]/index.html',
+ 'http://www.example.com/'),
+
+ ('http://username:[email protected]:80/foo/bar?query=value#somefrag',
+ 'http://www.example.com/'),
+
+ ('http://username:[email protected]:8008/foo/bar?query=value#somefrag',
+ 'http://www.example.com:8008/'),
+
+ ('https://username:[email protected]:443/index.html',
+ 'https://www.example.com/'),
+ ]:
+ self.assertEqual(strip_url(i, origin_only=True), o)
+ self.assertEqual(strip_url(urlparse(i), origin_only=True), o)
+
if __name__ == "__main__":
unittest.main()