Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add tests for stripping userinfo with percent-encoded delimiters

Paul Tremberth0a0b60a59f75
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 1f9845d82..9182d0fda 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -290,6 +290,26 @@ class StripUrl(unittest.TestCase):
self.assertEqual(strip_url(i, strip_credentials=True), o)
self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o)
+ def test_credentials_encoded_delims(self):
+ for i, o in [
+ # user: "username@"
+ # password: none
+ ('http://username%[email protected]/index.html?somekey=somevalue#section',
+ 'http://www.example.com/index.html?somekey=somevalue'),
+
+ # user: "username:pass"
+ # password: ""
+ ('https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section',
+ 'https://www.example.com/index.html?somekey=somevalue'),
+
+ # user: "me"
+ # password: "[email protected]"
+ ('ftp://me:user%[email protected]/index.html?somekey=somevalue#section',
+ 'ftp://www.example.com/index.html?somekey=somevalue'),
+ ]:
+ self.assertEqual(strip_url(i, strip_credentials=True), o)
+ self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o)
+
def test_default_ports_creds_off(self):
for i, o in [
('http://username:[email protected]:80/index.html?somekey=somevalue#section',