Scrapy, a fast high-level web crawling & scraping framework for Python.
Regex-based guess_scheme() + refactor tests
Paul Tremberth7a51d370f3a5
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex 25ceb5f99..5201feb42 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -3,7 +3,7 @@ Scrapy ShellSee documentation in docs/topics/shell.rst"""-+import refrom six.moves.urllib.parse import urlparsefrom threading import Threadfrom w3lib.url import any_to_uri@@ -21,16 +21,22 @@ def guess_scheme(url):otherwise returns an HTTP URL"""parts = urlparse(url)- if not parts.scheme:- if "." not in parts.path.split("/", 1)[0]:- url = any_to_uri(url)-- for pattern in ["/", "./", "../"]:- if url.startswith(pattern):- url = any_to_uri(url)- break- url = add_http_if_no_scheme(url)- return url+ if parts.scheme:+ return url+ # Note: this does not match Windows filepath+ if re.match(r'''^ # start with...+ (+ \. # ...a single dot,+ (+ \. | [^/\.]+ # optionally followed by+ )? # either a second dot or some characters+ )? # optional match of ".", ".." or ".blabla"+ / # at least one "/" for a file path,+ . # and something after the "/"+ ''', parts.path, flags=re.VERBOSE):+ return any_to_uri(url)+ else:+ return add_http_if_no_scheme(url)class Command(ScrapyCommand):diff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex 37c6b8c90..a61d520fa 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -11,67 +11,61 @@ from tests import tests_datadirclass ShellURLTest(unittest.TestCase):-- def test_file_uri_relative001(self):- # FIXME: 'index.html' is interpreted as a domain name- # is this correct?- url = guess_scheme('index.html')- assert url.startswith('http://')-- def test_file_uri_relative002(self):- url = guess_scheme('./index.html')- assert url.startswith('file://')-- def test_file_uri_relative003(self):- url = guess_scheme('../data/index.html')- assert url.startswith('file://')-- def test_file_uri_relative004(self):- url = guess_scheme('subdir/index.html')- assert url.startswith('file://')-- def test_file_uri_absolute001(self):- """Absolute file paths get prepended with "file://" scheme"""- iurl = '/home/user/www/index.html'- url = guess_scheme(iurl)- self.assertEquals(url, 'file://'+iurl)-- def test_file_uri_scheme(self):- """Output File URI does not change if "file://" scheme is set"""- iurl = 'file:///home/user/www/index.html'- url = guess_scheme(iurl)- self.assertEquals(url, iurl)-- def test_file_uri_windows(self):- raise unittest.SkipTest("Windows filepath are not supported for scrapy shell")- url = guess_scheme('C:\absolute\path\to\a\file.html')- assert url.startswith('file://')-- def test_http_url_001(self):- url = guess_scheme('index.html')- assert url.startswith('http://')-- def test_http_url_002(self):- url = guess_scheme('example.com')- assert url.startswith('http://')-- def test_http_url_003(self):- url = guess_scheme('www.example.com')- assert url.startswith('http://')-- def test_http_url_004(self):- url = guess_scheme('www.example.com/index')- assert url.startswith('http://')-- def test_http_url_005(self):- url = guess_scheme('www.example.com/index.html')- assert url.startswith('http://')-- def test_http_url_scheme(self):- """An full HTTP URL is unaltered"""- iurl = 'http://www.example.com/index.html'- url = guess_scheme(iurl)- self.assertEquals(url, iurl)+ pass++def create_guess_scheme_t(args):+ def do_expected(self):+ url = guess_scheme(args[0])+ assert url.startswith(args[1]), \+ 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (+ args[0], url, args[1])+ return do_expected++def create_skipped_scheme_t(args):+ def do_expected(self):+ raise unittest.SkipTest(args[2])+ url = guess_scheme(args[0])+ assert url.startswith(args[1])+ return do_expected++for k, args in enumerate ([+ ('/index', 'file://'),+ ('/index.html', 'file://'),+ ('./index.html', 'file://'),+ ('../index.html', 'file://'),+ ('../../index.html', 'file://'),+ ('./data/index.html', 'file://'),+ ('.hidden/data/index.html', 'file://'),+ ('/home/user/www/index.html', 'file://'),+ ('//home/user/www/index.html', 'file://'),+ ('file:///home/user/www/index.html', 'file://'),++ ('index.html', 'http://'),+ ('example.com', 'http://'),+ ('www.example.com', 'http://'),+ ('www.example.com/index.html', 'http://'),+ ('http://example.com', 'http://'),+ ('http://example.com/index.html', 'http://'),+ ('localhost', 'http://'),+ ('localhost/index.html', 'http://'),++ # some corner cases (default to http://)+ ('/', 'http://'),+ ('.../test', 'http://'),++ ], start=1):+ t_method = create_guess_scheme_t(args)+ t_method.__name__ = 'test_uri_%03d' % k+ setattr (ShellURLTest, t_method.__name__, t_method)++# TODO: the following tests do not pass with current implementation+for k, args in enumerate ([+ ('C:\absolute\path\to\a\file.html', 'file://',+ 'Windows filepath are not supported for scrapy shell'),+ ], start=1):+ t_method = create_skipped_scheme_t(args)+ t_method.__name__ = 'test_uri_skipped_%03d' % k+ setattr (ShellURLTest, t_method.__name__, t_method)class ShellTest(ProcessTest, SiteTest, unittest.TestCase):