Scrapy, a fast high-level web crawling & scraping framework for Python.
Refactor guess_scheme
Adrián Chavesd54c4496ee57
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex 60e2b50eb..1e431d3bf 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -83,26 +83,54 @@ def add_http_if_no_scheme(url):return url+def _is_posix_path(string):+ return bool(+ re.match(+ r'''+ ^ # start with...+ (+ \. # ...a single dot,+ (+ \. | [^/\.]+ # optionally followed by+ )? # either a second dot or some characters+ |+ ~ # $HOME+ )? # optional match of ".", ".." or ".blabla"+ / # at least one "/" for a file path,+ . # and something after the "/"+ ''',+ string,+ flags=re.VERBOSE,+ )+ )+++def _is_windows_path(string):+ return bool(+ re.match(+ r'''+ ^+ (+ [a-z]:\\+ | \\\\+ )+ ''',+ string,+ flags=re.IGNORECASE | re.VERBOSE,+ )+ )+++def _is_path(string):+ return _is_posix_path(string) or _is_windows_path(string)++def guess_scheme(url):"""Add an URL scheme if missing: file:// for filepath-like input orhttp:// otherwise."""- # POSIX path- if re.match(r'''^ # start with...- (- \. # ...a single dot,- (- \. | [^/\.]+ # optionally followed by- )? # either a second dot or some characters- )? # optional match of ".", ".." or ".blabla"- / # at least one "/" for a file path,- . # and something after the "/"- ''', url, flags=re.VERBOSE):- return any_to_uri(url)- # Windows drive-letter path- elif re.match(r'''^[a-z]:\\''', url, flags=re.IGNORECASE):+ if _is_path(url):return any_to_uri(url)- else:- return add_http_if_no_scheme(url)+ return add_http_if_no_scheme(url)def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex 09a6d6c70..6a5254d54 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -1,8 +1,14 @@import unittestfrom scrapy.spiders import Spider-from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,- add_http_if_no_scheme, guess_scheme, strip_url)+from scrapy.utils.url import (+ add_http_if_no_scheme,+ guess_scheme,+ _is_path,+ strip_url,+ url_is_from_any_domain,+ url_is_from_spider,+)__doctests__ = ['scrapy.utils.url']@@ -434,5 +440,28 @@ class StripUrl(unittest.TestCase):self.assertEqual(strip_url(i, origin_only=True), o)+class IsPathTestCase(unittest.TestCase):++ def test_path(self):+ for input_value, output_value in (+ # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell+ # Unix-like OS, Microsoft Windows / cmd.exe+ ("/home/user/docs/Letter.txt", True),+ ("./inthisdir", True),+ ("../../greatgrandparent", True),+ ("~/.rcinfo", True),+ (r"C:\user\docs\Letter.txt", True),+ ("/user/docs/Letter.txt", True),+ (r"C:\Letter.txt", True),+ (r"\\Server01\user\docs\Letter.txt", True),+ (r"\\?\UNC\Server01\user\docs\Letter.txt", True),+ (r"\\?\C:\user\docs\Letter.txt", True),+ (r"C:\user\docs\somefile.ext:alternate_stream_name", True),++ (r"https://example.com", False),+ ):+ self.assertEqual(_is_path(input_value), output_value, input_value)++if __name__ == "__main__":unittest.main()