Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Refactor guess_scheme

Adrián Chavesd54c4496ee57
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 60e2b50eb..1e431d3bf 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -83,26 +83,54 @@ def add_http_if_no_scheme(url):
return url
+def _is_posix_path(string):
+ return bool(
+ re.match(
+ r'''
+ ^ # start with...
+ (
+ \. # ...a single dot,
+ (
+ \. | [^/\.]+ # optionally followed by
+ )? # either a second dot or some characters
+ |
+ ~ # $HOME
+ )? # optional match of ".", ".." or ".blabla"
+ / # at least one "/" for a file path,
+ . # and something after the "/"
+ ''',
+ string,
+ flags=re.VERBOSE,
+ )
+ )
+
+
+def _is_windows_path(string):
+ return bool(
+ re.match(
+ r'''
+ ^
+ (
+ [a-z]:\\
+ | \\\\
+ )
+ ''',
+ string,
+ flags=re.IGNORECASE | re.VERBOSE,
+ )
+ )
+
+
+def _is_path(string):
+ return _is_posix_path(string) or _is_windows_path(string)
+
+
def guess_scheme(url):
"""Add an URL scheme if missing: file:// for filepath-like input or
http:// otherwise."""
- # POSIX path
- if re.match(r'''^ # start with...
- (
- \. # ...a single dot,
- (
- \. | [^/\.]+ # optionally followed by
- )? # either a second dot or some characters
- )? # optional match of ".", ".." or ".blabla"
- / # at least one "/" for a file path,
- . # and something after the "/"
- ''', url, flags=re.VERBOSE):
- return any_to_uri(url)
- # Windows drive-letter path
- elif re.match(r'''^[a-z]:\\''', url, flags=re.IGNORECASE):
+ if _is_path(url):
return any_to_uri(url)
- else:
- return add_http_if_no_scheme(url)
+ return add_http_if_no_scheme(url)
def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 09a6d6c70..6a5254d54 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,8 +1,14 @@
import unittest
from scrapy.spiders import Spider
-from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
- add_http_if_no_scheme, guess_scheme, strip_url)
+from scrapy.utils.url import (
+ add_http_if_no_scheme,
+ guess_scheme,
+ _is_path,
+ strip_url,
+ url_is_from_any_domain,
+ url_is_from_spider,
+)
__doctests__ = ['scrapy.utils.url']
@@ -434,5 +440,28 @@ class StripUrl(unittest.TestCase):
self.assertEqual(strip_url(i, origin_only=True), o)
+class IsPathTestCase(unittest.TestCase):
+
+ def test_path(self):
+ for input_value, output_value in (
+ # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
+ # Unix-like OS, Microsoft Windows / cmd.exe
+ ("/home/user/docs/Letter.txt", True),
+ ("./inthisdir", True),
+ ("../../greatgrandparent", True),
+ ("~/.rcinfo", True),
+ (r"C:\user\docs\Letter.txt", True),
+ ("/user/docs/Letter.txt", True),
+ (r"C:\Letter.txt", True),
+ (r"\\Server01\user\docs\Letter.txt", True),
+ (r"\\?\UNC\Server01\user\docs\Letter.txt", True),
+ (r"\\?\C:\user\docs\Letter.txt", True),
+ (r"C:\user\docs\somefile.ext:alternate_stream_name", True),
+
+ (r"https://example.com", False),
+ ):
+ self.assertEqual(_is_path(input_value), output_value, input_value)
+
+
if __name__ == "__main__":
unittest.main()