Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Update schemaless URI support

ajaymittur287fc80671a841
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 73e56c87d..15de8cdbd 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -7,7 +7,7 @@ import warnings
from contextlib import suppress
from io import BytesIO
from time import time
-from urllib.parse import urldefrag, urlparse
+from urllib.parse import urldefrag, urlunparse
from twisted.internet import defer, protocol, ssl
from twisted.internet.endpoints import TCP4ClientEndpoint
@@ -255,7 +255,7 @@ class ScrapyProxyAgent(Agent):
bindAddress=bindAddress,
pool=pool,
)
- self._proxyURI = URI.fromBytes(urlparse(proxyURI)._replace(scheme=b'http').geturl())
+ self._proxyURI = URI.fromBytes(proxyURI)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
@@ -297,7 +297,7 @@ class ScrapyAgent:
bindaddress = request.meta.get('bindaddress') or self._bindAddress
proxy = request.meta.get('proxy')
if proxy:
- _, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
+ proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
@@ -319,9 +319,13 @@ class ScrapyAgent:
pool=self._pool,
)
else:
+ proxyScheme = b'http' if not proxyScheme else proxyScheme
+ proxyHost = to_bytes(proxyHost, encoding='ascii')
+ proxyPort = to_bytes(str(proxyPort), encoding='ascii')
+ proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
return self._ProxyAgent(
reactor=reactor,
- proxyURI=to_bytes(proxy, encoding='ascii'),
+ proxyURI=to_bytes(proxyURI, encoding='ascii'),
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 355045d74..af49e78ce 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,5 +1,6 @@
from time import time
from urllib.parse import urlparse, urlunparse, urldefrag
+from re import match
from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
@@ -32,6 +33,8 @@ def _parse(url):
and is ascii-only.
"""
url = url.strip()
+ if not match(r'^\w+://', url):
+ url = '//' + url
parsed = urlparse(url)
return _parsed_url_args(parsed)