Scrapy, a fast high-level web crawling & scraping framework for Python.
Update schemaless URI support
ajaymittur287fc80671a841
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 73e56c87d..15de8cdbd 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -7,7 +7,7 @@ import warningsfrom contextlib import suppressfrom io import BytesIOfrom time import time-from urllib.parse import urldefrag, urlparse+from urllib.parse import urldefrag, urlunparsefrom twisted.internet import defer, protocol, sslfrom twisted.internet.endpoints import TCP4ClientEndpoint@@ -255,7 +255,7 @@ class ScrapyProxyAgent(Agent):bindAddress=bindAddress,pool=pool,)- self._proxyURI = URI.fromBytes(urlparse(proxyURI)._replace(scheme=b'http').geturl())+ self._proxyURI = URI.fromBytes(proxyURI)def request(self, method, uri, headers=None, bodyProducer=None):"""@@ -297,7 +297,7 @@ class ScrapyAgent:bindaddress = request.meta.get('bindaddress') or self._bindAddressproxy = request.meta.get('proxy')if proxy:- _, _, proxyHost, proxyPort, proxyParams = _parse(proxy)+ proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)scheme = _parse(request.url)[0]proxyHost = to_unicode(proxyHost)omitConnectTunnel = b'noconnect' in proxyParams@@ -319,9 +319,13 @@ class ScrapyAgent:pool=self._pool,)else:+ proxyScheme = b'http' if not proxyScheme else proxyScheme+ proxyHost = to_bytes(proxyHost, encoding='ascii')+ proxyPort = to_bytes(str(proxyPort), encoding='ascii')+ proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))return self._ProxyAgent(reactor=reactor,- proxyURI=to_bytes(proxy, encoding='ascii'),+ proxyURI=to_bytes(proxyURI, encoding='ascii'),connectTimeout=timeout,bindAddress=bindaddress,pool=self._pool,diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.pyindex 355045d74..af49e78ce 100644--- a/scrapy/core/downloader/webclient.py+++ b/scrapy/core/downloader/webclient.py@@ -1,5 +1,6 @@from time import timefrom urllib.parse import urlparse, urlunparse, urldefrag+from re import matchfrom twisted.web.client import HTTPClientFactoryfrom twisted.web.http import HTTPClient@@ -32,6 +33,8 @@ def _parse(url):and is ascii-only."""url = url.strip()+ if not match(r'^\w+://', url):+ url = '//' + urlparsed = urlparse(url)return _parsed_url_args(parsed)