Scrapy, a fast high-level web crawling & scraping framework for Python.
Add test for Referer header on HTTP redirections
Paul Tremberth8226e77010d7
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex f2910a0c8..bb184cc12 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -4,6 +4,8 @@ originated it."""from six.moves.urllib.parse import urlparse+from w3lib.url import safe_url_string+from scrapy.http import Request, Responsefrom scrapy.exceptions import NotConfiguredfrom scrapy import signals@@ -300,8 +302,7 @@ class RefererMiddleware(object):if isinstance(resp_or_url, Response):policy_name = to_native_str(resp_or_url.headers.get('Referrer-Policy', '').decode('latin1'))-- cls = _policy_classes.get(policy_name.lower(), self.default_policy)+ cls = _policy_classes.get(policy_name.lower()) if policy_name else self.default_policyreturn cls()def process_spider_output(self, response, result, spider):@@ -320,9 +321,14 @@ class RefererMiddleware(object):request_referrer = request.headers.get('Referer')# we don't patch the referrer value if there is noneif request_referrer is not None:- initial_url = redirected_urls[0]- policy_referrer = self.policy(initial_url,- request).referrer(orig_url, request.url)+ # the request's referrer header value acts as a surrogate+ # for the parent response URL+ #+ # Note: if the 3xx response contained a Referrer-Policy header,+ # the information is not available using this hook+ parent_url = safe_url_string(request_referrer)+ policy_referrer = self.policy(parent_url, request).referrer(+ parent_url, request.url)if policy_referrer != request_referrer:if policy_referrer is None:request.headers.pop('Referer')diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 39bbaab5d..28c694169 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -5,6 +5,7 @@ from scrapy.exceptions import NotConfiguredfrom scrapy.http import Response, Requestfrom scrapy.settings import Settingsfrom scrapy.spiders import Spider+from scrapy.downloadermiddlewares.redirect import RedirectMiddlewarefrom scrapy.spidermiddlewares.referer import RefererMiddleware, \POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \@@ -13,6 +14,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \DefaultReferrerPolicy, \NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \OriginWhenCrossOriginPolicy, OriginPolicy, \+ StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy@@ -289,35 +291,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):# --- Tests using settings to set policy using class path-class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):+class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}-class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):+class TestSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}-class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):+class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}-class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware):+class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}-class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):+class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}-class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):+class TestSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}-class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):+class TestSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}-class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):+class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}@@ -334,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy):return b'http://python.org/'-class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware):+class TestSettingsCustomPolicy(TestRefererMiddleware):settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}scenarii = [('https://example.com/', 'https://scrapy.org/', b'https://python.org/'),@@ -347,58 +349,58 @@ class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware):]# --- Tests using Request meta dict to set policy-class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware):+class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}-class TestRefererMiddlewareNoReferrer(MixinNoReferrer, TestRefererMiddleware):+class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_NO_REFERRER}-class TestRefererMiddlewareNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):+class TestRequestMetaNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}-class TestRefererMiddlewareSameOrigin(MixinSameOrigin, TestRefererMiddleware):+class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_SAME_ORIGIN}-class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware):+class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_ORIGIN}-class TestRefererMiddlewareSrictOrigin(MixinStrictOrigin, TestRefererMiddleware):+class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN}-class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):+class TestRequestMetaOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}-class TestRefererMiddlewareStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):+class TestRequestMetaStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}-class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):+class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_UNSAFE_URL}-class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):+class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}req_meta = {'referrer_policy': POLICY_UNSAFE_URL}-class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):+class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}req_meta = {'referrer_policy': POLICY_NO_REFERRER}-class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):+class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}req_meta = {'referrer_policy': POLICY_UNSAFE_URL}-class TestRefererMiddlewareSettingsPolicyByName(TestCase):+class TestSettingsPolicyByName(TestCase):def test_valid_name(self):for s, p in [@@ -407,7 +409,9 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),(POLICY_SAME_ORIGIN, SameOriginPolicy),(POLICY_ORIGIN, OriginPolicy),+ (POLICY_STRICT_ORIGIN, StrictOriginPolicy),(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),+ (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),(POLICY_UNSAFE_URL, UnsafeUrlPolicy),]:settings = Settings({'REFERRER_POLICY': s})@@ -421,7 +425,9 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),(POLICY_SAME_ORIGIN, SameOriginPolicy),(POLICY_ORIGIN, OriginPolicy),+ (POLICY_STRICT_ORIGIN, StrictOriginPolicy),(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),+ (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),(POLICY_UNSAFE_URL, UnsafeUrlPolicy),]:settings = Settings({'REFERRER_POLICY': s.upper()})@@ -434,52 +440,354 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):mw = RefererMiddleware(settings)-class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):+class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}-class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):+class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}-class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):+class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}-class TestReferrerPolicyOnRedirect(TestRefererMiddleware):+class TestReferrerOnRedirect(TestRefererMiddleware):- req_meta = {}- resp_headers = {}- #settings = {}settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}scenarii = [- ( # origin- 'http://scrapytest.org/1',-- # target + redirection- ['http://scrapytest.org/2',- 'http://scrapytest.org/3',],-+ ( 'http://scrapytest.org/1', # parent+ 'http://scrapytest.org/2', # target+ (+ # redirections: code, URL+ (301, 'http://scrapytest.org/3'),+ (301, 'http://scrapytest.org/4'),+ ),b'http://scrapytest.org/1', # expected initial refererb'http://scrapytest.org/1', # expected referer for the redirection request-+ ),+ ( 'https://scrapytest.org/1',+ 'https://scrapytest.org/2',+ (+ # redirecting to non-secure URL+ (301, 'http://scrapytest.org/3'),+ ),+ b'https://scrapytest.org/1',+ b'https://scrapytest.org/1',+ ),+ ( 'https://scrapytest.org/1',+ 'https://scrapytest.com/2',+ (+ # redirecting to non-secure URL: different origin+ (301, 'http://scrapytest.com/3'),+ ),+ b'https://scrapytest.org/1',+ b'https://scrapytest.org/1',),]def setUp(self):self.spider = Spider('foo')settings = Settings(self.settings)- self.mw = RefererMiddleware(settings)-- def test_(self):+ self.referrermw = RefererMiddleware(settings)+ self.redirectmw = RedirectMiddleware(settings)- for origin, target_chain, init_referrer, final_referrer in self.scenarii:- response = self.get_response(origin)- request = self.get_request(target_chain.pop())+ def test(self):+ for parent, target, redirections, init_referrer, final_referrer in self.scenarii:+ response = self.get_response(parent)+ request = self.get_request(target)- out = list(self.mw.process_spider_output(response, [request], self.spider))+ out = list(self.referrermw.process_spider_output(response, [request], self.spider))self.assertEquals(out[0].headers.get('Referer'), init_referrer)- request.meta['redirected_urls'] = target_chain+ for status, url in redirections:+ response = Response(request.url, headers={'Location': url}, status=status)+ request = self.redirectmw.process_response(request, response, self.spider)+ self.referrermw.request_scheduled(request, self.spider)++ assert isinstance(request, Request)+ self.assertEquals(request.headers.get('Referer'), final_referrer)+++class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):+ """+ No Referrer policy never sets the "Referer" header.+ HTTP redirections should not change that.+ """+ settings = {'REFERRER_POLICY': 'no-referrer'}+ scenarii = [+ ( 'http://scrapytest.org/1', # parent+ 'http://scrapytest.org/2', # target+ (+ # redirections: code, URL+ (301, 'http://scrapytest.org/3'),+ (301, 'http://scrapytest.org/4'),+ ),+ None, # expected initial "Referer"+ None, # expected "Referer" for the redirection request+ ),+ ( 'https://scrapytest.org/1',+ 'https://scrapytest.org/2',+ (+ (301, 'http://scrapytest.org/3'),+ ),+ None,+ None,+ ),+ ( 'https://scrapytest.org/1',+ 'https://example.com/2', # different origin+ (+ (301, 'http://scrapytest.com/3'),+ ),+ None,+ None,+ ),+ ]+++class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):+ """+ Same Origin policy sends the full URL as "Referer" if the target origin+ is the same as the parent response (same protocol, same domain, same port).++ HTTP redirections to a different domain or a lower secure level+ should have the "Referer" removed.+ """+ settings = {'REFERRER_POLICY': 'same-origin'}+ scenarii = [+ ( 'http://scrapytest.org/101', # origin+ 'http://scrapytest.org/102', # target+ (+ # redirections: code, URL+ (301, 'http://scrapytest.org/103'),+ (301, 'http://scrapytest.org/104'),+ ),+ b'http://scrapytest.org/101', # expected initial "Referer"+ b'http://scrapytest.org/101', # expected referer for the redirection request+ ),+ ( 'https://scrapytest.org/201',+ 'https://scrapytest.org/202',+ (+ # redirecting from secure to non-secure URL == different origin+ (301, 'http://scrapytest.org/203'),+ ),+ b'https://scrapytest.org/201',+ None,+ ),+ ( 'https://scrapytest.org/301',+ 'https://scrapytest.org/302',+ (+ # different domain == different origin+ (301, 'http://example.com/303'),+ ),+ b'https://scrapytest.org/301',+ None,+ ),+ ]+++class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):+ """+ Strict Origin policy will always send the "origin" as referrer+ (think of it as the parent URL without the path part),+ unless the security level is lower and no "Referer" is sent.++ Redirections from secure to non-secure URLs should have the+ "Referrer" header removed if necessary.+ """+ settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN}+ scenarii = [+ ( 'http://scrapytest.org/101',+ 'http://scrapytest.org/102',+ (+ (301, 'http://scrapytest.org/103'),+ (301, 'http://scrapytest.org/104'),+ ),+ b'http://scrapytest.org/', # send origin+ b'http://scrapytest.org/', # redirects to same origin: send origin+ ),+ ( 'https://scrapytest.org/201',+ 'https://scrapytest.org/202',+ (+ # redirecting to non-secure URL: no referrer+ (301, 'http://scrapytest.org/203'),+ ),+ b'https://scrapytest.org/',+ None,+ ),+ ( 'https://scrapytest.org/301',+ 'https://scrapytest.org/302',+ (+ # redirecting to non-secure URL (different domain): no referrer+ (301, 'http://example.com/303'),+ ),+ b'https://scrapytest.org/',+ None,+ ),+ ( 'http://scrapy.org/401',+ 'http://example.com/402',+ (+ (301, 'http://scrapytest.org/403'),+ ),+ b'http://scrapy.org/',+ b'http://scrapy.org/',+ ),+ ( 'https://scrapy.org/501',+ 'https://example.com/502',+ (+ # HTTPS all along, so origin referrer is kept as-is+ (301, 'https://google.com/503'),+ (301, 'https://facebook.com/504'),+ ),+ b'https://scrapy.org/',+ b'https://scrapy.org/',+ ),+ ( 'https://scrapytest.org/601',+ 'http://scrapytest.org/602', # TLS to non-TLS: no referrer+ (+ (301, 'https://scrapytest.org/603'), # TLS URL again: (still) no referrer+ ),+ None,+ None,+ ),+ ]+++class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):+ """+ Origin When Cross-Origin policy sends the full URL as "Referer",+ unless the target's origin is different (different domain, different protocol)+ in which case only the origin is sent.++ Redirections to a different origin should strip the "Referer"+ to the parent origin.+ """+ settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}+ scenarii = [+ ( 'http://scrapytest.org/101', # origin+ 'http://scrapytest.org/102', # target + redirection+ (+ # redirections: code, URL+ (301, 'http://scrapytest.org/103'),+ (301, 'http://scrapytest.org/104'),+ ),+ b'http://scrapytest.org/101', # expected initial referer+ b'http://scrapytest.org/101', # expected referer for the redirection request+ ),+ ( 'https://scrapytest.org/201',+ 'https://scrapytest.org/202',+ (+ # redirecting to non-secure URL: send origin+ (301, 'http://scrapytest.org/203'),+ ),+ b'https://scrapytest.org/201',+ b'https://scrapytest.org/',+ ),+ ( 'https://scrapytest.org/301',+ 'https://scrapytest.org/302',+ (+ # redirecting to non-secure URL (different domain): send origin+ (301, 'http://example.com/303'),+ ),+ b'https://scrapytest.org/301',+ b'https://scrapytest.org/',+ ),+ ( 'http://scrapy.org/401',+ 'http://example.com/402',+ (+ (301, 'http://scrapytest.org/403'),+ ),+ b'http://scrapy.org/',+ b'http://scrapy.org/',+ ),+ ( 'https://scrapy.org/501',+ 'https://example.com/502',+ (+ # all different domains: send origin+ (301, 'https://google.com/503'),+ (301, 'https://facebook.com/504'),+ ),+ b'https://scrapy.org/',+ b'https://scrapy.org/',+ ),+ ( 'https://scrapytest.org/301',+ 'http://scrapytest.org/302', # TLS to non-TLS: send origin+ (+ (301, 'https://scrapytest.org/303'), # TLS URL again: send origin (also)+ ),+ b'https://scrapytest.org/',+ b'https://scrapytest.org/',+ ),+ ]+++class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):+ """+ Strict Origin When Cross-Origin policy sends the full URL as "Referer",+ unless the target's origin is different (different domain, different protocol)+ in which case only the origin is sent...+ Unless there's also a downgrade in security and then the "Referer" header+ is not sent.++ Redirections to a different origin should strip the "Referer" to the parent origin,+ and from https:// to http:// will remove the "Referer" header.+ """+ settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}+ scenarii = [+ ( 'http://scrapytest.org/101', # origin+ 'http://scrapytest.org/102', # target + redirection+ (+ # redirections: code, URL+ (301, 'http://scrapytest.org/103'),+ (301, 'http://scrapytest.org/104'),+ ),+ b'http://scrapytest.org/101', # expected initial referer+ b'http://scrapytest.org/101', # expected referer for the redirection request+ ),+ ( 'https://scrapytest.org/201',+ 'https://scrapytest.org/202',+ (+ # redirecting to non-secure URL: do not send the "Referer" header+ (301, 'http://scrapytest.org/203'),+ ),+ b'https://scrapytest.org/201',+ None,+ ),+ ( 'https://scrapytest.org/301',+ 'https://scrapytest.org/302',+ (+ # redirecting to non-secure URL (different domain): send origin+ (301, 'http://example.com/303'),+ ),+ b'https://scrapytest.org/301',+ None,+ ),+ ( 'http://scrapy.org/401',+ 'http://example.com/402',+ (+ (301, 'http://scrapytest.org/403'),+ ),+ b'http://scrapy.org/',+ b'http://scrapy.org/',+ ),+ ( 'https://scrapy.org/501',+ 'https://example.com/502',+ (+ # all different domains: send origin+ (301, 'https://google.com/503'),+ (301, 'https://facebook.com/504'),+ ),+ b'https://scrapy.org/',+ b'https://scrapy.org/',+ ),+ ( 'https://scrapytest.org/601',+ 'http://scrapytest.org/602', # TLS to non-TLS: do not send "Referer"+ (+ (301, 'https://scrapytest.org/603'), # TLS URL again: (still) send nothing+ ),+ None,+ None,+ ),+ ]