Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove support for using HttpAuthMiddleware without http_auth_domain

Laerte Pereiraa2b9351f04f0
diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py
index de5a81388..5228db786 100644
--- a/scrapy/downloadermiddlewares/httpauth.py
+++ b/scrapy/downloadermiddlewares/httpauth.py
@@ -3,13 +3,10 @@ HTTP basic auth downloader middleware
See documentation in docs/topics/downloader-middleware.rst
"""
-import warnings
from w3lib.http import basic_auth_header
from scrapy import signals
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.url import url_is_from_any_domain
@@ -28,25 +25,10 @@ class HttpAuthMiddleware:
pwd = getattr(spider, "http_pass", "")
if usr or pwd:
self.auth = basic_auth_header(usr, pwd)
- if not hasattr(spider, "http_auth_domain"):
- warnings.warn(
- "Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security "
- "problems if the spider makes requests to several different domains. http_auth_domain "
- "will be set to the domain of the first request, please set it to the correct value "
- "explicitly.",
- category=ScrapyDeprecationWarning,
- )
- self.domain_unset = True
- else:
- self.domain = spider.http_auth_domain
- self.domain_unset = False
+ self.domain = spider.http_auth_domain
def process_request(self, request, spider):
auth = getattr(self, "auth", None)
if auth and b"Authorization" not in request.headers:
- domain = urlparse_cached(request).hostname
- if self.domain_unset:
- self.domain = domain
- self.domain_unset = False
if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
request.headers[b"Authorization"] = auth
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index 6b79234d0..fc110e6cc 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -1,10 +1,8 @@
import unittest
-import pytest
from w3lib.http import basic_auth_header
from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.spiders import Spider
@@ -31,39 +29,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
self.spider = TestSpiderLegacy("foo")
def test_auth(self):
- with pytest.warns(
- ScrapyDeprecationWarning,
- match="Using HttpAuthMiddleware without http_auth_domain is deprecated",
- ):
+ with self.assertRaises(AttributeError):
mw = HttpAuthMiddleware()
mw.spider_opened(self.spider)
- # initial request, sets the domain and sends the header
- req = Request("http://example.com/")
- assert mw.process_request(req, self.spider) is None
- self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
-
- # subsequent request to the same domain, should send the header
- req = Request("http://example.com/")
- assert mw.process_request(req, self.spider) is None
- self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
-
- # subsequent request to a different domain, shouldn't send the header
- req = Request("http://example-noauth.com/")
- assert mw.process_request(req, self.spider) is None
- self.assertNotIn("Authorization", req.headers)
-
- def test_auth_already_set(self):
- with pytest.warns(
- ScrapyDeprecationWarning,
- match="Using HttpAuthMiddleware without http_auth_domain is deprecated",
- ):
- mw = HttpAuthMiddleware()
- mw.spider_opened(self.spider)
- req = Request("http://example.com/", headers=dict(Authorization="Digest 123"))
- assert mw.process_request(req, self.spider) is None
- self.assertEqual(req.headers["Authorization"], b"Digest 123")
-
class HttpAuthMiddlewareTest(unittest.TestCase):
def setUp(self):