Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

prioritize default headers over user agent

Raul Gallegos49ac7de23162
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 7f49aacdb..c845c59b9 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -459,9 +459,9 @@ Default::
'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
- 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,
- 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
- 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,
+ 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
+ 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
+ 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8f064f81e..e563e56aa 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -93,9 +93,9 @@ DOWNLOADER_MIDDLEWARES_BASE = {
'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
- 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,
- 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
- 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,
+ 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
+ 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
+ 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,