Scrapy, a fast high-level web crawling & scraping framework for Python.
prioritize default headers over user agent
Raul Gallegos49ac7de23162
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 7f49aacdb..c845c59b9 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -459,9 +459,9 @@ Default::'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,- 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,- 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,- 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,+ 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,+ 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,+ 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 8f064f81e..e563e56aa 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -93,9 +93,9 @@ DOWNLOADER_MIDDLEWARES_BASE = {'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,- 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,- 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,- 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,+ 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,+ 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,+ 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,