Scrapy, a fast high-level web crawling & scraping framework for Python.
Added: HTTP status code 522/524 to retry.
Pengyu Chen11a1f970b7b7
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 87dbf6974..697314b7f 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -234,7 +234,7 @@ REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'RETRY_ENABLED = TrueRETRY_TIMES = 2 # initial response + 2 retries = 3 requests-RETRY_HTTP_CODES = [500, 502, 503, 504, 408]+RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]RETRY_PRIORITY_ADJUST = -1ROBOTSTXT_OBEY = False