Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Added: HTTP status code 522/524 to retry.

Pengyu Chen11a1f970b7b7
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 87dbf6974..697314b7f 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -234,7 +234,7 @@ REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
-RETRY_HTTP_CODES = [500, 502, 503, 504, 408]
+RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]
RETRY_PRIORITY_ADJUST = -1
ROBOTSTXT_OBEY = False