Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add 429 to RETRY_HTTP_CODES

Capi Etheriel0ee2284fcc23
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 9986827d8..2afa7b321 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -238,7 +238,7 @@ REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
-RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]
+RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
RETRY_PRIORITY_ADJUST = -1
ROBOTSTXT_OBEY = False