Skip to content

A platform for community discussion. Free, open, simple.

FIX: Honour Retry-After from upstreams instead of giving up

Régis Hanola67ab1b2d943
diff --git a/app/jobs/regular/pull_hotlinked_images.rb b/app/jobs/regular/pull_hotlinked_images.rb
index 15c2cd8eb61..11167c3cbb9 100644
--- a/app/jobs/regular/pull_hotlinked_images.rb
+++ b/app/jobs/regular/pull_hotlinked_images.rb
@@ -4,12 +4,16 @@ module Jobs
class PullHotlinkedImages < ::Jobs::Base
sidekiq_options queue: "low"
+ MAX_RATE_LIMIT_RETRIES = 3
+
def execute(args)
disable_if_low_on_disk_space
@post_id = args[:post_id]
raise Discourse::InvalidParameters.new(:post_id) if @post_id.blank?
+ @rate_limit_retries = args[:rate_limit_retries].to_i
+
# in test we have no choice cause we don't want to cause a deadlock
if Jobs.run_immediately?
pull_hotlinked_images
@@ -27,6 +31,7 @@ module Jobs
hotlinked_map = post.post_hotlinked_media.index_by { |r| r.url }
changed_hotlink_records = false
+ retry_after = nil
HotlinkedMedia
.extract_candidates(post.cooked)
@@ -55,6 +60,9 @@ module Jobs
changed_hotlink_records = true
hotlink_record.save!
end
+ rescue HotlinkedMediaDownloader::RateLimitedError => e
+ retry_after = [retry_after.to_i, e.retry_after].max
+ log(:warn, "Rate limited (#{download_src}) post: #{@post_id}, retry in #{e.retry_after}s")
rescue => e
raise e if Rails.env.test?
log(
@@ -76,6 +84,15 @@ module Jobs
update_raw_delay = SiteSetting.editing_grace_period + 1
Jobs.enqueue_in(update_raw_delay, :update_hotlinked_raw, post_id: post.id)
end
+
+ return if retry_after.nil? || @rate_limit_retries >= MAX_RATE_LIMIT_RETRIES
+
+ Jobs.enqueue_in(
+ retry_after,
+ :pull_hotlinked_images,
+ post_id: @post_id,
+ rate_limit_retries: @rate_limit_retries + 1,
+ )
end
# Error classes live on HotlinkedMediaDownloader now; these aliases keep the
diff --git a/lib/file_helper.rb b/lib/file_helper.rb
index d94b84cda30..2384d8f1e37 100644
--- a/lib/file_helper.rb
+++ b/lib/file_helper.rb
@@ -56,7 +56,7 @@ class FileHelper
end
class FakeIO
- attr_accessor :status
+ attr_accessor :status, :meta
end
def self.download(
@@ -97,6 +97,7 @@ class FileHelper
# attempt error API compatibility
io = FakeIO.new
io.status = [response.code, ""]
+ io.meta = response.each_header.to_h
raise OpenURI::HTTPError.new("#{response.code} Error", io)
else
log(:error, "FinalDestination did not work for: #{url}") if verbose
diff --git a/lib/hotlinked_media_downloader.rb b/lib/hotlinked_media_downloader.rb
index 268c41301c9..1ca8b7de144 100644
--- a/lib/hotlinked_media_downloader.rb
+++ b/lib/hotlinked_media_downloader.rb
@@ -5,6 +5,10 @@
# chat-side Jobs::Chat::PullHotlinkedImages so the download/retry/secure-upload
# handling lives in one place.
class HotlinkedMediaDownloader
+ TOO_MANY_REQUESTS = 429
+ MIN_RETRY_AFTER_SECONDS = 1.minute.to_i
+ MAX_RETRY_AFTER_SECONDS = 1.hour.to_i
+
class ImageTooLargeError < StandardError
end
@@ -14,6 +18,15 @@ class HotlinkedMediaDownloader
class UploadCreateError < StandardError
end
+ class RateLimitedError < StandardError
+ attr_reader :retry_after
+
+ def initialize(retry_after)
+ @retry_after = retry_after.to_i.clamp(MIN_RETRY_AFTER_SECONDS, MAX_RETRY_AFTER_SECONDS)
+ super("Rate limited, retry in #{@retry_after}s")
+ end
+ end
+
# Downloads +src+ and creates an Upload owned by +user_id+.
# Returns the persisted Upload or raises one of the typed errors above.
def self.download(src, user_id, tmp_file_name:)
@@ -72,6 +85,10 @@ class HotlinkedMediaDownloader
Rails.logger.warn("Verbose Upload Logging: Error '#{e.message}' while downloading #{src}")
end
+ if e.is_a?(OpenURI::HTTPError) && e.io&.status&.first.to_i == TOO_MANY_REQUESTS
+ raise RateLimitedError.new(e.io.meta.to_h["retry-after"])
+ end
+
if (retries -= 1) > 0 && !Rails.env.test?
sleep 1
retry
diff --git a/plugins/discourse-ai/lib/embeddings/provider_health.rb b/plugins/discourse-ai/lib/embeddings/provider_health.rb
index 19a3856108f..89b965d35fe 100644
--- a/plugins/discourse-ai/lib/embeddings/provider_health.rb
+++ b/plugins/discourse-ai/lib/embeddings/provider_health.rb
@@ -11,6 +11,7 @@ module DiscourseAi
class ProviderHealth
PAUSE_BACKOFFS = [30.seconds, 3.minutes, 20.minutes, 1.hour, 6.hours].freeze
BACKOFF_STATE_TTL = 1.day
+ MIN_RATE_LIMIT_PAUSE = 30.seconds
class << self
def paused?(definition)
@@ -25,7 +26,7 @@ module DiscourseAi
Discourse.redis.del(backoff_key(definition.id)) if state&.last
result
rescue DiscourseAi::Inference::EmbeddingInferenceError => error
- pause!(definition, error) if error.terminal?
+ note_failure!(definition, error)
raise
end
@@ -42,32 +43,42 @@ module DiscourseAi
Discourse.redis.mget(pause_key(definition.id), backoff_key(definition.id))
end
- def pause!(definition, error)
+ def note_failure!(definition, error)
return if !definition&.persisted?
+ return pause!(definition, error) if error.terminal?
+ return if !error.rate_limited?
+
+ hold!(definition, error, error.retry_after.to_i.clamp(MIN_RATE_LIMIT_PAUSE.to_i..))
+ end
+ def pause!(definition, error)
level = [
Discourse.redis.get(backoff_key(definition.id)).to_i,
PAUSE_BACKOFFS.length - 1,
].min
- ttl = PAUSE_BACKOFFS[level]
+ hold!(definition, error, PAUSE_BACKOFFS[level].to_i, mode: :overwrite)
+
+ Discourse.redis.set(
+ backoff_key(definition.id),
+ [level + 1, PAUSE_BACKOFFS.length - 1].min,
+ ex: BACKOFF_STATE_TTL.to_i,
+ )
+ end
+
+ def hold!(definition, error, ttl, mode: :if_absent)
paused =
Discourse.redis.set(
pause_key(definition.id),
error.category.to_s,
- nx: true,
- ex: ttl.to_i,
+ nx: mode == :if_absent,
+ ex: ttl,
)
return if !paused
- Discourse.redis.set(
- backoff_key(definition.id),
- [level + 1, PAUSE_BACKOFFS.length - 1].min,
- ex: BACKOFF_STATE_TTL.to_i,
- )
Rails.logger.warn(
"Discourse AI embedding provider paused definition_id=#{definition.id} " \
"provider=#{definition.provider} category=#{error.category} " \
- "duration=#{ttl.to_i}s status=#{error.http_status.inspect} " \
+ "duration=#{ttl}s status=#{error.http_status.inspect} " \
"code=#{error.provider_error_code.inspect}",
)
end
diff --git a/plugins/discourse-ai/lib/inference/embedding_inference_error.rb b/plugins/discourse-ai/lib/inference/embedding_inference_error.rb
index 567b8565182..59b0e5a9c3a 100644
--- a/plugins/discourse-ai/lib/inference/embedding_inference_error.rb
+++ b/plugins/discourse-ai/lib/inference/embedding_inference_error.rb
@@ -65,6 +65,10 @@ module DiscourseAi
RETRYABLE_CATEGORIES.include?(category)
end
+ def rate_limited?
+ category == :rate_limited
+ end
+
class << self
private
diff --git a/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rb b/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rb
index c852756f2d8..13b910a90aa 100644
--- a/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rb
@@ -5,12 +5,13 @@ RSpec.describe DiscourseAi::Embeddings::ProviderHealth do
after { described_class.clear!(definition) }
- def error(category)
+ def error(category, retry_after: nil)
DiscourseAi::Inference::EmbeddingInferenceError.new(
provider: definition.provider,
category: category,
http_status: 429,
provider_error_code: category,
+ retry_after: retry_after,
)
end
@@ -68,12 +69,24 @@ RSpec.describe DiscourseAi::Embeddings::ProviderHealth do
expect(pause_ttl).to be_between(29, 30)
end
- it "does not pause temporary rate limits" do
+ it "pauses rate limits for the advertised Retry-After" do
+ expect {
+ described_class.request!(definition) { raise error(:rate_limited, retry_after: 90) }
+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)
+
+ expect(described_class).to be_paused(definition)
+ expect(pause_ttl).to be_between(89, 90)
+ end
+
+ it "pauses rate limits for the minimum delay when there is no Retry-After" do
expect { described_class.request!(definition) { raise error(:rate_limited) } }.to raise_error(
DiscourseAi::Inference::EmbeddingInferenceError,
)
- expect(described_class).not_to be_paused(definition)
+ expect(pause_ttl).to be_between(
+ described_class::MIN_RATE_LIMIT_PAUSE.to_i - 1,
+ described_class::MIN_RATE_LIMIT_PAUSE.to_i,
+ )
end
it "does not share health state with unsaved definitions" do
diff --git a/spec/jobs/pull_hotlinked_images_spec.rb b/spec/jobs/pull_hotlinked_images_spec.rb
index e98e2863154..9897a203597 100644
--- a/spec/jobs/pull_hotlinked_images_spec.rb
+++ b/spec/jobs/pull_hotlinked_images_spec.rb
@@ -4,6 +4,7 @@ RSpec.describe Jobs::PullHotlinkedImages do
let(:image_url) { "http://wiki.mozilla.org/images/2/2e/Longcat1.gif" }
let(:broken_image_url) { "http://wiki.mozilla.org/images/2/2e/Longcat2.png" }
let(:large_image_url) { "http://wiki.mozilla.org/images/2/2e/Longcat3.png" }
+ let(:rate_limited_image_url) { "http://wiki.mozilla.org/images/2/2e/Longcat4.png" }
let(:encoded_image_url) { "https://example.com/אלחוט-.jpg" }
let(:gif) do
Base64.decode64(
@@ -30,6 +31,12 @@ RSpec.describe Jobs::PullHotlinkedImages do
},
)
stub_request(:get, broken_image_url).to_return(status: 404)
+ stub_request(:get, rate_limited_image_url).to_return(
+ status: 429,
+ headers: {
+ "Retry-After" => "120",
+ },
+ )
stub_request(:get, large_image_url).to_return(
body: large_png,
headers: {
@@ -103,6 +110,53 @@ RSpec.describe Jobs::PullHotlinkedImages do
) { Jobs::PullHotlinkedImages.new.execute(post_id: post.id) }
end
+ it "retries at Retry-After instead of recording a failure when rate limited" do
+ Jobs.run_later!
+
+ post = Fabricate(:post, user: user, raw: "<img src='#{rate_limited_image_url}'>")
+
+ freeze_time
+ expect_enqueued_with(
+ job: :pull_hotlinked_images,
+ args: {
+ post_id: post.id,
+ rate_limit_retries: 1,
+ },
+ at: 120.seconds.from_now,
+ ) { Jobs::PullHotlinkedImages.new.execute(post_id: post.id) }
+
+ expect(post.reload.post_hotlinked_media).to be_empty
+ end
+
+ it "still pulls the remaining images when one is rate limited" do
+ Jobs.run_later!
+
+ post =
+ Fabricate(
+ :post,
+ user: user,
+ raw: "<img src='#{rate_limited_image_url}'>\n<img src='#{encoded_image_url}'>",
+ )
+ stub_image_size
+
+ Jobs::PullHotlinkedImages.new.execute(post_id: post.id)
+
+ expect(post.reload.post_hotlinked_media.map(&:status)).to contain_exactly("downloaded")
+ end
+
+ it "stops retrying rate limited images once the retry limit is reached" do
+ Jobs.run_later!
+
+ post = Fabricate(:post, user: user, raw: "<img src='#{rate_limited_image_url}'>")
+
+ expect_not_enqueued_with(job: :pull_hotlinked_images) do
+ Jobs::PullHotlinkedImages.new.execute(
+ post_id: post.id,
+ rate_limit_retries: described_class::MAX_RATE_LIMIT_RETRIES,
+ )
+ end
+ end
+
it "removes downloaded images when they are no longer needed" do
post = Fabricate(:post, user: user, raw: "<img src='#{image_url}'>")
stub_image_size
diff --git a/spec/lib/file_helper_spec.rb b/spec/lib/file_helper_spec.rb
index 52cd446535c..70b096a3b9d 100644
--- a/spec/lib/file_helper_spec.rb
+++ b/spec/lib/file_helper_spec.rb
@@ -29,6 +29,18 @@ RSpec.describe FileHelper do
end.to raise_error(OpenURI::HTTPError, "404 Error")
end
+ it "exposes the response headers on the raised error" do
+ url = "http://toomany.com/429"
+ stub_request(:get, url).to_return(status: 429, headers: { "Retry-After" => "120" })
+
+ expect do
+ FileHelper.download(url, max_file_size: 10_000, tmp_file_name: "trouttmp")
+ rescue => e
+ expect(e.io.meta["retry-after"]).to eq("120")
+ raise
+ end.to raise_error(OpenURI::HTTPError, "429 Error")
+ end
+
it "does not follow redirects if instructed not to" do
url2 = "https://test.com/image.png"
stub_request(:get, url).to_return(status: 302, body: "", headers: { location: url2 })
diff --git a/spec/lib/hotlinked_media_downloader_spec.rb b/spec/lib/hotlinked_media_downloader_spec.rb
index 86de3ba838d..3cc7a87fbd4 100644
--- a/spec/lib/hotlinked_media_downloader_spec.rb
+++ b/spec/lib/hotlinked_media_downloader_spec.rb
@@ -25,6 +25,28 @@ RSpec.describe HotlinkedMediaDownloader do
}.to raise_error(described_class::ImageBrokenError)
end
+ it "raises RateLimitedError honouring Retry-After when the host rate limits" do
+ stub_request(:get, image_url).to_return(status: 429, headers: { "Retry-After" => "120" })
+
+ expect do
+ described_class.download(image_url, user.id, tmp_file_name: "test-hotlinked")
+ rescue => error
+ expect(error.retry_after).to eq(120)
+ raise
+ end.to raise_error(described_class::RateLimitedError)
+ end
+
+ it "raises RateLimitedError with the minimum delay when there is no Retry-After" do
+ stub_request(:get, image_url).to_return(status: 429)
+
+ expect do
+ described_class.download(image_url, user.id, tmp_file_name: "test-hotlinked")
+ rescue => error
+ expect(error.retry_after).to eq(described_class::MIN_RETRY_AFTER_SECONDS)
+ raise
+ end.to raise_error(described_class::RateLimitedError)
+ end
+
it "raises ImageTooLargeError when the file exceeds the limit" do
huge = "a" * (SiteSetting.max_image_size_kb * 1024 * 2)
stub_request(:get, image_url).to_return(body: huge, headers: { "Content-Type" => "image/png" })