A platform for community discussion. Free, open, simple.
FIX: Honour Retry-After from upstreams instead of giving up
Régis Hanola67ab1b2d943
diff --git a/app/jobs/regular/pull_hotlinked_images.rb b/app/jobs/regular/pull_hotlinked_images.rbindex 15c2cd8eb61..11167c3cbb9 100644--- a/app/jobs/regular/pull_hotlinked_images.rb+++ b/app/jobs/regular/pull_hotlinked_images.rb@@ -4,12 +4,16 @@ module Jobsclass PullHotlinkedImages < ::Jobs::Basesidekiq_options queue: "low"+ MAX_RATE_LIMIT_RETRIES = 3+def execute(args)disable_if_low_on_disk_space@post_id = args[:post_id]raise Discourse::InvalidParameters.new(:post_id) if @post_id.blank?+ @rate_limit_retries = args[:rate_limit_retries].to_i+# in test we have no choice cause we don't want to cause a deadlockif Jobs.run_immediately?pull_hotlinked_images@@ -27,6 +31,7 @@ module Jobshotlinked_map = post.post_hotlinked_media.index_by { |r| r.url }changed_hotlink_records = false+ retry_after = nilHotlinkedMedia.extract_candidates(post.cooked)@@ -55,6 +60,9 @@ module Jobschanged_hotlink_records = truehotlink_record.save!end+ rescue HotlinkedMediaDownloader::RateLimitedError => e+ retry_after = [retry_after.to_i, e.retry_after].max+ log(:warn, "Rate limited (#{download_src}) post: #{@post_id}, retry in #{e.retry_after}s")rescue => eraise e if Rails.env.test?log(@@ -76,6 +84,15 @@ module Jobsupdate_raw_delay = SiteSetting.editing_grace_period + 1Jobs.enqueue_in(update_raw_delay, :update_hotlinked_raw, post_id: post.id)end++ return if retry_after.nil? || @rate_limit_retries >= MAX_RATE_LIMIT_RETRIES++ Jobs.enqueue_in(+ retry_after,+ :pull_hotlinked_images,+ post_id: @post_id,+ rate_limit_retries: @rate_limit_retries + 1,+ )end# Error classes live on HotlinkedMediaDownloader now; these aliases keep thediff --git a/lib/file_helper.rb b/lib/file_helper.rbindex d94b84cda30..2384d8f1e37 100644--- a/lib/file_helper.rb+++ b/lib/file_helper.rb@@ -56,7 +56,7 @@ class FileHelperendclass FakeIO- attr_accessor :status+ attr_accessor :status, :metaenddef self.download(@@ -97,6 +97,7 @@ class FileHelper# attempt error API compatibilityio = FakeIO.newio.status = [response.code, ""]+ io.meta = response.each_header.to_hraise OpenURI::HTTPError.new("#{response.code} Error", io)elselog(:error, "FinalDestination did not work for: #{url}") if verbosediff --git a/lib/hotlinked_media_downloader.rb b/lib/hotlinked_media_downloader.rbindex 268c41301c9..1ca8b7de144 100644--- a/lib/hotlinked_media_downloader.rb+++ b/lib/hotlinked_media_downloader.rb@@ -5,6 +5,10 @@# chat-side Jobs::Chat::PullHotlinkedImages so the download/retry/secure-upload# handling lives in one place.class HotlinkedMediaDownloader+ TOO_MANY_REQUESTS = 429+ MIN_RETRY_AFTER_SECONDS = 1.minute.to_i+ MAX_RETRY_AFTER_SECONDS = 1.hour.to_i+class ImageTooLargeError < StandardErrorend@@ -14,6 +18,15 @@ class HotlinkedMediaDownloaderclass UploadCreateError < StandardErrorend+ class RateLimitedError < StandardError+ attr_reader :retry_after++ def initialize(retry_after)+ @retry_after = retry_after.to_i.clamp(MIN_RETRY_AFTER_SECONDS, MAX_RETRY_AFTER_SECONDS)+ super("Rate limited, retry in #{@retry_after}s")+ end+ end+# Downloads +src+ and creates an Upload owned by +user_id+.# Returns the persisted Upload or raises one of the typed errors above.def self.download(src, user_id, tmp_file_name:)@@ -72,6 +85,10 @@ class HotlinkedMediaDownloaderRails.logger.warn("Verbose Upload Logging: Error '#{e.message}' while downloading #{src}")end+ if e.is_a?(OpenURI::HTTPError) && e.io&.status&.first.to_i == TOO_MANY_REQUESTS+ raise RateLimitedError.new(e.io.meta.to_h["retry-after"])+ end+if (retries -= 1) > 0 && !Rails.env.test?sleep 1retrydiff --git a/plugins/discourse-ai/lib/embeddings/provider_health.rb b/plugins/discourse-ai/lib/embeddings/provider_health.rbindex 19a3856108f..89b965d35fe 100644--- a/plugins/discourse-ai/lib/embeddings/provider_health.rb+++ b/plugins/discourse-ai/lib/embeddings/provider_health.rb@@ -11,6 +11,7 @@ module DiscourseAiclass ProviderHealthPAUSE_BACKOFFS = [30.seconds, 3.minutes, 20.minutes, 1.hour, 6.hours].freezeBACKOFF_STATE_TTL = 1.day+ MIN_RATE_LIMIT_PAUSE = 30.secondsclass << selfdef paused?(definition)@@ -25,7 +26,7 @@ module DiscourseAiDiscourse.redis.del(backoff_key(definition.id)) if state&.lastresultrescue DiscourseAi::Inference::EmbeddingInferenceError => error- pause!(definition, error) if error.terminal?+ note_failure!(definition, error)raiseend@@ -42,32 +43,42 @@ module DiscourseAiDiscourse.redis.mget(pause_key(definition.id), backoff_key(definition.id))end- def pause!(definition, error)+ def note_failure!(definition, error)return if !definition&.persisted?+ return pause!(definition, error) if error.terminal?+ return if !error.rate_limited?++ hold!(definition, error, error.retry_after.to_i.clamp(MIN_RATE_LIMIT_PAUSE.to_i..))+ end+ def pause!(definition, error)level = [Discourse.redis.get(backoff_key(definition.id)).to_i,PAUSE_BACKOFFS.length - 1,].min- ttl = PAUSE_BACKOFFS[level]+ hold!(definition, error, PAUSE_BACKOFFS[level].to_i, mode: :overwrite)++ Discourse.redis.set(+ backoff_key(definition.id),+ [level + 1, PAUSE_BACKOFFS.length - 1].min,+ ex: BACKOFF_STATE_TTL.to_i,+ )+ end++ def hold!(definition, error, ttl, mode: :if_absent)paused =Discourse.redis.set(pause_key(definition.id),error.category.to_s,- nx: true,- ex: ttl.to_i,+ nx: mode == :if_absent,+ ex: ttl,)return if !paused- Discourse.redis.set(- backoff_key(definition.id),- [level + 1, PAUSE_BACKOFFS.length - 1].min,- ex: BACKOFF_STATE_TTL.to_i,- )Rails.logger.warn("Discourse AI embedding provider paused definition_id=#{definition.id} " \"provider=#{definition.provider} category=#{error.category} " \- "duration=#{ttl.to_i}s status=#{error.http_status.inspect} " \+ "duration=#{ttl}s status=#{error.http_status.inspect} " \"code=#{error.provider_error_code.inspect}",)enddiff --git a/plugins/discourse-ai/lib/inference/embedding_inference_error.rb b/plugins/discourse-ai/lib/inference/embedding_inference_error.rbindex 567b8565182..59b0e5a9c3a 100644--- a/plugins/discourse-ai/lib/inference/embedding_inference_error.rb+++ b/plugins/discourse-ai/lib/inference/embedding_inference_error.rb@@ -65,6 +65,10 @@ module DiscourseAiRETRYABLE_CATEGORIES.include?(category)end+ def rate_limited?+ category == :rate_limited+ end+class << selfprivatediff --git a/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rb b/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rbindex c852756f2d8..13b910a90aa 100644--- a/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rb@@ -5,12 +5,13 @@ RSpec.describe DiscourseAi::Embeddings::ProviderHealth doafter { described_class.clear!(definition) }- def error(category)+ def error(category, retry_after: nil)DiscourseAi::Inference::EmbeddingInferenceError.new(provider: definition.provider,category: category,http_status: 429,provider_error_code: category,+ retry_after: retry_after,)end@@ -68,12 +69,24 @@ RSpec.describe DiscourseAi::Embeddings::ProviderHealth doexpect(pause_ttl).to be_between(29, 30)end- it "does not pause temporary rate limits" do+ it "pauses rate limits for the advertised Retry-After" do+ expect {+ described_class.request!(definition) { raise error(:rate_limited, retry_after: 90) }+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)++ expect(described_class).to be_paused(definition)+ expect(pause_ttl).to be_between(89, 90)+ end++ it "pauses rate limits for the minimum delay when there is no Retry-After" doexpect { described_class.request!(definition) { raise error(:rate_limited) } }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError,)- expect(described_class).not_to be_paused(definition)+ expect(pause_ttl).to be_between(+ described_class::MIN_RATE_LIMIT_PAUSE.to_i - 1,+ described_class::MIN_RATE_LIMIT_PAUSE.to_i,+ )endit "does not share health state with unsaved definitions" dodiff --git a/spec/jobs/pull_hotlinked_images_spec.rb b/spec/jobs/pull_hotlinked_images_spec.rbindex e98e2863154..9897a203597 100644--- a/spec/jobs/pull_hotlinked_images_spec.rb+++ b/spec/jobs/pull_hotlinked_images_spec.rb@@ -4,6 +4,7 @@ RSpec.describe Jobs::PullHotlinkedImages dolet(:image_url) { "http://wiki.mozilla.org/images/2/2e/Longcat1.gif" }let(:broken_image_url) { "http://wiki.mozilla.org/images/2/2e/Longcat2.png" }let(:large_image_url) { "http://wiki.mozilla.org/images/2/2e/Longcat3.png" }+ let(:rate_limited_image_url) { "http://wiki.mozilla.org/images/2/2e/Longcat4.png" }let(:encoded_image_url) { "https://example.com/אלחוט-.jpg" }let(:gif) doBase64.decode64(@@ -30,6 +31,12 @@ RSpec.describe Jobs::PullHotlinkedImages do},)stub_request(:get, broken_image_url).to_return(status: 404)+ stub_request(:get, rate_limited_image_url).to_return(+ status: 429,+ headers: {+ "Retry-After" => "120",+ },+ )stub_request(:get, large_image_url).to_return(body: large_png,headers: {@@ -103,6 +110,53 @@ RSpec.describe Jobs::PullHotlinkedImages do) { Jobs::PullHotlinkedImages.new.execute(post_id: post.id) }end+ it "retries at Retry-After instead of recording a failure when rate limited" do+ Jobs.run_later!++ post = Fabricate(:post, user: user, raw: "<img src='#{rate_limited_image_url}'>")++ freeze_time+ expect_enqueued_with(+ job: :pull_hotlinked_images,+ args: {+ post_id: post.id,+ rate_limit_retries: 1,+ },+ at: 120.seconds.from_now,+ ) { Jobs::PullHotlinkedImages.new.execute(post_id: post.id) }++ expect(post.reload.post_hotlinked_media).to be_empty+ end++ it "still pulls the remaining images when one is rate limited" do+ Jobs.run_later!++ post =+ Fabricate(+ :post,+ user: user,+ raw: "<img src='#{rate_limited_image_url}'>\n<img src='#{encoded_image_url}'>",+ )+ stub_image_size++ Jobs::PullHotlinkedImages.new.execute(post_id: post.id)++ expect(post.reload.post_hotlinked_media.map(&:status)).to contain_exactly("downloaded")+ end++ it "stops retrying rate limited images once the retry limit is reached" do+ Jobs.run_later!++ post = Fabricate(:post, user: user, raw: "<img src='#{rate_limited_image_url}'>")++ expect_not_enqueued_with(job: :pull_hotlinked_images) do+ Jobs::PullHotlinkedImages.new.execute(+ post_id: post.id,+ rate_limit_retries: described_class::MAX_RATE_LIMIT_RETRIES,+ )+ end+ end+it "removes downloaded images when they are no longer needed" dopost = Fabricate(:post, user: user, raw: "<img src='#{image_url}'>")stub_image_sizediff --git a/spec/lib/file_helper_spec.rb b/spec/lib/file_helper_spec.rbindex 52cd446535c..70b096a3b9d 100644--- a/spec/lib/file_helper_spec.rb+++ b/spec/lib/file_helper_spec.rb@@ -29,6 +29,18 @@ RSpec.describe FileHelper doend.to raise_error(OpenURI::HTTPError, "404 Error")end+ it "exposes the response headers on the raised error" do+ url = "http://toomany.com/429"+ stub_request(:get, url).to_return(status: 429, headers: { "Retry-After" => "120" })++ expect do+ FileHelper.download(url, max_file_size: 10_000, tmp_file_name: "trouttmp")+ rescue => e+ expect(e.io.meta["retry-after"]).to eq("120")+ raise+ end.to raise_error(OpenURI::HTTPError, "429 Error")+ end+it "does not follow redirects if instructed not to" dourl2 = "https://test.com/image.png"stub_request(:get, url).to_return(status: 302, body: "", headers: { location: url2 })diff --git a/spec/lib/hotlinked_media_downloader_spec.rb b/spec/lib/hotlinked_media_downloader_spec.rbindex 86de3ba838d..3cc7a87fbd4 100644--- a/spec/lib/hotlinked_media_downloader_spec.rb+++ b/spec/lib/hotlinked_media_downloader_spec.rb@@ -25,6 +25,28 @@ RSpec.describe HotlinkedMediaDownloader do}.to raise_error(described_class::ImageBrokenError)end+ it "raises RateLimitedError honouring Retry-After when the host rate limits" do+ stub_request(:get, image_url).to_return(status: 429, headers: { "Retry-After" => "120" })++ expect do+ described_class.download(image_url, user.id, tmp_file_name: "test-hotlinked")+ rescue => error+ expect(error.retry_after).to eq(120)+ raise+ end.to raise_error(described_class::RateLimitedError)+ end++ it "raises RateLimitedError with the minimum delay when there is no Retry-After" do+ stub_request(:get, image_url).to_return(status: 429)++ expect do+ described_class.download(image_url, user.id, tmp_file_name: "test-hotlinked")+ rescue => error+ expect(error.retry_after).to eq(described_class::MIN_RETRY_AFTER_SECONDS)+ raise+ end.to raise_error(described_class::RateLimitedError)+ end+it "raises ImageTooLargeError when the file exceeds the limit" dohuge = "a" * (SiteSetting.max_image_size_kb * 1024 * 2)stub_request(:get, image_url).to_return(body: huge, headers: { "Content-Type" => "image/png" })