A platform for community discussion. Free, open, simple.
DEV: Read remote image size and animation in one streamed request
Régis Hanole7486257f3cd
diff --git a/lib/cooked_post_processor.rb b/lib/cooked_post_processor.rbindex f2f1f7b6e82..01b8b997429 100644--- a/lib/cooked_post_processor.rb+++ b/lib/cooked_post_processor.rb@@ -25,7 +25,6 @@ class CookedPostProcessorcooked = post.cook(post.raw, @cooking_options)@doc = Loofah.html5_fragment(cooked)@has_oneboxes = post.post_analyzer.found_oneboxes?- @size_cache = {}@disable_dominant_color = !!opts[:disable_dominant_color]@omit_nofollow = post.omit_nofollow?diff --git a/lib/cooked_processor_mixin.rb b/lib/cooked_processor_mixin.rbindex 072f814fed7..05cbaa66070 100644--- a/lib/cooked_processor_mixin.rb+++ b/lib/cooked_processor_mixin.rb@@ -188,23 +188,27 @@ module CookedProcessorMixinnilend- def add_to_size_cache(url, w, h)- @size_cache[url] = [w, h]+ def get_size(url)+ image_info(url)&.firstend- def get_size(url)- return @size_cache[url] if @size_cache.has_key?(url)+ # [[width, height], animated]+ def image_info(url)+ @image_info_cache ||= {}+ @image_info_cache.fetch(url) { @image_info_cache[url] = fetch_image_info(url) }+ end+ def fetch_image_info(url)absolute_url = urlabsolute_url = Discourse.base_url_no_prefix + absolute_url if absolute_url =~ %r{\A/[^/]}return unless absolute_url- # FastImage fails when there's no scheme+ # fetching fails when there's no schemeabsolute_url = SiteSetting.scheme + ":" + absolute_url if absolute_url.start_with?("//")- # we can't direct FastImage to our secure-uploads url because it bounces- # anonymous requests with a 404 error+ # we can't fetch our secure-uploads url because it bounces anonymous+ # requests with a 404 errorif url && Upload.secure_uploads_url?(url)absolute_url =Upload.signed_url_from_secure_uploads_url(absolute_url, include_content_disposition: false)@@ -214,12 +218,12 @@ module CookedProcessorMixinupload = Upload.get_from_url(absolute_url)if upload && upload.width && upload.width > 0- @size_cache[url] = [upload.width, upload.height]+ [[upload.width, upload.height], upload.animated]else- @size_cache[url] = FastImage.size(absolute_url)+ RemoteImage.size_and_animated(absolute_url)end- rescue Zlib::BufError, URI::Error, OpenSSL::SSL::SSLError- # FastImage.size raises BufError for some gifs, leave it.+ rescue URI::Error+ # the secure uploads url helpers raise on malformed urlsenddef get_filename(upload, src)diff --git a/lib/localized_cooked_post_processor.rb b/lib/localized_cooked_post_processor.rbindex 35df412f5b2..4b30c117c84 100644--- a/lib/localized_cooked_post_processor.rb+++ b/lib/localized_cooked_post_processor.rb@@ -14,7 +14,6 @@ class LocalizedCookedPostProcessor@model = @post@category_id = @post&.topic&.category_id@omit_nofollow = @post.omit_nofollow?- @size_cache = {}enddef post_processdiff --git a/lib/remote_image.rb b/lib/remote_image.rbnew file mode 100644index 00000000000..baec8e48e57--- /dev/null+++ b/lib/remote_image.rb@@ -0,0 +1,77 @@+# frozen_string_literal: true++# Reads the size of a remote image and whether it is animated with a single+# request, streaming only as many bytes as needed to answer both.+module RemoteImage+ ANIMATABLE_TYPES = %i[gif png webp avif]+ MIN_BYTES_FOR_TYPE = 1.kilobyte+ READ_TIMEOUT = 2++ # FinalDestination#get already handles timeouts and TLS errors+ FETCH_ERRORS = [+ ArgumentError, # malformed or too long urls and redirects+ URI::Error,+ SocketError,+ SystemCallError,+ IOError,+ Net::HTTPBadResponse,+ Zlib::Error,+ ]++ Result = Data.define(:size, :animated, :answered)++ # Returns `[[width, height], animated]`. `animated` is nil when the format+ # can't be animated, and both are nil when the image can't be read.+ def self.size_and_animated(url)+ max_bytes = SiteSetting.max_image_size_kb.kilobytes+ deadline =+ Process.clock_gettime(Process::CLOCK_MONOTONIC) + FinalDestination::MAX_REQUEST_TIME_SECONDS+ buffer = +"".b+ parse_at = 0+ result = nil++ begin+ # validation is skipped to keep fetching images on non-standard ports,+ # FinalDestination::HTTP still guards against SSRF+ FinalDestination+ .new(url, validate_uri: false, timeout: READ_TIMEOUT)+ .get do |_response, chunk, _uri|+ next if chunk.nil?+ buffer << chunk+ throw :done if buffer.bytesize >= max_bytes+ throw :done if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline++ # parsing restarts from the first byte, so only retry once the buffer has+ # doubled to keep large GIFs (read past their first frame) linear+ next if buffer.bytesize < parse_at+ parse_at = buffer.bytesize * 2++ result = parse(buffer)+ throw :done if result.answered+ end+ rescue *FETCH_ERRORS+ # keep whatever was read before the request failed+ end++ result = parse(buffer) if !result&.answered+ [result.size, result.animated]+ end++ def self.parse(buffer)+ image = FastImage.new(StringIO.new(buffer))+ size = image.size+ type = image.type++ if !type+ Result.new(size: nil, animated: nil, answered: buffer.bytesize >= MIN_BYTES_FOR_TYPE)+ elsif !size+ Result.new(size: nil, animated: nil, answered: false)+ elsif ANIMATABLE_TYPES.exclude?(type)+ Result.new(size:, animated: nil, answered: true)+ else+ animated = image.animated+ Result.new(size:, animated:, answered: !animated.nil?)+ end+ end+ private_class_method :parse+enddiff --git a/plugins/chat/lib/chat/message_processor.rb b/plugins/chat/lib/chat/message_processor.rbindex c08da21c2a1..e14a066754a 100644--- a/plugins/chat/lib/chat/message_processor.rb+++ b/plugins/chat/lib/chat/message_processor.rb@@ -11,7 +11,6 @@ module Chat@model = chat_message@previous_cooked = (chat_message.cooked || "").dup@should_secure_uploads = false- @size_cache = {}@opts = opts@used_hotlinked_media_ids = Set.new@stale_hotlinked_media_ids = []diff --git a/spec/lib/cooked_post_processor_spec.rb b/spec/lib/cooked_post_processor_spec.rbindex 9a1ee05efb9..69efbb4f617 100644--- a/spec/lib/cooked_post_processor_spec.rb+++ b/spec/lib/cooked_post_processor_spec.rb@@ -325,7 +325,6 @@ RSpec.describe CookedPostProcessor docpp = CookedPostProcessor.new(post)- cpp.add_to_size_cache(upload.url, 2000, 1500)cpp.post_processhtml = cpp.html@@ -341,7 +340,6 @@ RSpec.describe CookedPostProcessor doset_cdn_url("http://cdn.localhost")cpp = CookedPostProcessor.new(post)- cpp.add_to_size_cache(upload.url, 2000, 1500)cpp.post_processhtml = cpp.html@@ -370,7 +368,6 @@ RSpec.describe CookedPostProcessor do)cpp = CookedPostProcessor.new(post)- cpp.add_to_size_cache(upload.url, 200, 4000)cpp.post_processexpect(cpp.html).to_not include('srcset="')@@ -515,7 +512,7 @@ RSpec.describe CookedPostProcessor doendit "does not add a lightbox" do- FastImage.expects(:size).returns([1750, 2000])+ stub_image_size(width: 1750, height: 2000)cpp.post_process@@ -535,7 +532,7 @@ RSpec.describe CookedPostProcessor doendit "does not add a lightbox" do- FastImage.expects(:size).returns([1750, 2000])+ stub_image_size(width: 1750, height: 2000)cpp.post_process@@ -555,7 +552,7 @@ RSpec.describe CookedPostProcessor doendit "does not add a lightbox" do- FastImage.expects(:size).returns([1750, 2000])+ stub_image_size(width: 1750, height: 2000)cpp.post_process@@ -658,7 +655,6 @@ RSpec.describe CookedPostProcessor docontext "when the upload is attached to a different post" dobefore do- FastImage.size(upload.url)upload.update(secure: true,access_control_post: Fabricate(:post, user: user_with_auto_groups),@@ -857,7 +853,6 @@ RSpec.describe CookedPostProcessor dolet(:cpp) { CookedPostProcessor.new(post) }it "adds a topic image if there's one in the first post" do- FastImage.stubs(:size)expect(post.topic.image_upload_id).to eq(nil)cpp.post_process@@ -866,8 +861,6 @@ RSpec.describe CookedPostProcessor doendit "removes image if post is edited and no longer has an image" do- FastImage.stubs(:size)-cpp.post_processpost.topic.reloadexpect(post.topic.image_upload_id).to be_present@@ -956,7 +949,6 @@ RSpec.describe CookedPostProcessor doit "clears the generated OG image when the first post has an image" doSiteSetting.generate_topic_og_image = true- FastImage.stubs(:size)old_upload = Fabricate(:upload)image_post = Fabricate(:post_with_uploaded_image, user: user_with_auto_groups)image_post.topic.update_column(:og_image_upload_id, old_upload.id)@@ -991,7 +983,6 @@ RSpec.describe CookedPostProcessor dolet(:cpp) { CookedPostProcessor.new(reply) }it "adds a post image if there's one in the post" do- FastImage.stubs(:size)expect(reply.image_upload_id).to eq(nil)cpp.post_processreply.reload@@ -1050,13 +1041,13 @@ RSpec.describe CookedPostProcessor doit "resizes when only width is specified" doimg = { "src" => "http://foo.bar/image3.png", "width" => 100 }- FastImage.expects(:size).returns([200, 400])+ stub_image_size(width: 200, height: 400)expect(cpp.get_size_from_attributes(img)).to eq([100, 200])endit "resizes when only height is specified" doimg = { "src" => "http://foo.bar/image3.png", "height" => 100 }- FastImage.expects(:size).returns([100, 300])+ stub_image_size(width: 100, height: 300)expect(cpp.get_size_from_attributes(img)).to eq([33, 100])end@@ -1098,7 +1089,12 @@ RSpec.describe CookedPostProcessor doendit "caches the results" do- FastImage.expects(:size).returns([200, 400])+ RemoteImage+ .expects(:size_and_animated)+ .with("http://foo.bar/image3.png")+ .returns([[200, 400], nil])+ .once+cpp.get_size("http://foo.bar/image3.png")expect(cpp.get_size("http://foo.bar/image3.png")).to eq([200, 400])enddiff --git a/spec/lib/remote_image_spec.rb b/spec/lib/remote_image_spec.rbnew file mode 100644index 00000000000..051c735d621--- /dev/null+++ b/spec/lib/remote_image_spec.rb@@ -0,0 +1,90 @@+# frozen_string_literal: true++RSpec.describe RemoteImage do+ describe ".size_and_animated" do+ def stub_image(fixture)+ url = "https://example.com/#{fixture}"+ stub_request(:get, url).to_return(status: 200, body: file_from_fixtures(fixture).read)+ url+ end++ # mimics FinalDestination#get streaming a response in chunks until told to stop+ def stream_in_chunks(fixture, chunk_size, then_raise: nil)+ chunks = file_from_fixtures(fixture).read.b.scan(/.{1,#{chunk_size}}/m)+ destination = Object.new+ destination.define_singleton_method(:get) do |&block|+ completed =+ chunks.all? do |chunk|+ catch(:done) do+ block.call(nil, chunk, nil)+ true+ end+ end+ raise then_raise if completed && then_raise+ end+ FinalDestination.stubs(:new).returns(destination)+ "https://example.com/#{fixture}"+ end++ it "reads the size and animation of animated images" do+ expect(described_class.size_and_animated(stub_image("animated.gif"))).to eq(+ [[320, 320], true],+ )+ expect(described_class.size_and_animated(stub_image("animated.webp"))).to eq(+ [[250, 250], true],+ )+ end++ it "returns a nil animated flag for formats that can't be animated" do+ expect(described_class.size_and_animated(stub_image("logo.jpg"))).to eq([[512, 512], nil])+ end++ it "reads static images whose stream ends between two parses" do+ url = stream_in_chunks("static.gif", 1)++ expect(described_class.size_and_animated(url)).to eq([[2, 1], false])+ end++ it "stops reading past the maximum image size or duration" do+ url = stream_in_chunks("animated.gif", 256)++ SiteSetting.max_image_size_kb = 1+ expect(described_class.size_and_animated(url)).to eq([[320, 320], nil])++ SiteSetting.max_image_size_kb = 10_240+ stub_const(FinalDestination, "MAX_REQUEST_TIME_SECONDS", -1) do+ expect(described_class.size_and_animated(url)).to eq([[320, 320], nil])+ end+ end++ it "keeps what was read before the connection dropped" do+ url = stream_in_chunks("static.gif", 40, then_raise: EOFError)++ expect(described_class.size_and_animated(url)).to eq([[2, 1], false])+ end++ it "returns nils when the image can't be fetched" do+ stub_request(:get, "https://example.com/missing.gif").to_return(status: 404)+ stub_request(:get, "https://example.com/refused.gif").to_raise(Errno::ECONNREFUSED)+ too_long = "https://example.com/#{"a" * UrlHelper::MAX_URL_LENGTH}.gif"++ expect(described_class.size_and_animated("https://example.com/missing.gif")).to eq([nil, nil])+ expect(described_class.size_and_animated("https://example.com/refused.gif")).to eq([nil, nil])+ expect(described_class.size_and_animated(too_long)).to eq([nil, nil])+ end++ it "doesn't swallow unexpected errors" do+ url = stub_image("animated.gif")+ FastImage.stubs(:new).raises(NoMethodError)++ expect { described_class.size_and_animated(url) }.to raise_error(NoMethodError)+ end++ it "stops reading responses that aren't images" do+ url = "https://example.com/page"+ stub_request(:get, url).to_return(status: 200, body: "<html>#{"a" * 2.kilobytes}</html>")++ expect(described_class.size_and_animated(url)).to eq([nil, nil])+ end+ end+enddiff --git a/spec/rails_helper.rb b/spec/rails_helper.rbindex 7462495ae5a..33e1da9d455 100644--- a/spec/rails_helper.rb+++ b/spec/rails_helper.rb@@ -134,7 +134,7 @@ RSpec.configure do |config|config.include UploadsHelpersconfig.include BackupsHelpersconfig.include OneboxHelpers- config.include FastImageHelpers+ config.include RemoteImageHelpersconfig.include ServiceMatchersconfig.include JsonApiKitMatchers, type: :resourceconfig.include I18nHelpersdiff --git a/spec/support/fast_image_helpers.rb b/spec/support/fast_image_helpers.rbdeleted file mode 100644index 55ad54ba422..00000000000--- a/spec/support/fast_image_helpers.rb+++ /dev/null@@ -1,7 +0,0 @@-# frozen_string_literal: true--module FastImageHelpers- def stub_image_size(width: nil, height: nil)- FastImage.stubs(:size).returns([width, height])- end-enddiff --git a/spec/support/remote_image_helpers.rb b/spec/support/remote_image_helpers.rbnew file mode 100644index 00000000000..cd15e870e91--- /dev/null+++ b/spec/support/remote_image_helpers.rb@@ -0,0 +1,7 @@+# frozen_string_literal: true++module RemoteImageHelpers+ def stub_image_size(width: nil, height: nil)+ RemoteImage.stubs(:size_and_animated).returns([[width, height], nil])+ end+end