A platform for community discussion. Free, open, simple.
PERF: Skip the chat hotlinked images job when nothing is hotlinked (#43089)
Renato Atilio04bceda85df2
diff --git a/lib/hotlinked_media.rb b/lib/hotlinked_media.rbindex 853ea7f5c42..56fa341fdba 100644--- a/lib/hotlinked_media.rb+++ b/lib/hotlinked_media.rb@@ -28,6 +28,26 @@ module HotlinkedMediasrcend+ # Whether +src+ points somewhere this site does not serve, and so is a+ # candidate for downloading. Cheap by design: no database or network access.+ def self.remote_src?(src)+ return false if src.blank?+ return false if src.downcase.start_with?("data:")+ return false if src.start_with?("/") && !src.start_with?("//")+ return false if Discourse.store.has_been_uploaded?(src)++ !PostHotlinkedMedia.normalize_src(src).start_with?(*local_bases)+ end++ def self.local_bases+ [+ Discourse.base_url.presence,+ Discourse.asset_host.presence,+ SiteSetting.external_emoji_url.presence,+ ].compact.map { |base| PostHotlinkedMedia.normalize_src(base) }+ end+ private_class_method :local_bases+# Downloads +src+ for +user_id+, returning [status, upload]. The status is the# one to record against the target; upload is nil unless it is :downloaded.def self.download(src, user_id, tmp_file_name:)diff --git a/plugins/chat/app/jobs/regular/chat/process_message.rb b/plugins/chat/app/jobs/regular/chat/process_message.rbindex 22081111650..18ea22ed083 100644--- a/plugins/chat/app/jobs/regular/chat/process_message.rb+++ b/plugins/chat/app/jobs/regular/chat/process_message.rb@@ -4,6 +4,8 @@ module Jobsmodule Chatclass ProcessMessage < ::Jobs::Basedef execute(args = {})+ hotlinked_media_pending = false+::DistributedMutex.synchronize("jobs_chat_process_message_#{args[:chat_message_id]}",validity: 10.minutes,@@ -28,10 +30,13 @@ module Jobs)end- chat_message- .hotlinked_media- .where.not(id: processor.used_hotlinked_media_ids.to_a)- .destroy_all+ stale_ids = processor.stale_hotlinked_media_ids+ chat_message.hotlinked_media.where(id: stale_ids).destroy_all if stale_ids.present?+ # the pull job's own re-cook comes back with skip_pull set, and the+ # check walks the doc and resolves every local upload URL+ if !args[:skip_pull_hotlinked_images]+ hotlinked_media_pending = processor.hotlinked_media_pending?+ end# we don't process mentions when creating/updating message so we always have to do itchat_message.upsert_mentions@@ -52,7 +57,7 @@ module Jobs# outside the mutex: the pull job re-cooks through this job, and would# block on the lock we still hold whenever jobs run inline- if !args[:skip_pull_hotlinked_images] && SiteSetting.download_remote_images_to_local? &&+ if hotlinked_media_pending && SiteSetting.download_remote_images_to_local? &&SiteSetting.chat_allow_uploads?::Jobs.enqueue(::Jobs::Chat::PullHotlinkedImages, chat_message_id: args[:chat_message_id])enddiff --git a/plugins/chat/app/jobs/regular/chat/pull_hotlinked_images.rb b/plugins/chat/app/jobs/regular/chat/pull_hotlinked_images.rbindex db2fcb46bba..002a35b6a8f 100644--- a/plugins/chat/app/jobs/regular/chat/pull_hotlinked_images.rb+++ b/plugins/chat/app/jobs/regular/chat/pull_hotlinked_images.rb@@ -10,8 +10,6 @@ module Jobsraise Discourse::InvalidParameters.new(:chat_message_id) if @chat_message_id.blank?return if !SiteSetting.chat_allow_uploads?- disable_if_low_on_disk_space-if Jobs.run_immediately?pullelse@@ -41,7 +39,7 @@ module Jobsnext if node.name != "img"download_src = HotlinkedMedia.download_src_for(node)- next if !should_download?(download_src)+ next if !::Chat::MessageHotlinkedMedia.downloadable?(download_src)normalized_src = ::Chat::MessageHotlinkedMedia.normalize_src(download_src)if (existing = hotlinked_map[normalized_src])@@ -51,6 +49,8 @@ module Jobsnextend+ next if !enough_disk_space?+status, upload =HotlinkedMedia.download(download_src,@@ -85,7 +85,9 @@ module Jobs<<~SQL,INSERT INTO chat_message_hotlinked_media (chat_message_id, url, status, upload_id, created_at, updated_at)VALUES (:chat_message_id, :url, :status, :upload_id, NOW(), NOW())- ON CONFLICT (chat_message_id, md5(url)) DO NOTHING+ ON CONFLICT (chat_message_id, md5(url)) DO UPDATE+ SET status = EXCLUDED.status, upload_id = EXCLUDED.upload_id, updated_at = NOW()+ WHERE chat_message_hotlinked_media.upload_id IS NULL AND EXCLUDED.upload_id IS NOT NULLSQLchat_message_id: chat_message.id,url: normalized_src,@@ -95,40 +97,13 @@ module Jobs::Chat::MessageHotlinkedMedia.find_by(chat_message_id: chat_message.id, url: normalized_src)end- def should_download?(src)- return false if src.blank?- return false if !SiteSetting.download_remote_images_to_local?- return false if !SiteSetting.chat_allow_uploads?- return false if src.start_with?("data:")- # Downloading these means signing an S3 path for the underlying upload.- # Posts gate that on the author seeing the access-control post; chat- # messages have none, so there is nothing to authorize against.- return false if Upload.secure_uploads_url?(src)-- local_bases =- [- Discourse.base_url,- Discourse.asset_host,- SiteSetting.external_emoji_url.presence,- ].compact.map { |s| ::Chat::MessageHotlinkedMedia.normalize_src(s) }-- if Discourse.store.has_been_uploaded?(src) ||- ::Chat::MessageHotlinkedMedia.normalize_src(src).start_with?(*local_bases) ||- src =~ %r{\A/[^/]}i- return false if src !~ %r{/uploads/}- # pass nil: chat messages have no access-control post to reuse against- upload = Upload.consider_for_reuse(Upload.get_from_url(src), nil)- return !upload.present?- end+ # `df` forks a process, so only pay for it once there is something to+ # download; the overwhelming majority of messages have nothing.+ def enough_disk_space?+ return @enough_disk_space if defined?(@enough_disk_space)- begin- uri = URI.parse(src)- rescue URI::Error- return false- end- return false if uri.hostname.blank?-- SiteSetting.should_download_images?(src)+ disable_if_low_on_disk_space+ @enough_disk_space = SiteSetting.download_remote_images_to_local?enddef disable_if_low_on_disk_spacediff --git a/plugins/chat/app/models/chat/message_hotlinked_media.rb b/plugins/chat/app/models/chat/message_hotlinked_media.rbindex 87e5b55c439..ab1f1a91f39 100644--- a/plugins/chat/app/models/chat/message_hotlinked_media.rb+++ b/plugins/chat/app/models/chat/message_hotlinked_media.rb@@ -19,6 +19,35 @@ module Chatdef self.normalize_src(src, reset_scheme: true)PostHotlinkedMedia.normalize_src(src, reset_scheme: reset_scheme)end++ # Whether the pull job would fetch +src+. The cook consults this too, to+ # decide whether handing the message to that job is worth a job slot.+ def self.downloadable?(src)+ return false if src.blank?+ return false if !SiteSetting.download_remote_images_to_local?+ return false if !SiteSetting.chat_allow_uploads?+ return false if src.downcase.start_with?("data:")+ # Host-agnostic on purpose: the downloader signs our own S3 path for any+ # URL shaped like this, whoever serves it. Posts gate that on the author+ # seeing the access-control post; chat messages have none, so there is+ # nothing to authorize against.+ return false if Upload.secure_uploads_url?(src)++ if !HotlinkedMedia.remote_src?(src)+ return false if src !~ %r{/uploads/}+ # pass nil: chat messages have no access-control post to reuse against+ return !Upload.consider_for_reuse(Upload.get_from_url(src), nil).present?+ end++ begin+ uri = URI.parse(src)+ rescue URI::Error+ return false+ end+ return false if uri.hostname.blank?++ SiteSetting.should_download_images?(src)+ endendenddiff --git a/plugins/chat/lib/chat/message_processor.rb b/plugins/chat/lib/chat/message_processor.rbindex 6b2dacef192..c08da21c2a1 100644--- a/plugins/chat/lib/chat/message_processor.rb+++ b/plugins/chat/lib/chat/message_processor.rb@@ -5,7 +5,7 @@ module Chatinclude ::CookedProcessorMixinIMG_FILETYPES = %w[jpg jpeg gif png heic heif webp]- attr_reader :used_hotlinked_media_ids+ attr_reader :stale_hotlinked_media_idsdef initialize(chat_message, opts = {})@model = chat_message@@ -14,6 +14,7 @@ module Chat@size_cache = {}@opts = opts@used_hotlinked_media_ids = Set.new+ @stale_hotlinked_media_ids = []cook_opts = {user_id: chat_message.last_editor_id,@@ -35,23 +36,46 @@ module Chat# Unlike posts, chat keeps the hotlinked src when a download fails or is too large.def process_hotlinked_image(img)- normalized_src =- Chat::MessageHotlinkedMedia.normalize_src(HotlinkedMedia.download_src_for(img))- info = hotlinked_media_map[normalized_src]- used_hotlinked_media_ids << info.id if info+ download_src = HotlinkedMedia.download_src_for(img)+ info = hotlinked_media_map[Chat::MessageHotlinkedMedia.normalize_src(download_src)]+ @used_hotlinked_media_ids << info.id if infoif info&.downloaded? && (upload = info.upload)img["src"] = UrlHelper.cook_url(upload.url, secure: upload.secure?)img["data-base62-sha1"] = upload.base62_sha1img["data-dominant-color"] = upload.dominant_color(calculate_if_missing: true).presenceimg.delete(PrettyText::BLOCKED_HOTLINKED_SRC_ATTR)+ img.delete(PrettyText::BLOCKED_HOTLINKED_SRCSET_ATTR)endtrueenddef process_hotlinked_images- extract_images.each { |img| process_hotlinked_image(img) }+ images = extract_images+ # snapshot the tracked rows here rather than at sweep time, so a row the+ # pull job inserts while we work is not carried away as unused+ tracked_ids = hotlinked_media_map.each_value.map(&:id)+ images.each { |img| process_hotlinked_image(img) }+ @stale_hotlinked_media_ids = tracked_ids - @used_hotlinked_media_ids.to_a+ end++ # The pull job costs a job slot, a lock and a disk check per message, so let+ # the cook decide whether there is anything for it to do. Mirrors the job's+ # own loop: anything looser enqueues for nothing, anything tighter leaves an+ # image hotlinked for good.+ def hotlinked_media_pending?+ HotlinkedMedia+ .extract_candidates(@doc)+ .any? do |node|+ next false if node.name != "img"++ download_src = HotlinkedMedia.download_src_for(node)+ next false if !Chat::MessageHotlinkedMedia.downloadable?(download_src)++ normalized_src = Chat::MessageHotlinkedMedia.normalize_src(download_src)+ !hotlinked_media_map.key?(normalized_src)+ endenddef process_thumbnails@@ -92,18 +116,19 @@ module Chat.css("img").each do |img|if img["class"]&.include?("emoji") || img["class"]&.include?("avatar") ||- img["data-base62-sha1"].blank? || img.ancestors(".onebox, .onebox-body").any?+ img["data-base62-sha1"].blank? || img.classes.include?("onebox") ||+ img.ancestors(".onebox, .onebox-body").any?nextend- sha1 = Upload.sha1_from_base62_encoded(img["data-base62-sha1"])- if upload = Upload.find_by(sha1: sha1)- img["data-large-src"] = UrlHelper.cook_url(upload.url, secure: upload.secure?)- img["data-download-href"] = upload.short_path- img["data-target-width"] = upload.width- img["data-target-height"] = upload.height- img["class"] = "#{img["class"]} lightbox".strip- end+ upload = upload_for_base62_sha1(img["data-base62-sha1"])+ next if upload.blank?++ img["data-large-src"] = UrlHelper.cook_url(upload.url, secure: upload.secure?)+ img["data-download-href"] = upload.short_path+ img["data-target-width"] = upload.width+ img["data-target-height"] = upload.height+ img["class"] = "#{img["class"]} lightbox".stripendend@@ -111,6 +136,17 @@ module Chat@hotlinked_media_map ||= @model.hotlinked_media.preload(:upload).index_by(&:url)end+ def upload_for_base62_sha1(base62_sha1)+ hotlinked_uploads.fetch(base62_sha1) do+ Upload.find_by(sha1: Upload.sha1_from_base62_encoded(base62_sha1))+ end+ end++ def hotlinked_uploads+ @hotlinked_uploads ||=+ hotlinked_media_map.each_value.filter_map(&:upload).index_by(&:base62_sha1)+ end+def large_images[]enddiff --git a/plugins/chat/spec/jobs/regular/chat/process_message_spec.rb b/plugins/chat/spec/jobs/regular/chat/process_message_spec.rbindex ebe458fcf7f..7f4607dcae9 100644--- a/plugins/chat/spec/jobs/regular/chat/process_message_spec.rb+++ b/plugins/chat/spec/jobs/regular/chat/process_message_spec.rb@@ -71,22 +71,87 @@ describe Jobs::Chat::ProcessMessage doenddescribe "pull hotlinked images" do+ let(:image_url) { "https://example.com/img.png" }+ fab!(:hotlinked_message) do+ Fabricate(:chat_message, message: "")+ end+it "enqueues the pull job when download_remote_images_to_local is enabled" doSiteSetting.download_remote_images_to_local = trueexpect_enqueued_with(job: Jobs::Chat::PullHotlinkedImages,args: {- chat_message_id: chat_message.id,+ chat_message_id: hotlinked_message.id,+ },+ ) { described_class.new.execute(chat_message_id: hotlinked_message.id) }+ end++ # the job costs a slot, a lock and a disk check, and the overwhelming+ # majority of messages have nothing external in them+ it "does not enqueue the pull job for a message with nothing to fetch" do+ SiteSetting.download_remote_images_to_local = true++ expect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) do+ described_class.new.execute(chat_message_id: chat_message.id)+ end+ end++ it "does not enqueue the pull job when tracked media is localized during processing" do+ SiteSetting.download_remote_images_to_local = true+ Chat::MessageHotlinkedMedia.create!(+ chat_message: hotlinked_message,+ url: Chat::MessageHotlinkedMedia.normalize_src(image_url),+ status: :downloaded,+ upload: Fabricate(:upload),+ )++ expect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) do+ described_class.new.execute(chat_message_id: hotlinked_message.id)+ end+ end++ it "does not enqueue the pull job for tracked media with a terminal failure" do+ SiteSetting.download_remote_images_to_local = true+ Chat::MessageHotlinkedMedia.create!(+ chat_message: hotlinked_message,+ url: Chat::MessageHotlinkedMedia.normalize_src(image_url),+ status: :too_large,+ )++ expect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) do+ described_class.new.execute(chat_message_id: hotlinked_message.id)+ end+ end++ it "enqueues the pull job for a local upload URL that resolves to nothing" do+ SiteSetting.download_remote_images_to_local = true+ orphan_url = "#{Discourse.base_url}/uploads/default/original/1X/#{SecureRandom.hex(20)}.png"+ message = Fabricate(:chat_message, message: "")++ expect_enqueued_with(+ job: Jobs::Chat::PullHotlinkedImages,+ args: {+ chat_message_id: message.id,},- ) { described_class.new.execute(chat_message_id: chat_message.id) }+ ) { described_class.new.execute(chat_message_id: message.id) }+ end++ it "does not enqueue the pull job for an image that is already a local upload" do+ SiteSetting.download_remote_images_to_local = true+ upload = Fabricate(:upload)+ message = Fabricate(:chat_message, message: "")++ expect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) do+ described_class.new.execute(chat_message_id: message.id)+ endendit "does not enqueue the pull job when download_remote_images_to_local is disabled" doSiteSetting.download_remote_images_to_local = falseexpect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) do- described_class.new.execute(chat_message_id: chat_message.id)+ described_class.new.execute(chat_message_id: hotlinked_message.id)endend@@ -95,7 +160,7 @@ describe Jobs::Chat::ProcessMessage doSiteSetting.chat_allow_uploads = falseexpect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) do- described_class.new.execute(chat_message_id: chat_message.id)+ described_class.new.execute(chat_message_id: hotlinked_message.id)endend@@ -127,10 +192,13 @@ describe Jobs::Chat::ProcessMessage do# the two jobs enqueueing each otherit "does not enqueue the pull job when it was the pull job that asked for the re-cook" doSiteSetting.download_remote_images_to_local = true+ # the check walks the doc and resolves every local upload URL, so don't+ # pay for it when the answer is already ignored+ Chat::MessageProcessor.any_instance.expects(:hotlinked_media_pending?).neverexpect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) dodescribed_class.new.execute(- chat_message_id: chat_message.id,+ chat_message_id: hotlinked_message.id,skip_pull_hotlinked_images: true,)end@@ -154,6 +222,33 @@ describe Jobs::Chat::ProcessMessage do}.from(true).to(false)end+ # the pull job inserts rows while we cook, so sweeping everything the cook did+ # not use would carry away a row that did not exist yet when it read them+ it "keeps hotlinked media inserted after the cook read the tracked rows" do+ image_url = "https://example.com/late.png"+ message = Fabricate(:chat_message, message: "")+ record = nil+ insert_late_row =+ Proc.new do+ record ||=+ Chat::MessageHotlinkedMedia.create!(+ chat_message: message,+ url: Chat::MessageHotlinkedMedia.normalize_src(image_url),+ status: :downloaded,+ upload: Fabricate(:upload),+ )+ end++ DiscourseEvent.on(:chat_message_processed, &insert_late_row)+ begin+ described_class.new.execute(chat_message_id: message.id)+ ensure+ DiscourseEvent.off(:chat_message_processed, &insert_late_row)+ end++ expect(Chat::MessageHotlinkedMedia.exists?(record.id)).to eq(true)+ end+describe "skip_notifications" dofab!(:user)fab!(:mentioned_user, :user)diff --git a/plugins/chat/spec/jobs/regular/chat/pull_hotlinked_images_spec.rb b/plugins/chat/spec/jobs/regular/chat/pull_hotlinked_images_spec.rbindex 7b88c76c0e7..e03179a7f56 100644--- a/plugins/chat/spec/jobs/regular/chat/pull_hotlinked_images_spec.rb+++ b/plugins/chat/spec/jobs/regular/chat/pull_hotlinked_images_spec.rb@@ -67,6 +67,35 @@ describe Jobs::Chat::PullHotlinkedImages doexpect(message.upload_references).to be_emptyend+ # a racing job can insert the row between our download and our insert; the+ # upload we just created has no other reference, so the row has to claim it+ it "claims the download when a racing job already recorded a failure" do+ stub_image_size+ message = fabricate_chat_message("")+ record =+ Chat::MessageHotlinkedMedia.create!(+ chat_message: message,+ url: Chat::MessageHotlinkedMedia.normalize_src(image_url),+ status: :download_failed,+ )+ # the pull reads its map before the racing row lands+ Chat::Message.any_instance.stubs(:hotlinked_media).returns(Chat::MessageHotlinkedMedia.none)++ described_class.new.execute(chat_message_id: message.id)++ record.reload+ expect(record).to be_downloaded+ expect(record.upload).to eq(Upload.last)+ expect(Chat::MessageHotlinkedMedia.where(chat_message_id: message.id).count).to eq(1)+ end++ it "does not run the disk space check when there is nothing to download" do+ message = fabricate_chat_message("just text")+ DiskSpace.expects(:percent_free).never++ described_class.new.execute(chat_message_id: message.id)+ end+it "records terminal failures without changing the message or retrying" doraw = ""message = fabricate_chat_message(raw)@@ -251,6 +280,29 @@ describe Jobs::Chat::PullHotlinkedImages doendexpect(WebMock).not_to have_requested(:get, /amazonaws\.com/)end++ # the downloader signs our own S3 path for anything shaped like a secure+ # upload URL, so a foreign host serving that path must be rejected too+ it "does not rehost a secure upload path served by another host" do+ global_setting :allow_unsecure_chat_uploads, true+ SiteSetting.chat_allow_uploads = true+ foreign_url =+ "https://attacker.example/secure-uploads/original/1X/1234567890abcdef1234567890abcdef12345678.png"+ stub_request(:get, foreign_url).to_return(+ body: gif,+ headers: {+ "Content-Type" => "image/gif",+ },+ )+ stub_image_size+ message = fabricate_chat_message("")++ expect { described_class.new.execute(chat_message_id: message.id) }.not_to change {+ Upload.count+ }+ expect(message.reload.hotlinked_media).to be_empty+ expect(WebMock).not_to have_requested(:get, /amazonaws\.com/)+ endendit "does not treat an attacker host with the local host prefix as local" dodiff --git a/plugins/chat/spec/lib/chat/message_processor_spec.rb b/plugins/chat/spec/lib/chat/message_processor_spec.rbindex 257b1845eb5..2ddbaf2148a 100644--- a/plugins/chat/spec/lib/chat/message_processor_spec.rb+++ b/plugins/chat/spec/lib/chat/message_processor_spec.rb@@ -105,6 +105,21 @@ RSpec.describe Chat::MessageProcessor doendend+ it "clears both blocked-hotlinked attributes once the image is localized" do+ create_downloaded_record(message)+ Chat::Message.stubs(:cook).returns(<<~HTML)+ <p><img #{PrettyText::BLOCKED_HOTLINKED_SRC_ATTR}="#{image_url}" #{PrettyText::BLOCKED_HOTLINKED_SRCSET_ATTR}="#{image_url} 2x"></p>+ HTML++ processor = described_class.new(message)+ processor.run!++ img = processor.instance_variable_get(:@doc).at_css("img")+ expect(img["src"]).to eq(UrlHelper.cook_url(upload.url, secure: upload.secure?))+ expect(img[PrettyText::BLOCKED_HOTLINKED_SRC_ATTR]).to be_nil+ expect(img[PrettyText::BLOCKED_HOTLINKED_SRCSET_ATTR]).to be_nil+ end+it "leaves the hotlinked src for failed or oversized downloads" doChat::MessageHotlinkedMedia.create!(chat_message: message,@@ -147,6 +162,19 @@ RSpec.describe Chat::MessageProcessor doexpect(img["data-target-height"]).to eq(upload.height.to_s)end+ it "does not lightbox an image the onebox engine already wrapped" do+ cooked_html =+ "<p><img class=\"onebox\" src=\"#{upload.url}\" data-base62-sha1=\"#{base62}\"></p>"++ Chat::Message.stubs(:cook).returns(cooked_html)+ processor = described_class.new(message)++ processor.run!++ img = processor.instance_variable_get(:@doc).at_css("img")+ expect(img["class"]).not_to include("lightbox")+ end+context "with secure uploads enabled" dobefore dosetup_s3