A platform for community discussion. Free, open, simple.
DEV: pull hotlinked images in chat messages (#40278)
Renato Atilio0958c6ff5f52
diff --git a/db/structure.sql b/db/structure.sqlindex 91501cf0b4b..0d315803a12 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -3057,6 +3057,40 @@ CREATE SEQUENCE public.chat_message_custom_prompts_id_seqALTER SEQUENCE public.chat_message_custom_prompts_id_seq OWNED BY public.chat_message_custom_prompts.id;+--+-- Name: chat_message_hotlinked_media; Type: TABLE; Schema: public; Owner: -+--++CREATE TABLE public.chat_message_hotlinked_media (+ id bigint NOT NULL,+ chat_message_id bigint NOT NULL,+ url character varying NOT NULL,+ status character varying NOT NULL,+ upload_id bigint,+ created_at timestamp(6) without time zone NOT NULL,+ updated_at timestamp(6) without time zone NOT NULL+);+++--+-- Name: chat_message_hotlinked_media_id_seq; Type: SEQUENCE; Schema: public; Owner: -+--++CREATE SEQUENCE public.chat_message_hotlinked_media_id_seq+ START WITH 1+ INCREMENT BY 1+ NO MINVALUE+ NO MAXVALUE+ CACHE 1;+++--+-- Name: chat_message_hotlinked_media_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -+--++ALTER SEQUENCE public.chat_message_hotlinked_media_id_seq OWNED BY public.chat_message_hotlinked_media.id;++---- Name: chat_message_interactions; Type: TABLE; Schema: public; Owner: ---@@ -13096,6 +13130,13 @@ ALTER TABLE ONLY public.chat_message_custom_fields ALTER COLUMN id SET DEFAULT nALTER TABLE ONLY public.chat_message_custom_prompts ALTER COLUMN id SET DEFAULT nextval('public.chat_message_custom_prompts_id_seq'::regclass);+--+-- Name: chat_message_hotlinked_media id; Type: DEFAULT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.chat_message_hotlinked_media ALTER COLUMN id SET DEFAULT nextval('public.chat_message_hotlinked_media_id_seq'::regclass);++---- Name: chat_message_interactions id; Type: DEFAULT; Schema: public; Owner: ---@@ -15448,6 +15489,14 @@ ALTER TABLE ONLY public.chat_message_custom_promptsADD CONSTRAINT chat_message_custom_prompts_pkey PRIMARY KEY (id);+--+-- Name: chat_message_hotlinked_media chat_message_hotlinked_media_pkey; Type: CONSTRAINT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.chat_message_hotlinked_media+ ADD CONSTRAINT chat_message_hotlinked_media_pkey PRIMARY KEY (id);++---- Name: chat_message_interactions chat_message_interactions_pkey; Type: CONSTRAINT; Schema: public; Owner: ---@@ -19359,6 +19408,20 @@ CREATE UNIQUE INDEX index_chat_message_custom_fields_on_message_id_and_name ON pCREATE UNIQUE INDEX index_chat_message_custom_prompts_on_message_id ON public.chat_message_custom_prompts USING btree (message_id);+--+-- Name: index_chat_message_hotlinked_media_on_message_and_url_md5; Type: INDEX; Schema: public; Owner: -+--++CREATE UNIQUE INDEX index_chat_message_hotlinked_media_on_message_and_url_md5 ON public.chat_message_hotlinked_media USING btree (chat_message_id, md5((url)::text));+++--+-- Name: index_chat_message_hotlinked_media_on_upload_id; Type: INDEX; Schema: public; Owner: -+--++CREATE INDEX index_chat_message_hotlinked_media_on_upload_id ON public.chat_message_hotlinked_media USING btree (upload_id);++---- Name: index_chat_message_interactions_on_chat_message_id; Type: INDEX; Schema: public; Owner: ---@@ -23477,6 +23540,7 @@ INSERT INTO "schema_migrations" (version) VALUES('20260602104726'),('20260601063855'),('20260601043020'),+('20260529220000'),('20260528074731'),('20260528074719'),('20260525105009'),diff --git a/plugins/chat/app/jobs/regular/chat/channel_delete.rb b/plugins/chat/app/jobs/regular/chat/channel_delete.rbindex 1c47cb99078..b417ae16439 100644--- a/plugins/chat/app/jobs/regular/chat/channel_delete.rb+++ b/plugins/chat/app/jobs/regular/chat/channel_delete.rb@@ -49,6 +49,10 @@ module Jobstarget_type: ::Chat::Message.polymorphic_name,).delete_all+ # these rows keep hotlinked uploads flagged as in use, so leaving+ # them on trashed messages would pin the uploads forever+ ::Chat::MessageHotlinkedMedia.where(chat_message_id: message_ids).delete_all+# only the messages and the channel are Trashable, everything else gets# permanently destroyedchat_messages.update_all(diff --git a/plugins/chat/app/jobs/regular/chat/process_message.rb b/plugins/chat/app/jobs/regular/chat/process_message.rbindex fc2ce806776..22081111650 100644--- a/plugins/chat/app/jobs/regular/chat/process_message.rb+++ b/plugins/chat/app/jobs/regular/chat/process_message.rb@@ -28,6 +28,11 @@ module Jobs)end+ chat_message+ .hotlinked_media+ .where.not(id: processor.used_hotlinked_media_ids.to_a)+ .destroy_all+# we don't process mentions when creating/updating message so we always have to do itchat_message.upsert_mentions@@ -44,6 +49,13 @@ module Jobs::Chat::Publisher.publish_processed!(chat_message)end++ # outside the mutex: the pull job re-cooks through this job, and would+ # block on the lock we still hold whenever jobs run inline+ if !args[:skip_pull_hotlinked_images] && SiteSetting.download_remote_images_to_local? &&+ SiteSetting.chat_allow_uploads?+ ::Jobs.enqueue(::Jobs::Chat::PullHotlinkedImages, chat_message_id: args[:chat_message_id])+ endendendenddiff --git a/plugins/chat/app/jobs/regular/chat/pull_hotlinked_images.rb b/plugins/chat/app/jobs/regular/chat/pull_hotlinked_images.rbnew file mode 100644index 00000000000..db2fcb46bba--- /dev/null+++ b/plugins/chat/app/jobs/regular/chat/pull_hotlinked_images.rb@@ -0,0 +1,157 @@+# frozen_string_literal: true++module Jobs+ module Chat+ class PullHotlinkedImages < ::Jobs::Base+ sidekiq_options queue: "low"++ def execute(args)+ @chat_message_id = args[:chat_message_id]+ raise Discourse::InvalidParameters.new(:chat_message_id) if @chat_message_id.blank?+ return if !SiteSetting.chat_allow_uploads?++ disable_if_low_on_disk_space++ if Jobs.run_immediately?+ pull+ else+ DistributedMutex.synchronize(+ "chat_pull_hotlinked_images_#{@chat_message_id}",+ validity: 2.minutes,+ ) { pull }+ end+ end++ private++ def pull+ chat_message = ::Chat::Message.find_by(id: @chat_message_id)+ return if chat_message.nil? || chat_message.cooked.blank?+ # Don't re-cook system/webhook messages whose cooked is hand-written.+ return if chat_message.message.blank?++ hotlinked_map = chat_message.hotlinked_media.preload(:upload).index_by(&:url)+ needs_recook = false++ HotlinkedMedia+ .extract_candidates(chat_message.cooked)+ .each do |node|+ # chat only localizes imgs; downloading other media would loop+ # forever, as the unused-row sweep erases the download history+ next if node.name != "img"++ download_src = HotlinkedMedia.download_src_for(node)+ next if !should_download?(download_src)++ normalized_src = ::Chat::MessageHotlinkedMedia.normalize_src(download_src)+ if (existing = hotlinked_map[normalized_src])+ # still external despite a downloaded row: the localizing re-cook+ # was lost — trigger it again; terminal failures aren't retried+ needs_recook = true if existing.downloaded? && existing.upload+ next+ end++ status, upload =+ HotlinkedMedia.download(+ download_src,+ chat_message.last_editor_id,+ tmp_file_name: "chat-hotlinked",+ )+ record = upsert_record(chat_message, normalized_src, status, upload)+ hotlinked_map[normalized_src] = record if record+ needs_recook = true if upload && record&.downloaded?+ rescue => e+ raise e if Rails.env.test?+ Discourse.warn_exception(+ e,+ message:+ "Failed to pull hotlinked image (#{download_src}) for chat message #{@chat_message_id}",+ )+ end++ return if !needs_recook++ # the re-cook localizes the images; skip_pull prevents an enqueue loop+ ::Jobs.enqueue(+ ::Jobs::Chat::ProcessMessage,+ chat_message_id: chat_message.id,+ skip_pull_hotlinked_images: true,+ skip_notifications: true,+ )+ end++ def upsert_record(chat_message, normalized_src, status, upload)+ DB.exec(+ <<~SQL,+ INSERT INTO chat_message_hotlinked_media (chat_message_id, url, status, upload_id, created_at, updated_at)+ VALUES (:chat_message_id, :url, :status, :upload_id, NOW(), NOW())+ ON CONFLICT (chat_message_id, md5(url)) DO NOTHING+ SQL+ chat_message_id: chat_message.id,+ url: normalized_src,+ status: status.to_s,+ upload_id: upload&.id,+ )+ ::Chat::MessageHotlinkedMedia.find_by(chat_message_id: chat_message.id, url: normalized_src)+ end++ def should_download?(src)+ return false if src.blank?+ return false if !SiteSetting.download_remote_images_to_local?+ return false if !SiteSetting.chat_allow_uploads?+ return false if src.start_with?("data:")+ # Downloading these means signing an S3 path for the underlying upload.+ # Posts gate that on the author seeing the access-control post; chat+ # messages have none, so there is nothing to authorize against.+ return false if Upload.secure_uploads_url?(src)++ local_bases =+ [+ Discourse.base_url,+ Discourse.asset_host,+ SiteSetting.external_emoji_url.presence,+ ].compact.map { |s| ::Chat::MessageHotlinkedMedia.normalize_src(s) }++ if Discourse.store.has_been_uploaded?(src) ||+ ::Chat::MessageHotlinkedMedia.normalize_src(src).start_with?(*local_bases) ||+ src =~ %r{\A/[^/]}i+ return false if src !~ %r{/uploads/}+ # pass nil: chat messages have no access-control post to reuse against+ upload = Upload.consider_for_reuse(Upload.get_from_url(src), nil)+ return !upload.present?+ end++ begin+ uri = URI.parse(src)+ rescue URI::Error+ return false+ end+ return false if uri.hostname.blank?++ SiteSetting.should_download_images?(src)+ end++ def disable_if_low_on_disk_space+ return if Discourse.store.external?+ return if !SiteSetting.download_remote_images_to_local+ return if available_disk_space >= SiteSetting.download_remote_images_threshold++ SiteSetting.download_remote_images_to_local = false+ StaffActionLogger.new(Discourse.system_user).log_site_setting_change(+ "download_remote_images_to_local",+ true,+ false,+ details: I18n.t("disable_remote_images_download_reason"),+ )+ SystemMessage.create_from_system_user(+ Discourse.site_contact_user,+ :download_remote_images_disabled,+ )+ end++ def available_disk_space+ 100 - DiskSpace.percent_free("#{Rails.public_path.join("uploads")}")+ end+ end+ end+enddiff --git a/plugins/chat/app/models/chat/message.rb b/plugins/chat/app/models/chat/message.rbindex 10ce10cf319..b823644f815 100644--- a/plugins/chat/app/models/chat/message.rb+++ b/plugins/chat/app/models/chat/message.rb@@ -63,6 +63,11 @@ module Chatforeign_key: :target_idhas_many :uploads, through: :upload_references, class_name: "::Upload"+ has_many :hotlinked_media,+ dependent: :destroy,+ foreign_key: :chat_message_id,+ class_name: "Chat::MessageHotlinkedMedia"+has_one :chat_webhook_event,dependent: :destroy,class_name: "Chat::WebhookEvent",diff --git a/plugins/chat/app/models/chat/message_hotlinked_media.rb b/plugins/chat/app/models/chat/message_hotlinked_media.rbnew file mode 100644index 00000000000..87e5b55c439--- /dev/null+++ b/plugins/chat/app/models/chat/message_hotlinked_media.rb@@ -0,0 +1,41 @@+# frozen_string_literal: true++module Chat+ class MessageHotlinkedMedia < ActiveRecord::Base+ self.table_name = "chat_message_hotlinked_media"++ belongs_to :chat_message, class_name: "Chat::Message"+ belongs_to :upload, optional: true++ enum :status,+ {+ downloaded: "downloaded",+ too_large: "too_large",+ download_failed: "download_failed",+ upload_create_failed: "upload_create_failed",+ },+ scopes: false++ def self.normalize_src(src, reset_scheme: true)+ PostHotlinkedMedia.normalize_src(src, reset_scheme: reset_scheme)+ end+ end+end++# == Schema Information+#+# Table name: chat_message_hotlinked_media+#+# id :bigint not null, primary key+# status :string not null+# url :string not null+# created_at :datetime not null+# updated_at :datetime not null+# chat_message_id :bigint not null+# upload_id :bigint+#+# Indexes+#+# index_chat_message_hotlinked_media_on_message_and_url_md5 (chat_message_id, md5((url)::text)) UNIQUE+# index_chat_message_hotlinked_media_on_upload_id (upload_id)+#diff --git a/plugins/chat/db/migrate/20260529220000_create_chat_message_hotlinked_media.rb b/plugins/chat/db/migrate/20260529220000_create_chat_message_hotlinked_media.rbnew file mode 100644index 00000000000..aa3c3471540--- /dev/null+++ b/plugins/chat/db/migrate/20260529220000_create_chat_message_hotlinked_media.rb@@ -0,0 +1,22 @@+# frozen_string_literal: true++class CreateChatMessageHotlinkedMedia < ActiveRecord::Migration[8.0]+ def change+ create_table :chat_message_hotlinked_media do |t|+ t.bigint :chat_message_id, null: false+ t.string :url, null: false+ t.string :status, null: false+ t.bigint :upload_id+ t.timestamps+ end++ # url can exceed the btree key limit, so index on its md5 digest.+ add_index :chat_message_hotlinked_media,+ "chat_message_id, md5(url)",+ unique: true,+ name: "index_chat_message_hotlinked_media_on_message_and_url_md5"++ # the upload-in-use hook looks rows up by upload+ add_index :chat_message_hotlinked_media, :upload_id+ end+enddiff --git a/plugins/chat/lib/chat/message_mover.rb b/plugins/chat/lib/chat/message_mover.rbindex 2b29cb533dd..a62c7074187 100644--- a/plugins/chat/lib/chat/message_mover.rb+++ b/plugins/chat/lib/chat/message_mover.rb@@ -214,6 +214,14 @@ module ChatFROM moved_chat_messages mmWHERE cweb.chat_message_id = mm.old_chat_message_idSQL++ # The destination inherits cooked, so keep its localization metadata with it.+ DB.exec(<<~SQL)+ UPDATE chat_message_hotlinked_media chm+ SET chat_message_id = mm.new_chat_message_id+ FROM moved_chat_messages mm+ WHERE chm.chat_message_id = mm.old_chat_message_id+ SQLenddef delete_source_messagesdiff --git a/plugins/chat/lib/chat/message_processor.rb b/plugins/chat/lib/chat/message_processor.rbindex e130d248392..6b2dacef192 100644--- a/plugins/chat/lib/chat/message_processor.rb+++ b/plugins/chat/lib/chat/message_processor.rb@@ -5,12 +5,15 @@ module Chatinclude ::CookedProcessorMixinIMG_FILETYPES = %w[jpg jpeg gif png heic heif webp]+ attr_reader :used_hotlinked_media_ids+def initialize(chat_message, opts = {})@model = chat_message@previous_cooked = (chat_message.cooked || "").dup@should_secure_uploads = false@size_cache = {}@opts = opts+ @used_hotlinked_media_ids = Set.newcook_opts = {user_id: chat_message.last_editor_id,@@ -23,12 +26,34 @@ module Chatdef run!post_process_oneboxes+ process_hotlinked_imagesprocess_thumbnailspost_process_videosadd_lightbox_to_imagesDiscourseEvent.trigger(:chat_message_processed, @doc, @model)end+ # Unlike posts, chat keeps the hotlinked src when a download fails or is too large.+ def process_hotlinked_image(img)+ normalized_src =+ Chat::MessageHotlinkedMedia.normalize_src(HotlinkedMedia.download_src_for(img))+ info = hotlinked_media_map[normalized_src]+ used_hotlinked_media_ids << info.id if info++ if info&.downloaded? && (upload = info.upload)+ img["src"] = UrlHelper.cook_url(upload.url, secure: upload.secure?)+ img["data-base62-sha1"] = upload.base62_sha1+ img["data-dominant-color"] = upload.dominant_color(calculate_if_missing: true).presence+ img.delete(PrettyText::BLOCKED_HOTLINKED_SRC_ATTR)+ end++ true+ end++ def process_hotlinked_images+ extract_images.each { |img| process_hotlinked_image(img) }+ end+def process_thumbnailsreturn if !SiteSetting.create_thumbnails@@ -67,7 +92,7 @@ module Chat.css("img").each do |img|if img["class"]&.include?("emoji") || img["class"]&.include?("avatar") ||- img["data-base62-sha1"].blank?+ img["data-base62-sha1"].blank? || img.ancestors(".onebox, .onebox-body").any?nextend@@ -82,6 +107,10 @@ module Chatendend+ def hotlinked_media_map+ @hotlinked_media_map ||= @model.hotlinked_media.preload(:upload).index_by(&:url)+ end+def large_images[]enddiff --git a/plugins/chat/lib/chat/upload_extension.rb b/plugins/chat/lib/chat/upload_extension.rbnew file mode 100644index 00000000000..606a34a604e--- /dev/null+++ b/plugins/chat/lib/chat/upload_extension.rb@@ -0,0 +1,13 @@+# frozen_string_literal: true++module Chat+ module UploadExtension+ extend ActiveSupport::Concern++ prepended do+ has_many :chat_message_hotlinked_media,+ class_name: "Chat::MessageHotlinkedMedia",+ dependent: :destroy+ end+ end+enddiff --git a/plugins/chat/plugin.rb b/plugins/chat/plugin.rbindex 4baecf7fe20..a212675a1ec 100644--- a/plugins/chat/plugin.rb+++ b/plugins/chat/plugin.rb@@ -178,6 +178,7 @@ after_initialize doCategory.prepend Chat::CategoryExtensionReviewable.prepend Chat::ReviewableExtensionBookmark.prepend Chat::BookmarkExtension+ Upload.prepend Chat::UploadExtensionUser.prepend Chat::UserExtensionGroup.prepend Chat::GroupExtensionPlugin::Instance.prepend Chat::PluginInstanceExtension@@ -203,7 +204,7 @@ after_initialize do"data LIKE ? OR data LIKE ?","%#{upload.sha1}%","%#{upload.base62_sha1}%",- ).exists?+ ).exists? || Chat::MessageHotlinkedMedia.where(upload_id: upload.id).exists?endadd_to_serializer(:user_card, :can_chat_user) dodiff --git a/plugins/chat/spec/jobs/regular/chat/channel_delete_spec.rb b/plugins/chat/spec/jobs/regular/chat/channel_delete_spec.rbindex db438306aae..761e7c9461c 100644--- a/plugins/chat/spec/jobs/regular/chat/channel_delete_spec.rb+++ b/plugins/chat/spec/jobs/regular/chat/channel_delete_spec.rb@@ -43,6 +43,15 @@ describe Jobs::Chat::ChannelDelete donew_message: revision_message.message,)+ 5.times do |i|+ Chat::MessageHotlinkedMedia.create!(+ chat_message: messages.sample,+ url: "//example.com/hotlinked-#{i}.png",+ status: "downloaded",+ upload: Fabricate(:upload, user: users.sample),+ )+ end+Chat::Draft.create(chat_channel: chat_channel, user: users.sample, data: "wow some draft")Fabricate(:user_chat_channel_membership, chat_channel: chat_channel, user: user1)@@ -66,6 +75,7 @@ describe Jobs::Chat::ChannelDelete dotarget_id: @message_ids,target_type: Chat::Message.polymorphic_name,).count,+ hotlinked_media: Chat::MessageHotlinkedMedia.where(chat_message_id: @message_ids).count,messages: Chat::Message.where(id: @message_ids).count,reactions: Chat::MessageReaction.where(chat_message_id: @message_ids).count,}@@ -83,6 +93,7 @@ describe Jobs::Chat::ChannelDelete doexpect(new_counts[:revisions]).to eq(initial_counts[:revisions] - 1)expect(new_counts[:mentions]).to eq(initial_counts[:mentions] - 1)expect(new_counts[:upload_references]).to eq(initial_counts[:upload_references] - 10)+ expect(new_counts[:hotlinked_media]).to eq(initial_counts[:hotlinked_media] - 5)expect(new_counts[:messages]).to eq(initial_counts[:messages] - 20)expect(new_counts[:reactions]).to eq(initial_counts[:reactions] - 10)enddiff --git a/plugins/chat/spec/jobs/regular/chat/process_message_spec.rb b/plugins/chat/spec/jobs/regular/chat/process_message_spec.rbindex ebb1cd05e78..ebe458fcf7f 100644--- a/plugins/chat/spec/jobs/regular/chat/process_message_spec.rb+++ b/plugins/chat/spec/jobs/regular/chat/process_message_spec.rb@@ -70,6 +70,90 @@ describe Jobs::Chat::ProcessMessage doendend+ describe "pull hotlinked images" do+ it "enqueues the pull job when download_remote_images_to_local is enabled" do+ SiteSetting.download_remote_images_to_local = true++ expect_enqueued_with(+ job: Jobs::Chat::PullHotlinkedImages,+ args: {+ chat_message_id: chat_message.id,+ },+ ) { described_class.new.execute(chat_message_id: chat_message.id) }+ end++ it "does not enqueue the pull job when download_remote_images_to_local is disabled" do+ SiteSetting.download_remote_images_to_local = false++ expect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) do+ described_class.new.execute(chat_message_id: chat_message.id)+ end+ end++ it "does not enqueue the pull job when chat uploads are disabled" do+ SiteSetting.download_remote_images_to_local = true+ SiteSetting.chat_allow_uploads = false++ expect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) do+ described_class.new.execute(chat_message_id: chat_message.id)+ end+ end++ # the pull job re-cooks through here, so the enqueue has to happen outside+ # our own mutex or the inline re-cook blocks on the lock we still hold+ it "does not block on its own lock when the pull job runs inline" do+ image_url = "http://wiki.mozilla.org/images/2/2e/Longcat1.gif"+ stub_request(:get, image_url).to_return(+ body:+ Base64.decode64(+ "R0lGODlhAQABALMAAAAAAIAAAACAAICAAAAAgIAAgACAgMDAwICAgP8AAAD/AP//AAAA//8A/wD//wBiZCH5BAEAAA8ALAAAAAABAAEAAAQC8EUAOw==",+ ),+ headers: {+ "Content-Type" => "image/gif",+ },+ )+ stub_image_size+ Jobs.run_immediately!+ SiteSetting.download_remote_images_to_local = true+ SiteSetting.download_remote_images_threshold = 0+ message = Fabricate(:chat_message, message: "")++ Timeout.timeout(10) { described_class.new.execute(chat_message_id: message.id) }++ expect(message.reload.cooked).not_to include(image_url)+ end++ # the pull job re-cooks through here, so honouring the flag is what stops+ # the two jobs enqueueing each other+ it "does not enqueue the pull job when it was the pull job that asked for the re-cook" do+ SiteSetting.download_remote_images_to_local = true++ expect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) do+ described_class.new.execute(+ chat_message_id: chat_message.id,+ skip_pull_hotlinked_images: true,+ )+ end+ end+ end++ it "removes hotlinked media that is no longer present after re-cooking" do+ image_url = "https://example.com/old.png"+ message = Fabricate(:chat_message, message: "")+ record =+ Chat::MessageHotlinkedMedia.create!(+ chat_message: message,+ url: Chat::MessageHotlinkedMedia.normalize_src(image_url),+ status: :downloaded,+ upload: Fabricate(:upload),+ )+ message.update_columns(message: "no image")++ expect { described_class.new.execute(chat_message_id: message.id) }.to change {+ Chat::MessageHotlinkedMedia.exists?(record.id)+ }.from(true).to(false)+ end+describe "skip_notifications" dofab!(:user)fab!(:mentioned_user, :user)diff --git a/plugins/chat/spec/jobs/regular/chat/pull_hotlinked_images_spec.rb b/plugins/chat/spec/jobs/regular/chat/pull_hotlinked_images_spec.rbnew file mode 100644index 00000000000..7b88c76c0e7--- /dev/null+++ b/plugins/chat/spec/jobs/regular/chat/pull_hotlinked_images_spec.rb@@ -0,0 +1,334 @@+# frozen_string_literal: true++describe Jobs::Chat::PullHotlinkedImages do+ let(:image_url) { "http://wiki.mozilla.org/images/2/2e/Longcat1.gif" }+ let(:broken_image_url) { "http://wiki.mozilla.org/images/2/2e/Longcat3.png" }+ let(:gif) do+ Base64.decode64(+ "R0lGODlhAQABALMAAAAAAIAAAACAAICAAAAAgIAAgACAgMDAwICAgP8AAAD/AP//AAAA//8A/wD//wBiZCH5BAEAAA8ALAAAAAABAAEAAAQC8EUAOw==",+ )+ end++ fab!(:user) { Fabricate(:user, refresh_auto_groups: true) }+ fab!(:chat_channel)++ before do+ Jobs.run_immediately!++ stub_request(:get, image_url).to_return(body: gif, headers: { "Content-Type" => "image/gif" })+ stub_request(:get, broken_image_url).to_return(status: 404)++ SiteSetting.download_remote_images_to_local = true+ SiteSetting.max_image_size_kb = 2+ SiteSetting.download_remote_images_threshold = 0+ end++ def fabricate_chat_message(raw)+ Fabricate(:chat_message, chat_channel: chat_channel, user: user, message: raw)+ end++ describe "#execute" do+ it "raises when chat_message_id is missing" do+ expect { described_class.new.execute({}) }.to raise_error(Discourse::InvalidParameters)+ end++ it "does nothing if the message no longer exists" do+ expect { described_class.new.execute(chat_message_id: -1) }.not_to change { Upload.count }+ end++ it "does nothing when the message cannot be recooked" do+ missing_cooked = fabricate_chat_message("placeholder body")+ missing_cooked.update_columns(cooked: nil)+ hand_written = fabricate_chat_message("placeholder")+ hand_written.update_columns(message: "", cooked: "<p><img src=\"#{image_url}\"></p>")++ expect {+ described_class.new.execute(chat_message_id: missing_cooked.id)+ described_class.new.execute(chat_message_id: hand_written.id)+ }.not_to change { Upload.count }+ expect(hand_written.reload.cooked).to include(image_url)+ end++ it "downloads an external image and localizes it in cooked, leaving raw untouched" do+ stub_image_size+ raw = ""+ message = fabricate_chat_message(raw)++ expect { described_class.new.execute(chat_message_id: message.id) }.to change {+ Upload.count+ }.by(1)++ message.reload+ upload = Upload.last+ expect(message.message).to eq(raw)+ expect(message.cooked).to include(upload.url)+ expect(message.cooked).to include("data-base62-sha1=\"#{upload.base62_sha1}\"")+ expect(message.cooked).not_to include(image_url)+ expect(message.upload_references).to be_empty+ end++ it "records terminal failures without changing the message or retrying" do+ raw = ""+ message = fabricate_chat_message(raw)+ original_cooked = message.cooked++ expect { described_class.new.execute(chat_message_id: message.id) }.not_to change {+ Upload.count+ }++ record = message.reload.hotlinked_media.first+ expect(record).to be_present+ expect(record.status).to eq("download_failed")+ expect(message.cooked).to eq(original_cooked)+ expect(message.message).to eq(raw)++ described_class.new.execute(chat_message_id: message.id)++ expect(message.hotlinked_media.count).to eq(1)+ expect(WebMock).to have_requested(:get, broken_image_url).once+ end++ it "classifies an oversize image as :too_large (not :download_failed)" do+ huge = "a" * (SiteSetting.max_image_size_kb * 1024 * 2)+ stub_request(:get, image_url).to_return(+ body: huge,+ headers: {+ "Content-Type" => "image/gif",+ },+ )+ message = fabricate_chat_message("")++ described_class.new.execute(chat_message_id: message.id)++ record = message.reload.hotlinked_media.first+ expect(record).to be_present+ expect(record.status).to eq("too_large")+ end++ it "localizes a re-introduced URL from the cached hotlinked media row without re-downloading" do+ stub_image_size+ first = fabricate_chat_message("first: ")+ described_class.new.execute(chat_message_id: first.id)+ upload = Upload.last++ first.update_columns(+ message: "again: ",+ cooked: "<p><img src=\"#{image_url}\"></p>",+ )++ expect { described_class.new.execute(chat_message_id: first.id) }.not_to change {+ Upload.count+ }+ first.reload+ expect(first.cooked).to include("data-base62-sha1=\"#{upload.base62_sha1}\"")+ expect(first.cooked).not_to include(image_url)+ end++ it "localizes an image that only exists in cooked, such as a onebox thumbnail" do+ stub_image_size+ message = fabricate_chat_message("https://example.com/interesting-page")+ onebox_cooked =+ "<aside class=\"onebox\"><img src=\"#{image_url}\" width=\"100\" height=\"100\"></aside>"+ Chat::Message.stubs(:cook).returns(onebox_cooked)+ message.update_columns(cooked: onebox_cooked)++ expect { described_class.new.execute(chat_message_id: message.id) }.to change {+ Upload.count+ }.by(1)++ message.reload+ upload = Upload.last+ expect(message.message).to eq("https://example.com/interesting-page")+ expect(message.cooked).to include(upload.url)+ expect(message.cooked).not_to include(image_url)+ expect(message.upload_references).to be_empty+ end++ it "does not destroy a pre-existing unreferenced upload the download dedups to" do+ stub_image_size+ first = fabricate_chat_message("first: ")+ described_class.new.execute(chat_message_id: first.id)+ upload = Upload.last++ first.destroy!+ expect(UploadReference.where(upload_id: upload.id)).to be_empty++ second = fabricate_chat_message("second: ")+ described_class.new.execute(chat_message_id: second.id)++ expect(Upload.exists?(upload.id)).to eq(true)+ expect(second.reload.hotlinked_media.first.upload_id).to eq(upload.id)+ expect(second.upload_references).to be_empty+ end++ it "does not re-cook when nothing was downloaded" do+ message = fabricate_chat_message("")+ Jobs.run_later!++ expect_not_enqueued_with(job: ::Jobs::Chat::ProcessMessage) do+ described_class.new.execute(chat_message_id: message.id)+ end+ end++ it "re-cooks with the pull skipped, so processing cannot enqueue us again" do+ stub_image_size+ message = fabricate_chat_message("")+ Jobs.run_later!++ described_class.new.execute(chat_message_id: message.id)++ expect(+ job_enqueued?(+ job: ::Jobs::Chat::ProcessMessage,+ args: {+ chat_message_id: message.id,+ skip_pull_hotlinked_images: true,+ },+ ),+ ).to eq(true)+ end++ it "is a no-op when the setting is disabled" do+ SiteSetting.download_remote_images_to_local = false+ raw = ""+ message = fabricate_chat_message(raw)++ expect { described_class.new.execute(chat_message_id: message.id) }.not_to change {+ Upload.count+ }+ expect(message.reload.message).to eq(raw)+ end++ it "is a no-op when chat uploads are disabled" do+ SiteSetting.chat_allow_uploads = false+ message = fabricate_chat_message("")++ expect { described_class.new.execute(chat_message_id: message.id) }.not_to change {+ Upload.count+ }+ expect(message.reload.hotlinked_media).to be_empty+ expect(WebMock).not_to have_requested(:get, image_url)+ end++ it "skips local /uploads URLs" do+ raw = ""+ message = fabricate_chat_message(raw)++ expect { described_class.new.execute(chat_message_id: message.id) }.not_to change {+ Upload.count+ }+ expect(message.reload.message).to eq(raw)+ end++ context "with secure upload proxy URLs" do+ before do+ setup_s3+ SiteSetting.secure_uploads = true+ stub_request(:get, /s3-upload-bucket.*amazonaws\.com/).to_return(+ body: gif,+ headers: {+ "Content-Type" => "image/gif",+ },+ )+ end++ it "does not rehost secure upload proxy URLs" do+ global_setting :allow_unsecure_chat_uploads, true+ SiteSetting.chat_allow_uploads = true++ # the multisite path is the one the local-upload check would let through+ %w[+ /secure-uploads/original/1X/1234567890abcdef1234567890abcdef12345678.png+ /secure-uploads/uploads/default/original/1X/1234567890abcdef1234567890abcdef12345678.png+ ].each do |path|+ stub_image_size+ message = fabricate_chat_message("")++ expect { described_class.new.execute(chat_message_id: message.id) }.not_to change {+ Upload.count+ }+ expect(message.reload.hotlinked_media).to be_empty+ end+ expect(WebMock).not_to have_requested(:get, /amazonaws\.com/)+ end+ end++ it "does not treat an attacker host with the local host prefix as local" do+ attacker_url = "#{Discourse.base_url}.attacker.example/foo.png"+ stub_request(:get, attacker_url).to_return(+ body: gif,+ headers: {+ "Content-Type" => "image/gif",+ },+ )+ stub_image_size+ message = fabricate_chat_message("")++ expect { described_class.new.execute(chat_message_id: message.id) }.to change {+ Upload.count+ }.by(1)+ end++ it "downloads protocol-relative URLs" do+ relative_url = "//wiki.mozilla.org/images/2/2e/Longcat1.gif"+ stub_image_size+ message = fabricate_chat_message("")++ expect { described_class.new.execute(chat_message_id: message.id) }.to change {+ Upload.count+ }.by(1)+ end++ it "downloads images that block_hotlinked_media moved to a data attribute" do+ stub_image_size+ message = fabricate_chat_message("")+ blocked_cooked = "<p><img #{PrettyText::BLOCKED_HOTLINKED_SRC_ATTR}=\"#{image_url}\"></p>"+ message.update_columns(cooked: blocked_cooked)++ expect { described_class.new.execute(chat_message_id: message.id) }.to change {+ Upload.count+ }.by(1)++ upload = Upload.last+ expect(message.reload.cooked).to include(upload.url)+ expect(message.cooked).not_to include(PrettyText::BLOCKED_HOTLINKED_SRC_ATTR)+ end++ it "ignores blocked non-image candidates that chat cannot localize" do+ stub_image_size+ message = fabricate_chat_message("https://example.com/video-page")+ blocked_cooked =+ "<p><video><source #{PrettyText::BLOCKED_HOTLINKED_SRC_ATTR}=\"#{image_url}\"></video></p>"+ Chat::Message.stubs(:cook).returns(blocked_cooked)+ message.update_columns(cooked: blocked_cooked)++ # two full pull/re-cook cycles: without the img filter, the unused-row+ # sweep in ProcessMessage erases the row each time and the second run+ # downloads the media from the remote host again+ expect {+ 2.times { described_class.new.execute(chat_message_id: message.id) }+ }.not_to change { Upload.count }++ expect(message.reload.hotlinked_media).to be_empty+ expect(WebMock).not_to have_requested(:get, image_url)+ end++ it "destroys tracking rows when the message is destroyed" do+ stub_image_size+ message = fabricate_chat_message("")+ described_class.new.execute(chat_message_id: message.id)+ expect(message.reload.hotlinked_media.count).to eq(1)++ expect { message.destroy! }.to change { Chat::MessageHotlinkedMedia.count }.by(-1)+ end++ it "destroys tracking rows when the upload is destroyed" do+ stub_image_size+ message = fabricate_chat_message("")+ described_class.new.execute(chat_message_id: message.id)+ upload = message.reload.hotlinked_media.first.upload++ expect { upload.destroy! }.to change { Chat::MessageHotlinkedMedia.count }.by(-1)+ end+ end+enddiff --git a/plugins/chat/spec/lib/chat/message_mover_spec.rb b/plugins/chat/spec/lib/chat/message_mover_spec.rbindex 158231d05f3..e9696add1b1 100644--- a/plugins/chat/spec/lib/chat/message_mover_spec.rb+++ b/plugins/chat/spec/lib/chat/message_mover_spec.rb@@ -117,6 +117,13 @@ describe Chat::MessageMover domention = Fabricate(:user_chat_mention, chat_message: message2, user: acting_user)revision = Fabricate(:chat_message_revision, chat_message: message3)webhook_event = Fabricate(:chat_webhook_event, chat_message: message3)+ hotlinked_media =+ Chat::MessageHotlinkedMedia.create!(+ chat_message: message1,+ url: "//example.com/cat.png",+ status: :downloaded,+ upload: Fabricate(:upload),+ )move!moved_messages =@@ -129,6 +136,7 @@ describe Chat::MessageMover doexpect(mention.reload.chat_message_id).to eq(moved_messages.second.id)expect(revision.reload.chat_message_id).to eq(moved_messages.third.id)expect(webhook_event.reload.chat_message_id).to eq(moved_messages.third.id)+ expect(hotlinked_media.reload.chat_message_id).to eq(moved_messages.first.id)endit "does not preserve reply chains using in_reply_to_id" dodiff --git a/plugins/chat/spec/lib/chat/message_processor_spec.rb b/plugins/chat/spec/lib/chat/message_processor_spec.rbindex 60b2e89241e..257b1845eb5 100644--- a/plugins/chat/spec/lib/chat/message_processor_spec.rb+++ b/plugins/chat/spec/lib/chat/message_processor_spec.rb@@ -39,6 +39,88 @@ RSpec.describe Chat::MessageProcessor doendend+ describe "#process_hotlinked_image" do+ fab!(:upload) { Fabricate(:upload, width: 100, height: 100, extension: "gif") }+ let(:image_url) { "http://example.com/image.gif" }++ def create_downloaded_record(chat_message)+ Chat::MessageHotlinkedMedia.create!(+ chat_message: chat_message,+ url: Chat::MessageHotlinkedMedia.normalize_src(image_url),+ status: :downloaded,+ upload: upload,+ )+ end++ it "swaps a downloaded external img src for the local upload" do+ create_downloaded_record(message)+ Chat::Message.stubs(:cook).returns("<p><img src=\"#{image_url}\"></p>")++ processor = described_class.new(message)+ processor.run!++ img = processor.instance_variable_get(:@doc).at_css("img")+ expect(img["src"]).to eq(UrlHelper.cook_url(upload.url, secure: false))+ expect(img["data-base62-sha1"]).to eq(upload.base62_sha1)+ expect(img["class"]).to include("lightbox")+ end++ it "swaps an image inside a onebox without lightboxing it" do+ create_downloaded_record(message)+ Chat::Message.stubs(:cook).returns(+ "<aside class=\"onebox\"><img src=\"#{image_url}\" width=\"100\" height=\"100\"></aside>",+ )++ processor = described_class.new(message)+ processor.run!++ img = processor.instance_variable_get(:@doc).at_css("img")+ expect(img["src"]).to eq(UrlHelper.cook_url(upload.url, secure: false))+ expect(img["class"].to_s).not_to include("lightbox")+ end++ context "with secure uploads enabled" do+ before do+ setup_s3+ SiteSetting.secure_uploads = true+ end++ it "localizes to the secure proxy URL when the upload is secure" do+ secure_upload = Fabricate(:secure_upload_s3, width: 100, height: 100)+ secure_upload.update_columns(dominant_color: "000000")+ Chat::MessageHotlinkedMedia.create!(+ chat_message: message,+ url: Chat::MessageHotlinkedMedia.normalize_src(image_url),+ status: :downloaded,+ upload: secure_upload,+ )+ Chat::Message.stubs(:cook).returns("<p><img src=\"#{image_url}\"></p>")++ processor = described_class.new(message)+ processor.run!++ img = processor.instance_variable_get(:@doc).at_css("img")+ expect(img["src"]).to include("secure-uploads")+ expect(img["src"]).not_to include("s3-upload-bucket")+ end+ end++ it "leaves the hotlinked src for failed or oversized downloads" do+ Chat::MessageHotlinkedMedia.create!(+ chat_message: message,+ url: Chat::MessageHotlinkedMedia.normalize_src(image_url),+ status: :too_large,+ )+ Chat::Message.stubs(:cook).returns("<p><img src=\"#{image_url}\"></p>")++ processor = described_class.new(message)+ processor.run!++ img = processor.instance_variable_get(:@doc).at_css("img")+ expect(img["src"]).to eq(image_url)+ end+ end+describe "#add_lightbox_to_images" dofab!(:upload) { Fabricate(:upload, width: 800, height: 600) }let(:base62) { Upload.base62_sha1(upload.sha1) }diff --git a/plugins/chat/spec/plugin_spec.rb b/plugins/chat/spec/plugin_spec.rbindex acd19843aee..1fa70ddd888 100644--- a/plugins/chat/spec/plugin_spec.rb+++ b/plugins/chat/spec/plugin_spec.rb@@ -66,6 +66,20 @@ describe Chat doexpect(Upload.exists?(id: draft_upload.id)).to eq(true)expect(Upload.exists?(id: unused_upload.id)).to eq(false)end++ it "marks uploads tracked as hotlinked media in use, even while unreferenced" do+ hotlinked_upload = Fabricate(:upload, user: user, created_at: 1.month.ago)+ Chat::MessageHotlinkedMedia.create!(+ chat_message: chat_message,+ url: "//example.com/image.png",+ status: :downloaded,+ upload: hotlinked_upload,+ )++ Jobs::CleanUpUploads.new.execute({})++ expect(Upload.exists?(id: hotlinked_upload.id)).to eq(true)+ endenddescribe "user card serializer extension #can_chat_user" do