Skip to content

A platform for community discussion. Free, open, simple.

DEV: pull hotlinked images in chat messages (#40278)

Renato Atilio0958c6ff5f52
diff --git a/db/structure.sql b/db/structure.sql
index 91501cf0b4b..0d315803a12 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -3057,6 +3057,40 @@ CREATE SEQUENCE public.chat_message_custom_prompts_id_seq
ALTER SEQUENCE public.chat_message_custom_prompts_id_seq OWNED BY public.chat_message_custom_prompts.id;
+--
+-- Name: chat_message_hotlinked_media; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.chat_message_hotlinked_media (
+ id bigint NOT NULL,
+ chat_message_id bigint NOT NULL,
+ url character varying NOT NULL,
+ status character varying NOT NULL,
+ upload_id bigint,
+ created_at timestamp(6) without time zone NOT NULL,
+ updated_at timestamp(6) without time zone NOT NULL
+);
+
+
+--
+-- Name: chat_message_hotlinked_media_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.chat_message_hotlinked_media_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: chat_message_hotlinked_media_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.chat_message_hotlinked_media_id_seq OWNED BY public.chat_message_hotlinked_media.id;
+
+
--
-- Name: chat_message_interactions; Type: TABLE; Schema: public; Owner: -
--
@@ -13096,6 +13130,13 @@ ALTER TABLE ONLY public.chat_message_custom_fields ALTER COLUMN id SET DEFAULT n
ALTER TABLE ONLY public.chat_message_custom_prompts ALTER COLUMN id SET DEFAULT nextval('public.chat_message_custom_prompts_id_seq'::regclass);
+--
+-- Name: chat_message_hotlinked_media id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.chat_message_hotlinked_media ALTER COLUMN id SET DEFAULT nextval('public.chat_message_hotlinked_media_id_seq'::regclass);
+
+
--
-- Name: chat_message_interactions id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -15448,6 +15489,14 @@ ALTER TABLE ONLY public.chat_message_custom_prompts
ADD CONSTRAINT chat_message_custom_prompts_pkey PRIMARY KEY (id);
+--
+-- Name: chat_message_hotlinked_media chat_message_hotlinked_media_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.chat_message_hotlinked_media
+ ADD CONSTRAINT chat_message_hotlinked_media_pkey PRIMARY KEY (id);
+
+
--
-- Name: chat_message_interactions chat_message_interactions_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -19359,6 +19408,20 @@ CREATE UNIQUE INDEX index_chat_message_custom_fields_on_message_id_and_name ON p
CREATE UNIQUE INDEX index_chat_message_custom_prompts_on_message_id ON public.chat_message_custom_prompts USING btree (message_id);
+--
+-- Name: index_chat_message_hotlinked_media_on_message_and_url_md5; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX index_chat_message_hotlinked_media_on_message_and_url_md5 ON public.chat_message_hotlinked_media USING btree (chat_message_id, md5((url)::text));
+
+
+--
+-- Name: index_chat_message_hotlinked_media_on_upload_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_chat_message_hotlinked_media_on_upload_id ON public.chat_message_hotlinked_media USING btree (upload_id);
+
+
--
-- Name: index_chat_message_interactions_on_chat_message_id; Type: INDEX; Schema: public; Owner: -
--
@@ -23477,6 +23540,7 @@ INSERT INTO "schema_migrations" (version) VALUES
('20260602104726'),
('20260601063855'),
('20260601043020'),
+('20260529220000'),
('20260528074731'),
('20260528074719'),
('20260525105009'),
diff --git a/plugins/chat/app/jobs/regular/chat/channel_delete.rb b/plugins/chat/app/jobs/regular/chat/channel_delete.rb
index 1c47cb99078..b417ae16439 100644
--- a/plugins/chat/app/jobs/regular/chat/channel_delete.rb
+++ b/plugins/chat/app/jobs/regular/chat/channel_delete.rb
@@ -49,6 +49,10 @@ module Jobs
target_type: ::Chat::Message.polymorphic_name,
).delete_all
+ # these rows keep hotlinked uploads flagged as in use, so leaving
+ # them on trashed messages would pin the uploads forever
+ ::Chat::MessageHotlinkedMedia.where(chat_message_id: message_ids).delete_all
+
# only the messages and the channel are Trashable, everything else gets
# permanently destroyed
chat_messages.update_all(
diff --git a/plugins/chat/app/jobs/regular/chat/process_message.rb b/plugins/chat/app/jobs/regular/chat/process_message.rb
index fc2ce806776..22081111650 100644
--- a/plugins/chat/app/jobs/regular/chat/process_message.rb
+++ b/plugins/chat/app/jobs/regular/chat/process_message.rb
@@ -28,6 +28,11 @@ module Jobs
)
end
+ chat_message
+ .hotlinked_media
+ .where.not(id: processor.used_hotlinked_media_ids.to_a)
+ .destroy_all
+
# we don't process mentions when creating/updating message so we always have to do it
chat_message.upsert_mentions
@@ -44,6 +49,13 @@ module Jobs
::Chat::Publisher.publish_processed!(chat_message)
end
+
+ # outside the mutex: the pull job re-cooks through this job, and would
+ # block on the lock we still hold whenever jobs run inline
+ if !args[:skip_pull_hotlinked_images] && SiteSetting.download_remote_images_to_local? &&
+ SiteSetting.chat_allow_uploads?
+ ::Jobs.enqueue(::Jobs::Chat::PullHotlinkedImages, chat_message_id: args[:chat_message_id])
+ end
end
end
end
diff --git a/plugins/chat/app/jobs/regular/chat/pull_hotlinked_images.rb b/plugins/chat/app/jobs/regular/chat/pull_hotlinked_images.rb
new file mode 100644
index 00000000000..db2fcb46bba
--- /dev/null
+++ b/plugins/chat/app/jobs/regular/chat/pull_hotlinked_images.rb
@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Jobs
+ module Chat
+ class PullHotlinkedImages < ::Jobs::Base
+ sidekiq_options queue: "low"
+
+ def execute(args)
+ @chat_message_id = args[:chat_message_id]
+ raise Discourse::InvalidParameters.new(:chat_message_id) if @chat_message_id.blank?
+ return if !SiteSetting.chat_allow_uploads?
+
+ disable_if_low_on_disk_space
+
+ if Jobs.run_immediately?
+ pull
+ else
+ DistributedMutex.synchronize(
+ "chat_pull_hotlinked_images_#{@chat_message_id}",
+ validity: 2.minutes,
+ ) { pull }
+ end
+ end
+
+ private
+
+ def pull
+ chat_message = ::Chat::Message.find_by(id: @chat_message_id)
+ return if chat_message.nil? || chat_message.cooked.blank?
+ # Don't re-cook system/webhook messages whose cooked is hand-written.
+ return if chat_message.message.blank?
+
+ hotlinked_map = chat_message.hotlinked_media.preload(:upload).index_by(&:url)
+ needs_recook = false
+
+ HotlinkedMedia
+ .extract_candidates(chat_message.cooked)
+ .each do |node|
+ # chat only localizes imgs; downloading other media would loop
+ # forever, as the unused-row sweep erases the download history
+ next if node.name != "img"
+
+ download_src = HotlinkedMedia.download_src_for(node)
+ next if !should_download?(download_src)
+
+ normalized_src = ::Chat::MessageHotlinkedMedia.normalize_src(download_src)
+ if (existing = hotlinked_map[normalized_src])
+ # still external despite a downloaded row: the localizing re-cook
+ # was lost — trigger it again; terminal failures aren't retried
+ needs_recook = true if existing.downloaded? && existing.upload
+ next
+ end
+
+ status, upload =
+ HotlinkedMedia.download(
+ download_src,
+ chat_message.last_editor_id,
+ tmp_file_name: "chat-hotlinked",
+ )
+ record = upsert_record(chat_message, normalized_src, status, upload)
+ hotlinked_map[normalized_src] = record if record
+ needs_recook = true if upload && record&.downloaded?
+ rescue => e
+ raise e if Rails.env.test?
+ Discourse.warn_exception(
+ e,
+ message:
+ "Failed to pull hotlinked image (#{download_src}) for chat message #{@chat_message_id}",
+ )
+ end
+
+ return if !needs_recook
+
+ # the re-cook localizes the images; skip_pull prevents an enqueue loop
+ ::Jobs.enqueue(
+ ::Jobs::Chat::ProcessMessage,
+ chat_message_id: chat_message.id,
+ skip_pull_hotlinked_images: true,
+ skip_notifications: true,
+ )
+ end
+
+ def upsert_record(chat_message, normalized_src, status, upload)
+ DB.exec(
+ <<~SQL,
+ INSERT INTO chat_message_hotlinked_media (chat_message_id, url, status, upload_id, created_at, updated_at)
+ VALUES (:chat_message_id, :url, :status, :upload_id, NOW(), NOW())
+ ON CONFLICT (chat_message_id, md5(url)) DO NOTHING
+ SQL
+ chat_message_id: chat_message.id,
+ url: normalized_src,
+ status: status.to_s,
+ upload_id: upload&.id,
+ )
+ ::Chat::MessageHotlinkedMedia.find_by(chat_message_id: chat_message.id, url: normalized_src)
+ end
+
+ def should_download?(src)
+ return false if src.blank?
+ return false if !SiteSetting.download_remote_images_to_local?
+ return false if !SiteSetting.chat_allow_uploads?
+ return false if src.start_with?("data:")
+ # Downloading these means signing an S3 path for the underlying upload.
+ # Posts gate that on the author seeing the access-control post; chat
+ # messages have none, so there is nothing to authorize against.
+ return false if Upload.secure_uploads_url?(src)
+
+ local_bases =
+ [
+ Discourse.base_url,
+ Discourse.asset_host,
+ SiteSetting.external_emoji_url.presence,
+ ].compact.map { |s| ::Chat::MessageHotlinkedMedia.normalize_src(s) }
+
+ if Discourse.store.has_been_uploaded?(src) ||
+ ::Chat::MessageHotlinkedMedia.normalize_src(src).start_with?(*local_bases) ||
+ src =~ %r{\A/[^/]}i
+ return false if src !~ %r{/uploads/}
+ # pass nil: chat messages have no access-control post to reuse against
+ upload = Upload.consider_for_reuse(Upload.get_from_url(src), nil)
+ return !upload.present?
+ end
+
+ begin
+ uri = URI.parse(src)
+ rescue URI::Error
+ return false
+ end
+ return false if uri.hostname.blank?
+
+ SiteSetting.should_download_images?(src)
+ end
+
+ def disable_if_low_on_disk_space
+ return if Discourse.store.external?
+ return if !SiteSetting.download_remote_images_to_local
+ return if available_disk_space >= SiteSetting.download_remote_images_threshold
+
+ SiteSetting.download_remote_images_to_local = false
+ StaffActionLogger.new(Discourse.system_user).log_site_setting_change(
+ "download_remote_images_to_local",
+ true,
+ false,
+ details: I18n.t("disable_remote_images_download_reason"),
+ )
+ SystemMessage.create_from_system_user(
+ Discourse.site_contact_user,
+ :download_remote_images_disabled,
+ )
+ end
+
+ def available_disk_space
+ 100 - DiskSpace.percent_free("#{Rails.public_path.join("uploads")}")
+ end
+ end
+ end
+end
diff --git a/plugins/chat/app/models/chat/message.rb b/plugins/chat/app/models/chat/message.rb
index 10ce10cf319..b823644f815 100644
--- a/plugins/chat/app/models/chat/message.rb
+++ b/plugins/chat/app/models/chat/message.rb
@@ -63,6 +63,11 @@ module Chat
foreign_key: :target_id
has_many :uploads, through: :upload_references, class_name: "::Upload"
+ has_many :hotlinked_media,
+ dependent: :destroy,
+ foreign_key: :chat_message_id,
+ class_name: "Chat::MessageHotlinkedMedia"
+
has_one :chat_webhook_event,
dependent: :destroy,
class_name: "Chat::WebhookEvent",
diff --git a/plugins/chat/app/models/chat/message_hotlinked_media.rb b/plugins/chat/app/models/chat/message_hotlinked_media.rb
new file mode 100644
index 00000000000..87e5b55c439
--- /dev/null
+++ b/plugins/chat/app/models/chat/message_hotlinked_media.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Chat
+ class MessageHotlinkedMedia < ActiveRecord::Base
+ self.table_name = "chat_message_hotlinked_media"
+
+ belongs_to :chat_message, class_name: "Chat::Message"
+ belongs_to :upload, optional: true
+
+ enum :status,
+ {
+ downloaded: "downloaded",
+ too_large: "too_large",
+ download_failed: "download_failed",
+ upload_create_failed: "upload_create_failed",
+ },
+ scopes: false
+
+ def self.normalize_src(src, reset_scheme: true)
+ PostHotlinkedMedia.normalize_src(src, reset_scheme: reset_scheme)
+ end
+ end
+end
+
+# == Schema Information
+#
+# Table name: chat_message_hotlinked_media
+#
+# id :bigint not null, primary key
+# status :string not null
+# url :string not null
+# created_at :datetime not null
+# updated_at :datetime not null
+# chat_message_id :bigint not null
+# upload_id :bigint
+#
+# Indexes
+#
+# index_chat_message_hotlinked_media_on_message_and_url_md5 (chat_message_id, md5((url)::text)) UNIQUE
+# index_chat_message_hotlinked_media_on_upload_id (upload_id)
+#
diff --git a/plugins/chat/db/migrate/20260529220000_create_chat_message_hotlinked_media.rb b/plugins/chat/db/migrate/20260529220000_create_chat_message_hotlinked_media.rb
new file mode 100644
index 00000000000..aa3c3471540
--- /dev/null
+++ b/plugins/chat/db/migrate/20260529220000_create_chat_message_hotlinked_media.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class CreateChatMessageHotlinkedMedia < ActiveRecord::Migration[8.0]
+ def change
+ create_table :chat_message_hotlinked_media do |t|
+ t.bigint :chat_message_id, null: false
+ t.string :url, null: false
+ t.string :status, null: false
+ t.bigint :upload_id
+ t.timestamps
+ end
+
+ # url can exceed the btree key limit, so index on its md5 digest.
+ add_index :chat_message_hotlinked_media,
+ "chat_message_id, md5(url)",
+ unique: true,
+ name: "index_chat_message_hotlinked_media_on_message_and_url_md5"
+
+ # the upload-in-use hook looks rows up by upload
+ add_index :chat_message_hotlinked_media, :upload_id
+ end
+end
diff --git a/plugins/chat/lib/chat/message_mover.rb b/plugins/chat/lib/chat/message_mover.rb
index 2b29cb533dd..a62c7074187 100644
--- a/plugins/chat/lib/chat/message_mover.rb
+++ b/plugins/chat/lib/chat/message_mover.rb
@@ -214,6 +214,14 @@ module Chat
FROM moved_chat_messages mm
WHERE cweb.chat_message_id = mm.old_chat_message_id
SQL
+
+ # The destination inherits cooked, so keep its localization metadata with it.
+ DB.exec(<<~SQL)
+ UPDATE chat_message_hotlinked_media chm
+ SET chat_message_id = mm.new_chat_message_id
+ FROM moved_chat_messages mm
+ WHERE chm.chat_message_id = mm.old_chat_message_id
+ SQL
end
def delete_source_messages
diff --git a/plugins/chat/lib/chat/message_processor.rb b/plugins/chat/lib/chat/message_processor.rb
index e130d248392..6b2dacef192 100644
--- a/plugins/chat/lib/chat/message_processor.rb
+++ b/plugins/chat/lib/chat/message_processor.rb
@@ -5,12 +5,15 @@ module Chat
include ::CookedProcessorMixin
IMG_FILETYPES = %w[jpg jpeg gif png heic heif webp]
+ attr_reader :used_hotlinked_media_ids
+
def initialize(chat_message, opts = {})
@model = chat_message
@previous_cooked = (chat_message.cooked || "").dup
@should_secure_uploads = false
@size_cache = {}
@opts = opts
+ @used_hotlinked_media_ids = Set.new
cook_opts = {
user_id: chat_message.last_editor_id,
@@ -23,12 +26,34 @@ module Chat
def run!
post_process_oneboxes
+ process_hotlinked_images
process_thumbnails
post_process_videos
add_lightbox_to_images
DiscourseEvent.trigger(:chat_message_processed, @doc, @model)
end
+ # Unlike posts, chat keeps the hotlinked src when a download fails or is too large.
+ def process_hotlinked_image(img)
+ normalized_src =
+ Chat::MessageHotlinkedMedia.normalize_src(HotlinkedMedia.download_src_for(img))
+ info = hotlinked_media_map[normalized_src]
+ used_hotlinked_media_ids << info.id if info
+
+ if info&.downloaded? && (upload = info.upload)
+ img["src"] = UrlHelper.cook_url(upload.url, secure: upload.secure?)
+ img["data-base62-sha1"] = upload.base62_sha1
+ img["data-dominant-color"] = upload.dominant_color(calculate_if_missing: true).presence
+ img.delete(PrettyText::BLOCKED_HOTLINKED_SRC_ATTR)
+ end
+
+ true
+ end
+
+ def process_hotlinked_images
+ extract_images.each { |img| process_hotlinked_image(img) }
+ end
+
def process_thumbnails
return if !SiteSetting.create_thumbnails
@@ -67,7 +92,7 @@ module Chat
.css("img")
.each do |img|
if img["class"]&.include?("emoji") || img["class"]&.include?("avatar") ||
- img["data-base62-sha1"].blank?
+ img["data-base62-sha1"].blank? || img.ancestors(".onebox, .onebox-body").any?
next
end
@@ -82,6 +107,10 @@ module Chat
end
end
+ def hotlinked_media_map
+ @hotlinked_media_map ||= @model.hotlinked_media.preload(:upload).index_by(&:url)
+ end
+
def large_images
[]
end
diff --git a/plugins/chat/lib/chat/upload_extension.rb b/plugins/chat/lib/chat/upload_extension.rb
new file mode 100644
index 00000000000..606a34a604e
--- /dev/null
+++ b/plugins/chat/lib/chat/upload_extension.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Chat
+ module UploadExtension
+ extend ActiveSupport::Concern
+
+ prepended do
+ has_many :chat_message_hotlinked_media,
+ class_name: "Chat::MessageHotlinkedMedia",
+ dependent: :destroy
+ end
+ end
+end
diff --git a/plugins/chat/plugin.rb b/plugins/chat/plugin.rb
index 4baecf7fe20..a212675a1ec 100644
--- a/plugins/chat/plugin.rb
+++ b/plugins/chat/plugin.rb
@@ -178,6 +178,7 @@ after_initialize do
Category.prepend Chat::CategoryExtension
Reviewable.prepend Chat::ReviewableExtension
Bookmark.prepend Chat::BookmarkExtension
+ Upload.prepend Chat::UploadExtension
User.prepend Chat::UserExtension
Group.prepend Chat::GroupExtension
Plugin::Instance.prepend Chat::PluginInstanceExtension
@@ -203,7 +204,7 @@ after_initialize do
"data LIKE ? OR data LIKE ?",
"%#{upload.sha1}%",
"%#{upload.base62_sha1}%",
- ).exists?
+ ).exists? || Chat::MessageHotlinkedMedia.where(upload_id: upload.id).exists?
end
add_to_serializer(:user_card, :can_chat_user) do
diff --git a/plugins/chat/spec/jobs/regular/chat/channel_delete_spec.rb b/plugins/chat/spec/jobs/regular/chat/channel_delete_spec.rb
index db438306aae..761e7c9461c 100644
--- a/plugins/chat/spec/jobs/regular/chat/channel_delete_spec.rb
+++ b/plugins/chat/spec/jobs/regular/chat/channel_delete_spec.rb
@@ -43,6 +43,15 @@ describe Jobs::Chat::ChannelDelete do
new_message: revision_message.message,
)
+ 5.times do |i|
+ Chat::MessageHotlinkedMedia.create!(
+ chat_message: messages.sample,
+ url: "//example.com/hotlinked-#{i}.png",
+ status: "downloaded",
+ upload: Fabricate(:upload, user: users.sample),
+ )
+ end
+
Chat::Draft.create(chat_channel: chat_channel, user: users.sample, data: "wow some draft")
Fabricate(:user_chat_channel_membership, chat_channel: chat_channel, user: user1)
@@ -66,6 +75,7 @@ describe Jobs::Chat::ChannelDelete do
target_id: @message_ids,
target_type: Chat::Message.polymorphic_name,
).count,
+ hotlinked_media: Chat::MessageHotlinkedMedia.where(chat_message_id: @message_ids).count,
messages: Chat::Message.where(id: @message_ids).count,
reactions: Chat::MessageReaction.where(chat_message_id: @message_ids).count,
}
@@ -83,6 +93,7 @@ describe Jobs::Chat::ChannelDelete do
expect(new_counts[:revisions]).to eq(initial_counts[:revisions] - 1)
expect(new_counts[:mentions]).to eq(initial_counts[:mentions] - 1)
expect(new_counts[:upload_references]).to eq(initial_counts[:upload_references] - 10)
+ expect(new_counts[:hotlinked_media]).to eq(initial_counts[:hotlinked_media] - 5)
expect(new_counts[:messages]).to eq(initial_counts[:messages] - 20)
expect(new_counts[:reactions]).to eq(initial_counts[:reactions] - 10)
end
diff --git a/plugins/chat/spec/jobs/regular/chat/process_message_spec.rb b/plugins/chat/spec/jobs/regular/chat/process_message_spec.rb
index ebb1cd05e78..ebe458fcf7f 100644
--- a/plugins/chat/spec/jobs/regular/chat/process_message_spec.rb
+++ b/plugins/chat/spec/jobs/regular/chat/process_message_spec.rb
@@ -70,6 +70,90 @@ describe Jobs::Chat::ProcessMessage do
end
end
+ describe "pull hotlinked images" do
+ it "enqueues the pull job when download_remote_images_to_local is enabled" do
+ SiteSetting.download_remote_images_to_local = true
+
+ expect_enqueued_with(
+ job: Jobs::Chat::PullHotlinkedImages,
+ args: {
+ chat_message_id: chat_message.id,
+ },
+ ) { described_class.new.execute(chat_message_id: chat_message.id) }
+ end
+
+ it "does not enqueue the pull job when download_remote_images_to_local is disabled" do
+ SiteSetting.download_remote_images_to_local = false
+
+ expect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) do
+ described_class.new.execute(chat_message_id: chat_message.id)
+ end
+ end
+
+ it "does not enqueue the pull job when chat uploads are disabled" do
+ SiteSetting.download_remote_images_to_local = true
+ SiteSetting.chat_allow_uploads = false
+
+ expect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) do
+ described_class.new.execute(chat_message_id: chat_message.id)
+ end
+ end
+
+ # the pull job re-cooks through here, so the enqueue has to happen outside
+ # our own mutex or the inline re-cook blocks on the lock we still hold
+ it "does not block on its own lock when the pull job runs inline" do
+ image_url = "http://wiki.mozilla.org/images/2/2e/Longcat1.gif"
+ stub_request(:get, image_url).to_return(
+ body:
+ Base64.decode64(
+ "R0lGODlhAQABALMAAAAAAIAAAACAAICAAAAAgIAAgACAgMDAwICAgP8AAAD/AP//AAAA//8A/wD//wBiZCH5BAEAAA8ALAAAAAABAAEAAAQC8EUAOw==",
+ ),
+ headers: {
+ "Content-Type" => "image/gif",
+ },
+ )
+ stub_image_size
+ Jobs.run_immediately!
+ SiteSetting.download_remote_images_to_local = true
+ SiteSetting.download_remote_images_threshold = 0
+ message = Fabricate(:chat_message, message: "![longcat](#{image_url})")
+
+ Timeout.timeout(10) { described_class.new.execute(chat_message_id: message.id) }
+
+ expect(message.reload.cooked).not_to include(image_url)
+ end
+
+ # the pull job re-cooks through here, so honouring the flag is what stops
+ # the two jobs enqueueing each other
+ it "does not enqueue the pull job when it was the pull job that asked for the re-cook" do
+ SiteSetting.download_remote_images_to_local = true
+
+ expect_not_enqueued_with(job: Jobs::Chat::PullHotlinkedImages) do
+ described_class.new.execute(
+ chat_message_id: chat_message.id,
+ skip_pull_hotlinked_images: true,
+ )
+ end
+ end
+ end
+
+ it "removes hotlinked media that is no longer present after re-cooking" do
+ image_url = "https://example.com/old.png"
+ message = Fabricate(:chat_message, message: "![](#{image_url})")
+ record =
+ Chat::MessageHotlinkedMedia.create!(
+ chat_message: message,
+ url: Chat::MessageHotlinkedMedia.normalize_src(image_url),
+ status: :downloaded,
+ upload: Fabricate(:upload),
+ )
+ message.update_columns(message: "no image")
+
+ expect { described_class.new.execute(chat_message_id: message.id) }.to change {
+ Chat::MessageHotlinkedMedia.exists?(record.id)
+ }.from(true).to(false)
+ end
+
describe "skip_notifications" do
fab!(:user)
fab!(:mentioned_user, :user)
diff --git a/plugins/chat/spec/jobs/regular/chat/pull_hotlinked_images_spec.rb b/plugins/chat/spec/jobs/regular/chat/pull_hotlinked_images_spec.rb
new file mode 100644
index 00000000000..7b88c76c0e7
--- /dev/null
+++ b/plugins/chat/spec/jobs/regular/chat/pull_hotlinked_images_spec.rb
@@ -0,0 +1,334 @@
+# frozen_string_literal: true
+
+describe Jobs::Chat::PullHotlinkedImages do
+ let(:image_url) { "http://wiki.mozilla.org/images/2/2e/Longcat1.gif" }
+ let(:broken_image_url) { "http://wiki.mozilla.org/images/2/2e/Longcat3.png" }
+ let(:gif) do
+ Base64.decode64(
+ "R0lGODlhAQABALMAAAAAAIAAAACAAICAAAAAgIAAgACAgMDAwICAgP8AAAD/AP//AAAA//8A/wD//wBiZCH5BAEAAA8ALAAAAAABAAEAAAQC8EUAOw==",
+ )
+ end
+
+ fab!(:user) { Fabricate(:user, refresh_auto_groups: true) }
+ fab!(:chat_channel)
+
+ before do
+ Jobs.run_immediately!
+
+ stub_request(:get, image_url).to_return(body: gif, headers: { "Content-Type" => "image/gif" })
+ stub_request(:get, broken_image_url).to_return(status: 404)
+
+ SiteSetting.download_remote_images_to_local = true
+ SiteSetting.max_image_size_kb = 2
+ SiteSetting.download_remote_images_threshold = 0
+ end
+
+ def fabricate_chat_message(raw)
+ Fabricate(:chat_message, chat_channel: chat_channel, user: user, message: raw)
+ end
+
+ describe "#execute" do
+ it "raises when chat_message_id is missing" do
+ expect { described_class.new.execute({}) }.to raise_error(Discourse::InvalidParameters)
+ end
+
+ it "does nothing if the message no longer exists" do
+ expect { described_class.new.execute(chat_message_id: -1) }.not_to change { Upload.count }
+ end
+
+ it "does nothing when the message cannot be recooked" do
+ missing_cooked = fabricate_chat_message("placeholder body")
+ missing_cooked.update_columns(cooked: nil)
+ hand_written = fabricate_chat_message("placeholder")
+ hand_written.update_columns(message: "", cooked: "<p><img src=\"#{image_url}\"></p>")
+
+ expect {
+ described_class.new.execute(chat_message_id: missing_cooked.id)
+ described_class.new.execute(chat_message_id: hand_written.id)
+ }.not_to change { Upload.count }
+ expect(hand_written.reload.cooked).to include(image_url)
+ end
+
+ it "downloads an external image and localizes it in cooked, leaving raw untouched" do
+ stub_image_size
+ raw = "![longcat](#{image_url})"
+ message = fabricate_chat_message(raw)
+
+ expect { described_class.new.execute(chat_message_id: message.id) }.to change {
+ Upload.count
+ }.by(1)
+
+ message.reload
+ upload = Upload.last
+ expect(message.message).to eq(raw)
+ expect(message.cooked).to include(upload.url)
+ expect(message.cooked).to include("data-base62-sha1=\"#{upload.base62_sha1}\"")
+ expect(message.cooked).not_to include(image_url)
+ expect(message.upload_references).to be_empty
+ end
+
+ it "records terminal failures without changing the message or retrying" do
+ raw = "![broken](#{broken_image_url})"
+ message = fabricate_chat_message(raw)
+ original_cooked = message.cooked
+
+ expect { described_class.new.execute(chat_message_id: message.id) }.not_to change {
+ Upload.count
+ }
+
+ record = message.reload.hotlinked_media.first
+ expect(record).to be_present
+ expect(record.status).to eq("download_failed")
+ expect(message.cooked).to eq(original_cooked)
+ expect(message.message).to eq(raw)
+
+ described_class.new.execute(chat_message_id: message.id)
+
+ expect(message.hotlinked_media.count).to eq(1)
+ expect(WebMock).to have_requested(:get, broken_image_url).once
+ end
+
+ it "classifies an oversize image as :too_large (not :download_failed)" do
+ huge = "a" * (SiteSetting.max_image_size_kb * 1024 * 2)
+ stub_request(:get, image_url).to_return(
+ body: huge,
+ headers: {
+ "Content-Type" => "image/gif",
+ },
+ )
+ message = fabricate_chat_message("![big](#{image_url})")
+
+ described_class.new.execute(chat_message_id: message.id)
+
+ record = message.reload.hotlinked_media.first
+ expect(record).to be_present
+ expect(record.status).to eq("too_large")
+ end
+
+ it "localizes a re-introduced URL from the cached hotlinked media row without re-downloading" do
+ stub_image_size
+ first = fabricate_chat_message("first: ![](#{image_url})")
+ described_class.new.execute(chat_message_id: first.id)
+ upload = Upload.last
+
+ first.update_columns(
+ message: "again: ![](#{image_url})",
+ cooked: "<p><img src=\"#{image_url}\"></p>",
+ )
+
+ expect { described_class.new.execute(chat_message_id: first.id) }.not_to change {
+ Upload.count
+ }
+ first.reload
+ expect(first.cooked).to include("data-base62-sha1=\"#{upload.base62_sha1}\"")
+ expect(first.cooked).not_to include(image_url)
+ end
+
+ it "localizes an image that only exists in cooked, such as a onebox thumbnail" do
+ stub_image_size
+ message = fabricate_chat_message("https://example.com/interesting-page")
+ onebox_cooked =
+ "<aside class=\"onebox\"><img src=\"#{image_url}\" width=\"100\" height=\"100\"></aside>"
+ Chat::Message.stubs(:cook).returns(onebox_cooked)
+ message.update_columns(cooked: onebox_cooked)
+
+ expect { described_class.new.execute(chat_message_id: message.id) }.to change {
+ Upload.count
+ }.by(1)
+
+ message.reload
+ upload = Upload.last
+ expect(message.message).to eq("https://example.com/interesting-page")
+ expect(message.cooked).to include(upload.url)
+ expect(message.cooked).not_to include(image_url)
+ expect(message.upload_references).to be_empty
+ end
+
+ it "does not destroy a pre-existing unreferenced upload the download dedups to" do
+ stub_image_size
+ first = fabricate_chat_message("first: ![](#{image_url})")
+ described_class.new.execute(chat_message_id: first.id)
+ upload = Upload.last
+
+ first.destroy!
+ expect(UploadReference.where(upload_id: upload.id)).to be_empty
+
+ second = fabricate_chat_message("second: ![](#{image_url})")
+ described_class.new.execute(chat_message_id: second.id)
+
+ expect(Upload.exists?(upload.id)).to eq(true)
+ expect(second.reload.hotlinked_media.first.upload_id).to eq(upload.id)
+ expect(second.upload_references).to be_empty
+ end
+
+ it "does not re-cook when nothing was downloaded" do
+ message = fabricate_chat_message("![broken](#{broken_image_url})")
+ Jobs.run_later!
+
+ expect_not_enqueued_with(job: ::Jobs::Chat::ProcessMessage) do
+ described_class.new.execute(chat_message_id: message.id)
+ end
+ end
+
+ it "re-cooks with the pull skipped, so processing cannot enqueue us again" do
+ stub_image_size
+ message = fabricate_chat_message("![longcat](#{image_url})")
+ Jobs.run_later!
+
+ described_class.new.execute(chat_message_id: message.id)
+
+ expect(
+ job_enqueued?(
+ job: ::Jobs::Chat::ProcessMessage,
+ args: {
+ chat_message_id: message.id,
+ skip_pull_hotlinked_images: true,
+ },
+ ),
+ ).to eq(true)
+ end
+
+ it "is a no-op when the setting is disabled" do
+ SiteSetting.download_remote_images_to_local = false
+ raw = "![longcat](#{image_url})"
+ message = fabricate_chat_message(raw)
+
+ expect { described_class.new.execute(chat_message_id: message.id) }.not_to change {
+ Upload.count
+ }
+ expect(message.reload.message).to eq(raw)
+ end
+
+ it "is a no-op when chat uploads are disabled" do
+ SiteSetting.chat_allow_uploads = false
+ message = fabricate_chat_message("![longcat](#{image_url})")
+
+ expect { described_class.new.execute(chat_message_id: message.id) }.not_to change {
+ Upload.count
+ }
+ expect(message.reload.hotlinked_media).to be_empty
+ expect(WebMock).not_to have_requested(:get, image_url)
+ end
+
+ it "skips local /uploads URLs" do
+ raw = "![local](/uploads/default/local.png)"
+ message = fabricate_chat_message(raw)
+
+ expect { described_class.new.execute(chat_message_id: message.id) }.not_to change {
+ Upload.count
+ }
+ expect(message.reload.message).to eq(raw)
+ end
+
+ context "with secure upload proxy URLs" do
+ before do
+ setup_s3
+ SiteSetting.secure_uploads = true
+ stub_request(:get, /s3-upload-bucket.*amazonaws\.com/).to_return(
+ body: gif,
+ headers: {
+ "Content-Type" => "image/gif",
+ },
+ )
+ end
+
+ it "does not rehost secure upload proxy URLs" do
+ global_setting :allow_unsecure_chat_uploads, true
+ SiteSetting.chat_allow_uploads = true
+
+ # the multisite path is the one the local-upload check would let through
+ %w[
+ /secure-uploads/original/1X/1234567890abcdef1234567890abcdef12345678.png
+ /secure-uploads/uploads/default/original/1X/1234567890abcdef1234567890abcdef12345678.png
+ ].each do |path|
+ stub_image_size
+ message = fabricate_chat_message("![](#{Discourse.base_url}#{path})")
+
+ expect { described_class.new.execute(chat_message_id: message.id) }.not_to change {
+ Upload.count
+ }
+ expect(message.reload.hotlinked_media).to be_empty
+ end
+ expect(WebMock).not_to have_requested(:get, /amazonaws\.com/)
+ end
+ end
+
+ it "does not treat an attacker host with the local host prefix as local" do
+ attacker_url = "#{Discourse.base_url}.attacker.example/foo.png"
+ stub_request(:get, attacker_url).to_return(
+ body: gif,
+ headers: {
+ "Content-Type" => "image/gif",
+ },
+ )
+ stub_image_size
+ message = fabricate_chat_message("![](#{attacker_url})")
+
+ expect { described_class.new.execute(chat_message_id: message.id) }.to change {
+ Upload.count
+ }.by(1)
+ end
+
+ it "downloads protocol-relative URLs" do
+ relative_url = "//wiki.mozilla.org/images/2/2e/Longcat1.gif"
+ stub_image_size
+ message = fabricate_chat_message("![](#{relative_url})")
+
+ expect { described_class.new.execute(chat_message_id: message.id) }.to change {
+ Upload.count
+ }.by(1)
+ end
+
+ it "downloads images that block_hotlinked_media moved to a data attribute" do
+ stub_image_size
+ message = fabricate_chat_message("![](#{image_url})")
+ blocked_cooked = "<p><img #{PrettyText::BLOCKED_HOTLINKED_SRC_ATTR}=\"#{image_url}\"></p>"
+ message.update_columns(cooked: blocked_cooked)
+
+ expect { described_class.new.execute(chat_message_id: message.id) }.to change {
+ Upload.count
+ }.by(1)
+
+ upload = Upload.last
+ expect(message.reload.cooked).to include(upload.url)
+ expect(message.cooked).not_to include(PrettyText::BLOCKED_HOTLINKED_SRC_ATTR)
+ end
+
+ it "ignores blocked non-image candidates that chat cannot localize" do
+ stub_image_size
+ message = fabricate_chat_message("https://example.com/video-page")
+ blocked_cooked =
+ "<p><video><source #{PrettyText::BLOCKED_HOTLINKED_SRC_ATTR}=\"#{image_url}\"></video></p>"
+ Chat::Message.stubs(:cook).returns(blocked_cooked)
+ message.update_columns(cooked: blocked_cooked)
+
+ # two full pull/re-cook cycles: without the img filter, the unused-row
+ # sweep in ProcessMessage erases the row each time and the second run
+ # downloads the media from the remote host again
+ expect {
+ 2.times { described_class.new.execute(chat_message_id: message.id) }
+ }.not_to change { Upload.count }
+
+ expect(message.reload.hotlinked_media).to be_empty
+ expect(WebMock).not_to have_requested(:get, image_url)
+ end
+
+ it "destroys tracking rows when the message is destroyed" do
+ stub_image_size
+ message = fabricate_chat_message("![longcat](#{image_url})")
+ described_class.new.execute(chat_message_id: message.id)
+ expect(message.reload.hotlinked_media.count).to eq(1)
+
+ expect { message.destroy! }.to change { Chat::MessageHotlinkedMedia.count }.by(-1)
+ end
+
+ it "destroys tracking rows when the upload is destroyed" do
+ stub_image_size
+ message = fabricate_chat_message("![longcat](#{image_url})")
+ described_class.new.execute(chat_message_id: message.id)
+ upload = message.reload.hotlinked_media.first.upload
+
+ expect { upload.destroy! }.to change { Chat::MessageHotlinkedMedia.count }.by(-1)
+ end
+ end
+end
diff --git a/plugins/chat/spec/lib/chat/message_mover_spec.rb b/plugins/chat/spec/lib/chat/message_mover_spec.rb
index 158231d05f3..e9696add1b1 100644
--- a/plugins/chat/spec/lib/chat/message_mover_spec.rb
+++ b/plugins/chat/spec/lib/chat/message_mover_spec.rb
@@ -117,6 +117,13 @@ describe Chat::MessageMover do
mention = Fabricate(:user_chat_mention, chat_message: message2, user: acting_user)
revision = Fabricate(:chat_message_revision, chat_message: message3)
webhook_event = Fabricate(:chat_webhook_event, chat_message: message3)
+ hotlinked_media =
+ Chat::MessageHotlinkedMedia.create!(
+ chat_message: message1,
+ url: "//example.com/cat.png",
+ status: :downloaded,
+ upload: Fabricate(:upload),
+ )
move!
moved_messages =
@@ -129,6 +136,7 @@ describe Chat::MessageMover do
expect(mention.reload.chat_message_id).to eq(moved_messages.second.id)
expect(revision.reload.chat_message_id).to eq(moved_messages.third.id)
expect(webhook_event.reload.chat_message_id).to eq(moved_messages.third.id)
+ expect(hotlinked_media.reload.chat_message_id).to eq(moved_messages.first.id)
end
it "does not preserve reply chains using in_reply_to_id" do
diff --git a/plugins/chat/spec/lib/chat/message_processor_spec.rb b/plugins/chat/spec/lib/chat/message_processor_spec.rb
index 60b2e89241e..257b1845eb5 100644
--- a/plugins/chat/spec/lib/chat/message_processor_spec.rb
+++ b/plugins/chat/spec/lib/chat/message_processor_spec.rb
@@ -39,6 +39,88 @@ RSpec.describe Chat::MessageProcessor do
end
end
+ describe "#process_hotlinked_image" do
+ fab!(:upload) { Fabricate(:upload, width: 100, height: 100, extension: "gif") }
+ let(:image_url) { "http://example.com/image.gif" }
+
+ def create_downloaded_record(chat_message)
+ Chat::MessageHotlinkedMedia.create!(
+ chat_message: chat_message,
+ url: Chat::MessageHotlinkedMedia.normalize_src(image_url),
+ status: :downloaded,
+ upload: upload,
+ )
+ end
+
+ it "swaps a downloaded external img src for the local upload" do
+ create_downloaded_record(message)
+ Chat::Message.stubs(:cook).returns("<p><img src=\"#{image_url}\"></p>")
+
+ processor = described_class.new(message)
+ processor.run!
+
+ img = processor.instance_variable_get(:@doc).at_css("img")
+ expect(img["src"]).to eq(UrlHelper.cook_url(upload.url, secure: false))
+ expect(img["data-base62-sha1"]).to eq(upload.base62_sha1)
+ expect(img["class"]).to include("lightbox")
+ end
+
+ it "swaps an image inside a onebox without lightboxing it" do
+ create_downloaded_record(message)
+ Chat::Message.stubs(:cook).returns(
+ "<aside class=\"onebox\"><img src=\"#{image_url}\" width=\"100\" height=\"100\"></aside>",
+ )
+
+ processor = described_class.new(message)
+ processor.run!
+
+ img = processor.instance_variable_get(:@doc).at_css("img")
+ expect(img["src"]).to eq(UrlHelper.cook_url(upload.url, secure: false))
+ expect(img["class"].to_s).not_to include("lightbox")
+ end
+
+ context "with secure uploads enabled" do
+ before do
+ setup_s3
+ SiteSetting.secure_uploads = true
+ end
+
+ it "localizes to the secure proxy URL when the upload is secure" do
+ secure_upload = Fabricate(:secure_upload_s3, width: 100, height: 100)
+ secure_upload.update_columns(dominant_color: "000000")
+ Chat::MessageHotlinkedMedia.create!(
+ chat_message: message,
+ url: Chat::MessageHotlinkedMedia.normalize_src(image_url),
+ status: :downloaded,
+ upload: secure_upload,
+ )
+ Chat::Message.stubs(:cook).returns("<p><img src=\"#{image_url}\"></p>")
+
+ processor = described_class.new(message)
+ processor.run!
+
+ img = processor.instance_variable_get(:@doc).at_css("img")
+ expect(img["src"]).to include("secure-uploads")
+ expect(img["src"]).not_to include("s3-upload-bucket")
+ end
+ end
+
+ it "leaves the hotlinked src for failed or oversized downloads" do
+ Chat::MessageHotlinkedMedia.create!(
+ chat_message: message,
+ url: Chat::MessageHotlinkedMedia.normalize_src(image_url),
+ status: :too_large,
+ )
+ Chat::Message.stubs(:cook).returns("<p><img src=\"#{image_url}\"></p>")
+
+ processor = described_class.new(message)
+ processor.run!
+
+ img = processor.instance_variable_get(:@doc).at_css("img")
+ expect(img["src"]).to eq(image_url)
+ end
+ end
+
describe "#add_lightbox_to_images" do
fab!(:upload) { Fabricate(:upload, width: 800, height: 600) }
let(:base62) { Upload.base62_sha1(upload.sha1) }
diff --git a/plugins/chat/spec/plugin_spec.rb b/plugins/chat/spec/plugin_spec.rb
index acd19843aee..1fa70ddd888 100644
--- a/plugins/chat/spec/plugin_spec.rb
+++ b/plugins/chat/spec/plugin_spec.rb
@@ -66,6 +66,20 @@ describe Chat do
expect(Upload.exists?(id: draft_upload.id)).to eq(true)
expect(Upload.exists?(id: unused_upload.id)).to eq(false)
end
+
+ it "marks uploads tracked as hotlinked media in use, even while unreferenced" do
+ hotlinked_upload = Fabricate(:upload, user: user, created_at: 1.month.ago)
+ Chat::MessageHotlinkedMedia.create!(
+ chat_message: chat_message,
+ url: "//example.com/image.png",
+ status: :downloaded,
+ upload: hotlinked_upload,
+ )
+
+ Jobs::CleanUpUploads.new.execute({})
+
+ expect(Upload.exists?(id: hotlinked_upload.id)).to eq(true)
+ end
end
describe "user card serializer extension #can_chat_user" do