Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Accept AVIF images, and say why the others are dropped (#43010)

Régis Hanolde1e8010fcbb
diff --git a/plugins/discourse-ai/lib/agents/agent.rb b/plugins/discourse-ai/lib/agents/agent.rb
index a939edf9b93..22c00e9d119 100644
--- a/plugins/discourse-ai/lib/agents/agent.rb
+++ b/plugins/discourse-ai/lib/agents/agent.rb
@@ -533,7 +533,10 @@ module DiscourseAi
content.filter_map do |part|
if part.is_a?(Hash) && part.key?(:upload_id)
upload = uploads_by_id[part[:upload_id].to_i]
- next part if upload.blank? || !image_upload?(upload)
+ if upload.blank? ||
+ !DiscourseAi::Completions::UploadEncoder.supported_image_upload?(upload)
+ next part
+ end
next if seen_upload_ids.include?(upload.id)
delegated_image_handle(upload, context, seen_upload_ids)
@@ -552,7 +555,10 @@ module DiscourseAi
content.gsub(DELEGATED_IMAGE_PATTERN) do |markdown|
sha1 = Upload.sha1_from_short_url(Regexp.last_match(1))
upload = uploads_by_sha1[sha1]
- next markdown if upload.blank? || !image_upload?(upload)
+ if upload.blank? ||
+ !DiscourseAi::Completions::UploadEncoder.supported_image_upload?(upload)
+ next markdown
+ end
next "" if seen_upload_ids.include?(upload.id)
delegated_image_handle(upload, context, seen_upload_ids) || "[Image unavailable]"
@@ -571,10 +577,6 @@ module DiscourseAi
context.image_guardian
end
- def image_upload?(upload)
- DiscourseAi::Completions::UploadEncoder.image_upload?(upload)
- end
-
def replace_placeholders(content, context)
replaced =
content.gsub(/\{(\w+)\}/) do |match|
diff --git a/plugins/discourse-ai/lib/agents/bot.rb b/plugins/discourse-ai/lib/agents/bot.rb
index c060777f62a..c84b4e9b261 100644
--- a/plugins/discourse-ai/lib/agents/bot.rb
+++ b/plugins/discourse-ai/lib/agents/bot.rb
@@ -623,7 +623,7 @@ module DiscourseAi
references_by_sha1.filter_map do |sha1, short_url|
upload = uploads_by_sha1[sha1]
next if upload.blank?
- next if !DiscourseAi::Completions::UploadEncoder.image_upload?(upload)
+ next if !DiscourseAi::Completions::UploadEncoder.supported_image_upload?(upload)
next if !guardian.can_see_upload?(upload)
if system_secure_upload_ids && upload.secure? &&
!system_secure_upload_ids.include?(upload.id)
diff --git a/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb b/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb
index 881077ddd3e..718e33c86a8 100644
--- a/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb
+++ b/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb
@@ -9,7 +9,6 @@ module DiscourseAi
COMPRESSED_CONTEXT_PREFIX = "<compressed_context>"
COMPRESSED_CONTEXT_SUFFIX = "</compressed_context>"
COMPRESSED_CONTEXT_ACK = "Understood, I have the context."
- IMAGE_UPLOAD_EXTENSIONS = %w[jpg jpeg png gif webp].freeze
attr_reader :chat_context_posts
attr_accessor :topic
@@ -341,8 +340,8 @@ module DiscourseAi
allowed_attachment_types: nil
)
type_allowed =
- if image_upload?(upload)
- include_image_uploads
+ if UploadEncoder.image?(upload)
+ include_image_uploads && UploadEncoder.supported_image_upload?(upload)
else
include_document_uploads &&
document_upload_allowed_for_prompt?(upload, allowed_attachment_types)
@@ -364,16 +363,6 @@ module DiscourseAi
allowed_attachment_types.include?(attachment_type)
end
- def self.image_upload?(upload)
- extension = upload.extension.to_s.delete_prefix(".").downcase
- return true if IMAGE_UPLOAD_EXTENSIONS.include?(extension)
-
- filename = upload.original_filename.to_s
- filename = "upload.#{extension}" if File.extname(filename).blank? && extension.present?
-
- MiniMime.lookup_by_filename(filename)&.content_type.to_s.start_with?("image/")
- end
-
def initialize
@raw_messages = []
@timestamps = {}
diff --git a/plugins/discourse-ai/lib/completions/upload_encoder.rb b/plugins/discourse-ai/lib/completions/upload_encoder.rb
index 0600110b9f0..2c1249714cb 100644
--- a/plugins/discourse-ai/lib/completions/upload_encoder.rb
+++ b/plugins/discourse-ai/lib/completions/upload_encoder.rb
@@ -5,13 +5,21 @@ module DiscourseAi
class UploadEncoder
extend UploadEncoding
- def self.image_upload?(upload)
- mime_type = MiniMime.lookup_by_filename(upload.original_filename)&.content_type
- mime_type&.start_with?("image/") == true
- end
+ JPEG_EXTENSIONS = %w[jpg jpeg].freeze
+ SUPPORTED_IMAGE_EXTENSIONS = (JPEG_EXTENSIONS + %w[png gif webp avif]).freeze
def self.supported_image_upload?(upload)
- image_upload?(upload) && %w[jpg jpeg png gif webp].include?(upload.extension&.downcase)
+ image_extension?(upload.extension&.downcase)
+ end
+
+ def self.image?(upload)
+ extension = upload.extension.to_s.delete_prefix(".").downcase
+ return true if FileHelper.supported_images.include?(extension)
+
+ filename = upload.original_filename.to_s
+ filename = "upload.#{extension}" if File.extname(filename).blank? && extension.present?
+
+ MiniMime.lookup_by_filename(filename)&.content_type.to_s.start_with?("image/")
end
def self.encode(
@@ -30,6 +38,14 @@ module DiscourseAi
extension = upload.extension&.downcase
kind = image_extension?(extension) ? :image : :document
+ if kind == :document && image?(upload)
+ log_image_upload_skip(
+ upload,
+ "unsupported image format, supported formats are: #{SUPPORTED_IMAGE_EXTENSIONS.join(", ")}",
+ )
+ next
+ end
+
next if allowed_kinds.exclude?(kind)
if kind == :document
@@ -43,17 +59,12 @@ module DiscourseAi
next DocumentEncoder.encode(upload, mime_type, attachment_type)
end
- next if upload.width.to_i == 0 || upload.height.to_i == 0
-
- desired_extension = upload.extension
- desired_extension = "png" if upload.extension == "gif"
- desired_extension = "png" if upload.extension == "webp"
- desired_extension = "jpeg" if upload.extension == "jpg"
-
- # this keeps it very simple format wise given everyone supports png and jpg
- next if !%w[jpeg png].include?(desired_extension)
+ if upload.width.to_i == 0 || upload.height.to_i == 0
+ log_image_upload_skip(upload, "image dimensions are unknown")
+ next
+ end
- encode_image(upload, desired_extension, max_pixels)
+ encode_image(upload, transcode_format(extension), max_pixels)
end
end
@@ -67,7 +78,18 @@ module DiscourseAi
end
def image_extension?(ext)
- %w[jpg jpeg png gif webp].include?(ext)
+ SUPPORTED_IMAGE_EXTENSIONS.include?(ext)
+ end
+
+ def transcode_format(extension)
+ JPEG_EXTENSIONS.include?(extension) ? "jpeg" : "png"
+ end
+
+ def log_image_upload_skip(upload, message)
+ Rails.logger.warn(
+ "Discourse AI: Skipping image upload " \
+ "(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",
+ )
end
def encode_image(upload, desired_extension, max_pixels)
@@ -82,17 +104,24 @@ module DiscourseAi
new_height = (ratio * upload.height).to_i
image = upload.get_optimized_image(new_width, new_height, format: desired_extension)
- elsif upload.extension != desired_extension
+ elsif upload.extension&.downcase != desired_extension
image =
upload.get_optimized_image(upload.width, upload.height, format: desired_extension)
end
- return if !image
+ if !image
+ log_image_upload_skip(upload, "could not be converted to #{desired_extension}")
+ return
+ end
mime_type = MiniMime.lookup_by_filename("test.#{desired_extension}").content_type
path = fetch_path(image)
- return if path.blank?
+
+ if path.blank?
+ log_image_upload_skip(upload, "file is not available in the store")
+ return
+ end
encoded = Base64.strict_encode64(File.binread(path))
diff --git a/plugins/discourse-ai/lib/post_image_captions.rb b/plugins/discourse-ai/lib/post_image_captions.rb
index 3ff39c8ccdf..4462032ad41 100644
--- a/plugins/discourse-ai/lib/post_image_captions.rb
+++ b/plugins/discourse-ai/lib/post_image_captions.rb
@@ -13,7 +13,6 @@ module DiscourseAi
RETRY_AFTER = 1.day
COOK_ENQUEUE_POST_MAX_AGE = 1.day
BACKFILL_RUNS_PER_HOUR = 4
- SUPPORTED_EXTENSIONS = %w[jpg jpeg png gif webp].freeze
MAX_CAPTION_LENGTH = 1_000
module_function
@@ -238,7 +237,9 @@ module DiscourseAi
max_age_days = SiteSetting.ai_post_image_captions_backfill_max_age_days.to_i
original_locale_sql = "COALESCE(NULLIF(posts.locale, ''), #{default_locale})"
supported_extensions =
- SUPPORTED_EXTENSIONS.map { |extension| connection.quote(extension) }.join(", ")
+ Completions::UploadEncoder::SUPPORTED_IMAGE_EXTENSIONS
+ .map { |extension| connection.quote(extension) }
+ .join(", ")
localized_targets_sql =
if SiteSetting.content_localization_enabled
<<~SQL
@@ -448,7 +449,7 @@ module DiscourseAi
def captionable_upload?(upload, post)
return false if upload.blank?
- return false if !SUPPORTED_EXTENSIONS.include?(upload.extension.to_s.downcase)
+ return false if !Completions::UploadEncoder.supported_image_upload?(upload)
return false if upload.secure? && upload.access_control_post_id != post.id
true
diff --git a/plugins/discourse-ai/spec/fixtures/images/100x100.avif b/plugins/discourse-ai/spec/fixtures/images/100x100.avif
new file mode 100644
index 00000000000..4d40fd10f71
Binary files /dev/null and b/plugins/discourse-ai/spec/fixtures/images/100x100.avif differ
diff --git a/plugins/discourse-ai/spec/fixtures/images/1x1.avif b/plugins/discourse-ai/spec/fixtures/images/1x1.avif
new file mode 100644
index 00000000000..1d5ac5b85e3
Binary files /dev/null and b/plugins/discourse-ai/spec/fixtures/images/1x1.avif differ
diff --git a/plugins/discourse-ai/spec/lib/agents/delegated_vision_spec.rb b/plugins/discourse-ai/spec/lib/agents/delegated_vision_spec.rb
index eaafbb4b852..c65a1f0fc94 100644
--- a/plugins/discourse-ai/spec/lib/agents/delegated_vision_spec.rb
+++ b/plugins/discourse-ai/spec/lib/agents/delegated_vision_spec.rb
@@ -46,6 +46,20 @@ RSpec.describe DelegatedVisionTestAgent do
before { enable_current_plugin }
+ it "does not offer view_image handles for formats view_image cannot serve" do
+ svg = Fabricate(:upload, original_filename: "logo.svg", extension: "svg")
+ context =
+ DiscourseAi::Agents::BotContext.new(
+ user: user,
+ messages: [{ type: :user, content: ["Compare this", { upload_id: svg.id }] }],
+ )
+
+ prompt = agent.craft_prompt(context, llm: delegated_model.to_llm)
+
+ expect(prompt.messages.last[:content]).to eq(["Compare this", { upload_id: svg.id }])
+ expect(context.image_upload_authorized?(svg.id)).to eq(false)
+ end
+
it "exposes view_image and replaces image parts with authorized handles" do
context =
DiscourseAi::Agents::BotContext.new(
diff --git a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
index b0924de15b7..d986b928e41 100644
--- a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
@@ -7,6 +7,8 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
let(:jpg) { plugin_file_from_fixtures("1x1.jpg") }
let(:large_jpg) { plugin_file_from_fixtures("100x100.jpg") }
let(:webp) { plugin_file_from_fixtures("1x1.webp") }
+ let(:avif) { plugin_file_from_fixtures("1x1.avif") }
+ let(:large_avif) { plugin_file_from_fixtures("100x100.avif") }
before { enable_current_plugin }
@@ -112,12 +114,111 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
expect(encoded[0][:mime_type]).to eq("image/jpeg")
end
+ it "automatically converts avif to pngs" do
+ upload = UploadCreator.new(avif, "1x1.avif").create_for(Discourse.system_user.id)
+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)
+ expect(encoded.length).to eq(1)
+ expect(encoded[0][:base64]).to be_present
+ expect(encoded[0][:mime_type]).to eq("image/png")
+ end
+
+ it "resizes avif to the configured pixel area" do
+ upload = UploadCreator.new(large_avif, "100x100.avif").create_for(Discourse.system_user.id)
+
+ described_class.encode(upload_ids: [upload.id], max_pixels: 2_500)
+
+ expect(upload.optimized_images.find_by(width: 50, height: 50, extension: ".png")).to be_present
+ end
+
+ it "encodes uploads whose extension is not lowercase" do
+ upload = UploadCreator.new(jpg, "1x1.jpg").create_for(Discourse.system_user.id)
+ upload.update!(extension: "JPG")
+
+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)
+
+ expect(encoded.length).to eq(1)
+ expect(encoded[0][:mime_type]).to eq("image/jpeg")
+ end
+
+ it "skips and logs images it cannot encode" do
+ upload = Fabricate(:upload, original_filename: "avatar.jxl", extension: "jxl")
+ allow(Rails.logger).to receive(:warn)
+
+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)
+
+ expect(encoded).to be_empty
+ expect(Rails.logger).to have_received(:warn).with(
+ a_string_including("Skipping image upload", "avatar.jxl", "unsupported image format"),
+ )
+ end
+
+ it "never routes an unsupported image through the document path" do
+ upload = Fabricate(:upload, original_filename: "avatar.jxl", extension: "jxl")
+
+ encoded =
+ described_class.encode(
+ upload_ids: [upload.id],
+ max_pixels: MAX_PIXELS,
+ allowed_kinds: %i[image document],
+ allowed_attachment_types: nil,
+ )
+
+ expect(encoded).to be_empty
+ end
+
+ it "skips and logs images with unknown dimensions" do
+ upload = Fabricate(:upload, original_filename: "avatar.png", extension: "png", width: 0)
+ allow(Rails.logger).to receive(:warn)
+
+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)
+
+ expect(encoded).to be_empty
+ expect(Rails.logger).to have_received(:warn).with(
+ a_string_including("Skipping image upload", "dimensions are unknown"),
+ )
+ end
+
+ it "skips and logs images that cannot be optimized" do
+ upload = UploadCreator.new(webp, "1x1.webp").create_for(Discourse.system_user.id)
+ allow(Rails.logger).to receive(:warn)
+ allow_any_instance_of(Upload).to receive(:get_optimized_image).and_return(nil)
+
+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)
+
+ expect(encoded).to be_empty
+ expect(Rails.logger).to have_received(:warn).with(
+ a_string_including("Skipping image upload", "could not be converted to png"),
+ )
+ end
+
it "does not raise when an upload no longer exists" do
missing_id = Upload.maximum(:id).to_i + 1
expect(described_class.encode(upload_ids: [missing_id], max_pixels: MAX_PIXELS)).to be_empty
end
+ it "only claims formats ImageMagick can decode" do
+ described_class::SUPPORTED_IMAGE_EXTENSIONS.each do |extension|
+ expect(extension).to match(OptimizedImage::IM_DECODERS)
+ end
+ end
+
+ describe ".supported_image_upload?" do
+ it "accepts every format the encoder can transcode" do
+ %w[jpg jpeg png gif webp avif].each do |extension|
+ upload = Fabricate.build(:upload, original_filename: "x.#{extension}", extension: extension)
+ expect(described_class.supported_image_upload?(upload)).to eq(true), extension
+ end
+ end
+
+ it "rejects formats the encoder cannot transcode" do
+ %w[jxl heic svg bmp pdf].each do |extension|
+ upload = Fabricate.build(:upload, original_filename: "x.#{extension}", extension: extension)
+ expect(described_class.supported_image_upload?(upload)).to eq(false), extension
+ end
+ end
+ end
+
describe ".doc, .docx, .xls, and .xlsx uploads" do
before { SiteSetting.authorized_extensions = "*" }