A platform for community discussion. Free, open, simple.
FEATURE: Accept AVIF images, and say why the others are dropped (#43010)
Régis Hanolde1e8010fcbb
diff --git a/plugins/discourse-ai/lib/agents/agent.rb b/plugins/discourse-ai/lib/agents/agent.rbindex a939edf9b93..22c00e9d119 100644--- a/plugins/discourse-ai/lib/agents/agent.rb+++ b/plugins/discourse-ai/lib/agents/agent.rb@@ -533,7 +533,10 @@ module DiscourseAicontent.filter_map do |part|if part.is_a?(Hash) && part.key?(:upload_id)upload = uploads_by_id[part[:upload_id].to_i]- next part if upload.blank? || !image_upload?(upload)+ if upload.blank? ||+ !DiscourseAi::Completions::UploadEncoder.supported_image_upload?(upload)+ next part+ endnext if seen_upload_ids.include?(upload.id)delegated_image_handle(upload, context, seen_upload_ids)@@ -552,7 +555,10 @@ module DiscourseAicontent.gsub(DELEGATED_IMAGE_PATTERN) do |markdown|sha1 = Upload.sha1_from_short_url(Regexp.last_match(1))upload = uploads_by_sha1[sha1]- next markdown if upload.blank? || !image_upload?(upload)+ if upload.blank? ||+ !DiscourseAi::Completions::UploadEncoder.supported_image_upload?(upload)+ next markdown+ endnext "" if seen_upload_ids.include?(upload.id)delegated_image_handle(upload, context, seen_upload_ids) || "[Image unavailable]"@@ -571,10 +577,6 @@ module DiscourseAicontext.image_guardianend- def image_upload?(upload)- DiscourseAi::Completions::UploadEncoder.image_upload?(upload)- end-def replace_placeholders(content, context)replaced =content.gsub(/\{(\w+)\}/) do |match|diff --git a/plugins/discourse-ai/lib/agents/bot.rb b/plugins/discourse-ai/lib/agents/bot.rbindex c060777f62a..c84b4e9b261 100644--- a/plugins/discourse-ai/lib/agents/bot.rb+++ b/plugins/discourse-ai/lib/agents/bot.rb@@ -623,7 +623,7 @@ module DiscourseAireferences_by_sha1.filter_map do |sha1, short_url|upload = uploads_by_sha1[sha1]next if upload.blank?- next if !DiscourseAi::Completions::UploadEncoder.image_upload?(upload)+ next if !DiscourseAi::Completions::UploadEncoder.supported_image_upload?(upload)next if !guardian.can_see_upload?(upload)if system_secure_upload_ids && upload.secure? &&!system_secure_upload_ids.include?(upload.id)diff --git a/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb b/plugins/discourse-ai/lib/completions/prompt_messages_builder.rbindex 881077ddd3e..718e33c86a8 100644--- a/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb+++ b/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb@@ -9,7 +9,6 @@ module DiscourseAiCOMPRESSED_CONTEXT_PREFIX = "<compressed_context>"COMPRESSED_CONTEXT_SUFFIX = "</compressed_context>"COMPRESSED_CONTEXT_ACK = "Understood, I have the context."- IMAGE_UPLOAD_EXTENSIONS = %w[jpg jpeg png gif webp].freezeattr_reader :chat_context_postsattr_accessor :topic@@ -341,8 +340,8 @@ module DiscourseAiallowed_attachment_types: nil)type_allowed =- if image_upload?(upload)- include_image_uploads+ if UploadEncoder.image?(upload)+ include_image_uploads && UploadEncoder.supported_image_upload?(upload)elseinclude_document_uploads &&document_upload_allowed_for_prompt?(upload, allowed_attachment_types)@@ -364,16 +363,6 @@ module DiscourseAiallowed_attachment_types.include?(attachment_type)end- def self.image_upload?(upload)- extension = upload.extension.to_s.delete_prefix(".").downcase- return true if IMAGE_UPLOAD_EXTENSIONS.include?(extension)-- filename = upload.original_filename.to_s- filename = "upload.#{extension}" if File.extname(filename).blank? && extension.present?-- MiniMime.lookup_by_filename(filename)&.content_type.to_s.start_with?("image/")- end-def initialize@raw_messages = []@timestamps = {}diff --git a/plugins/discourse-ai/lib/completions/upload_encoder.rb b/plugins/discourse-ai/lib/completions/upload_encoder.rbindex 0600110b9f0..2c1249714cb 100644--- a/plugins/discourse-ai/lib/completions/upload_encoder.rb+++ b/plugins/discourse-ai/lib/completions/upload_encoder.rb@@ -5,13 +5,21 @@ module DiscourseAiclass UploadEncoderextend UploadEncoding- def self.image_upload?(upload)- mime_type = MiniMime.lookup_by_filename(upload.original_filename)&.content_type- mime_type&.start_with?("image/") == true- end+ JPEG_EXTENSIONS = %w[jpg jpeg].freeze+ SUPPORTED_IMAGE_EXTENSIONS = (JPEG_EXTENSIONS + %w[png gif webp avif]).freezedef self.supported_image_upload?(upload)- image_upload?(upload) && %w[jpg jpeg png gif webp].include?(upload.extension&.downcase)+ image_extension?(upload.extension&.downcase)+ end++ def self.image?(upload)+ extension = upload.extension.to_s.delete_prefix(".").downcase+ return true if FileHelper.supported_images.include?(extension)++ filename = upload.original_filename.to_s+ filename = "upload.#{extension}" if File.extname(filename).blank? && extension.present?++ MiniMime.lookup_by_filename(filename)&.content_type.to_s.start_with?("image/")enddef self.encode(@@ -30,6 +38,14 @@ module DiscourseAiextension = upload.extension&.downcasekind = image_extension?(extension) ? :image : :document+ if kind == :document && image?(upload)+ log_image_upload_skip(+ upload,+ "unsupported image format, supported formats are: #{SUPPORTED_IMAGE_EXTENSIONS.join(", ")}",+ )+ next+ end+next if allowed_kinds.exclude?(kind)if kind == :document@@ -43,17 +59,12 @@ module DiscourseAinext DocumentEncoder.encode(upload, mime_type, attachment_type)end- next if upload.width.to_i == 0 || upload.height.to_i == 0-- desired_extension = upload.extension- desired_extension = "png" if upload.extension == "gif"- desired_extension = "png" if upload.extension == "webp"- desired_extension = "jpeg" if upload.extension == "jpg"-- # this keeps it very simple format wise given everyone supports png and jpg- next if !%w[jpeg png].include?(desired_extension)+ if upload.width.to_i == 0 || upload.height.to_i == 0+ log_image_upload_skip(upload, "image dimensions are unknown")+ next+ end- encode_image(upload, desired_extension, max_pixels)+ encode_image(upload, transcode_format(extension), max_pixels)endend@@ -67,7 +78,18 @@ module DiscourseAienddef image_extension?(ext)- %w[jpg jpeg png gif webp].include?(ext)+ SUPPORTED_IMAGE_EXTENSIONS.include?(ext)+ end++ def transcode_format(extension)+ JPEG_EXTENSIONS.include?(extension) ? "jpeg" : "png"+ end++ def log_image_upload_skip(upload, message)+ Rails.logger.warn(+ "Discourse AI: Skipping image upload " \+ "(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",+ )enddef encode_image(upload, desired_extension, max_pixels)@@ -82,17 +104,24 @@ module DiscourseAinew_height = (ratio * upload.height).to_iimage = upload.get_optimized_image(new_width, new_height, format: desired_extension)- elsif upload.extension != desired_extension+ elsif upload.extension&.downcase != desired_extensionimage =upload.get_optimized_image(upload.width, upload.height, format: desired_extension)end- return if !image+ if !image+ log_image_upload_skip(upload, "could not be converted to #{desired_extension}")+ return+ endmime_type = MiniMime.lookup_by_filename("test.#{desired_extension}").content_typepath = fetch_path(image)- return if path.blank?++ if path.blank?+ log_image_upload_skip(upload, "file is not available in the store")+ return+ endencoded = Base64.strict_encode64(File.binread(path))diff --git a/plugins/discourse-ai/lib/post_image_captions.rb b/plugins/discourse-ai/lib/post_image_captions.rbindex 3ff39c8ccdf..4462032ad41 100644--- a/plugins/discourse-ai/lib/post_image_captions.rb+++ b/plugins/discourse-ai/lib/post_image_captions.rb@@ -13,7 +13,6 @@ module DiscourseAiRETRY_AFTER = 1.dayCOOK_ENQUEUE_POST_MAX_AGE = 1.dayBACKFILL_RUNS_PER_HOUR = 4- SUPPORTED_EXTENSIONS = %w[jpg jpeg png gif webp].freezeMAX_CAPTION_LENGTH = 1_000module_function@@ -238,7 +237,9 @@ module DiscourseAimax_age_days = SiteSetting.ai_post_image_captions_backfill_max_age_days.to_ioriginal_locale_sql = "COALESCE(NULLIF(posts.locale, ''), #{default_locale})"supported_extensions =- SUPPORTED_EXTENSIONS.map { |extension| connection.quote(extension) }.join(", ")+ Completions::UploadEncoder::SUPPORTED_IMAGE_EXTENSIONS+ .map { |extension| connection.quote(extension) }+ .join(", ")localized_targets_sql =if SiteSetting.content_localization_enabled<<~SQL@@ -448,7 +449,7 @@ module DiscourseAidef captionable_upload?(upload, post)return false if upload.blank?- return false if !SUPPORTED_EXTENSIONS.include?(upload.extension.to_s.downcase)+ return false if !Completions::UploadEncoder.supported_image_upload?(upload)return false if upload.secure? && upload.access_control_post_id != post.idtruediff --git a/plugins/discourse-ai/spec/fixtures/images/100x100.avif b/plugins/discourse-ai/spec/fixtures/images/100x100.avifnew file mode 100644index 00000000000..4d40fd10f71Binary files /dev/null and b/plugins/discourse-ai/spec/fixtures/images/100x100.avif differdiff --git a/plugins/discourse-ai/spec/fixtures/images/1x1.avif b/plugins/discourse-ai/spec/fixtures/images/1x1.avifnew file mode 100644index 00000000000..1d5ac5b85e3Binary files /dev/null and b/plugins/discourse-ai/spec/fixtures/images/1x1.avif differdiff --git a/plugins/discourse-ai/spec/lib/agents/delegated_vision_spec.rb b/plugins/discourse-ai/spec/lib/agents/delegated_vision_spec.rbindex eaafbb4b852..c65a1f0fc94 100644--- a/plugins/discourse-ai/spec/lib/agents/delegated_vision_spec.rb+++ b/plugins/discourse-ai/spec/lib/agents/delegated_vision_spec.rb@@ -46,6 +46,20 @@ RSpec.describe DelegatedVisionTestAgent dobefore { enable_current_plugin }+ it "does not offer view_image handles for formats view_image cannot serve" do+ svg = Fabricate(:upload, original_filename: "logo.svg", extension: "svg")+ context =+ DiscourseAi::Agents::BotContext.new(+ user: user,+ messages: [{ type: :user, content: ["Compare this", { upload_id: svg.id }] }],+ )++ prompt = agent.craft_prompt(context, llm: delegated_model.to_llm)++ expect(prompt.messages.last[:content]).to eq(["Compare this", { upload_id: svg.id }])+ expect(context.image_upload_authorized?(svg.id)).to eq(false)+ end+it "exposes view_image and replaces image parts with authorized handles" docontext =DiscourseAi::Agents::BotContext.new(diff --git a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rbindex b0924de15b7..d986b928e41 100644--- a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb@@ -7,6 +7,8 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder dolet(:jpg) { plugin_file_from_fixtures("1x1.jpg") }let(:large_jpg) { plugin_file_from_fixtures("100x100.jpg") }let(:webp) { plugin_file_from_fixtures("1x1.webp") }+ let(:avif) { plugin_file_from_fixtures("1x1.avif") }+ let(:large_avif) { plugin_file_from_fixtures("100x100.avif") }before { enable_current_plugin }@@ -112,12 +114,111 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doexpect(encoded[0][:mime_type]).to eq("image/jpeg")end+ it "automatically converts avif to pngs" do+ upload = UploadCreator.new(avif, "1x1.avif").create_for(Discourse.system_user.id)+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)+ expect(encoded.length).to eq(1)+ expect(encoded[0][:base64]).to be_present+ expect(encoded[0][:mime_type]).to eq("image/png")+ end++ it "resizes avif to the configured pixel area" do+ upload = UploadCreator.new(large_avif, "100x100.avif").create_for(Discourse.system_user.id)++ described_class.encode(upload_ids: [upload.id], max_pixels: 2_500)++ expect(upload.optimized_images.find_by(width: 50, height: 50, extension: ".png")).to be_present+ end++ it "encodes uploads whose extension is not lowercase" do+ upload = UploadCreator.new(jpg, "1x1.jpg").create_for(Discourse.system_user.id)+ upload.update!(extension: "JPG")++ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)++ expect(encoded.length).to eq(1)+ expect(encoded[0][:mime_type]).to eq("image/jpeg")+ end++ it "skips and logs images it cannot encode" do+ upload = Fabricate(:upload, original_filename: "avatar.jxl", extension: "jxl")+ allow(Rails.logger).to receive(:warn)++ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)++ expect(encoded).to be_empty+ expect(Rails.logger).to have_received(:warn).with(+ a_string_including("Skipping image upload", "avatar.jxl", "unsupported image format"),+ )+ end++ it "never routes an unsupported image through the document path" do+ upload = Fabricate(:upload, original_filename: "avatar.jxl", extension: "jxl")++ encoded =+ described_class.encode(+ upload_ids: [upload.id],+ max_pixels: MAX_PIXELS,+ allowed_kinds: %i[image document],+ allowed_attachment_types: nil,+ )++ expect(encoded).to be_empty+ end++ it "skips and logs images with unknown dimensions" do+ upload = Fabricate(:upload, original_filename: "avatar.png", extension: "png", width: 0)+ allow(Rails.logger).to receive(:warn)++ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)++ expect(encoded).to be_empty+ expect(Rails.logger).to have_received(:warn).with(+ a_string_including("Skipping image upload", "dimensions are unknown"),+ )+ end++ it "skips and logs images that cannot be optimized" do+ upload = UploadCreator.new(webp, "1x1.webp").create_for(Discourse.system_user.id)+ allow(Rails.logger).to receive(:warn)+ allow_any_instance_of(Upload).to receive(:get_optimized_image).and_return(nil)++ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)++ expect(encoded).to be_empty+ expect(Rails.logger).to have_received(:warn).with(+ a_string_including("Skipping image upload", "could not be converted to png"),+ )+ end+it "does not raise when an upload no longer exists" domissing_id = Upload.maximum(:id).to_i + 1expect(described_class.encode(upload_ids: [missing_id], max_pixels: MAX_PIXELS)).to be_emptyend+ it "only claims formats ImageMagick can decode" do+ described_class::SUPPORTED_IMAGE_EXTENSIONS.each do |extension|+ expect(extension).to match(OptimizedImage::IM_DECODERS)+ end+ end++ describe ".supported_image_upload?" do+ it "accepts every format the encoder can transcode" do+ %w[jpg jpeg png gif webp avif].each do |extension|+ upload = Fabricate.build(:upload, original_filename: "x.#{extension}", extension: extension)+ expect(described_class.supported_image_upload?(upload)).to eq(true), extension+ end+ end++ it "rejects formats the encoder cannot transcode" do+ %w[jxl heic svg bmp pdf].each do |extension|+ upload = Fabricate.build(:upload, original_filename: "x.#{extension}", extension: extension)+ expect(described_class.supported_image_upload?(upload)).to eq(false), extension+ end+ end+ end+describe ".doc, .docx, .xls, and .xlsx uploads" dobefore { SiteSetting.authorized_extensions = "*" }