A platform for community discussion. Free, open, simple.
DEV: Let callers collect the reasons uploads were dropped (#43012)
Régis Hanol768a4ed1cd8e
diff --git a/plugins/discourse-ai/lib/completions/document_encoder.rb b/plugins/discourse-ai/lib/completions/document_encoder.rbindex 719a861a9c8..9d7649b9346 100644--- a/plugins/discourse-ai/lib/completions/document_encoder.rb+++ b/plugins/discourse-ai/lib/completions/document_encoder.rb@@ -43,16 +43,21 @@ module DiscourseAiMAX_TEXT_FILE_BYTES = 1 * 1024 * 1024MAX_RAW_DOCUMENT_BYTES = 10 * 1024 * 1024- def self.encode(upload, mime_type, attachment_type)+ def self.encode(upload, mime_type, attachment_type, skips)path = fetch_path(upload)if path.blank?- log_document_upload_skip(upload, attachment_type, "file is not available in the store")+ log_document_upload_skip(+ skips,+ upload,+ attachment_type,+ "file is not available in the store",+ )returnend- extract_text_payload(upload, path, attachment_type) ||- raw_document_payload(upload, path, mime_type, attachment_type)+ extract_text_payload(upload, path, attachment_type, skips) ||+ raw_document_payload(upload, path, mime_type, attachment_type, skips)enddef self.attachment_type_for(extension, mime_type)@@ -68,7 +73,7 @@ module DiscourseAiclass << selfprivate- def extract_text_payload(upload, path, attachment_type)+ def extract_text_payload(upload, path, attachment_type, skips)converter = TEXT_CONVERTERS[attachment_type]return if converter.nil? && PLAIN_TEXT_ATTACHMENT_TYPES.exclude?(attachment_type)@@ -77,6 +82,7 @@ module DiscourseAiif text.blank?log_document_conversion_failure(+ skips,upload,attachment_type,"#{attachment_type.upcase} converter returned blank output",@@ -86,13 +92,19 @@ module DiscourseAitext_document_payload(upload, path, text, converted_from: attachment_type)rescue StandardError => e- log_document_conversion_failure(upload, attachment_type, "#{e.class}: #{e.message}")+ log_document_conversion_failure(+ skips,+ upload,+ attachment_type,+ "#{e.class}: #{e.message}",+ )nilend- def raw_document_payload(upload, path, mime_type, attachment_type)+ def raw_document_payload(upload, path, mime_type, attachment_type, skips)if RAW_DOCUMENT_ATTACHMENT_TYPES.exclude?(attachment_type)log_document_upload_skip(+ skips,upload,attachment_type,"raw upload is not supported for this attachment type; it must be converted to text",@@ -103,6 +115,7 @@ module DiscourseAibytesize = File.size(path)if bytesize > MAX_RAW_DOCUMENT_BYTESlog_document_upload_skip(+ skips,upload,attachment_type,"raw upload size #{ActiveSupport::NumberHelper.number_to_human_size(bytesize)} " \@@ -118,7 +131,7 @@ module DiscourseAifilename: upload.original_filename,}rescue SystemCallError => e- log_document_upload_skip(upload, attachment_type, "#{e.class}: #{e.message}")+ log_document_upload_skip(skips, upload, attachment_type, "#{e.class}: #{e.message}")nilend@@ -161,14 +174,18 @@ module DiscourseAi"Uploaded document: #{filename} (#{ActiveSupport::NumberHelper.number_to_human_size(filesize)})\n\n"end- def log_document_conversion_failure(upload, extension, message)+ def log_document_conversion_failure(skips, upload, extension, message)+ record_skip(skips, upload, message)+Rails.logger.warn("Discourse AI: Failed to convert .#{extension} upload to text " \"(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",)end- def log_document_upload_skip(upload, extension, message)+ def log_document_upload_skip(skips, upload, extension, message)+ record_skip(skips, upload, message)+Rails.logger.warn("Discourse AI: Skipping .#{extension} upload " \"(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",diff --git a/plugins/discourse-ai/lib/completions/execution_context.rb b/plugins/discourse-ai/lib/completions/execution_context.rbindex 78c697223d6..918f1eb59df 100644--- a/plugins/discourse-ai/lib/completions/execution_context.rb+++ b/plugins/discourse-ai/lib/completions/execution_context.rb@@ -4,12 +4,13 @@ module DiscourseAimodule Completionsclass ExecutionContextattr_accessor :token_usage_tracker- attr_reader :audit_logger, :structured_audit_logger+ attr_reader :audit_logger, :structured_audit_logger, :upload_skipsdef initialize(token_usage_tracker: nil, audit_logger: nil, structured_audit_logger: nil)@token_usage_tracker = token_usage_tracker@audit_logger = audit_logger@structured_audit_logger = structured_audit_logger+ @upload_skips = []endendenddiff --git a/plugins/discourse-ai/lib/completions/llm.rb b/plugins/discourse-ai/lib/completions/llm.rbindex 10a25f26242..3cba24cf503 100644--- a/plugins/discourse-ai/lib/completions/llm.rb+++ b/plugins/discourse-ai/lib/completions/llm.rb@@ -249,6 +249,8 @@ module DiscourseAimodel_params = gateway.prepare_model_params(model_params) if gateway.respond_to?(:prepare_model_params,)+ prompt.upload_skips ||= execution_context&.upload_skips+dialect = dialect_klass.new(prompt, llm_model, opts: model_params)gateway.perform_completion!(diff --git a/plugins/discourse-ai/lib/completions/prompt.rb b/plugins/discourse-ai/lib/completions/prompt.rbindex 6c2078ebb46..7c58770b440 100644--- a/plugins/discourse-ai/lib/completions/prompt.rb+++ b/plugins/discourse-ai/lib/completions/prompt.rb@@ -6,7 +6,13 @@ module DiscourseAiINVALID_TURN = Class.new(StandardError)attr_reader :messages, :tools, :system_message_text- attr_accessor :topic_id, :post_id, :max_pixels, :tool_choice, :skip_trim, :native_tools+ attr_accessor :topic_id,+ :post_id,+ :max_pixels,+ :tool_choice,+ :skip_trim,+ :native_tools,+ :upload_skipsdef self.text_only(message)if message[:content].is_a?(Array)@@ -194,12 +200,7 @@ module DiscourseAi.compactif !upload_ids.empty?encoded_uploads.concat(- UploadEncoder.encode(- upload_ids: upload_ids,- max_pixels: max_pixels,- allowed_kinds: allowed_kinds,- allowed_attachment_types: allowed_attachment_types,- ),+ encode_upload_ids(upload_ids, allowed_kinds, allowed_attachment_types),)endreturn encoded_uploads@@ -218,12 +219,17 @@ module DiscourseAiallowed_upload_kinds(allow_images: allow_images, allow_documents: allow_documents)return if allowed_kinds.empty?+ encode_upload_ids([upload_id], allowed_kinds, allowed_attachment_types).first+ end++ def encode_upload_ids(upload_ids, allowed_kinds, allowed_attachment_types)UploadEncoder.encode(- upload_ids: [upload_id],+ upload_ids: upload_ids,max_pixels: max_pixels,allowed_kinds: allowed_kinds,allowed_attachment_types: allowed_attachment_types,- ).first+ skips: upload_skips,+ )enddef content_with_encoded_uploads(diff --git a/plugins/discourse-ai/lib/completions/upload_encoder.rb b/plugins/discourse-ai/lib/completions/upload_encoder.rbindex 2c1249714cb..6df791dd253 100644--- a/plugins/discourse-ai/lib/completions/upload_encoder.rb+++ b/plugins/discourse-ai/lib/completions/upload_encoder.rb@@ -26,9 +26,11 @@ module DiscourseAiupload_ids:,max_pixels:,allowed_kinds: [:image],- allowed_attachment_types: nil+ allowed_attachment_types: nil,+ skips: nil)allowed_attachment_types = normalize_attachment_types(allowed_attachment_types)+ skips ||= []uploads_by_id = Upload.where(id: upload_ids).index_by(&:id)upload_ids.filter_map do |upload_id|@@ -40,6 +42,7 @@ module DiscourseAiif kind == :document && image?(upload)log_image_upload_skip(+ skips,upload,"unsupported image format, supported formats are: #{SUPPORTED_IMAGE_EXTENSIONS.join(", ")}",)@@ -56,15 +59,15 @@ module DiscourseAiattachment_type = DocumentEncoder.attachment_type_for(upload.extension, mime_type)next if allowed_attachment_types&.exclude?(attachment_type)- next DocumentEncoder.encode(upload, mime_type, attachment_type)+ next DocumentEncoder.encode(upload, mime_type, attachment_type, skips)endif upload.width.to_i == 0 || upload.height.to_i == 0- log_image_upload_skip(upload, "image dimensions are unknown")+ log_image_upload_skip(skips, upload, "image dimensions are unknown")nextend- encode_image(upload, transcode_format(extension), max_pixels)+ encode_image(upload, transcode_format(extension), max_pixels, skips)endend@@ -85,14 +88,16 @@ module DiscourseAiJPEG_EXTENSIONS.include?(extension) ? "jpeg" : "png"end- def log_image_upload_skip(upload, message)+ def log_image_upload_skip(skips, upload, message)+ record_skip(skips, upload, message)+Rails.logger.warn("Discourse AI: Skipping image upload " \"(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",)end- def encode_image(upload, desired_extension, max_pixels)+ def encode_image(upload, desired_extension, max_pixels, skips)original_pixels = upload.width * upload.heightimage = upload@@ -110,7 +115,7 @@ module DiscourseAiendif !image- log_image_upload_skip(upload, "could not be converted to #{desired_extension}")+ log_image_upload_skip(skips, upload, "could not be converted to #{desired_extension}")returnend@@ -119,7 +124,7 @@ module DiscourseAipath = fetch_path(image)if path.blank?- log_image_upload_skip(upload, "file is not available in the store")+ log_image_upload_skip(skips, upload, "file is not available in the store")returnenddiff --git a/plugins/discourse-ai/lib/completions/upload_encoding.rb b/plugins/discourse-ai/lib/completions/upload_encoding.rbindex 14d0125f7f8..d4279fd9e55 100644--- a/plugins/discourse-ai/lib/completions/upload_encoding.rb+++ b/plugins/discourse-ai/lib/completions/upload_encoding.rb@@ -13,6 +13,12 @@ module DiscourseAipathend++ def record_skip(skips, upload, message)+ return if skips.any? { |skip| skip[:upload_id] == upload.id }++ skips << { upload_id: upload.id, filename: upload.original_filename, message: message }+ endendendenddiff --git a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb b/plugins/discourse-ai/spec/lib/completions/llm_spec.rbindex 0f10acd496f..35ed1ac3014 100644--- a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb@@ -1,6 +1,42 @@# frozen_string_literal: trueRSpec.describe DiscourseAi::Completions::Llm do+ describe "upload skips" do+ fab!(:vision_model) { Fabricate(:anthropic_model, vision_enabled: true) }++ it "hands the execution context's collector to the prompt it is about to translate" do+ upload = Fabricate(:upload, original_filename: "avatar.jxl", extension: "jxl")+ execution_context = DiscourseAi::Completions::ExecutionContext.new+ prompt =+ DiscourseAi::Completions::Prompt.new(+ "system",+ messages: [{ type: :user, content: ["look", { upload_id: upload.id }] }],+ )++ stub_request(:post, "https://api.anthropic.com/v1/messages").to_return(+ body: {+ id: "msg_1",+ type: "message",+ role: "assistant",+ content: [{ type: "text", text: "done" }],+ model: "claude-3-opus",+ usage: {+ input_tokens: 1,+ output_tokens: 1,+ },+ }.to_json,+ )++ vision_model.to_llm.generate(+ prompt,+ user: Discourse.system_user,+ execution_context: execution_context,+ )++ expect(execution_context.upload_skips.map { |skip| skip[:upload_id] }).to eq([upload.id])+ end+ end+fab!(:user)fab!(:model, :llm_model)diff --git a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rbindex d986b928e41..ec7ec092e70 100644--- a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb@@ -197,6 +197,34 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doexpect(described_class.encode(upload_ids: [missing_id], max_pixels: MAX_PIXELS)).to be_emptyend+ it "records skipped images into a caller supplied collector" do+ upload = Fabricate(:upload, original_filename: "avatar.jxl", extension: "jxl")+ skips = []++ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS, skips: skips)++ expect(encoded).to be_empty+ expect(skips.map { |skip| skip[:upload_id] }).to eq([upload.id])+ expect(skips.first[:filename]).to eq("avatar.jxl")+ expect(skips.first[:message]).to include("unsupported image format")+ end++ it "reaches the collector a prompt was handed" do+ upload = Fabricate(:upload, original_filename: "avatar.jxl", extension: "jxl")+ execution_context = DiscourseAi::Completions::ExecutionContext.new++ prompt =+ DiscourseAi::Completions::Prompt.new(+ "system",+ messages: [{ type: :user, content: ["look", { upload_id: upload.id }] }],+ )+ prompt.upload_skips = execution_context.upload_skips++ prompt.encoded_uploads(prompt.messages.last)++ expect(execution_context.upload_skips.map { |skip| skip[:upload_id] }).to eq([upload.id])+ end+it "only claims formats ImageMagick can decode" dodescribed_class::SUPPORTED_IMAGE_EXTENSIONS.each do |extension|expect(extension).to match(OptimizedImage::IM_DECODERS)