Skip to content

A platform for community discussion. Free, open, simple.

DEV: Let callers collect the reasons uploads were dropped (#43012)

Régis Hanol768a4ed1cd8e
diff --git a/plugins/discourse-ai/lib/completions/document_encoder.rb b/plugins/discourse-ai/lib/completions/document_encoder.rb
index 719a861a9c8..9d7649b9346 100644
--- a/plugins/discourse-ai/lib/completions/document_encoder.rb
+++ b/plugins/discourse-ai/lib/completions/document_encoder.rb
@@ -43,16 +43,21 @@ module DiscourseAi
MAX_TEXT_FILE_BYTES = 1 * 1024 * 1024
MAX_RAW_DOCUMENT_BYTES = 10 * 1024 * 1024
- def self.encode(upload, mime_type, attachment_type)
+ def self.encode(upload, mime_type, attachment_type, skips)
path = fetch_path(upload)
if path.blank?
- log_document_upload_skip(upload, attachment_type, "file is not available in the store")
+ log_document_upload_skip(
+ skips,
+ upload,
+ attachment_type,
+ "file is not available in the store",
+ )
return
end
- extract_text_payload(upload, path, attachment_type) ||
- raw_document_payload(upload, path, mime_type, attachment_type)
+ extract_text_payload(upload, path, attachment_type, skips) ||
+ raw_document_payload(upload, path, mime_type, attachment_type, skips)
end
def self.attachment_type_for(extension, mime_type)
@@ -68,7 +73,7 @@ module DiscourseAi
class << self
private
- def extract_text_payload(upload, path, attachment_type)
+ def extract_text_payload(upload, path, attachment_type, skips)
converter = TEXT_CONVERTERS[attachment_type]
return if converter.nil? && PLAIN_TEXT_ATTACHMENT_TYPES.exclude?(attachment_type)
@@ -77,6 +82,7 @@ module DiscourseAi
if text.blank?
log_document_conversion_failure(
+ skips,
upload,
attachment_type,
"#{attachment_type.upcase} converter returned blank output",
@@ -86,13 +92,19 @@ module DiscourseAi
text_document_payload(upload, path, text, converted_from: attachment_type)
rescue StandardError => e
- log_document_conversion_failure(upload, attachment_type, "#{e.class}: #{e.message}")
+ log_document_conversion_failure(
+ skips,
+ upload,
+ attachment_type,
+ "#{e.class}: #{e.message}",
+ )
nil
end
- def raw_document_payload(upload, path, mime_type, attachment_type)
+ def raw_document_payload(upload, path, mime_type, attachment_type, skips)
if RAW_DOCUMENT_ATTACHMENT_TYPES.exclude?(attachment_type)
log_document_upload_skip(
+ skips,
upload,
attachment_type,
"raw upload is not supported for this attachment type; it must be converted to text",
@@ -103,6 +115,7 @@ module DiscourseAi
bytesize = File.size(path)
if bytesize > MAX_RAW_DOCUMENT_BYTES
log_document_upload_skip(
+ skips,
upload,
attachment_type,
"raw upload size #{ActiveSupport::NumberHelper.number_to_human_size(bytesize)} " \
@@ -118,7 +131,7 @@ module DiscourseAi
filename: upload.original_filename,
}
rescue SystemCallError => e
- log_document_upload_skip(upload, attachment_type, "#{e.class}: #{e.message}")
+ log_document_upload_skip(skips, upload, attachment_type, "#{e.class}: #{e.message}")
nil
end
@@ -161,14 +174,18 @@ module DiscourseAi
"Uploaded document: #{filename} (#{ActiveSupport::NumberHelper.number_to_human_size(filesize)})\n\n"
end
- def log_document_conversion_failure(upload, extension, message)
+ def log_document_conversion_failure(skips, upload, extension, message)
+ record_skip(skips, upload, message)
+
Rails.logger.warn(
"Discourse AI: Failed to convert .#{extension} upload to text " \
"(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",
)
end
- def log_document_upload_skip(upload, extension, message)
+ def log_document_upload_skip(skips, upload, extension, message)
+ record_skip(skips, upload, message)
+
Rails.logger.warn(
"Discourse AI: Skipping .#{extension} upload " \
"(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",
diff --git a/plugins/discourse-ai/lib/completions/execution_context.rb b/plugins/discourse-ai/lib/completions/execution_context.rb
index 78c697223d6..918f1eb59df 100644
--- a/plugins/discourse-ai/lib/completions/execution_context.rb
+++ b/plugins/discourse-ai/lib/completions/execution_context.rb
@@ -4,12 +4,13 @@ module DiscourseAi
module Completions
class ExecutionContext
attr_accessor :token_usage_tracker
- attr_reader :audit_logger, :structured_audit_logger
+ attr_reader :audit_logger, :structured_audit_logger, :upload_skips
def initialize(token_usage_tracker: nil, audit_logger: nil, structured_audit_logger: nil)
@token_usage_tracker = token_usage_tracker
@audit_logger = audit_logger
@structured_audit_logger = structured_audit_logger
+ @upload_skips = []
end
end
end
diff --git a/plugins/discourse-ai/lib/completions/llm.rb b/plugins/discourse-ai/lib/completions/llm.rb
index 10a25f26242..3cba24cf503 100644
--- a/plugins/discourse-ai/lib/completions/llm.rb
+++ b/plugins/discourse-ai/lib/completions/llm.rb
@@ -249,6 +249,8 @@ module DiscourseAi
model_params = gateway.prepare_model_params(model_params) if gateway.respond_to?(
:prepare_model_params,
)
+ prompt.upload_skips ||= execution_context&.upload_skips
+
dialect = dialect_klass.new(prompt, llm_model, opts: model_params)
gateway.perform_completion!(
diff --git a/plugins/discourse-ai/lib/completions/prompt.rb b/plugins/discourse-ai/lib/completions/prompt.rb
index 6c2078ebb46..7c58770b440 100644
--- a/plugins/discourse-ai/lib/completions/prompt.rb
+++ b/plugins/discourse-ai/lib/completions/prompt.rb
@@ -6,7 +6,13 @@ module DiscourseAi
INVALID_TURN = Class.new(StandardError)
attr_reader :messages, :tools, :system_message_text
- attr_accessor :topic_id, :post_id, :max_pixels, :tool_choice, :skip_trim, :native_tools
+ attr_accessor :topic_id,
+ :post_id,
+ :max_pixels,
+ :tool_choice,
+ :skip_trim,
+ :native_tools,
+ :upload_skips
def self.text_only(message)
if message[:content].is_a?(Array)
@@ -194,12 +200,7 @@ module DiscourseAi
.compact
if !upload_ids.empty?
encoded_uploads.concat(
- UploadEncoder.encode(
- upload_ids: upload_ids,
- max_pixels: max_pixels,
- allowed_kinds: allowed_kinds,
- allowed_attachment_types: allowed_attachment_types,
- ),
+ encode_upload_ids(upload_ids, allowed_kinds, allowed_attachment_types),
)
end
return encoded_uploads
@@ -218,12 +219,17 @@ module DiscourseAi
allowed_upload_kinds(allow_images: allow_images, allow_documents: allow_documents)
return if allowed_kinds.empty?
+ encode_upload_ids([upload_id], allowed_kinds, allowed_attachment_types).first
+ end
+
+ def encode_upload_ids(upload_ids, allowed_kinds, allowed_attachment_types)
UploadEncoder.encode(
- upload_ids: [upload_id],
+ upload_ids: upload_ids,
max_pixels: max_pixels,
allowed_kinds: allowed_kinds,
allowed_attachment_types: allowed_attachment_types,
- ).first
+ skips: upload_skips,
+ )
end
def content_with_encoded_uploads(
diff --git a/plugins/discourse-ai/lib/completions/upload_encoder.rb b/plugins/discourse-ai/lib/completions/upload_encoder.rb
index 2c1249714cb..6df791dd253 100644
--- a/plugins/discourse-ai/lib/completions/upload_encoder.rb
+++ b/plugins/discourse-ai/lib/completions/upload_encoder.rb
@@ -26,9 +26,11 @@ module DiscourseAi
upload_ids:,
max_pixels:,
allowed_kinds: [:image],
- allowed_attachment_types: nil
+ allowed_attachment_types: nil,
+ skips: nil
)
allowed_attachment_types = normalize_attachment_types(allowed_attachment_types)
+ skips ||= []
uploads_by_id = Upload.where(id: upload_ids).index_by(&:id)
upload_ids.filter_map do |upload_id|
@@ -40,6 +42,7 @@ module DiscourseAi
if kind == :document && image?(upload)
log_image_upload_skip(
+ skips,
upload,
"unsupported image format, supported formats are: #{SUPPORTED_IMAGE_EXTENSIONS.join(", ")}",
)
@@ -56,15 +59,15 @@ module DiscourseAi
attachment_type = DocumentEncoder.attachment_type_for(upload.extension, mime_type)
next if allowed_attachment_types&.exclude?(attachment_type)
- next DocumentEncoder.encode(upload, mime_type, attachment_type)
+ next DocumentEncoder.encode(upload, mime_type, attachment_type, skips)
end
if upload.width.to_i == 0 || upload.height.to_i == 0
- log_image_upload_skip(upload, "image dimensions are unknown")
+ log_image_upload_skip(skips, upload, "image dimensions are unknown")
next
end
- encode_image(upload, transcode_format(extension), max_pixels)
+ encode_image(upload, transcode_format(extension), max_pixels, skips)
end
end
@@ -85,14 +88,16 @@ module DiscourseAi
JPEG_EXTENSIONS.include?(extension) ? "jpeg" : "png"
end
- def log_image_upload_skip(upload, message)
+ def log_image_upload_skip(skips, upload, message)
+ record_skip(skips, upload, message)
+
Rails.logger.warn(
"Discourse AI: Skipping image upload " \
"(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",
)
end
- def encode_image(upload, desired_extension, max_pixels)
+ def encode_image(upload, desired_extension, max_pixels, skips)
original_pixels = upload.width * upload.height
image = upload
@@ -110,7 +115,7 @@ module DiscourseAi
end
if !image
- log_image_upload_skip(upload, "could not be converted to #{desired_extension}")
+ log_image_upload_skip(skips, upload, "could not be converted to #{desired_extension}")
return
end
@@ -119,7 +124,7 @@ module DiscourseAi
path = fetch_path(image)
if path.blank?
- log_image_upload_skip(upload, "file is not available in the store")
+ log_image_upload_skip(skips, upload, "file is not available in the store")
return
end
diff --git a/plugins/discourse-ai/lib/completions/upload_encoding.rb b/plugins/discourse-ai/lib/completions/upload_encoding.rb
index 14d0125f7f8..d4279fd9e55 100644
--- a/plugins/discourse-ai/lib/completions/upload_encoding.rb
+++ b/plugins/discourse-ai/lib/completions/upload_encoding.rb
@@ -13,6 +13,12 @@ module DiscourseAi
path
end
+
+ def record_skip(skips, upload, message)
+ return if skips.any? { |skip| skip[:upload_id] == upload.id }
+
+ skips << { upload_id: upload.id, filename: upload.original_filename, message: message }
+ end
end
end
end
diff --git a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
index 0f10acd496f..35ed1ac3014 100644
--- a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
@@ -1,6 +1,42 @@
# frozen_string_literal: true
RSpec.describe DiscourseAi::Completions::Llm do
+ describe "upload skips" do
+ fab!(:vision_model) { Fabricate(:anthropic_model, vision_enabled: true) }
+
+ it "hands the execution context's collector to the prompt it is about to translate" do
+ upload = Fabricate(:upload, original_filename: "avatar.jxl", extension: "jxl")
+ execution_context = DiscourseAi::Completions::ExecutionContext.new
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "system",
+ messages: [{ type: :user, content: ["look", { upload_id: upload.id }] }],
+ )
+
+ stub_request(:post, "https://api.anthropic.com/v1/messages").to_return(
+ body: {
+ id: "msg_1",
+ type: "message",
+ role: "assistant",
+ content: [{ type: "text", text: "done" }],
+ model: "claude-3-opus",
+ usage: {
+ input_tokens: 1,
+ output_tokens: 1,
+ },
+ }.to_json,
+ )
+
+ vision_model.to_llm.generate(
+ prompt,
+ user: Discourse.system_user,
+ execution_context: execution_context,
+ )
+
+ expect(execution_context.upload_skips.map { |skip| skip[:upload_id] }).to eq([upload.id])
+ end
+ end
+
fab!(:user)
fab!(:model, :llm_model)
diff --git a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
index d986b928e41..ec7ec092e70 100644
--- a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
@@ -197,6 +197,34 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
expect(described_class.encode(upload_ids: [missing_id], max_pixels: MAX_PIXELS)).to be_empty
end
+ it "records skipped images into a caller supplied collector" do
+ upload = Fabricate(:upload, original_filename: "avatar.jxl", extension: "jxl")
+ skips = []
+
+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS, skips: skips)
+
+ expect(encoded).to be_empty
+ expect(skips.map { |skip| skip[:upload_id] }).to eq([upload.id])
+ expect(skips.first[:filename]).to eq("avatar.jxl")
+ expect(skips.first[:message]).to include("unsupported image format")
+ end
+
+ it "reaches the collector a prompt was handed" do
+ upload = Fabricate(:upload, original_filename: "avatar.jxl", extension: "jxl")
+ execution_context = DiscourseAi::Completions::ExecutionContext.new
+
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "system",
+ messages: [{ type: :user, content: ["look", { upload_id: upload.id }] }],
+ )
+ prompt.upload_skips = execution_context.upload_skips
+
+ prompt.encoded_uploads(prompt.messages.last)
+
+ expect(execution_context.upload_skips.map { |skip| skip[:upload_id] }).to eq([upload.id])
+ end
+
it "only claims formats ImageMagick can decode" do
described_class::SUPPORTED_IMAGE_EXTENSIONS.each do |extension|
expect(extension).to match(OptimizedImage::IM_DECODERS)