A platform for community discussion. Free, open, simple.
FIX: AI: AWS Bedrock Converse image uploads were doubly base64-encoded (#39880)
Takao Yokoyamacea4780cb38a
diff --git a/plugins/discourse-ai/lib/completions/dialects/converse.rb b/plugins/discourse-ai/lib/completions/dialects/converse.rbindex 33752bd73eb..fb5aefc58d5 100644--- a/plugins/discourse-ai/lib/completions/dialects/converse.rb+++ b/plugins/discourse-ai/lib/completions/dialects/converse.rb@@ -137,7 +137,11 @@ module DiscourseAiimage: {format: details[:format] || detect_format(details[:mime_type]),source: {- bytes: details[:base64],+ # AWS SDK for Ruby expects raw bytes here and will base64-encode them+ # on the wire. Passing the already-base64-encoded string causes Bedrock+ # to receive doubly-encoded data and respond with+ # "Could not process image".+ bytes: Base64.decode64(details[:base64]),},},}diff --git a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rbindex 52b52077802..5e387c824a2 100644--- a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb+++ b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb@@ -82,7 +82,15 @@ module DiscourseAilog = nilsdk_params = build_converse_params(prompt, model_params, dialect)- request_body = sdk_params.to_json+ # Image bytes in sdk_params are raw binary (ASCII-8BIT) and cannot be+ # encoded as UTF-8 JSON for logging. Fall back to a placeholder so the+ # request can still proceed when binary payloads are present.+ request_body =+ begin+ sdk_params.to_json+ rescue EncodingError, JSON::GeneratorError+ "[converse params contained binary payload, omitted from log]"+ endlog =start_log(diff --git a/plugins/discourse-ai/spec/lib/completions/dialects/converse_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/converse_spec.rbindex be03fa2e56c..2cf3c4545c0 100644--- a/plugins/discourse-ai/spec/lib/completions/dialects/converse_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/dialects/converse_spec.rb@@ -59,5 +59,30 @@ RSpec.describe DiscourseAi::Completions::Dialects::Converse doexpect(user_message[:content]).not_to include(hash_including(image: anything))expect(user_message[:content]).not_to include(hash_including(document: anything))end++ it "passes raw bytes for image uploads, not the base64-encoded string" do+ model.update!(vision_enabled: true)+ raw_bytes = "\x89PNG\r\n\x1a\nbinary".b+ prompt =+ DiscourseAi::Completions::Prompt.new(+ nil,+ messages: [{ type: :user, content: ["Describe: ", { upload_id: 456 }] }],+ )++ allow(DiscourseAi::Completions::UploadEncoder).to receive(:encode).and_return(+ [{ kind: :image, mime_type: "image/png", base64: Base64.strict_encode64(raw_bytes) }],+ )++ translated = described_class.new(prompt, model).translate+ user_message = translated.messages.find { |msg| msg[:role] == "user" }+ image_block = user_message[:content].find { |c| c[:image] }++ expect(image_block).to be_present+ expect(image_block.dig(:image, :format)).to eq("png")+ # AWS SDK for Ruby expects raw bytes; it will base64-encode on the wire.+ # Passing the base64 string would cause double-encoding and Bedrock would+ # return "Could not process image".+ expect(image_block.dig(:image, :source, :bytes)).to eq(raw_bytes)+ endendenddiff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rbindex 4468bafeb46..29e17da8f91 100644--- a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb@@ -181,6 +181,33 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse doexpect(params[:system]).to be_presentendend++ it "logs a placeholder when image bytes are binary" do+ model.update!(vision_enabled: true)+ raw_bytes = "\x89PNG\r\n\x1a\nbinary".b+ response = mock_converse_response+ client = stub_sdk_client(response: response)+ prompt =+ DiscourseAi::Completions::Prompt.new(+ nil,+ messages: [{ type: :user, content: ["Describe: ", { upload_id: 456 }] }],+ )++ allow(DiscourseAi::Completions::UploadEncoder).to receive(:encode).and_return(+ [{ kind: :image, mime_type: "image/png", base64: Base64.strict_encode64(raw_bytes) }],+ )++ llm = DiscourseAi::Completions::Llm.proxy("custom:#{model.id}")+ result = llm.generate(prompt, user: user)++ expect(result).to eq("Hello world")+ expect(AiApiAuditLog.last.raw_request_payload).to eq(+ "[converse params contained binary payload, omitted from log]",+ )+ expect(client).to have_received(:converse) do |params|+ expect(params.dig(:messages, 0, :content, 1, :image, :source, :bytes)).to eq(raw_bytes)+ end+ endenddescribe "streaming completion" do