A platform for community discussion. Free, open, simple.
FIX: Count image tokens in prompt capacity (#44619)
Sam16cb540dc7d5
diff --git a/plugins/discourse-ai/lib/completions/llm.rb b/plugins/discourse-ai/lib/completions/llm.rbindex df50c501e81..3e8d8b3946c 100644--- a/plugins/discourse-ai/lib/completions/llm.rb+++ b/plugins/discourse-ai/lib/completions/llm.rb@@ -705,17 +705,23 @@ module DiscourseAi(attachment && %w[data base64].include?(key.to_s)) ||(binary_source && key.to_s == "bytes") ||(- %w[url file_data].include?(key.to_s) &&+ %w[url image_url file_data].include?(key.to_s) &&part.match?(%r{\Adata:(?:image/|application/pdf).*;base64,}) &&(- parent_key.to_s == "image_url" || value.key?(:filename) ||- value.key?("filename")+ parent_key.to_s == "image_url" ||+ (+ key.to_s == "image_url" &&+ (value[:type] || value["type"]) == "input_image"+ ) || value.key?(:filename) || value.key?("filename"))))content =if blob"[binary upload]"+ elsif key.to_s == "images" && (value[:role] || value["role"]) == "user" &&+ part.is_a?(Array)+ part.map { "[binary upload]" }elsecapacity_payload(part,diff --git a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb b/plugins/discourse-ai/spec/lib/completions/llm_spec.rbindex 3de4b4cfa32..3f69994fec6 100644--- a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb@@ -355,6 +355,60 @@ RSpec.describe DiscourseAi::Completions::Llm doenddescribe "#prompt_capacity" do+ it "counts Responses API images by their image tokens rather than their base64 text" do+ model.update!(vision_enabled: true)+ image = file_from_fixtures("logo.png")+ prompt =+ DiscourseAi::Completions::Prompt.new(+ "Instructions",+ messages: [+ {+ type: :user,+ content: [+ "Review this image",+ {+ encoded_upload: {+ kind: :image,+ mime_type: "image/png",+ base64: Base64.strict_encode64(File.binread(image.path)),+ },+ },+ ],+ },+ ],+ )+ chat_size = model.to_llm.prompt_capacity(prompt).first++ model.update!(url: "https://api.openai.com/v1/responses")+ responses_size = model.to_llm.prompt_capacity(prompt).first++ expect(responses_size).to be >= 256+ expect(responses_size).to be_within(100).of(chat_size)+ end++ it "counts Ollama images by their image tokens rather than their base64 text" do+ ollama_model = Fabricate(:ollama_model, vision_enabled: true)+ measured_llm = ollama_model.to_llm+ image = file_from_fixtures("logo.png")+ prompt =+ DiscourseAi::Completions::Prompt.new(+ "Instructions",+ messages: [{ type: :user, content: ["Review this image"] }],+ )+ text_size = measured_llm.prompt_capacity(prompt).first+ prompt.messages.last[:content] << {+ encoded_upload: {+ kind: :image,+ mime_type: "image/png",+ base64: Base64.strict_encode64(File.binread(image.path)),+ },+ }++ image_size = measured_llm.prompt_capacity(prompt).first++ expect(image_size - text_size).to be_within(100).of(8192)+ end+it "admits native PDFs on supported providers and reserves rendered pages plus extracted text" dopdf = file_from_fixtures("2-page.pdf", "rag", "plugins/discourse-ai/spec/fixtures")encoded = {