Skip to content

A platform for community discussion. Free, open, simple.

FIX: Count image tokens in prompt capacity (#44619)

Sam16cb540dc7d5
diff --git a/plugins/discourse-ai/lib/completions/llm.rb b/plugins/discourse-ai/lib/completions/llm.rb
index df50c501e81..3e8d8b3946c 100644
--- a/plugins/discourse-ai/lib/completions/llm.rb
+++ b/plugins/discourse-ai/lib/completions/llm.rb
@@ -705,17 +705,23 @@ module DiscourseAi
(attachment && %w[data base64].include?(key.to_s)) ||
(binary_source && key.to_s == "bytes") ||
(
- %w[url file_data].include?(key.to_s) &&
+ %w[url image_url file_data].include?(key.to_s) &&
part.match?(%r{\Adata:(?:image/|application/pdf).*;base64,}) &&
(
- parent_key.to_s == "image_url" || value.key?(:filename) ||
- value.key?("filename")
+ parent_key.to_s == "image_url" ||
+ (
+ key.to_s == "image_url" &&
+ (value[:type] || value["type"]) == "input_image"
+ ) || value.key?(:filename) || value.key?("filename")
)
)
)
content =
if blob
"[binary upload]"
+ elsif key.to_s == "images" && (value[:role] || value["role"]) == "user" &&
+ part.is_a?(Array)
+ part.map { "[binary upload]" }
else
capacity_payload(
part,
diff --git a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
index 3de4b4cfa32..3f69994fec6 100644
--- a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
@@ -355,6 +355,60 @@ RSpec.describe DiscourseAi::Completions::Llm do
end
describe "#prompt_capacity" do
+ it "counts Responses API images by their image tokens rather than their base64 text" do
+ model.update!(vision_enabled: true)
+ image = file_from_fixtures("logo.png")
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "Instructions",
+ messages: [
+ {
+ type: :user,
+ content: [
+ "Review this image",
+ {
+ encoded_upload: {
+ kind: :image,
+ mime_type: "image/png",
+ base64: Base64.strict_encode64(File.binread(image.path)),
+ },
+ },
+ ],
+ },
+ ],
+ )
+ chat_size = model.to_llm.prompt_capacity(prompt).first
+
+ model.update!(url: "https://api.openai.com/v1/responses")
+ responses_size = model.to_llm.prompt_capacity(prompt).first
+
+ expect(responses_size).to be >= 256
+ expect(responses_size).to be_within(100).of(chat_size)
+ end
+
+ it "counts Ollama images by their image tokens rather than their base64 text" do
+ ollama_model = Fabricate(:ollama_model, vision_enabled: true)
+ measured_llm = ollama_model.to_llm
+ image = file_from_fixtures("logo.png")
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "Instructions",
+ messages: [{ type: :user, content: ["Review this image"] }],
+ )
+ text_size = measured_llm.prompt_capacity(prompt).first
+ prompt.messages.last[:content] << {
+ encoded_upload: {
+ kind: :image,
+ mime_type: "image/png",
+ base64: Base64.strict_encode64(File.binread(image.path)),
+ },
+ }
+
+ image_size = measured_llm.prompt_capacity(prompt).first
+
+ expect(image_size - text_size).to be_within(100).of(8192)
+ end
+
it "admits native PDFs on supported providers and reserves rendered pages plus extracted text" do
pdf = file_from_fixtures("2-page.pdf", "rag", "plugins/discourse-ai/spec/fixtures")
encoded = {