Skip to content

A platform for community discussion. Free, open, simple.

DEV: Update models in config and the DE runner (#41082)

Natalie Tayb12015e49092
diff --git a/plugins/discourse-ai/config/eval-llms.yml b/plugins/discourse-ai/config/eval-llms.yml
index c8ac592b5e4..92797f2b70e 100644
--- a/plugins/discourse-ai/config/eval-llms.yml
+++ b/plugins/discourse-ai/config/eval-llms.yml
@@ -1,60 +1,82 @@
llms:
- gpt-5-nano:
- display_name: GPT-5-Nano
- name: gpt-5-nano
+ gpt-5.5:
+ display_name: GPT-5.5
+ name: gpt-5.5
tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizer
api_key_env: OPENAI_API_KEY
provider: open_ai
- url: https://api.openai.com/v1/chat/completions
- max_prompt_tokens: 131072
+ url: https://api.openai.com/v1/responses
+ max_prompt_tokens: 1050000
vision_enabled: true
- gpt-5-nano-thinking:
- display_name: GPT-5-Nano Thinking
- name: gpt-5-nano
+ gpt-5.5-thinking:
+ display_name: GPT-5.5 Thinking
+ name: gpt-5.5
tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizer
api_key_env: OPENAI_API_KEY
provider: open_ai
url: https://api.openai.com/v1/responses
- max_prompt_tokens: 131072
+ max_prompt_tokens: 1050000
vision_enabled: true
provider_params:
- reasoning_effort: low
+ reasoning_effort: high
- gpt-5.2:
- display_name: GPT-5.2
- name: gpt-5.2
+ gpt-5.4:
+ display_name: GPT-5.4
+ name: gpt-5.4
tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizer
api_key_env: OPENAI_API_KEY
provider: open_ai
- url: https://api.openai.com/v1/chat/completions
- max_prompt_tokens: 131072
+ url: https://api.openai.com/v1/responses
+ max_prompt_tokens: 400000
vision_enabled: true
- claude-4.5-haiku:
- display_name: Claude 4.5 Haiku
- name: claude-haiku-4-5
+ gpt-5.4-nano:
+ display_name: GPT-5.4 Nano
+ name: gpt-5.4-nano
+ tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizer
+ api_key_env: OPENAI_API_KEY
+ provider: open_ai
+ url: https://api.openai.com/v1/responses
+ max_prompt_tokens: 400000
+ vision_enabled: true
+
+ gpt-5.4-nano-thinking:
+ display_name: GPT-5.4 Nano Thinking
+ name: gpt-5.4-nano
+ tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizer
+ api_key_env: OPENAI_API_KEY
+ provider: open_ai
+ url: https://api.openai.com/v1/responses
+ max_prompt_tokens: 400000
+ vision_enabled: true
+ provider_params:
+ reasoning_effort: low
+
+ claude-4.7-opus:
+ display_name: Claude 4.7 Opus
+ name: claude-opus-4-7
tokenizer: DiscourseAi::Tokenizer::AnthropicTokenizer
api_key_env: ANTHROPIC_API_KEY
provider: anthropic
url: https://api.anthropic.com/v1/messages
- max_prompt_tokens: 200000
- vision_enabled: false
+ max_prompt_tokens: 1000000
+ vision_enabled: true
- claude-4.5-haiku-thinking:
- display_name: Claude 4.5 Haiku
- name: claude-haiku-4-5
+ claude-4.7-opus-thinking:
+ display_name: Claude 4.7 Opus Thinking
+ name: claude-opus-4-7
tokenizer: DiscourseAi::Tokenizer::AnthropicTokenizer
api_key_env: ANTHROPIC_API_KEY
provider: anthropic
url: https://api.anthropic.com/v1/messages
- max_prompt_tokens: 200000
+ max_prompt_tokens: 1000000
vision_enabled: true
provider_params:
disable_top_p: true
disable_temperature: true
enable_reasoning: true
- reasoning_tokens: 1024
+ reasoning_tokens: 2048
claude-4.6-sonnet:
display_name: Claude 4.6 Sonnet
@@ -63,50 +85,57 @@ llms:
api_key_env: ANTHROPIC_API_KEY
provider: anthropic
url: https://api.anthropic.com/v1/messages
- max_prompt_tokens: 200000
+ max_prompt_tokens: 1000000
vision_enabled: true
claude-4.6-sonnet-thinking:
- display_name: Claude 4.6 Sonnet
+ display_name: Claude 4.6 Sonnet Thinking
name: claude-sonnet-4-6
tokenizer: DiscourseAi::Tokenizer::AnthropicTokenizer
api_key_env: ANTHROPIC_API_KEY
provider: anthropic
url: https://api.anthropic.com/v1/messages
- max_prompt_tokens: 200000
+ max_prompt_tokens: 1000000
vision_enabled: true
provider_params:
disable_top_p: true
disable_temperature: true
enable_reasoning: true
- reasoning_tokens: 1024
+ reasoning_tokens: 2048
- gemini-2.0-flash:
- display_name: Gemini 2.0 Flash
- name: gemini-2-0-flash
+ claude-4.5-haiku:
+ display_name: Claude 4.5 Haiku
+ name: claude-haiku-4-5
+ tokenizer: DiscourseAi::Tokenizer::AnthropicTokenizer
+ api_key_env: ANTHROPIC_API_KEY
+ provider: anthropic
+ url: https://api.anthropic.com/v1/messages
+ max_prompt_tokens: 200000
+ vision_enabled: true
+
+ gemini-3.1-pro:
+ display_name: Gemini 3.1 Pro
+ name: gemini-3.1-pro
tokenizer: DiscourseAi::Tokenizer::GeminiTokenizer
api_key_env: GEMINI_API_KEY
provider: google
- url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash
+ url: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-pro-preview
max_prompt_tokens: 1000000
vision_enabled: true
- gemini-2.5-flash:
- display_name: Gemini 2.5 Flash
- name: gemini-2-5-flash
+ gemini-3-flash:
+ display_name: Gemini 3 Flash
+ name: gemini-3-flash
tokenizer: DiscourseAi::Tokenizer::GeminiTokenizer
api_key_env: GEMINI_API_KEY
provider: google
- url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash
+ url: https://generativelanguage.googleapis.com/v1beta/models/gemini-3-flash-preview
max_prompt_tokens: 1000000
vision_enabled: true
- provider_params:
- disable_top_p: true
- disable_temperature: true
gemini-3-flash-thinking:
- display_name: Gemini 3.0 Flash
- name: gemini-3-0-flash
+ display_name: Gemini 3 Flash Thinking
+ name: gemini-3-flash
tokenizer: DiscourseAi::Tokenizer::GeminiTokenizer
api_key_env: GEMINI_API_KEY
provider: google
@@ -117,24 +146,4 @@ llms:
disable_top_p: true
disable_temperature: true
enable_thinking: true
- thinking_tokens: 1024
-
- gemini-3-pro:
- display_name: Gemini 3.0 pro
- name: gemini-3-pro-preview
- tokenizer: DiscourseAi::Tokenizer::GeminiTokenizer
- api_key_env: GEMINI_API_KEY
- provider: google
- url: https://generativelanguage.googleapis.com/v1beta/models/gemini-3-pro-preview
- max_prompt_tokens: 1000000
- vision_enabled: true
-
- gemini-3.1-flash-lite:
- display_name: Gemini 3.1 Flash Lite
- name: gemini-3.1-flash-lite
- tokenizer: DiscourseAi::Tokenizer::GeminiTokenizer
- api_key_env: GEMINI_API_KEY
- provider: google
- url: https://generativelanguage.googleapis.com/v1beta/models/gemini-3-flash-preview
- max_prompt_tokens: 1000000
- vision_enabled: true
+ thinking_tokens: 2048
diff --git a/plugins/discourse-ai/evals/lib/runners/data_explorer.rb b/plugins/discourse-ai/evals/lib/runners/data_explorer.rb
index a625da11fd7..7d4ff1911a5 100644
--- a/plugins/discourse-ai/evals/lib/runners/data_explorer.rb
+++ b/plugins/discourse-ai/evals/lib/runners/data_explorer.rb
@@ -6,8 +6,6 @@ module DiscourseAi
module Evals
module Runners
class DataExplorer < Base
- STRUCTURED_KEYS = %i[name description sql].freeze
-
def self.can_handle?(full_feature_name)
full_feature_name&.start_with?("data_explorer:")
end
@@ -26,9 +24,12 @@ module DiscourseAi
)
bot = DiscourseAi::Agents::Bot.as(user, agent: agent, model: llm)
- captured = capture_structured_fields(bot, context, execution_context:)
+ bot.reply(context, execution_context:)
+
+ captured =
+ context.feature_context[DiscourseDataExplorer::Tools::SubmitQuery::CONTEXT_KEY] || {}
- sql = captured[:sql].to_s.strip
+ sql = captured[:sql].to_s.chomp(";").strip
metadata = {
feature: feature_name,
name: captured[:name].to_s.strip,
@@ -37,25 +38,6 @@ module DiscourseAi
wrap_result(sql, metadata)
end
-
- private
-
- def capture_structured_fields(bot, context, execution_context:)
- buffers = STRUCTURED_KEYS.index_with { +"" }
-
- bot.reply(context, execution_context:) do |partial, _, type|
- if type == :structured_output
- STRUCTURED_KEYS.each do |key|
- chunk = partial.read_buffered_property(key)
- buffers[key] << chunk.to_s if chunk
- end
- elsif type.blank?
- buffers[:sql] << partial.to_s
- end
- end
-
- buffers
- end
end
end
end
diff --git a/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rb b/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rb
index f34fa902ceb..003871ee120 100644
--- a/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rb
+++ b/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rb
@@ -9,22 +9,22 @@ RSpec.describe DiscourseAi::Evals::Runners::DataExplorer do
let(:runner) { described_class.new("query_generation") }
let(:eval_case) { OpenStruct.new(args: { input: "Show me signups by month" }) }
- def stub_structured_response(name:, description:, sql:)
- payload = { name: name, description: description, sql: sql }
- structured =
- instance_double(DiscourseAi::Completions::StructuredOutput).tap do |double|
- allow(double).to receive(:read_buffered_property) { |key| payload[key] }
- end
+ def stub_submitted_query(name:, description:, sql:)
+ submission = { name: name, description: description, sql: sql }
+ bot = instance_double(DiscourseAi::Agents::Bot)
- stub_runner_bot { |blk| blk.call(structured, nil, :structured_output) }
+ allow(DiscourseAi::Agents::Bot).to receive(:as).and_return(bot)
+ allow(bot).to receive(:reply) do |context, execution_context: nil|
+ context.feature_context[DiscourseDataExplorer::Tools::SubmitQuery::CONTEXT_KEY] = submission
+ end
end
describe "#run" do
- it "captures name, description, and sql separately" do
- stub_structured_response(
+ it "captures the query submitted through the Data Explorer submit tool" do
+ stub_submitted_query(
name: "Signups by month",
description: "Counts user signups grouped by month.",
- sql: "SELECT date_trunc('month', created_at) AS month FROM users GROUP BY month",
+ sql: "SELECT date_trunc('month', created_at) AS month FROM users GROUP BY month;",
)
result = runner.run(eval_case, llm, execution_context: execution_context)
@@ -33,6 +33,7 @@ RSpec.describe DiscourseAi::Evals::Runners::DataExplorer do
expect(result[:metadata][:name]).to eq("Signups by month")
expect(result[:metadata][:description]).to eq("Counts user signups grouped by month.")
expect(result[:metadata][:feature]).to eq("query_generation")
+ expect(result[:raw]).not_to end_with(";")
end
end
end