A platform for community discussion. Free, open, simple.
DEV: Update models in config and the DE runner (#41082)
Natalie Tayb12015e49092
diff --git a/plugins/discourse-ai/config/eval-llms.yml b/plugins/discourse-ai/config/eval-llms.ymlindex c8ac592b5e4..92797f2b70e 100644--- a/plugins/discourse-ai/config/eval-llms.yml+++ b/plugins/discourse-ai/config/eval-llms.yml@@ -1,60 +1,82 @@llms:- gpt-5-nano:- display_name: GPT-5-Nano- name: gpt-5-nano+ gpt-5.5:+ display_name: GPT-5.5+ name: gpt-5.5tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizerapi_key_env: OPENAI_API_KEYprovider: open_ai- url: https://api.openai.com/v1/chat/completions- max_prompt_tokens: 131072+ url: https://api.openai.com/v1/responses+ max_prompt_tokens: 1050000vision_enabled: true- gpt-5-nano-thinking:- display_name: GPT-5-Nano Thinking- name: gpt-5-nano+ gpt-5.5-thinking:+ display_name: GPT-5.5 Thinking+ name: gpt-5.5tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizerapi_key_env: OPENAI_API_KEYprovider: open_aiurl: https://api.openai.com/v1/responses- max_prompt_tokens: 131072+ max_prompt_tokens: 1050000vision_enabled: trueprovider_params:- reasoning_effort: low+ reasoning_effort: high- gpt-5.2:- display_name: GPT-5.2- name: gpt-5.2+ gpt-5.4:+ display_name: GPT-5.4+ name: gpt-5.4tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizerapi_key_env: OPENAI_API_KEYprovider: open_ai- url: https://api.openai.com/v1/chat/completions- max_prompt_tokens: 131072+ url: https://api.openai.com/v1/responses+ max_prompt_tokens: 400000vision_enabled: true- claude-4.5-haiku:- display_name: Claude 4.5 Haiku- name: claude-haiku-4-5+ gpt-5.4-nano:+ display_name: GPT-5.4 Nano+ name: gpt-5.4-nano+ tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizer+ api_key_env: OPENAI_API_KEY+ provider: open_ai+ url: https://api.openai.com/v1/responses+ max_prompt_tokens: 400000+ vision_enabled: true++ gpt-5.4-nano-thinking:+ display_name: GPT-5.4 Nano Thinking+ name: gpt-5.4-nano+ tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizer+ api_key_env: OPENAI_API_KEY+ provider: open_ai+ url: https://api.openai.com/v1/responses+ max_prompt_tokens: 400000+ vision_enabled: true+ provider_params:+ reasoning_effort: low++ claude-4.7-opus:+ display_name: Claude 4.7 Opus+ name: claude-opus-4-7tokenizer: DiscourseAi::Tokenizer::AnthropicTokenizerapi_key_env: ANTHROPIC_API_KEYprovider: anthropicurl: https://api.anthropic.com/v1/messages- max_prompt_tokens: 200000- vision_enabled: false+ max_prompt_tokens: 1000000+ vision_enabled: true- claude-4.5-haiku-thinking:- display_name: Claude 4.5 Haiku- name: claude-haiku-4-5+ claude-4.7-opus-thinking:+ display_name: Claude 4.7 Opus Thinking+ name: claude-opus-4-7tokenizer: DiscourseAi::Tokenizer::AnthropicTokenizerapi_key_env: ANTHROPIC_API_KEYprovider: anthropicurl: https://api.anthropic.com/v1/messages- max_prompt_tokens: 200000+ max_prompt_tokens: 1000000vision_enabled: trueprovider_params:disable_top_p: truedisable_temperature: trueenable_reasoning: true- reasoning_tokens: 1024+ reasoning_tokens: 2048claude-4.6-sonnet:display_name: Claude 4.6 Sonnet@@ -63,50 +85,57 @@ llms:api_key_env: ANTHROPIC_API_KEYprovider: anthropicurl: https://api.anthropic.com/v1/messages- max_prompt_tokens: 200000+ max_prompt_tokens: 1000000vision_enabled: trueclaude-4.6-sonnet-thinking:- display_name: Claude 4.6 Sonnet+ display_name: Claude 4.6 Sonnet Thinkingname: claude-sonnet-4-6tokenizer: DiscourseAi::Tokenizer::AnthropicTokenizerapi_key_env: ANTHROPIC_API_KEYprovider: anthropicurl: https://api.anthropic.com/v1/messages- max_prompt_tokens: 200000+ max_prompt_tokens: 1000000vision_enabled: trueprovider_params:disable_top_p: truedisable_temperature: trueenable_reasoning: true- reasoning_tokens: 1024+ reasoning_tokens: 2048- gemini-2.0-flash:- display_name: Gemini 2.0 Flash- name: gemini-2-0-flash+ claude-4.5-haiku:+ display_name: Claude 4.5 Haiku+ name: claude-haiku-4-5+ tokenizer: DiscourseAi::Tokenizer::AnthropicTokenizer+ api_key_env: ANTHROPIC_API_KEY+ provider: anthropic+ url: https://api.anthropic.com/v1/messages+ max_prompt_tokens: 200000+ vision_enabled: true++ gemini-3.1-pro:+ display_name: Gemini 3.1 Pro+ name: gemini-3.1-protokenizer: DiscourseAi::Tokenizer::GeminiTokenizerapi_key_env: GEMINI_API_KEYprovider: google- url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash+ url: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-pro-previewmax_prompt_tokens: 1000000vision_enabled: true- gemini-2.5-flash:- display_name: Gemini 2.5 Flash- name: gemini-2-5-flash+ gemini-3-flash:+ display_name: Gemini 3 Flash+ name: gemini-3-flashtokenizer: DiscourseAi::Tokenizer::GeminiTokenizerapi_key_env: GEMINI_API_KEYprovider: google- url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash+ url: https://generativelanguage.googleapis.com/v1beta/models/gemini-3-flash-previewmax_prompt_tokens: 1000000vision_enabled: true- provider_params:- disable_top_p: true- disable_temperature: truegemini-3-flash-thinking:- display_name: Gemini 3.0 Flash- name: gemini-3-0-flash+ display_name: Gemini 3 Flash Thinking+ name: gemini-3-flashtokenizer: DiscourseAi::Tokenizer::GeminiTokenizerapi_key_env: GEMINI_API_KEYprovider: google@@ -117,24 +146,4 @@ llms:disable_top_p: truedisable_temperature: trueenable_thinking: true- thinking_tokens: 1024-- gemini-3-pro:- display_name: Gemini 3.0 pro- name: gemini-3-pro-preview- tokenizer: DiscourseAi::Tokenizer::GeminiTokenizer- api_key_env: GEMINI_API_KEY- provider: google- url: https://generativelanguage.googleapis.com/v1beta/models/gemini-3-pro-preview- max_prompt_tokens: 1000000- vision_enabled: true-- gemini-3.1-flash-lite:- display_name: Gemini 3.1 Flash Lite- name: gemini-3.1-flash-lite- tokenizer: DiscourseAi::Tokenizer::GeminiTokenizer- api_key_env: GEMINI_API_KEY- provider: google- url: https://generativelanguage.googleapis.com/v1beta/models/gemini-3-flash-preview- max_prompt_tokens: 1000000- vision_enabled: true+ thinking_tokens: 2048diff --git a/plugins/discourse-ai/evals/lib/runners/data_explorer.rb b/plugins/discourse-ai/evals/lib/runners/data_explorer.rbindex a625da11fd7..7d4ff1911a5 100644--- a/plugins/discourse-ai/evals/lib/runners/data_explorer.rb+++ b/plugins/discourse-ai/evals/lib/runners/data_explorer.rb@@ -6,8 +6,6 @@ module DiscourseAimodule Evalsmodule Runnersclass DataExplorer < Base- STRUCTURED_KEYS = %i[name description sql].freeze-def self.can_handle?(full_feature_name)full_feature_name&.start_with?("data_explorer:")end@@ -26,9 +24,12 @@ module DiscourseAi)bot = DiscourseAi::Agents::Bot.as(user, agent: agent, model: llm)- captured = capture_structured_fields(bot, context, execution_context:)+ bot.reply(context, execution_context:)++ captured =+ context.feature_context[DiscourseDataExplorer::Tools::SubmitQuery::CONTEXT_KEY] || {}- sql = captured[:sql].to_s.strip+ sql = captured[:sql].to_s.chomp(";").stripmetadata = {feature: feature_name,name: captured[:name].to_s.strip,@@ -37,25 +38,6 @@ module DiscourseAiwrap_result(sql, metadata)end-- private-- def capture_structured_fields(bot, context, execution_context:)- buffers = STRUCTURED_KEYS.index_with { +"" }-- bot.reply(context, execution_context:) do |partial, _, type|- if type == :structured_output- STRUCTURED_KEYS.each do |key|- chunk = partial.read_buffered_property(key)- buffers[key] << chunk.to_s if chunk- end- elsif type.blank?- buffers[:sql] << partial.to_s- end- end-- buffers- endendendenddiff --git a/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rb b/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rbindex f34fa902ceb..003871ee120 100644--- a/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rb+++ b/plugins/discourse-ai/spec/evals/runners/data_explorer_spec.rb@@ -9,22 +9,22 @@ RSpec.describe DiscourseAi::Evals::Runners::DataExplorer dolet(:runner) { described_class.new("query_generation") }let(:eval_case) { OpenStruct.new(args: { input: "Show me signups by month" }) }- def stub_structured_response(name:, description:, sql:)- payload = { name: name, description: description, sql: sql }- structured =- instance_double(DiscourseAi::Completions::StructuredOutput).tap do |double|- allow(double).to receive(:read_buffered_property) { |key| payload[key] }- end+ def stub_submitted_query(name:, description:, sql:)+ submission = { name: name, description: description, sql: sql }+ bot = instance_double(DiscourseAi::Agents::Bot)- stub_runner_bot { |blk| blk.call(structured, nil, :structured_output) }+ allow(DiscourseAi::Agents::Bot).to receive(:as).and_return(bot)+ allow(bot).to receive(:reply) do |context, execution_context: nil|+ context.feature_context[DiscourseDataExplorer::Tools::SubmitQuery::CONTEXT_KEY] = submission+ endenddescribe "#run" do- it "captures name, description, and sql separately" do- stub_structured_response(+ it "captures the query submitted through the Data Explorer submit tool" do+ stub_submitted_query(name: "Signups by month",description: "Counts user signups grouped by month.",- sql: "SELECT date_trunc('month', created_at) AS month FROM users GROUP BY month",+ sql: "SELECT date_trunc('month', created_at) AS month FROM users GROUP BY month;",)result = runner.run(eval_case, llm, execution_context: execution_context)@@ -33,6 +33,7 @@ RSpec.describe DiscourseAi::Evals::Runners::DataExplorer doexpect(result[:metadata][:name]).to eq("Signups by month")expect(result[:metadata][:description]).to eq("Counts user signups grouped by month.")expect(result[:metadata][:feature]).to eq("query_generation")+ expect(result[:raw]).not_to end_with(";")endendend