A platform for community discussion. Free, open, simple.
DEV: exercise streaming in LLM model Run Test (#39983)
Rafael dos Santos Silva0dfc9e997f42
diff --git a/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-llm-editor-form.gjs b/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-llm-editor-form.gjsindex 7b7f1fd7637..f9398fb884b 100644--- a/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-llm-editor-form.gjs+++ b/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-llm-editor-form.gjs@@ -37,6 +37,7 @@ export default class AiLlmEditorForm extends Component {@tracked testResult = null;@tracked testError = null;@tracked testValidationErrors = null;+ @tracked testFailedMode = null;@cachedget formData() {@@ -139,9 +140,15 @@ export default class AiLlmEditorForm extends Component {get testErrorMessage() {if (this.testValidationErrors?.length > 0) {return i18n("discourse_ai.llms.tests.invalid_config");- } else {- return i18n("discourse_ai.llms.tests.failure", { error: this.testError });}++ if (this.testFailedMode) {+ return i18n(`discourse_ai.llms.tests.failure_${this.testFailedMode}`, {+ error: this.testError,+ });+ }++ return i18n("discourse_ai.llms.tests.failure", { error: this.testError });}get displayTestResult() {@@ -257,9 +264,11 @@ export default class AiLlmEditorForm extends Component {if (this.testResult) {this.testError = null;this.testValidationErrors = null;+ this.testFailedMode = null;} else {this.testError = configTestResult.error;this.testValidationErrors = configTestResult.validation_errors;+ this.testFailedMode = configTestResult.failed_mode;}} catch (e) {popupAjaxError(e);diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_llms_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_llms_controller.rbindex a8fa776fd8d..561b88c93c9 100644--- a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_llms_controller.rb+++ b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_llms_controller.rb@@ -147,13 +147,15 @@ module DiscourseAienddef test- RateLimiter.new(current_user, "llm_test_#{current_user.id}", 3, 1.minute).performed!+ RateLimiter.new(current_user, "llm_test_#{current_user.id}", 6, 1.minute).performed!++ validator = DiscourseAi::Configuration::LlmValidator.new# For seeded models, test the existing model directly since provider/url/api_key are hiddenif params.dig(:ai_llm, :id).present?existing_model = LlmModel.find_by(id: params[:ai_llm][:id])if existing_model&.seeded?- DiscourseAi::Configuration::LlmValidator.new.run_test(existing_model)+ validator.run_test(existing_model)return render json: { success: true }endend@@ -162,13 +164,13 @@ module DiscourseAillm_model = LlmModel.new(ai_llm_params.merge(display_name: "LLM test"))if llm_model.valid?- DiscourseAi::Configuration::LlmValidator.new.run_test(llm_model)+ validator.run_test(llm_model)render json: { success: true }elserender json: { success: false, validation_errors: llm_model.errors.full_messages }endrescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e- render json: { success: false, error: e.message }+ render json: { success: false, error: e.message, failed_mode: validator&.last_failed_mode }endprivatediff --git a/plugins/discourse-ai/config/locales/client.en.yml b/plugins/discourse-ai/config/locales/client.en.ymlindex 5eb3ae329d1..28454744e32 100644--- a/plugins/discourse-ai/config/locales/client.en.yml+++ b/plugins/discourse-ai/config/locales/client.en.yml@@ -954,6 +954,8 @@ en:success: "Success!"invalid_config: "Fix the following errors in your configuration first:"failure: "Trying to contact the model returned this error: %{error}"+ failure_non_streaming: "Non-streaming request failed: %{error}"+ failure_streaming: "Streaming request failed: %{error}"hints:max_prompt_tokens: "The maximum number of tokens the model can process in a single request"diff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.ymlindex 9c57ca2d2b2..931dacc4789 100644--- a/plugins/discourse-ai/config/locales/server.en.yml+++ b/plugins/discourse-ai/config/locales/server.en.yml@@ -884,6 +884,7 @@ en:disable_modules_first: "You must disable these modules first: %{settings}"set_llm_first: "Set %{setting} first"model_unreachable: "We couldn't get a response from this model. Check your settings first."+ empty_response: "The model returned an empty response."invalid_seeded_model: "You can't use this model with this feature"invalid_agent_response_format: "The selected agent must have a response format with a boolean field names \"spam\""must_select_model: "You must select a LLM first"diff --git a/plugins/discourse-ai/lib/configuration/llm_validator.rb b/plugins/discourse-ai/lib/configuration/llm_validator.rbindex 800cabc2adc..71187882b20 100644--- a/plugins/discourse-ai/lib/configuration/llm_validator.rb+++ b/plugins/discourse-ai/lib/configuration/llm_validator.rb@@ -3,6 +3,10 @@module DiscourseAimodule Configurationclass LlmValidator+ TEST_PROMPT = "How much is 1 + 1?"++ attr_reader :last_failed_mode+def initialize(opts = {})@opts = optsend@@ -31,16 +35,18 @@ module DiscourseAienddef run_test(val)- DiscourseAi::Completions::Llm- .proxy(val)- .generate(- "How much is 1 + 1?",- user: @opts[:user] || Discourse.system_user,- feature_name: "llm_validator",- temperature: 0.7,- top_p: 0.9,- )- .present?+ llm = DiscourseAi::Completions::Llm.proxy(val)++ @last_failed_mode = :non_streaming+ raise empty_response_error if probe(llm).blank?++ @last_failed_mode = :streaming+ streamed = +""+ probe(llm) { |partial| streamed << partial.to_s if partial.is_a?(String) }+ raise empty_response_error if streamed.blank?++ @last_failed_mode = nil+ trueenddef is_using(llm_model)@@ -74,6 +80,25 @@ module DiscourseAiai_translation_enabled]end++ private++ def probe(llm, &blk)+ llm.generate(+ TEST_PROMPT,+ user: @opts[:user] || Discourse.system_user,+ feature_name: "llm_validator",+ temperature: 0.7,+ top_p: 0.9,+ &blk+ )+ end++ def empty_response_error+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed.new(+ I18n.t("discourse_ai.llm.configuration.empty_response"),+ )+ endendendenddiff --git a/plugins/discourse-ai/spec/configuration/llm_validator_spec.rb b/plugins/discourse-ai/spec/configuration/llm_validator_spec.rbindex 6825ac04e13..572a45528bb 100644--- a/plugins/discourse-ai/spec/configuration/llm_validator_spec.rb+++ b/plugins/discourse-ai/spec/configuration/llm_validator_spec.rb@@ -36,9 +36,45 @@ RSpec.describe DiscourseAi::Configuration::LlmValidator doSiteSetting.ai_helper_enabled = trueSiteSetting.ai_summarization_enabled = true- DiscourseAi::Completions::Llm.with_prepared_responses([true]) do+ DiscourseAi::Completions::Llm.with_prepared_responses(%w[ok ok]) doexpect(validator.valid_value?(llm_model)).to eq(true)endendend++ describe "#run_test" do+ let(:validator) { described_class.new }+ fab!(:llm_model)++ it "exercises both non-streaming and streaming completions" do+ prompts =+ DiscourseAi::Completions::Llm.with_prepared_responses(%w[ok ok]) do |_, _, p|+ validator.run_test(llm_model)+ p+ end++ expect(prompts.length).to eq(2)+ expect(validator.last_failed_mode).to be_nil+ end++ it "marks non-streaming failure when the first probe returns nothing" do+ DiscourseAi::Completions::Llm.with_prepared_responses(["", "ok"]) do+ expect { validator.run_test(llm_model) }.to raise_error(+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,+ )+ end++ expect(validator.last_failed_mode).to eq(:non_streaming)+ end++ it "marks streaming failure when the streaming probe returns nothing" do+ DiscourseAi::Completions::Llm.with_prepared_responses(["ok", ""]) do+ expect { validator.run_test(llm_model) }.to raise_error(+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,+ )+ end++ expect(validator.last_failed_mode).to eq(:streaming)+ end+ endenddiff --git a/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rbindex 5d7844c8a66..eaf3e550b66 100644--- a/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb+++ b/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb@@ -538,7 +538,7 @@ RSpec.describe DiscourseAi::Admin::AiLlmsController doendit "does not route via GET to prevent CSRF" do- DiscourseAi::Completions::Llm.with_prepared_responses(["a response"]) do+ DiscourseAi::Completions::Llm.with_prepared_responses(%w[a a]) doget "/admin/plugins/discourse-ai/ai-llms/test.json", params: { ai_llm: test_attrs }expect(response.status).to eq(404)end@@ -546,7 +546,7 @@ RSpec.describe DiscourseAi::Admin::AiLlmsController docontext "when we can contact the model" doit "returns a success true flag" do- DiscourseAi::Completions::Llm.with_prepared_responses(["a response"]) do+ DiscourseAi::Completions::Llm.with_prepared_responses(%w[a a]) dopost "/admin/plugins/discourse-ai/ai-llms/test.json", params: { ai_llm: test_attrs }expect(response).to be_successful@@ -572,6 +572,20 @@ RSpec.describe DiscourseAi::Admin::AiLlmsController doexpect(response).to be_successfulexpect(response.parsed_body["success"]).to eq(false)expect(response.parsed_body["error"]).to eq(error_message.to_json)+ expect(response.parsed_body["failed_mode"]).to eq("non_streaming")+ end+ end++ it "reports a streaming failure when only the streaming probe fails" do+ error = DiscourseAi::Completions::Endpoints::Base::CompletionFailed.new("stream broken")++ DiscourseAi::Completions::Llm.with_prepared_responses(["a", error]) do+ post "/admin/plugins/discourse-ai/ai-llms/test.json", params: { ai_llm: test_attrs }++ expect(response).to be_successful+ expect(response.parsed_body["success"]).to eq(false)+ expect(response.parsed_body["error"]).to eq("stream broken")+ expect(response.parsed_body["failed_mode"]).to eq("streaming")endendend@@ -595,7 +609,7 @@ RSpec.describe DiscourseAi::Admin::AiLlmsController dofab!(:seeded_llm) { Fabricate(:fake_model, id: -200) }it "tests the existing model directly instead of using params" do- DiscourseAi::Completions::Llm.with_prepared_responses(["a response"]) do+ DiscourseAi::Completions::Llm.with_prepared_responses(%w[a a]) dopost "/admin/plugins/discourse-ai/ai-llms/test.json",params: {ai_llm: {