Skip to content

A platform for community discussion. Free, open, simple.

DEV: exercise streaming in LLM model Run Test (#39983)

Rafael dos Santos Silva0dfc9e997f42
diff --git a/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-llm-editor-form.gjs b/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-llm-editor-form.gjs
index 7b7f1fd7637..f9398fb884b 100644
--- a/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-llm-editor-form.gjs
+++ b/plugins/discourse-ai/admin/assets/javascripts/discourse/components/ai-llm-editor-form.gjs
@@ -37,6 +37,7 @@ export default class AiLlmEditorForm extends Component {
@tracked testResult = null;
@tracked testError = null;
@tracked testValidationErrors = null;
+ @tracked testFailedMode = null;
@cached
get formData() {
@@ -139,9 +140,15 @@ export default class AiLlmEditorForm extends Component {
get testErrorMessage() {
if (this.testValidationErrors?.length > 0) {
return i18n("discourse_ai.llms.tests.invalid_config");
- } else {
- return i18n("discourse_ai.llms.tests.failure", { error: this.testError });
}
+
+ if (this.testFailedMode) {
+ return i18n(`discourse_ai.llms.tests.failure_${this.testFailedMode}`, {
+ error: this.testError,
+ });
+ }
+
+ return i18n("discourse_ai.llms.tests.failure", { error: this.testError });
}
get displayTestResult() {
@@ -257,9 +264,11 @@ export default class AiLlmEditorForm extends Component {
if (this.testResult) {
this.testError = null;
this.testValidationErrors = null;
+ this.testFailedMode = null;
} else {
this.testError = configTestResult.error;
this.testValidationErrors = configTestResult.validation_errors;
+ this.testFailedMode = configTestResult.failed_mode;
}
} catch (e) {
popupAjaxError(e);
diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_llms_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_llms_controller.rb
index a8fa776fd8d..561b88c93c9 100644
--- a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_llms_controller.rb
+++ b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_llms_controller.rb
@@ -147,13 +147,15 @@ module DiscourseAi
end
def test
- RateLimiter.new(current_user, "llm_test_#{current_user.id}", 3, 1.minute).performed!
+ RateLimiter.new(current_user, "llm_test_#{current_user.id}", 6, 1.minute).performed!
+
+ validator = DiscourseAi::Configuration::LlmValidator.new
# For seeded models, test the existing model directly since provider/url/api_key are hidden
if params.dig(:ai_llm, :id).present?
existing_model = LlmModel.find_by(id: params[:ai_llm][:id])
if existing_model&.seeded?
- DiscourseAi::Configuration::LlmValidator.new.run_test(existing_model)
+ validator.run_test(existing_model)
return render json: { success: true }
end
end
@@ -162,13 +164,13 @@ module DiscourseAi
llm_model = LlmModel.new(ai_llm_params.merge(display_name: "LLM test"))
if llm_model.valid?
- DiscourseAi::Configuration::LlmValidator.new.run_test(llm_model)
+ validator.run_test(llm_model)
render json: { success: true }
else
render json: { success: false, validation_errors: llm_model.errors.full_messages }
end
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
- render json: { success: false, error: e.message }
+ render json: { success: false, error: e.message, failed_mode: validator&.last_failed_mode }
end
private
diff --git a/plugins/discourse-ai/config/locales/client.en.yml b/plugins/discourse-ai/config/locales/client.en.yml
index 5eb3ae329d1..28454744e32 100644
--- a/plugins/discourse-ai/config/locales/client.en.yml
+++ b/plugins/discourse-ai/config/locales/client.en.yml
@@ -954,6 +954,8 @@ en:
success: "Success!"
invalid_config: "Fix the following errors in your configuration first:"
failure: "Trying to contact the model returned this error: %{error}"
+ failure_non_streaming: "Non-streaming request failed: %{error}"
+ failure_streaming: "Streaming request failed: %{error}"
hints:
max_prompt_tokens: "The maximum number of tokens the model can process in a single request"
diff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.yml
index 9c57ca2d2b2..931dacc4789 100644
--- a/plugins/discourse-ai/config/locales/server.en.yml
+++ b/plugins/discourse-ai/config/locales/server.en.yml
@@ -884,6 +884,7 @@ en:
disable_modules_first: "You must disable these modules first: %{settings}"
set_llm_first: "Set %{setting} first"
model_unreachable: "We couldn't get a response from this model. Check your settings first."
+ empty_response: "The model returned an empty response."
invalid_seeded_model: "You can't use this model with this feature"
invalid_agent_response_format: "The selected agent must have a response format with a boolean field names \"spam\""
must_select_model: "You must select a LLM first"
diff --git a/plugins/discourse-ai/lib/configuration/llm_validator.rb b/plugins/discourse-ai/lib/configuration/llm_validator.rb
index 800cabc2adc..71187882b20 100644
--- a/plugins/discourse-ai/lib/configuration/llm_validator.rb
+++ b/plugins/discourse-ai/lib/configuration/llm_validator.rb
@@ -3,6 +3,10 @@
module DiscourseAi
module Configuration
class LlmValidator
+ TEST_PROMPT = "How much is 1 + 1?"
+
+ attr_reader :last_failed_mode
+
def initialize(opts = {})
@opts = opts
end
@@ -31,16 +35,18 @@ module DiscourseAi
end
def run_test(val)
- DiscourseAi::Completions::Llm
- .proxy(val)
- .generate(
- "How much is 1 + 1?",
- user: @opts[:user] || Discourse.system_user,
- feature_name: "llm_validator",
- temperature: 0.7,
- top_p: 0.9,
- )
- .present?
+ llm = DiscourseAi::Completions::Llm.proxy(val)
+
+ @last_failed_mode = :non_streaming
+ raise empty_response_error if probe(llm).blank?
+
+ @last_failed_mode = :streaming
+ streamed = +""
+ probe(llm) { |partial| streamed << partial.to_s if partial.is_a?(String) }
+ raise empty_response_error if streamed.blank?
+
+ @last_failed_mode = nil
+ true
end
def is_using(llm_model)
@@ -74,6 +80,25 @@ module DiscourseAi
ai_translation_enabled
]
end
+
+ private
+
+ def probe(llm, &blk)
+ llm.generate(
+ TEST_PROMPT,
+ user: @opts[:user] || Discourse.system_user,
+ feature_name: "llm_validator",
+ temperature: 0.7,
+ top_p: 0.9,
+ &blk
+ )
+ end
+
+ def empty_response_error
+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed.new(
+ I18n.t("discourse_ai.llm.configuration.empty_response"),
+ )
+ end
end
end
end
diff --git a/plugins/discourse-ai/spec/configuration/llm_validator_spec.rb b/plugins/discourse-ai/spec/configuration/llm_validator_spec.rb
index 6825ac04e13..572a45528bb 100644
--- a/plugins/discourse-ai/spec/configuration/llm_validator_spec.rb
+++ b/plugins/discourse-ai/spec/configuration/llm_validator_spec.rb
@@ -36,9 +36,45 @@ RSpec.describe DiscourseAi::Configuration::LlmValidator do
SiteSetting.ai_helper_enabled = true
SiteSetting.ai_summarization_enabled = true
- DiscourseAi::Completions::Llm.with_prepared_responses([true]) do
+ DiscourseAi::Completions::Llm.with_prepared_responses(%w[ok ok]) do
expect(validator.valid_value?(llm_model)).to eq(true)
end
end
end
+
+ describe "#run_test" do
+ let(:validator) { described_class.new }
+ fab!(:llm_model)
+
+ it "exercises both non-streaming and streaming completions" do
+ prompts =
+ DiscourseAi::Completions::Llm.with_prepared_responses(%w[ok ok]) do |_, _, p|
+ validator.run_test(llm_model)
+ p
+ end
+
+ expect(prompts.length).to eq(2)
+ expect(validator.last_failed_mode).to be_nil
+ end
+
+ it "marks non-streaming failure when the first probe returns nothing" do
+ DiscourseAi::Completions::Llm.with_prepared_responses(["", "ok"]) do
+ expect { validator.run_test(llm_model) }.to raise_error(
+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,
+ )
+ end
+
+ expect(validator.last_failed_mode).to eq(:non_streaming)
+ end
+
+ it "marks streaming failure when the streaming probe returns nothing" do
+ DiscourseAi::Completions::Llm.with_prepared_responses(["ok", ""]) do
+ expect { validator.run_test(llm_model) }.to raise_error(
+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,
+ )
+ end
+
+ expect(validator.last_failed_mode).to eq(:streaming)
+ end
+ end
end
diff --git a/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb
index 5d7844c8a66..eaf3e550b66 100644
--- a/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb
+++ b/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb
@@ -538,7 +538,7 @@ RSpec.describe DiscourseAi::Admin::AiLlmsController do
end
it "does not route via GET to prevent CSRF" do
- DiscourseAi::Completions::Llm.with_prepared_responses(["a response"]) do
+ DiscourseAi::Completions::Llm.with_prepared_responses(%w[a a]) do
get "/admin/plugins/discourse-ai/ai-llms/test.json", params: { ai_llm: test_attrs }
expect(response.status).to eq(404)
end
@@ -546,7 +546,7 @@ RSpec.describe DiscourseAi::Admin::AiLlmsController do
context "when we can contact the model" do
it "returns a success true flag" do
- DiscourseAi::Completions::Llm.with_prepared_responses(["a response"]) do
+ DiscourseAi::Completions::Llm.with_prepared_responses(%w[a a]) do
post "/admin/plugins/discourse-ai/ai-llms/test.json", params: { ai_llm: test_attrs }
expect(response).to be_successful
@@ -572,6 +572,20 @@ RSpec.describe DiscourseAi::Admin::AiLlmsController do
expect(response).to be_successful
expect(response.parsed_body["success"]).to eq(false)
expect(response.parsed_body["error"]).to eq(error_message.to_json)
+ expect(response.parsed_body["failed_mode"]).to eq("non_streaming")
+ end
+ end
+
+ it "reports a streaming failure when only the streaming probe fails" do
+ error = DiscourseAi::Completions::Endpoints::Base::CompletionFailed.new("stream broken")
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(["a", error]) do
+ post "/admin/plugins/discourse-ai/ai-llms/test.json", params: { ai_llm: test_attrs }
+
+ expect(response).to be_successful
+ expect(response.parsed_body["success"]).to eq(false)
+ expect(response.parsed_body["error"]).to eq("stream broken")
+ expect(response.parsed_body["failed_mode"]).to eq("streaming")
end
end
end
@@ -595,7 +609,7 @@ RSpec.describe DiscourseAi::Admin::AiLlmsController do
fab!(:seeded_llm) { Fabricate(:fake_model, id: -200) }
it "tests the existing model directly instead of using params" do
- DiscourseAi::Completions::Llm.with_prepared_responses(["a response"]) do
+ DiscourseAi::Completions::Llm.with_prepared_responses(%w[a a]) do
post "/admin/plugins/discourse-ai/ai-llms/test.json",
params: {
ai_llm: {