A platform for community discussion. Free, open, simple.
FEATURE: Add agent-backed sentiment classification (#40136)
Rafael dos Santos Silva2be96e209814
diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/sentiment/sentiment_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/sentiment/sentiment_controller.rbindex 4702a667a9e..110960d0bc2 100644--- a/plugins/discourse-ai/app/controllers/discourse_ai/sentiment/sentiment_controller.rb+++ b/plugins/discourse-ai/app/controllers/discourse_ai/sentiment/sentiment_controller.rb@@ -15,6 +15,7 @@ module DiscourseAistart_date = params[:start_date].presenceend_date = params[:end_date]threshold = SENTIMENT_THRESHOLD+ model_name = DiscourseAi::Sentiment::PostClassification.active_model_name_for(:sentiment)raise Discourse::InvalidParameters if %i[category tag].exclude?(group_by)@@ -61,7 +62,7 @@ module DiscourseAi#{grouping_clause} = :group_value ANDt.archetype = 'regular' ANDp.user_id > 0 AND- cr.model_used = 'cardiffnlp/twitter-roberta-base-sentiment-latest' AND+ cr.model_used = :model_name AND((:start_date IS NULL OR p.created_at > :start_date) AND (:end_date IS NULL OR p.created_at < :end_date))AND p.deleted_at IS NULLAND t.deleted_at IS NULL@@ -76,6 +77,7 @@ module DiscourseAilimit: limit + 1,offset: offset,allowed_category_ids: guardian.allowed_category_ids,+ model_name: model_name,)has_more = posts.length > limitdiff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.ymlindex 26857056e87..36ca3d7c933 100644--- a/plugins/discourse-ai/config/locales/server.en.yml+++ b/plugins/discourse-ai/config/locales/server.en.yml@@ -70,7 +70,12 @@ en:ai_sentiment_enabled: "Enable the sentiment module."ai_sentiment_inference_service_api_endpoint: "URL where the API is running for the sentiment module"ai_sentiment_inference_service_api_key: "API key for the sentiment API"- ai_sentiment_models: "Models to use for inference. Sentiment classifies post on the positive/neutral/negative space. Emotion classifies on the anger/disgust/fear/joy/neutral/sadness/surprise space."+ ai_sentiment_model_configs: "Classification model endpoints to use for inference. Sentiment classifies posts as positive, neutral, or negative. Emotion classifies posts across supported emotion labels."+ ai_sentiment_models: "Classification model endpoints to use for inference. Sentiment classifies posts as positive, neutral, or negative. Emotion classifies posts across supported emotion labels."+ ai_sentiment_sentiment_classification_strategy: "Sentiment classification backend."+ ai_sentiment_sentiment_agent: "Agent to use for sentiment classification when the sentiment backend is agent."+ ai_sentiment_emotion_classification_strategy: "Emotion classification backend."+ ai_sentiment_emotion_agent: "Agent to use for emotion classification when the emotion backend is agent."ai_nsfw_detection_enabled: "Enable the NSFW module."ai_nsfw_inference_service_api_endpoint: "URL where the API is running for the NSFW module"@@ -495,6 +500,12 @@ en:chat_thread_titler:name: "Chat Thread Titler"description: "Default agent powering automatic chat thread title generation"+ sentiment_classifier:+ name: "Sentiment classifier"+ description: "Default agent powering sentiment classification"+ emotion_classifier:+ name: "Emotion classifier"+ description: "Default agent powering emotion classification"topic_not_found: "Summary unavailable, topic not found!"summarizing: "Summarizing topic"diff --git a/plugins/discourse-ai/config/settings.yml b/plugins/discourse-ai/config/settings.ymlindex 35dc8ab9a59..94269e30ada 100644--- a/plugins/discourse-ai/config/settings.yml+++ b/plugins/discourse-ai/config/settings.yml@@ -25,6 +25,26 @@ discourse_ai:default: ""secret: truejson_schema: DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema+ ai_sentiment_sentiment_classification_strategy:+ default: "classification_model"+ type: enum+ choices:+ - classification_model+ - agent+ ai_sentiment_sentiment_agent:+ default: "-36"+ type: enum+ enum: "DiscourseAi::Configuration::AgentEnumerator"+ ai_sentiment_emotion_classification_strategy:+ default: "classification_model"+ type: enum+ choices:+ - classification_model+ - agent+ ai_sentiment_emotion_agent:+ default: "-37"+ type: enum+ enum: "DiscourseAi::Configuration::AgentEnumerator"ai_sentiment_backfill_maximum_posts_per_hour:default: 2500min: 0diff --git a/plugins/discourse-ai/lib/agents/agent.rb b/plugins/discourse-ai/lib/agents/agent.rbindex 950858e8d82..daef9c298e6 100644--- a/plugins/discourse-ai/lib/agents/agent.rb+++ b/plugins/discourse-ai/lib/agents/agent.rb@@ -236,6 +236,8 @@ module DiscourseAiReportRunner => -33,Discover => -34,ChatThreadTitler => -35,+ SentimentClassifier => -36,+ EmotionClassifier => -37,}.freezeendenddiff --git a/plugins/discourse-ai/lib/agents/emotion_classifier.rb b/plugins/discourse-ai/lib/agents/emotion_classifier.rbnew file mode 100644index 00000000000..e7d5cdbba0d--- /dev/null+++ b/plugins/discourse-ai/lib/agents/emotion_classifier.rb@@ -0,0 +1,117 @@+# frozen_string_literal: true++module DiscourseAi+ module Agents+ class EmotionClassifier < Agent+ def self.default_enabled+ false+ end++ def system_prompt+ <<~PROMPT+ You classify Discourse forum posts by emotion.++ Return a probability score for every supported emotion label. Each score must be a float from 0 to 1. The scores must sum to 1.0. Higher scores mean the label is more applicable to the post.++ Classify the author's emotions, not quoted text or replies from other people. Ignore markup, links, signatures, logs, and code blocks unless they clearly express emotion. Interpret sarcasm and irony by their implied meaning.++ Multiple emotions can share probability mass, but the scores must still sum to 1.0. Use neutral as the highest score for factual announcements, questions, or technical debugging with little affect.++ Use the full post content provided by the user. Reply with valid JSON only. Do not include explanations, confidence fields, or any keys except the supported emotion labels.+ PROMPT+ end++ def response_format+ DiscourseAi::Sentiment::Emotions::LIST.map do |label|+ { "key" => label, "type" => "number" }+ end+ end++ def examples+ [+ [+ "Thank you for the detailed walkthrough. This solved my issue and saved me a lot of time.",+ emotion_scores(+ gratitude: 0.72,+ admiration: 0.14,+ approval: 0.08,+ relief: 0.04,+ neutral: 0.02,+ ),+ ],+ [+ "This keeps failing after every deploy. I am frustrated that the same bug came back again.",+ emotion_scores(+ annoyance: 0.42,+ anger: 0.25,+ disappointment: 0.18,+ disapproval: 0.1,+ neutral: 0.05,+ ),+ ],+ [+ "Does anyone know why this setting disappeared after the migration?",+ emotion_scores(+ curiosity: 0.46,+ confusion: 0.32,+ surprise: 0.12,+ neutral: 0.08,+ realization: 0.02,+ ),+ ],+ [+ "I am sorry for deleting the topic by mistake. I should have checked before clicking.",+ emotion_scores(+ remorse: 0.68,+ embarrassment: 0.16,+ sadness: 0.08,+ disappointment: 0.05,+ neutral: 0.03,+ ),+ ],+ [+ "Great, now the backup is gone too. That is exactly what I needed today.",+ emotion_scores(+ annoyance: 0.35,+ anger: 0.22,+ disappointment: 0.2,+ disapproval: 0.12,+ sadness: 0.06,+ neutral: 0.05,+ ),+ ],+ [+ "I cannot believe how quickly this was fixed. The new workflow is a joy to use.",+ emotion_scores(+ joy: 0.42,+ admiration: 0.2,+ approval: 0.16,+ surprise: 0.12,+ excitement: 0.08,+ neutral: 0.02,+ ),+ ],+ [+ "I am worried the importer may lose attachments if we run it before the patch is ready.",+ emotion_scores(+ fear: 0.38,+ nervousness: 0.24,+ curiosity: 0.12,+ confusion: 0.1,+ neutral: 0.1,+ sadness: 0.06,+ ),+ ],+ ]+ end++ private++ def emotion_scores(overrides)+ DiscourseAi::Sentiment::Emotions::LIST+ .index_with { |label| overrides[label.to_sym] || 0.0 }+ .to_json+ end+ end+ end+enddiff --git a/plugins/discourse-ai/lib/agents/sentiment_classifier.rb b/plugins/discourse-ai/lib/agents/sentiment_classifier.rbnew file mode 100644index 00000000000..46f3d6ddfff--- /dev/null+++ b/plugins/discourse-ai/lib/agents/sentiment_classifier.rb@@ -0,0 +1,51 @@+# frozen_string_literal: true++module DiscourseAi+ module Agents+ class SentimentClassifier < Agent+ def self.default_enabled+ false+ end++ def system_prompt+ <<~PROMPT+ You classify Discourse forum posts by sentiment.++ Return a probability score for each sentiment label:+ - negative+ - neutral+ - positive++ Each score must be a float from 0 to 1. The scores must sum to 1.0. Higher scores mean the label is more applicable to the post.++ Classify the author's sentiment, not quoted text or replies from other people. Ignore markup, links, signatures, logs, and code blocks unless they clearly express sentiment. Interpret sarcasm and irony by their implied meaning.++ Use neutral as the highest score for factual announcements, questions, or technical debugging with little affect. For mixed sentiment, distribute probability across labels instead of forcing a single winner.++ Use the full post content provided by the user. Reply with valid JSON only. Do not include explanations, confidence fields, or any keys except the sentiment labels.+ PROMPT+ end++ def response_format+ %w[negative neutral positive].map { |label| { "key" => label, "type" => "number" } }+ end++ def examples+ [+ [+ "This release is fantastic. The new editor is fast, polished, and much easier to use.",+ { negative: 0.02, neutral: 0.08, positive: 0.9 }.to_json,+ ],+ [+ "The upgrade broke search for our team and the workaround did not help.",+ { negative: 0.88, neutral: 0.09, positive: 0.03 }.to_json,+ ],+ [+ "Version 3.2 is now available in the admin updates page.",+ { negative: 0.03, neutral: 0.92, positive: 0.05 }.to_json,+ ],+ ]+ end+ end+ end+enddiff --git a/plugins/discourse-ai/lib/sentiment/constants.rb b/plugins/discourse-ai/lib/sentiment/constants.rbindex 42d21db5418..91f348c3569 100644--- a/plugins/discourse-ai/lib/sentiment/constants.rb+++ b/plugins/discourse-ai/lib/sentiment/constants.rb@@ -3,6 +3,12 @@module DiscourseAimodule Sentimentmodule Constants+ SENTIMENT_MODEL = "cardiffnlp/twitter-roberta-base-sentiment-latest"+ EMOTION_MODEL = "SamLowe/roberta-base-go_emotions"+ SENTIMENT_AGENT_MODEL = "discourse-ai/sentiment-agent"+ EMOTION_AGENT_MODEL = "discourse-ai/emotion-agent"+ CLASSIFICATION_MODEL_STRATEGY = "classification_model"+ AGENT_STRATEGY = "agent"SENTIMENT_THRESHOLD = 0.6endenddiff --git a/plugins/discourse-ai/lib/sentiment/emotion_dashboard_report.rb b/plugins/discourse-ai/lib/sentiment/emotion_dashboard_report.rbindex 11c190c507c..10bff1a6e65 100644--- a/plugins/discourse-ai/lib/sentiment/emotion_dashboard_report.rb+++ b/plugins/discourse-ai/lib/sentiment/emotion_dashboard_report.rb@@ -15,7 +15,9 @@ module DiscourseAienddef self.fetch_data(report)- DB.query(<<~SQL, end: report.end_date, start: report.start_date)+ model_name = DiscourseAi::Sentiment::PostClassification.active_model_name_for(:emotion)++ DB.query(<<~SQL, end: report.end_date, start: report.start_date, model_name: model_name)SELECTposts.created_at::DATE AS day,#{@@ -37,7 +39,7 @@ module DiscourseAitopics.deleted_at IS NULLWHEREclassification_results.target_type = 'Post' AND- classification_results.model_used = 'SamLowe/roberta-base-go_emotions'+ classification_results.model_used = :model_nameGROUP BY 1ORDER BY 1 ASCSQLdiff --git a/plugins/discourse-ai/lib/sentiment/emotion_filter_order.rb b/plugins/discourse-ai/lib/sentiment/emotion_filter_order.rbindex 0f1b3b62ae3..b2ad58f88ba 100644--- a/plugins/discourse-ai/lib/sentiment/emotion_filter_order.rb+++ b/plugins/discourse-ai/lib/sentiment/emotion_filter_order.rb@@ -28,6 +28,10 @@ module DiscourseAiemotion_clause = <<~SQLCOUNT(*) FILTER (WHERE (classification_results.classification::jsonb->'#{emotion}')::float > 0.1)SQL+ model_name =+ ActiveRecord::Base.connection.quote(+ DiscourseAi::Sentiment::PostClassification.active_model_name_for(:emotion),+ )# TODO: This is slow, we will need to materialize this in the futurewith_clause = <<~SQL@@ -42,7 +46,7 @@ module DiscourseAiclassification_results ONclassification_results.target_id = posts.id ANDclassification_results.target_type = 'Post' AND- classification_results.model_used = 'SamLowe/roberta-base-go_emotions'+ classification_results.model_used = #{model_name}WHEREtopics.archetype = 'regular'AND topics.deleted_at IS NULLdiff --git a/plugins/discourse-ai/lib/sentiment/post_classification.rb b/plugins/discourse-ai/lib/sentiment/post_classification.rbindex f4dd526aaff..41ecb44fe46 100644--- a/plugins/discourse-ai/lib/sentiment/post_classification.rb+++ b/plugins/discourse-ai/lib/sentiment/post_classification.rb@@ -3,12 +3,16 @@module DiscourseAimodule Sentimentclass PostClassification+ include Constants+def self.backfill_query(from_post_id: nil, max_age_days: nil)- available_classifier_names =- DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema.values.map { it.model_name }+ available_classifier_names = active_classifier_names+ return Post.none if available_classifier_names.blank?queries =available_classifier_names.map do |classifier_name|+ quoted_classifier_name = ActiveRecord::Base.connection.quote(classifier_name)+base_query =Post.includes(:sentiment_classifications)@@ -22,7 +26,7 @@ module DiscourseAiON crs.target_id = posts.idAND crs.target_type = 'Post'AND crs.classification_type = 'sentiment'- AND crs.model_used = '#{classifier_name}'+ AND crs.model_used = #{quoted_classifier_name}SQL.where("crs.id IS NULL")@@ -44,19 +48,101 @@ module DiscourseAiPost.from(Arel.sql("(#{unioned_queries}) as posts"))end- CONCURRENT_CLASSFICATIONS = 40+ def self.active_classifier_names+ new.classifiers.map { |classifier| classifier[:model_name] }+ end- def bulk_classify!(relation)- pool =- Scheduler::ThreadPool.new(- min_threads: 0,- max_threads: CONCURRENT_CLASSFICATIONS,- idle_time: 30,+ def self.active_model_name_for(classification_type)+ classification_type = classification_type.to_s++ if strategy_for(classification_type) == Constants::AGENT_STRATEGY+ return(+ if classification_type == "sentiment"+ Constants::SENTIMENT_AGENT_MODEL+ else+ Constants::EMOTION_AGENT_MODEL+ end)+ end++ configured_model_name_for(classification_type) ||+ default_model_name_for(classification_type)+ end++ def self.configured_model_name_for(classification_type)+ configs = DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema.values+ return if configs.blank?+ classification_type = classification_type.to_s++ explicitly_typed_config =+ configs.find do |config|+ config.respond_to?(:classification_type) &&+ config.classification_type.to_s == classification_type+ end+ return explicitly_typed_config.model_name if explicitly_typed_config.present?++ default_config =+ configs.find { |config| config.model_name == default_model_name_for(classification_type) }+ return default_config.model_name if default_config.present?++ configs+ .find { |config| classification_type_for(config) == classification_type }+ &.model_name || untyped_custom_model_name(configs, classification_type)+ end++ def self.untyped_custom_model_name(configs, classification_type)+ return if classification_type != "sentiment"++ untyped_configs = configs.select { |config| classification_type_for(config).blank? }+ untyped_configs.one? ? untyped_configs.first.model_name : nil+ end++ def self.default_model_name_for(classification_type)+ if classification_type.to_s == "sentiment"+ Constants::SENTIMENT_MODEL+ else+ Constants::EMOTION_MODEL+ end+ end++ def self.classification_type_for(config)+ if config.respond_to?(:classification_type) && config.classification_type.present?+ return config.classification_type.to_s+ end++ return "sentiment" if config.model_name == Constants::SENTIMENT_MODEL+ return "emotion" if config.model_name == Constants::EMOTION_MODEL++ nil+ end++ def self.strategy_for(classification_type)+ if classification_type.to_s == "sentiment"+ SiteSetting.ai_sentiment_sentiment_classification_strategy+ else+ SiteSetting.ai_sentiment_emotion_classification_strategy+ end+ end++ CONCURRENT_CLASSFICATIONS = 40+ CONCURRENT_AGENT_CLASSIFICATIONS = 5++ def bulk_classify!(relation)available_classifiers = classifiersreturn if available_classifiers.blank?+ max_threads =+ (+ if available_classifiers.any? { |c| c[:provider] == :agent }+ CONCURRENT_AGENT_CLASSIFICATIONS+ else+ CONCURRENT_CLASSFICATIONS+ end+ )++ pool = Scheduler::ThreadPool.new(min_threads: 0, max_threads: max_threads, idle_time: 30)+results = Queue.newqueued = 0@@ -72,7 +158,7 @@ module DiscourseAipool.post doresult = { target: record, classifier: classifier, text: text }begin- result[:classification] = request_with(classifier[:client], text)+ result[:classification] = request_with(classifier, record, text)rescue StandardError => eresult[:error] = eend@@ -88,7 +174,7 @@ module DiscourseAiresult = results.popif result[:error]errors << result- else+ elsif result[:classification].present?store_classification(result[:target],[[result[:classifier][:model_name], result[:classification]]],@@ -106,8 +192,10 @@ module DiscourseAi)endensure- pool.shutdown- pool.wait_for_termination(timeout: 30)+ if pool+ pool.shutdown+ pool.wait_for_termination(timeout: 30)+ endenddef classify!(target)@@ -123,16 +211,37 @@ module DiscourseAiavailable_classifiers.reject { |ac| already_classified.include?(ac[:model_name]) }results =- classifiers_for_target.reduce({}) do |memo, cft|- memo[cft[:model_name]] = request_with(cft[:client], to_classify)- memo+ classifiers_for_target.each_with_object({}) do |cft, memo|+ classification = request_with(cft, target, to_classify)+ memo[cft[:model_name]] = classification if classification.present?end- store_classification(target, results)+ store_classification(target, results) if results.present?enddef classifiers- DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema.values.map do |config|+ hugging_face_classifiers + agent_classifiers+ end++ def has_classifiers?+ classifiers.present?+ end++ private++ def hugging_face_classifiers+ return [] if agent_strategy_for?(:sentiment) && agent_strategy_for?(:emotion)++ configs = DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema.values+ legacy_sentiment_model = self.class.untyped_custom_model_name(configs, "sentiment")++ configs.filter_map do |config|+ classification_type = classification_type_for(config)+ effective_type =+ classification_type.presence ||+ ("sentiment" if config.model_name == legacy_sentiment_model)+ next if effective_type.present? && agent_strategy_for?(effective_type)+api_endpoint = config.endpointif api_endpoint.present? && api_endpoint.start_with?("srv://")@@ -141,18 +250,58 @@ module DiscourseAiend{+ classification_type: classification_type,model_name: config.model_name,client:DiscourseAi::Inference::HuggingFaceTextEmbeddings.new(api_endpoint, config.api_key),+ provider: :classification_model,}endend- def has_classifiers?- classifiers.present?+ def agent_classifiers+ [+ agent_classifier(+ :sentiment,+ SiteSetting.ai_sentiment_sentiment_agent,+ Constants::SENTIMENT_AGENT_MODEL,+ ),+ agent_classifier(+ :emotion,+ SiteSetting.ai_sentiment_emotion_agent,+ Constants::EMOTION_AGENT_MODEL,+ ),+ ].compactend- private+ def agent_classifier(classification_type, agent_id, model_name)+ return if !agent_strategy_for?(classification_type)++ ai_agent = AiAgent.find_by_id_from_cache(agent_id)+ return if ai_agent.blank?++ agent_klass = ai_agent.class_instance+ model_id = agent_klass.default_llm_id || SiteSetting.ai_default_llm_model+ model = model_id.present? ? LlmModel.find_by(id: model_id) : LlmModel.last+ return if model.blank?++ {+ classification_type: classification_type.to_s,+ model_name: model_name,+ agent: agent_klass.new,+ user: ai_agent.user || Discourse.system_user,+ model: model,+ provider: :agent,+ }+ end++ def classification_type_for(config)+ self.class.classification_type_for(config)+ end++ def agent_strategy_for?(classification_type)+ self.class.strategy_for(classification_type) == Constants::AGENT_STRATEGY+ enddef prepare_text(target)content =@@ -169,8 +318,10 @@ module DiscourseAi)end- def request_with(client, content)- result = client.classify_by_sentiment!(content)+ def request_with(classifier, target, content)+ return request_with_agent(classifier, target, content) if classifier[:provider] == :agent++ result = classifier[:client].classify_by_sentiment!(content)transform_result(result)end@@ -181,6 +332,68 @@ module DiscourseAihash_resultend+ def request_with_agent(classifier, target, content)+ context =+ DiscourseAi::Agents::BotContext.new(+ post: target,+ messages: [{ type: :user, content: content }],+ user: classifier[:user],+ skip_show_thinking: true,+ feature_name: "sentiment",+ )++ bot =+ DiscourseAi::Agents::Bot.as(+ classifier[:user],+ agent: classifier[:agent],+ model: classifier[:model],+ )++ structured_output = nil+ raw_result = +""+ bot.reply(context) do |partial, _, type|+ if type == :structured_output+ structured_output = partial+ else+ raw_result << partial.to_s+ end+ end++ transform_agent_result(+ structured_output,+ raw_result,+ labels_for(classifier[:classification_type]),+ )+ end++ def transform_agent_result(structured_output, raw_result, labels)+ parsed_result = parse_raw_agent_result(raw_result)++ result =+ labels.index_with do |label|+ value =+ structured_output&.read_buffered_property(label.to_sym) || parsed_result[label] ||+ parsed_result[label.to_sym] || 0++ value.to_f.clamp(0.0, 1.0)+ end++ return nil if result.values.all?(&:zero?)+ result+ end++ def parse_raw_agent_result(raw_result)+ return {} if raw_result.blank?++ JSON.parse(raw_result)+ rescue JSON::ParserError+ {}+ end++ def labels_for(classification_type)+ classification_type.to_s == "sentiment" ? %w[negative neutral positive] : Emotions::LIST+ end+def store_classification(target, classification)attrs =classification.map do |model_name, classifications|diff --git a/plugins/discourse-ai/lib/sentiment/sentiment_analysis_report.rb b/plugins/discourse-ai/lib/sentiment/sentiment_analysis_report.rbindex b820cc28782..4b1e4391367 100644--- a/plugins/discourse-ai/lib/sentiment/sentiment_analysis_report.rb+++ b/plugins/discourse-ai/lib/sentiment/sentiment_analysis_report.rb@@ -64,6 +64,7 @@ module DiscourseAidef self.fetch_data(report, opts)threshold = SENTIMENT_THRESHOLD+ model_name = DiscourseAi::Sentiment::PostClassification.active_model_name_for(:sentiment)grouping = (report.filters.dig(:group_by) || GROUP_BY_FILTER_DEFAULT).to_symsorting = (report.filters.dig(:sort_by) || SORT_BY_FILTER_DEFAULT).to_sym@@ -162,7 +163,7 @@ module DiscourseAiWHEREt.archetype = 'regular' ANDp.user_id > 0 AND- cr.model_used = 'cardiffnlp/twitter-roberta-base-sentiment-latest' AND+ cr.model_used = :model_name AND(p.created_at > :report_start AND p.created_at < :report_end)#{where_clause}#{group_by_clause}@@ -173,6 +174,7 @@ module DiscourseAithreshold: threshold,category_filter: category_filter,tag_filter: tag_filter,+ model_name: model_name,)grouped_sentimentsdiff --git a/plugins/discourse-ai/lib/sentiment/sentiment_dashboard_report.rb b/plugins/discourse-ai/lib/sentiment/sentiment_dashboard_report.rbindex 124e14b8ba8..93809abfccd 100644--- a/plugins/discourse-ai/lib/sentiment/sentiment_dashboard_report.rb+++ b/plugins/discourse-ai/lib/sentiment/sentiment_dashboard_report.rb@@ -9,6 +9,7 @@ module DiscourseAiplugin.add_report("overall_sentiment") do |report|report.modes = [:stacked_chart]threshold = SENTIMENT_THRESHOLD+ model_name = DiscourseAi::Sentiment::PostClassification.active_model_name_for(:sentiment)sentiment_count_sql = Proc.new { |sentiment| <<~SQL }COUNT(@@ -30,7 +31,7 @@ module DiscourseAiWHEREt.archetype = 'regular' ANDp.user_id > 0 AND- cr.model_used = 'cardiffnlp/twitter-roberta-base-sentiment-latest' AND+ cr.model_used = :model_name AND(p.created_at > :report_start AND p.created_at < :report_end)GROUP BY DATE_TRUNC('day', p.created_at)ORDER BY 1 ASC@@ -38,6 +39,7 @@ module DiscourseAireport_start: report.start_date,report_end: report.end_date,threshold: threshold,+ model_name: model_name,)return report if grouped_sentiments.empty?diff --git a/plugins/discourse-ai/lib/sentiment/sentiment_site_setting_json_schema.rb b/plugins/discourse-ai/lib/sentiment/sentiment_site_setting_json_schema.rbindex c3003028369..80b52bdd3ab 100644--- a/plugins/discourse-ai/lib/sentiment/sentiment_site_setting_json_schema.rb+++ b/plugins/discourse-ai/lib/sentiment/sentiment_site_setting_json_schema.rb@@ -11,6 +11,10 @@ module DiscourseAiformat: "table",title: "model",properties: {+ classification_type: {+ type: "string",+ enum: %w[sentiment emotion],+ },model_name: {type: "string",},@@ -21,7 +25,7 @@ module DiscourseAitype: "string",},},- required: %w[model_name endpoint api_key],+ required: %w[classification_type model_name endpoint api_key],},}enddiff --git a/plugins/discourse-ai/lib/tasks/modules/sentiment/populate.rake b/plugins/discourse-ai/lib/tasks/modules/sentiment/populate.rakeindex b632431d826..72677b24bef 100644--- a/plugins/discourse-ai/lib/tasks/modules/sentiment/populate.rake+++ b/plugins/discourse-ai/lib/tasks/modules/sentiment/populate.rake@@ -4,13 +4,16 @@ desc "Creates sample sentiment / emotion data"task "ai:sentiment:populate", [:start_post] => [:environment] do |_, args|raise "Don't run this task in production!" if Rails.env.production?+ sentiment_model = DiscourseAi::Sentiment::PostClassification.active_model_name_for(:sentiment)+ emotion_model = DiscourseAi::Sentiment::PostClassification.active_model_name_for(:emotion)+Post- .joins(<<~SQL)- LEFT JOIN classification_results ON- posts.id = classification_results.target_id AND- classification_results.target_type = 'Post' AND- model_used = 'cardiffnlp/twitter-roberta-base-sentiment-latest'- SQL+ .joins(ActiveRecord::Base.sanitize_sql_array([<<~SQL, sentiment_model]))+ LEFT JOIN classification_results ON+ posts.id = classification_results.target_id AND+ classification_results.target_type = 'Post' AND+ model_used = ?+ SQL.where("classification_results.id IS NULL").where("posts.id > ?", args[:start_post].to_i || 0).find_each do |post|@@ -20,7 +23,7 @@ task "ai:sentiment:populate", [:start_post] => [:environment] do |_, args|ClassificationResult.create!(target_id: post.id,- model_used: "cardiffnlp/twitter-roberta-base-sentiment-latest",+ model_used: sentiment_model,classification_type: "sentiment",target_type: "Post",classification: {@@ -32,12 +35,12 @@ task "ai:sentiment:populate", [:start_post] => [:environment] do |_, args|endPost- .joins(<<~SQL)- LEFT JOIN classification_results ON- posts.id = classification_results.target_id AND- classification_results.target_type = 'Post' AND- classification_results.model_used = 'SamLowe/roberta-base-go_emotions'- SQL+ .joins(ActiveRecord::Base.sanitize_sql_array([<<~SQL, emotion_model]))+ LEFT JOIN classification_results ON+ posts.id = classification_results.target_id AND+ classification_results.target_type = 'Post' AND+ classification_results.model_used = ?+ SQL.where("classification_results.id IS NULL").where("posts.id > ?", args[:start_post].to_i || 0).find_each do |post|@@ -53,7 +56,7 @@ task "ai:sentiment:populate", [:start_post] => [:environment] do |_, args|ClassificationResult.create!(target_id: post.id,- model_used: "SamLowe/roberta-base-go_emotions",+ model_used: emotion_model,classification_type: "sentiment",target_type: "Post",classification: emotions,diff --git a/plugins/discourse-ai/spec/lib/agents/emotion_classifier_spec.rb b/plugins/discourse-ai/spec/lib/agents/emotion_classifier_spec.rbnew file mode 100644index 00000000000..2c8329856e0--- /dev/null+++ b/plugins/discourse-ai/spec/lib/agents/emotion_classifier_spec.rb@@ -0,0 +1,29 @@+# frozen_string_literal: true++RSpec.describe DiscourseAi::Agents::EmotionClassifier do+ subject(:agent) { described_class.new }++ before { enable_current_plugin }++ describe "#system_prompt" do+ it "describes probability score constraints" do+ prompt = agent.system_prompt++ expect(prompt).to include("float from 0 to 1")+ expect(prompt).to include("sum to 1.0")+ end+ end++ describe "#examples" do+ it "uses valid emotion probability distributions" do+ labels = DiscourseAi::Sentiment::Emotions::LIST++ agent.examples.each do |_input, output|+ parsed_output = JSON.parse(output)++ expect(parsed_output.keys).to contain_exactly(*labels)+ expect(parsed_output.values.sum).to be_within(0.001).of(1.0)+ end+ end+ end+enddiff --git a/plugins/discourse-ai/spec/lib/agents/sentiment_classifier_spec.rb b/plugins/discourse-ai/spec/lib/agents/sentiment_classifier_spec.rbnew file mode 100644index 00000000000..965b94378da--- /dev/null+++ b/plugins/discourse-ai/spec/lib/agents/sentiment_classifier_spec.rb@@ -0,0 +1,29 @@+# frozen_string_literal: true++RSpec.describe DiscourseAi::Agents::SentimentClassifier do+ subject(:agent) { described_class.new }++ before { enable_current_plugin }++ describe "#system_prompt" do+ it "describes probability score constraints" do+ prompt = agent.system_prompt++ expect(prompt).to include("float from 0 to 1")+ expect(prompt).to include("sum to 1.0")+ end+ end++ describe "#examples" do+ it "uses valid sentiment probability distributions" do+ labels = %w[negative neutral positive]++ agent.examples.each do |_input, output|+ parsed_output = JSON.parse(output)++ expect(parsed_output.keys).to contain_exactly(*labels)+ expect(parsed_output.values.sum).to be_within(0.001).of(1.0)+ end+ end+ end+enddiff --git a/plugins/discourse-ai/spec/lib/modules/sentiment/emotion_filter_order_spec.rb b/plugins/discourse-ai/spec/lib/modules/sentiment/emotion_filter_order_spec.rbindex 1d9a0d08051..efb31e14a8f 100644--- a/plugins/discourse-ai/spec/lib/modules/sentiment/emotion_filter_order_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/sentiment/emotion_filter_order_spec.rb@@ -190,6 +190,27 @@ RSpec.describe DiscourseAi::Sentiment::EmotionFilterOrder doexpect(result.to_sql).to include("ORDER BY topic_emotion.emotion_joy desc")end+ context "when emotion classification uses an agent" do+ let(:model_used) { DiscourseAi::Sentiment::Constants::EMOTION_AGENT_MODEL }++ before { SiteSetting.ai_sentiment_emotion_classification_strategy = "agent" }++ it "filters topics using the stable emotion agent model key" do+ emotion = "joy"+ filter =+ DiscoursePluginRegistry+ .custom_filter_mappings+ .find { it.keys.include? "order:emotion_#{emotion}" }+ .values+ .first+ result = filter.call(Topic.all, "desc", guardian)++ expect(result.to_sql).to include(+ "classification_results.model_used = '#{DiscourseAi::Sentiment::Constants::EMOTION_AGENT_MODEL}'",+ )+ end+ end+it "sorts emotion in ascending order" doexpect(TopicsFilter.new(guardian:).filter_from_query_string("order:emotion_love-asc").pluck(:id),diff --git a/plugins/discourse-ai/spec/lib/modules/sentiment/post_classification_spec.rb b/plugins/discourse-ai/spec/lib/modules/sentiment/post_classification_spec.rbindex f630c85615b..10e0a04bcfd 100644--- a/plugins/discourse-ai/spec/lib/modules/sentiment/post_classification_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/sentiment/post_classification_spec.rb@@ -22,6 +22,48 @@ RSpec.describe DiscourseAi::Sentiment::PostClassification doexpect(result.classification.keys).to contain_exactly("negative", "neutral", "positive")end+ def agent_for(response_format, llm_model)+ Fabricate(+ :ai_agent,+ default_llm: llm_model,+ response_format: response_format,+ system_prompt: "Classify this post",+ )+ end++ describe ".active_model_name_for" do+ it "uses an explicitly typed custom sentiment classifier for read paths" do+ SiteSetting.ai_sentiment_model_configs = [+ {+ classification_type: "sentiment",+ model_name: "custom/sentiment-model",+ endpoint: "https://sentiment.example.com",+ api_key: "123",+ },+ ].to_json++ expect(described_class.active_model_name_for(:sentiment)).to eq("custom/sentiment-model")+ end++ it "uses a single legacy untyped custom classifier for sentiment read paths" do+ SiteSetting.ai_sentiment_model_configs = [+ {+ model_name: "custom/classifier",+ endpoint: "https://sentiment.example.com",+ api_key: "123",+ },+ ].to_json++ expect(described_class.active_model_name_for(:sentiment)).to eq("custom/classifier")+ end++ it "prefers the default emotion classifier when multiple legacy emotion classifiers are configured" do+ expect(described_class.active_model_name_for(:emotion)).to eq(+ DiscourseAi::Sentiment::Constants::EMOTION_MODEL,+ )+ end+ end+describe "#classify!" dofab!(:post_1) { Fabricate(:post, post_number: 2) }@@ -74,6 +116,129 @@ RSpec.describe DiscourseAi::Sentiment::PostClassification donew_classifications = ClassificationResult.where("created_at > ?", first_classified_at).countexpect(new_classifications).to eq(1)end++ it "classifies sentiment through a configured agent" do+ llm_model = Fabricate(:fake_model)+ ai_agent =+ agent_for(+ %w[negative neutral positive].map { |label| { "key" => label, "type" => "number" } },+ llm_model,+ )+ SiteSetting.ai_sentiment_model_configs = ""+ SiteSetting.ai_sentiment_sentiment_classification_strategy = "agent"+ SiteSetting.ai_sentiment_sentiment_agent = ai_agent.id++ DiscourseAi::Completions::Llm.with_prepared_responses(+ [{ negative: 0.1, neutral: 0.2, positive: 0.7 }],+ ) { post_classification.classify!(post_1) }++ result =+ ClassificationResult.find_by!(+ model_used: DiscourseAi::Sentiment::Constants::SENTIMENT_AGENT_MODEL,+ target: post_1,+ )++ expect(result.classification).to eq("negative" => 0.1, "neutral" => 0.2, "positive" => 0.7)+ end++ it "classifies emotion through a configured agent" do+ llm_model = Fabricate(:fake_model)+ ai_agent =+ agent_for(+ DiscourseAi::Sentiment::Emotions::LIST.map do |label|+ { "key" => label, "type" => "number" }+ end,+ llm_model,+ )+ SiteSetting.ai_sentiment_model_configs = ""+ SiteSetting.ai_sentiment_emotion_classification_strategy = "agent"+ SiteSetting.ai_sentiment_emotion_agent = ai_agent.id++ DiscourseAi::Completions::Llm.with_prepared_responses(+ [{ anger: 0.7, joy: 0.2, neutral: 0.1 }],+ ) { post_classification.classify!(post_1) }++ result =+ ClassificationResult.find_by!(+ model_used: DiscourseAi::Sentiment::Constants::EMOTION_AGENT_MODEL,+ target: post_1,+ )++ expect(result.classification.keys).to contain_exactly(*DiscourseAi::Sentiment::Emotions::LIST)+ expect(result.classification.slice("anger", "joy", "neutral")).to eq(+ "anger" => 0.7,+ "joy" => 0.2,+ "neutral" => 0.1,+ )+ end++ it "skips a lone legacy untyped sentiment config when sentiment strategy is agent" do+ llm_model = Fabricate(:fake_model)+ ai_agent =+ agent_for(+ %w[negative neutral positive].map { |label| { "key" => label, "type" => "number" } },+ llm_model,+ )+ SiteSetting.ai_sentiment_model_configs = [+ { model_name: "custom/legacy", endpoint: "https://legacy.example.com", api_key: "123" },+ ].to_json+ SiteSetting.ai_sentiment_sentiment_classification_strategy = "agent"+ SiteSetting.ai_sentiment_sentiment_agent = ai_agent.id++ model_names = post_classification.classifiers.map { |c| c[:model_name] }+ expect(model_names).to include(DiscourseAi::Sentiment::Constants::SENTIMENT_AGENT_MODEL)+ expect(model_names).not_to include("custom/legacy")+ end++ it "skips storing when the agent returns no usable classification" do+ llm_model = Fabricate(:fake_model)+ ai_agent =+ agent_for(+ %w[negative neutral positive].map { |label| { "key" => label, "type" => "number" } },+ llm_model,+ )+ SiteSetting.ai_sentiment_model_configs = ""+ SiteSetting.ai_sentiment_sentiment_classification_strategy = "agent"+ SiteSetting.ai_sentiment_sentiment_agent = ai_agent.id++ DiscourseAi::Completions::Llm.with_prepared_responses(["not json"]) do+ post_classification.classify!(post_1)+ end++ expect(+ ClassificationResult.where(+ model_used: DiscourseAi::Sentiment::Constants::SENTIMENT_AGENT_MODEL,+ target: post_1,+ ),+ ).to be_empty+ end++ it "falls back to the newest LLM model when the configured agent does not set a default LLM" do+ Fabricate(:fake_model, name: "older-model", created_at: 1.day.ago)+ llm_model = Fabricate(:fake_model, name: "newer-model")+ ai_agent =+ Fabricate(+ :ai_agent,+ default_llm: nil,+ response_format:+ %w[negative neutral positive].map { |label| { "key" => label, "type" => "number" } },+ system_prompt: "Classify this post",+ )+ SiteSetting.ai_default_llm_model = ""+ SiteSetting.ai_sentiment_model_configs = ""+ SiteSetting.ai_sentiment_sentiment_classification_strategy = "agent"+ SiteSetting.ai_sentiment_sentiment_agent = ai_agent.id++ classifier =+ post_classification.send(+ :agent_classifier,+ :sentiment,+ ai_agent.id,+ DiscourseAi::Sentiment::Constants::SENTIMENT_AGENT_MODEL,+ )++ expect(classifier[:model]).to eq(llm_model)+ endenddescribe "#classify_bulk!" do@@ -172,6 +337,32 @@ RSpec.describe DiscourseAi::Sentiment::PostClassification doexpect(posts).to contain_exactly(classified_post)end+ it "does not reclassify agent results when the agent LLM changes" do+ llm_model = Fabricate(:fake_model)+ ai_agent =+ agent_for(+ %w[negative neutral positive].map { |label| { "key" => label, "type" => "number" } },+ llm_model,+ )+ SiteSetting.ai_sentiment_model_configs = ""+ SiteSetting.ai_sentiment_sentiment_classification_strategy = "agent"+ SiteSetting.ai_sentiment_emotion_classification_strategy = "agent"+ SiteSetting.ai_sentiment_sentiment_agent = ai_agent.id+ SiteSetting.ai_sentiment_emotion_agent = "0"++ classified_post = Fabricate(:post)+ Fabricate(+ :classification_result,+ target: classified_post,+ model_used: DiscourseAi::Sentiment::Constants::SENTIMENT_AGENT_MODEL,+ )++ ai_agent.update!(default_llm: Fabricate(:fake_model, name: "new-model"))++ posts = described_class.backfill_query+ expect(posts).to be_empty+ end+it "excludes deleted posts" doFabricate(:post, deleted_at: 1.hour.ago)