Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Add agent-backed sentiment classification (#40136)

Rafael dos Santos Silva2be96e209814
diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/sentiment/sentiment_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/sentiment/sentiment_controller.rb
index 4702a667a9e..110960d0bc2 100644
--- a/plugins/discourse-ai/app/controllers/discourse_ai/sentiment/sentiment_controller.rb
+++ b/plugins/discourse-ai/app/controllers/discourse_ai/sentiment/sentiment_controller.rb
@@ -15,6 +15,7 @@ module DiscourseAi
start_date = params[:start_date].presence
end_date = params[:end_date]
threshold = SENTIMENT_THRESHOLD
+ model_name = DiscourseAi::Sentiment::PostClassification.active_model_name_for(:sentiment)
raise Discourse::InvalidParameters if %i[category tag].exclude?(group_by)
@@ -61,7 +62,7 @@ module DiscourseAi
#{grouping_clause} = :group_value AND
t.archetype = 'regular' AND
p.user_id > 0 AND
- cr.model_used = 'cardiffnlp/twitter-roberta-base-sentiment-latest' AND
+ cr.model_used = :model_name AND
((:start_date IS NULL OR p.created_at > :start_date) AND (:end_date IS NULL OR p.created_at < :end_date))
AND p.deleted_at IS NULL
AND t.deleted_at IS NULL
@@ -76,6 +77,7 @@ module DiscourseAi
limit: limit + 1,
offset: offset,
allowed_category_ids: guardian.allowed_category_ids,
+ model_name: model_name,
)
has_more = posts.length > limit
diff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.yml
index 26857056e87..36ca3d7c933 100644
--- a/plugins/discourse-ai/config/locales/server.en.yml
+++ b/plugins/discourse-ai/config/locales/server.en.yml
@@ -70,7 +70,12 @@ en:
ai_sentiment_enabled: "Enable the sentiment module."
ai_sentiment_inference_service_api_endpoint: "URL where the API is running for the sentiment module"
ai_sentiment_inference_service_api_key: "API key for the sentiment API"
- ai_sentiment_models: "Models to use for inference. Sentiment classifies post on the positive/neutral/negative space. Emotion classifies on the anger/disgust/fear/joy/neutral/sadness/surprise space."
+ ai_sentiment_model_configs: "Classification model endpoints to use for inference. Sentiment classifies posts as positive, neutral, or negative. Emotion classifies posts across supported emotion labels."
+ ai_sentiment_models: "Classification model endpoints to use for inference. Sentiment classifies posts as positive, neutral, or negative. Emotion classifies posts across supported emotion labels."
+ ai_sentiment_sentiment_classification_strategy: "Sentiment classification backend."
+ ai_sentiment_sentiment_agent: "Agent to use for sentiment classification when the sentiment backend is agent."
+ ai_sentiment_emotion_classification_strategy: "Emotion classification backend."
+ ai_sentiment_emotion_agent: "Agent to use for emotion classification when the emotion backend is agent."
ai_nsfw_detection_enabled: "Enable the NSFW module."
ai_nsfw_inference_service_api_endpoint: "URL where the API is running for the NSFW module"
@@ -495,6 +500,12 @@ en:
chat_thread_titler:
name: "Chat Thread Titler"
description: "Default agent powering automatic chat thread title generation"
+ sentiment_classifier:
+ name: "Sentiment classifier"
+ description: "Default agent powering sentiment classification"
+ emotion_classifier:
+ name: "Emotion classifier"
+ description: "Default agent powering emotion classification"
topic_not_found: "Summary unavailable, topic not found!"
summarizing: "Summarizing topic"
diff --git a/plugins/discourse-ai/config/settings.yml b/plugins/discourse-ai/config/settings.yml
index 35dc8ab9a59..94269e30ada 100644
--- a/plugins/discourse-ai/config/settings.yml
+++ b/plugins/discourse-ai/config/settings.yml
@@ -25,6 +25,26 @@ discourse_ai:
default: ""
secret: true
json_schema: DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema
+ ai_sentiment_sentiment_classification_strategy:
+ default: "classification_model"
+ type: enum
+ choices:
+ - classification_model
+ - agent
+ ai_sentiment_sentiment_agent:
+ default: "-36"
+ type: enum
+ enum: "DiscourseAi::Configuration::AgentEnumerator"
+ ai_sentiment_emotion_classification_strategy:
+ default: "classification_model"
+ type: enum
+ choices:
+ - classification_model
+ - agent
+ ai_sentiment_emotion_agent:
+ default: "-37"
+ type: enum
+ enum: "DiscourseAi::Configuration::AgentEnumerator"
ai_sentiment_backfill_maximum_posts_per_hour:
default: 2500
min: 0
diff --git a/plugins/discourse-ai/lib/agents/agent.rb b/plugins/discourse-ai/lib/agents/agent.rb
index 950858e8d82..daef9c298e6 100644
--- a/plugins/discourse-ai/lib/agents/agent.rb
+++ b/plugins/discourse-ai/lib/agents/agent.rb
@@ -236,6 +236,8 @@ module DiscourseAi
ReportRunner => -33,
Discover => -34,
ChatThreadTitler => -35,
+ SentimentClassifier => -36,
+ EmotionClassifier => -37,
}.freeze
end
end
diff --git a/plugins/discourse-ai/lib/agents/emotion_classifier.rb b/plugins/discourse-ai/lib/agents/emotion_classifier.rb
new file mode 100644
index 00000000000..e7d5cdbba0d
--- /dev/null
+++ b/plugins/discourse-ai/lib/agents/emotion_classifier.rb
@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+ module Agents
+ class EmotionClassifier < Agent
+ def self.default_enabled
+ false
+ end
+
+ def system_prompt
+ <<~PROMPT
+ You classify Discourse forum posts by emotion.
+
+ Return a probability score for every supported emotion label. Each score must be a float from 0 to 1. The scores must sum to 1.0. Higher scores mean the label is more applicable to the post.
+
+ Classify the author's emotions, not quoted text or replies from other people. Ignore markup, links, signatures, logs, and code blocks unless they clearly express emotion. Interpret sarcasm and irony by their implied meaning.
+
+ Multiple emotions can share probability mass, but the scores must still sum to 1.0. Use neutral as the highest score for factual announcements, questions, or technical debugging with little affect.
+
+ Use the full post content provided by the user. Reply with valid JSON only. Do not include explanations, confidence fields, or any keys except the supported emotion labels.
+ PROMPT
+ end
+
+ def response_format
+ DiscourseAi::Sentiment::Emotions::LIST.map do |label|
+ { "key" => label, "type" => "number" }
+ end
+ end
+
+ def examples
+ [
+ [
+ "Thank you for the detailed walkthrough. This solved my issue and saved me a lot of time.",
+ emotion_scores(
+ gratitude: 0.72,
+ admiration: 0.14,
+ approval: 0.08,
+ relief: 0.04,
+ neutral: 0.02,
+ ),
+ ],
+ [
+ "This keeps failing after every deploy. I am frustrated that the same bug came back again.",
+ emotion_scores(
+ annoyance: 0.42,
+ anger: 0.25,
+ disappointment: 0.18,
+ disapproval: 0.1,
+ neutral: 0.05,
+ ),
+ ],
+ [
+ "Does anyone know why this setting disappeared after the migration?",
+ emotion_scores(
+ curiosity: 0.46,
+ confusion: 0.32,
+ surprise: 0.12,
+ neutral: 0.08,
+ realization: 0.02,
+ ),
+ ],
+ [
+ "I am sorry for deleting the topic by mistake. I should have checked before clicking.",
+ emotion_scores(
+ remorse: 0.68,
+ embarrassment: 0.16,
+ sadness: 0.08,
+ disappointment: 0.05,
+ neutral: 0.03,
+ ),
+ ],
+ [
+ "Great, now the backup is gone too. That is exactly what I needed today.",
+ emotion_scores(
+ annoyance: 0.35,
+ anger: 0.22,
+ disappointment: 0.2,
+ disapproval: 0.12,
+ sadness: 0.06,
+ neutral: 0.05,
+ ),
+ ],
+ [
+ "I cannot believe how quickly this was fixed. The new workflow is a joy to use.",
+ emotion_scores(
+ joy: 0.42,
+ admiration: 0.2,
+ approval: 0.16,
+ surprise: 0.12,
+ excitement: 0.08,
+ neutral: 0.02,
+ ),
+ ],
+ [
+ "I am worried the importer may lose attachments if we run it before the patch is ready.",
+ emotion_scores(
+ fear: 0.38,
+ nervousness: 0.24,
+ curiosity: 0.12,
+ confusion: 0.1,
+ neutral: 0.1,
+ sadness: 0.06,
+ ),
+ ],
+ ]
+ end
+
+ private
+
+ def emotion_scores(overrides)
+ DiscourseAi::Sentiment::Emotions::LIST
+ .index_with { |label| overrides[label.to_sym] || 0.0 }
+ .to_json
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/lib/agents/sentiment_classifier.rb b/plugins/discourse-ai/lib/agents/sentiment_classifier.rb
new file mode 100644
index 00000000000..46f3d6ddfff
--- /dev/null
+++ b/plugins/discourse-ai/lib/agents/sentiment_classifier.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+ module Agents
+ class SentimentClassifier < Agent
+ def self.default_enabled
+ false
+ end
+
+ def system_prompt
+ <<~PROMPT
+ You classify Discourse forum posts by sentiment.
+
+ Return a probability score for each sentiment label:
+ - negative
+ - neutral
+ - positive
+
+ Each score must be a float from 0 to 1. The scores must sum to 1.0. Higher scores mean the label is more applicable to the post.
+
+ Classify the author's sentiment, not quoted text or replies from other people. Ignore markup, links, signatures, logs, and code blocks unless they clearly express sentiment. Interpret sarcasm and irony by their implied meaning.
+
+ Use neutral as the highest score for factual announcements, questions, or technical debugging with little affect. For mixed sentiment, distribute probability across labels instead of forcing a single winner.
+
+ Use the full post content provided by the user. Reply with valid JSON only. Do not include explanations, confidence fields, or any keys except the sentiment labels.
+ PROMPT
+ end
+
+ def response_format
+ %w[negative neutral positive].map { |label| { "key" => label, "type" => "number" } }
+ end
+
+ def examples
+ [
+ [
+ "This release is fantastic. The new editor is fast, polished, and much easier to use.",
+ { negative: 0.02, neutral: 0.08, positive: 0.9 }.to_json,
+ ],
+ [
+ "The upgrade broke search for our team and the workaround did not help.",
+ { negative: 0.88, neutral: 0.09, positive: 0.03 }.to_json,
+ ],
+ [
+ "Version 3.2 is now available in the admin updates page.",
+ { negative: 0.03, neutral: 0.92, positive: 0.05 }.to_json,
+ ],
+ ]
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/lib/sentiment/constants.rb b/plugins/discourse-ai/lib/sentiment/constants.rb
index 42d21db5418..91f348c3569 100644
--- a/plugins/discourse-ai/lib/sentiment/constants.rb
+++ b/plugins/discourse-ai/lib/sentiment/constants.rb
@@ -3,6 +3,12 @@
module DiscourseAi
module Sentiment
module Constants
+ SENTIMENT_MODEL = "cardiffnlp/twitter-roberta-base-sentiment-latest"
+ EMOTION_MODEL = "SamLowe/roberta-base-go_emotions"
+ SENTIMENT_AGENT_MODEL = "discourse-ai/sentiment-agent"
+ EMOTION_AGENT_MODEL = "discourse-ai/emotion-agent"
+ CLASSIFICATION_MODEL_STRATEGY = "classification_model"
+ AGENT_STRATEGY = "agent"
SENTIMENT_THRESHOLD = 0.6
end
end
diff --git a/plugins/discourse-ai/lib/sentiment/emotion_dashboard_report.rb b/plugins/discourse-ai/lib/sentiment/emotion_dashboard_report.rb
index 11c190c507c..10bff1a6e65 100644
--- a/plugins/discourse-ai/lib/sentiment/emotion_dashboard_report.rb
+++ b/plugins/discourse-ai/lib/sentiment/emotion_dashboard_report.rb
@@ -15,7 +15,9 @@ module DiscourseAi
end
def self.fetch_data(report)
- DB.query(<<~SQL, end: report.end_date, start: report.start_date)
+ model_name = DiscourseAi::Sentiment::PostClassification.active_model_name_for(:emotion)
+
+ DB.query(<<~SQL, end: report.end_date, start: report.start_date, model_name: model_name)
SELECT
posts.created_at::DATE AS day,
#{
@@ -37,7 +39,7 @@ module DiscourseAi
topics.deleted_at IS NULL
WHERE
classification_results.target_type = 'Post' AND
- classification_results.model_used = 'SamLowe/roberta-base-go_emotions'
+ classification_results.model_used = :model_name
GROUP BY 1
ORDER BY 1 ASC
SQL
diff --git a/plugins/discourse-ai/lib/sentiment/emotion_filter_order.rb b/plugins/discourse-ai/lib/sentiment/emotion_filter_order.rb
index 0f1b3b62ae3..b2ad58f88ba 100644
--- a/plugins/discourse-ai/lib/sentiment/emotion_filter_order.rb
+++ b/plugins/discourse-ai/lib/sentiment/emotion_filter_order.rb
@@ -28,6 +28,10 @@ module DiscourseAi
emotion_clause = <<~SQL
COUNT(*) FILTER (WHERE (classification_results.classification::jsonb->'#{emotion}')::float > 0.1)
SQL
+ model_name =
+ ActiveRecord::Base.connection.quote(
+ DiscourseAi::Sentiment::PostClassification.active_model_name_for(:emotion),
+ )
# TODO: This is slow, we will need to materialize this in the future
with_clause = <<~SQL
@@ -42,7 +46,7 @@ module DiscourseAi
classification_results ON
classification_results.target_id = posts.id AND
classification_results.target_type = 'Post' AND
- classification_results.model_used = 'SamLowe/roberta-base-go_emotions'
+ classification_results.model_used = #{model_name}
WHERE
topics.archetype = 'regular'
AND topics.deleted_at IS NULL
diff --git a/plugins/discourse-ai/lib/sentiment/post_classification.rb b/plugins/discourse-ai/lib/sentiment/post_classification.rb
index f4dd526aaff..41ecb44fe46 100644
--- a/plugins/discourse-ai/lib/sentiment/post_classification.rb
+++ b/plugins/discourse-ai/lib/sentiment/post_classification.rb
@@ -3,12 +3,16 @@
module DiscourseAi
module Sentiment
class PostClassification
+ include Constants
+
def self.backfill_query(from_post_id: nil, max_age_days: nil)
- available_classifier_names =
- DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema.values.map { it.model_name }
+ available_classifier_names = active_classifier_names
+ return Post.none if available_classifier_names.blank?
queries =
available_classifier_names.map do |classifier_name|
+ quoted_classifier_name = ActiveRecord::Base.connection.quote(classifier_name)
+
base_query =
Post
.includes(:sentiment_classifications)
@@ -22,7 +26,7 @@ module DiscourseAi
ON crs.target_id = posts.id
AND crs.target_type = 'Post'
AND crs.classification_type = 'sentiment'
- AND crs.model_used = '#{classifier_name}'
+ AND crs.model_used = #{quoted_classifier_name}
SQL
.where("crs.id IS NULL")
@@ -44,19 +48,101 @@ module DiscourseAi
Post.from(Arel.sql("(#{unioned_queries}) as posts"))
end
- CONCURRENT_CLASSFICATIONS = 40
+ def self.active_classifier_names
+ new.classifiers.map { |classifier| classifier[:model_name] }
+ end
- def bulk_classify!(relation)
- pool =
- Scheduler::ThreadPool.new(
- min_threads: 0,
- max_threads: CONCURRENT_CLASSFICATIONS,
- idle_time: 30,
+ def self.active_model_name_for(classification_type)
+ classification_type = classification_type.to_s
+
+ if strategy_for(classification_type) == Constants::AGENT_STRATEGY
+ return(
+ if classification_type == "sentiment"
+ Constants::SENTIMENT_AGENT_MODEL
+ else
+ Constants::EMOTION_AGENT_MODEL
+ end
)
+ end
+
+ configured_model_name_for(classification_type) ||
+ default_model_name_for(classification_type)
+ end
+
+ def self.configured_model_name_for(classification_type)
+ configs = DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema.values
+ return if configs.blank?
+ classification_type = classification_type.to_s
+
+ explicitly_typed_config =
+ configs.find do |config|
+ config.respond_to?(:classification_type) &&
+ config.classification_type.to_s == classification_type
+ end
+ return explicitly_typed_config.model_name if explicitly_typed_config.present?
+
+ default_config =
+ configs.find { |config| config.model_name == default_model_name_for(classification_type) }
+ return default_config.model_name if default_config.present?
+
+ configs
+ .find { |config| classification_type_for(config) == classification_type }
+ &.model_name || untyped_custom_model_name(configs, classification_type)
+ end
+
+ def self.untyped_custom_model_name(configs, classification_type)
+ return if classification_type != "sentiment"
+
+ untyped_configs = configs.select { |config| classification_type_for(config).blank? }
+ untyped_configs.one? ? untyped_configs.first.model_name : nil
+ end
+
+ def self.default_model_name_for(classification_type)
+ if classification_type.to_s == "sentiment"
+ Constants::SENTIMENT_MODEL
+ else
+ Constants::EMOTION_MODEL
+ end
+ end
+
+ def self.classification_type_for(config)
+ if config.respond_to?(:classification_type) && config.classification_type.present?
+ return config.classification_type.to_s
+ end
+
+ return "sentiment" if config.model_name == Constants::SENTIMENT_MODEL
+ return "emotion" if config.model_name == Constants::EMOTION_MODEL
+
+ nil
+ end
+
+ def self.strategy_for(classification_type)
+ if classification_type.to_s == "sentiment"
+ SiteSetting.ai_sentiment_sentiment_classification_strategy
+ else
+ SiteSetting.ai_sentiment_emotion_classification_strategy
+ end
+ end
+
+ CONCURRENT_CLASSFICATIONS = 40
+ CONCURRENT_AGENT_CLASSIFICATIONS = 5
+
+ def bulk_classify!(relation)
available_classifiers = classifiers
return if available_classifiers.blank?
+ max_threads =
+ (
+ if available_classifiers.any? { |c| c[:provider] == :agent }
+ CONCURRENT_AGENT_CLASSIFICATIONS
+ else
+ CONCURRENT_CLASSFICATIONS
+ end
+ )
+
+ pool = Scheduler::ThreadPool.new(min_threads: 0, max_threads: max_threads, idle_time: 30)
+
results = Queue.new
queued = 0
@@ -72,7 +158,7 @@ module DiscourseAi
pool.post do
result = { target: record, classifier: classifier, text: text }
begin
- result[:classification] = request_with(classifier[:client], text)
+ result[:classification] = request_with(classifier, record, text)
rescue StandardError => e
result[:error] = e
end
@@ -88,7 +174,7 @@ module DiscourseAi
result = results.pop
if result[:error]
errors << result
- else
+ elsif result[:classification].present?
store_classification(
result[:target],
[[result[:classifier][:model_name], result[:classification]]],
@@ -106,8 +192,10 @@ module DiscourseAi
)
end
ensure
- pool.shutdown
- pool.wait_for_termination(timeout: 30)
+ if pool
+ pool.shutdown
+ pool.wait_for_termination(timeout: 30)
+ end
end
def classify!(target)
@@ -123,16 +211,37 @@ module DiscourseAi
available_classifiers.reject { |ac| already_classified.include?(ac[:model_name]) }
results =
- classifiers_for_target.reduce({}) do |memo, cft|
- memo[cft[:model_name]] = request_with(cft[:client], to_classify)
- memo
+ classifiers_for_target.each_with_object({}) do |cft, memo|
+ classification = request_with(cft, target, to_classify)
+ memo[cft[:model_name]] = classification if classification.present?
end
- store_classification(target, results)
+ store_classification(target, results) if results.present?
end
def classifiers
- DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema.values.map do |config|
+ hugging_face_classifiers + agent_classifiers
+ end
+
+ def has_classifiers?
+ classifiers.present?
+ end
+
+ private
+
+ def hugging_face_classifiers
+ return [] if agent_strategy_for?(:sentiment) && agent_strategy_for?(:emotion)
+
+ configs = DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema.values
+ legacy_sentiment_model = self.class.untyped_custom_model_name(configs, "sentiment")
+
+ configs.filter_map do |config|
+ classification_type = classification_type_for(config)
+ effective_type =
+ classification_type.presence ||
+ ("sentiment" if config.model_name == legacy_sentiment_model)
+ next if effective_type.present? && agent_strategy_for?(effective_type)
+
api_endpoint = config.endpoint
if api_endpoint.present? && api_endpoint.start_with?("srv://")
@@ -141,18 +250,58 @@ module DiscourseAi
end
{
+ classification_type: classification_type,
model_name: config.model_name,
client:
DiscourseAi::Inference::HuggingFaceTextEmbeddings.new(api_endpoint, config.api_key),
+ provider: :classification_model,
}
end
end
- def has_classifiers?
- classifiers.present?
+ def agent_classifiers
+ [
+ agent_classifier(
+ :sentiment,
+ SiteSetting.ai_sentiment_sentiment_agent,
+ Constants::SENTIMENT_AGENT_MODEL,
+ ),
+ agent_classifier(
+ :emotion,
+ SiteSetting.ai_sentiment_emotion_agent,
+ Constants::EMOTION_AGENT_MODEL,
+ ),
+ ].compact
end
- private
+ def agent_classifier(classification_type, agent_id, model_name)
+ return if !agent_strategy_for?(classification_type)
+
+ ai_agent = AiAgent.find_by_id_from_cache(agent_id)
+ return if ai_agent.blank?
+
+ agent_klass = ai_agent.class_instance
+ model_id = agent_klass.default_llm_id || SiteSetting.ai_default_llm_model
+ model = model_id.present? ? LlmModel.find_by(id: model_id) : LlmModel.last
+ return if model.blank?
+
+ {
+ classification_type: classification_type.to_s,
+ model_name: model_name,
+ agent: agent_klass.new,
+ user: ai_agent.user || Discourse.system_user,
+ model: model,
+ provider: :agent,
+ }
+ end
+
+ def classification_type_for(config)
+ self.class.classification_type_for(config)
+ end
+
+ def agent_strategy_for?(classification_type)
+ self.class.strategy_for(classification_type) == Constants::AGENT_STRATEGY
+ end
def prepare_text(target)
content =
@@ -169,8 +318,10 @@ module DiscourseAi
)
end
- def request_with(client, content)
- result = client.classify_by_sentiment!(content)
+ def request_with(classifier, target, content)
+ return request_with_agent(classifier, target, content) if classifier[:provider] == :agent
+
+ result = classifier[:client].classify_by_sentiment!(content)
transform_result(result)
end
@@ -181,6 +332,68 @@ module DiscourseAi
hash_result
end
+ def request_with_agent(classifier, target, content)
+ context =
+ DiscourseAi::Agents::BotContext.new(
+ post: target,
+ messages: [{ type: :user, content: content }],
+ user: classifier[:user],
+ skip_show_thinking: true,
+ feature_name: "sentiment",
+ )
+
+ bot =
+ DiscourseAi::Agents::Bot.as(
+ classifier[:user],
+ agent: classifier[:agent],
+ model: classifier[:model],
+ )
+
+ structured_output = nil
+ raw_result = +""
+ bot.reply(context) do |partial, _, type|
+ if type == :structured_output
+ structured_output = partial
+ else
+ raw_result << partial.to_s
+ end
+ end
+
+ transform_agent_result(
+ structured_output,
+ raw_result,
+ labels_for(classifier[:classification_type]),
+ )
+ end
+
+ def transform_agent_result(structured_output, raw_result, labels)
+ parsed_result = parse_raw_agent_result(raw_result)
+
+ result =
+ labels.index_with do |label|
+ value =
+ structured_output&.read_buffered_property(label.to_sym) || parsed_result[label] ||
+ parsed_result[label.to_sym] || 0
+
+ value.to_f.clamp(0.0, 1.0)
+ end
+
+ return nil if result.values.all?(&:zero?)
+ result
+ end
+
+ def parse_raw_agent_result(raw_result)
+ return {} if raw_result.blank?
+
+ JSON.parse(raw_result)
+ rescue JSON::ParserError
+ {}
+ end
+
+ def labels_for(classification_type)
+ classification_type.to_s == "sentiment" ? %w[negative neutral positive] : Emotions::LIST
+ end
+
def store_classification(target, classification)
attrs =
classification.map do |model_name, classifications|
diff --git a/plugins/discourse-ai/lib/sentiment/sentiment_analysis_report.rb b/plugins/discourse-ai/lib/sentiment/sentiment_analysis_report.rb
index b820cc28782..4b1e4391367 100644
--- a/plugins/discourse-ai/lib/sentiment/sentiment_analysis_report.rb
+++ b/plugins/discourse-ai/lib/sentiment/sentiment_analysis_report.rb
@@ -64,6 +64,7 @@ module DiscourseAi
def self.fetch_data(report, opts)
threshold = SENTIMENT_THRESHOLD
+ model_name = DiscourseAi::Sentiment::PostClassification.active_model_name_for(:sentiment)
grouping = (report.filters.dig(:group_by) || GROUP_BY_FILTER_DEFAULT).to_sym
sorting = (report.filters.dig(:sort_by) || SORT_BY_FILTER_DEFAULT).to_sym
@@ -162,7 +163,7 @@ module DiscourseAi
WHERE
t.archetype = 'regular' AND
p.user_id > 0 AND
- cr.model_used = 'cardiffnlp/twitter-roberta-base-sentiment-latest' AND
+ cr.model_used = :model_name AND
(p.created_at > :report_start AND p.created_at < :report_end)
#{where_clause}
#{group_by_clause}
@@ -173,6 +174,7 @@ module DiscourseAi
threshold: threshold,
category_filter: category_filter,
tag_filter: tag_filter,
+ model_name: model_name,
)
grouped_sentiments
diff --git a/plugins/discourse-ai/lib/sentiment/sentiment_dashboard_report.rb b/plugins/discourse-ai/lib/sentiment/sentiment_dashboard_report.rb
index 124e14b8ba8..93809abfccd 100644
--- a/plugins/discourse-ai/lib/sentiment/sentiment_dashboard_report.rb
+++ b/plugins/discourse-ai/lib/sentiment/sentiment_dashboard_report.rb
@@ -9,6 +9,7 @@ module DiscourseAi
plugin.add_report("overall_sentiment") do |report|
report.modes = [:stacked_chart]
threshold = SENTIMENT_THRESHOLD
+ model_name = DiscourseAi::Sentiment::PostClassification.active_model_name_for(:sentiment)
sentiment_count_sql = Proc.new { |sentiment| <<~SQL }
COUNT(
@@ -30,7 +31,7 @@ module DiscourseAi
WHERE
t.archetype = 'regular' AND
p.user_id > 0 AND
- cr.model_used = 'cardiffnlp/twitter-roberta-base-sentiment-latest' AND
+ cr.model_used = :model_name AND
(p.created_at > :report_start AND p.created_at < :report_end)
GROUP BY DATE_TRUNC('day', p.created_at)
ORDER BY 1 ASC
@@ -38,6 +39,7 @@ module DiscourseAi
report_start: report.start_date,
report_end: report.end_date,
threshold: threshold,
+ model_name: model_name,
)
return report if grouped_sentiments.empty?
diff --git a/plugins/discourse-ai/lib/sentiment/sentiment_site_setting_json_schema.rb b/plugins/discourse-ai/lib/sentiment/sentiment_site_setting_json_schema.rb
index c3003028369..80b52bdd3ab 100644
--- a/plugins/discourse-ai/lib/sentiment/sentiment_site_setting_json_schema.rb
+++ b/plugins/discourse-ai/lib/sentiment/sentiment_site_setting_json_schema.rb
@@ -11,6 +11,10 @@ module DiscourseAi
format: "table",
title: "model",
properties: {
+ classification_type: {
+ type: "string",
+ enum: %w[sentiment emotion],
+ },
model_name: {
type: "string",
},
@@ -21,7 +25,7 @@ module DiscourseAi
type: "string",
},
},
- required: %w[model_name endpoint api_key],
+ required: %w[classification_type model_name endpoint api_key],
},
}
end
diff --git a/plugins/discourse-ai/lib/tasks/modules/sentiment/populate.rake b/plugins/discourse-ai/lib/tasks/modules/sentiment/populate.rake
index b632431d826..72677b24bef 100644
--- a/plugins/discourse-ai/lib/tasks/modules/sentiment/populate.rake
+++ b/plugins/discourse-ai/lib/tasks/modules/sentiment/populate.rake
@@ -4,13 +4,16 @@ desc "Creates sample sentiment / emotion data"
task "ai:sentiment:populate", [:start_post] => [:environment] do |_, args|
raise "Don't run this task in production!" if Rails.env.production?
+ sentiment_model = DiscourseAi::Sentiment::PostClassification.active_model_name_for(:sentiment)
+ emotion_model = DiscourseAi::Sentiment::PostClassification.active_model_name_for(:emotion)
+
Post
- .joins(<<~SQL)
- LEFT JOIN classification_results ON
- posts.id = classification_results.target_id AND
- classification_results.target_type = 'Post' AND
- model_used = 'cardiffnlp/twitter-roberta-base-sentiment-latest'
- SQL
+ .joins(ActiveRecord::Base.sanitize_sql_array([<<~SQL, sentiment_model]))
+ LEFT JOIN classification_results ON
+ posts.id = classification_results.target_id AND
+ classification_results.target_type = 'Post' AND
+ model_used = ?
+ SQL
.where("classification_results.id IS NULL")
.where("posts.id > ?", args[:start_post].to_i || 0)
.find_each do |post|
@@ -20,7 +23,7 @@ task "ai:sentiment:populate", [:start_post] => [:environment] do |_, args|
ClassificationResult.create!(
target_id: post.id,
- model_used: "cardiffnlp/twitter-roberta-base-sentiment-latest",
+ model_used: sentiment_model,
classification_type: "sentiment",
target_type: "Post",
classification: {
@@ -32,12 +35,12 @@ task "ai:sentiment:populate", [:start_post] => [:environment] do |_, args|
end
Post
- .joins(<<~SQL)
- LEFT JOIN classification_results ON
- posts.id = classification_results.target_id AND
- classification_results.target_type = 'Post' AND
- classification_results.model_used = 'SamLowe/roberta-base-go_emotions'
- SQL
+ .joins(ActiveRecord::Base.sanitize_sql_array([<<~SQL, emotion_model]))
+ LEFT JOIN classification_results ON
+ posts.id = classification_results.target_id AND
+ classification_results.target_type = 'Post' AND
+ classification_results.model_used = ?
+ SQL
.where("classification_results.id IS NULL")
.where("posts.id > ?", args[:start_post].to_i || 0)
.find_each do |post|
@@ -53,7 +56,7 @@ task "ai:sentiment:populate", [:start_post] => [:environment] do |_, args|
ClassificationResult.create!(
target_id: post.id,
- model_used: "SamLowe/roberta-base-go_emotions",
+ model_used: emotion_model,
classification_type: "sentiment",
target_type: "Post",
classification: emotions,
diff --git a/plugins/discourse-ai/spec/lib/agents/emotion_classifier_spec.rb b/plugins/discourse-ai/spec/lib/agents/emotion_classifier_spec.rb
new file mode 100644
index 00000000000..2c8329856e0
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/agents/emotion_classifier_spec.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Agents::EmotionClassifier do
+ subject(:agent) { described_class.new }
+
+ before { enable_current_plugin }
+
+ describe "#system_prompt" do
+ it "describes probability score constraints" do
+ prompt = agent.system_prompt
+
+ expect(prompt).to include("float from 0 to 1")
+ expect(prompt).to include("sum to 1.0")
+ end
+ end
+
+ describe "#examples" do
+ it "uses valid emotion probability distributions" do
+ labels = DiscourseAi::Sentiment::Emotions::LIST
+
+ agent.examples.each do |_input, output|
+ parsed_output = JSON.parse(output)
+
+ expect(parsed_output.keys).to contain_exactly(*labels)
+ expect(parsed_output.values.sum).to be_within(0.001).of(1.0)
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/spec/lib/agents/sentiment_classifier_spec.rb b/plugins/discourse-ai/spec/lib/agents/sentiment_classifier_spec.rb
new file mode 100644
index 00000000000..965b94378da
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/agents/sentiment_classifier_spec.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Agents::SentimentClassifier do
+ subject(:agent) { described_class.new }
+
+ before { enable_current_plugin }
+
+ describe "#system_prompt" do
+ it "describes probability score constraints" do
+ prompt = agent.system_prompt
+
+ expect(prompt).to include("float from 0 to 1")
+ expect(prompt).to include("sum to 1.0")
+ end
+ end
+
+ describe "#examples" do
+ it "uses valid sentiment probability distributions" do
+ labels = %w[negative neutral positive]
+
+ agent.examples.each do |_input, output|
+ parsed_output = JSON.parse(output)
+
+ expect(parsed_output.keys).to contain_exactly(*labels)
+ expect(parsed_output.values.sum).to be_within(0.001).of(1.0)
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/spec/lib/modules/sentiment/emotion_filter_order_spec.rb b/plugins/discourse-ai/spec/lib/modules/sentiment/emotion_filter_order_spec.rb
index 1d9a0d08051..efb31e14a8f 100644
--- a/plugins/discourse-ai/spec/lib/modules/sentiment/emotion_filter_order_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/sentiment/emotion_filter_order_spec.rb
@@ -190,6 +190,27 @@ RSpec.describe DiscourseAi::Sentiment::EmotionFilterOrder do
expect(result.to_sql).to include("ORDER BY topic_emotion.emotion_joy desc")
end
+ context "when emotion classification uses an agent" do
+ let(:model_used) { DiscourseAi::Sentiment::Constants::EMOTION_AGENT_MODEL }
+
+ before { SiteSetting.ai_sentiment_emotion_classification_strategy = "agent" }
+
+ it "filters topics using the stable emotion agent model key" do
+ emotion = "joy"
+ filter =
+ DiscoursePluginRegistry
+ .custom_filter_mappings
+ .find { it.keys.include? "order:emotion_#{emotion}" }
+ .values
+ .first
+ result = filter.call(Topic.all, "desc", guardian)
+
+ expect(result.to_sql).to include(
+ "classification_results.model_used = '#{DiscourseAi::Sentiment::Constants::EMOTION_AGENT_MODEL}'",
+ )
+ end
+ end
+
it "sorts emotion in ascending order" do
expect(
TopicsFilter.new(guardian:).filter_from_query_string("order:emotion_love-asc").pluck(:id),
diff --git a/plugins/discourse-ai/spec/lib/modules/sentiment/post_classification_spec.rb b/plugins/discourse-ai/spec/lib/modules/sentiment/post_classification_spec.rb
index f630c85615b..10e0a04bcfd 100644
--- a/plugins/discourse-ai/spec/lib/modules/sentiment/post_classification_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/sentiment/post_classification_spec.rb
@@ -22,6 +22,48 @@ RSpec.describe DiscourseAi::Sentiment::PostClassification do
expect(result.classification.keys).to contain_exactly("negative", "neutral", "positive")
end
+ def agent_for(response_format, llm_model)
+ Fabricate(
+ :ai_agent,
+ default_llm: llm_model,
+ response_format: response_format,
+ system_prompt: "Classify this post",
+ )
+ end
+
+ describe ".active_model_name_for" do
+ it "uses an explicitly typed custom sentiment classifier for read paths" do
+ SiteSetting.ai_sentiment_model_configs = [
+ {
+ classification_type: "sentiment",
+ model_name: "custom/sentiment-model",
+ endpoint: "https://sentiment.example.com",
+ api_key: "123",
+ },
+ ].to_json
+
+ expect(described_class.active_model_name_for(:sentiment)).to eq("custom/sentiment-model")
+ end
+
+ it "uses a single legacy untyped custom classifier for sentiment read paths" do
+ SiteSetting.ai_sentiment_model_configs = [
+ {
+ model_name: "custom/classifier",
+ endpoint: "https://sentiment.example.com",
+ api_key: "123",
+ },
+ ].to_json
+
+ expect(described_class.active_model_name_for(:sentiment)).to eq("custom/classifier")
+ end
+
+ it "prefers the default emotion classifier when multiple legacy emotion classifiers are configured" do
+ expect(described_class.active_model_name_for(:emotion)).to eq(
+ DiscourseAi::Sentiment::Constants::EMOTION_MODEL,
+ )
+ end
+ end
+
describe "#classify!" do
fab!(:post_1) { Fabricate(:post, post_number: 2) }
@@ -74,6 +116,129 @@ RSpec.describe DiscourseAi::Sentiment::PostClassification do
new_classifications = ClassificationResult.where("created_at > ?", first_classified_at).count
expect(new_classifications).to eq(1)
end
+
+ it "classifies sentiment through a configured agent" do
+ llm_model = Fabricate(:fake_model)
+ ai_agent =
+ agent_for(
+ %w[negative neutral positive].map { |label| { "key" => label, "type" => "number" } },
+ llm_model,
+ )
+ SiteSetting.ai_sentiment_model_configs = ""
+ SiteSetting.ai_sentiment_sentiment_classification_strategy = "agent"
+ SiteSetting.ai_sentiment_sentiment_agent = ai_agent.id
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ [{ negative: 0.1, neutral: 0.2, positive: 0.7 }],
+ ) { post_classification.classify!(post_1) }
+
+ result =
+ ClassificationResult.find_by!(
+ model_used: DiscourseAi::Sentiment::Constants::SENTIMENT_AGENT_MODEL,
+ target: post_1,
+ )
+
+ expect(result.classification).to eq("negative" => 0.1, "neutral" => 0.2, "positive" => 0.7)
+ end
+
+ it "classifies emotion through a configured agent" do
+ llm_model = Fabricate(:fake_model)
+ ai_agent =
+ agent_for(
+ DiscourseAi::Sentiment::Emotions::LIST.map do |label|
+ { "key" => label, "type" => "number" }
+ end,
+ llm_model,
+ )
+ SiteSetting.ai_sentiment_model_configs = ""
+ SiteSetting.ai_sentiment_emotion_classification_strategy = "agent"
+ SiteSetting.ai_sentiment_emotion_agent = ai_agent.id
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ [{ anger: 0.7, joy: 0.2, neutral: 0.1 }],
+ ) { post_classification.classify!(post_1) }
+
+ result =
+ ClassificationResult.find_by!(
+ model_used: DiscourseAi::Sentiment::Constants::EMOTION_AGENT_MODEL,
+ target: post_1,
+ )
+
+ expect(result.classification.keys).to contain_exactly(*DiscourseAi::Sentiment::Emotions::LIST)
+ expect(result.classification.slice("anger", "joy", "neutral")).to eq(
+ "anger" => 0.7,
+ "joy" => 0.2,
+ "neutral" => 0.1,
+ )
+ end
+
+ it "skips a lone legacy untyped sentiment config when sentiment strategy is agent" do
+ llm_model = Fabricate(:fake_model)
+ ai_agent =
+ agent_for(
+ %w[negative neutral positive].map { |label| { "key" => label, "type" => "number" } },
+ llm_model,
+ )
+ SiteSetting.ai_sentiment_model_configs = [
+ { model_name: "custom/legacy", endpoint: "https://legacy.example.com", api_key: "123" },
+ ].to_json
+ SiteSetting.ai_sentiment_sentiment_classification_strategy = "agent"
+ SiteSetting.ai_sentiment_sentiment_agent = ai_agent.id
+
+ model_names = post_classification.classifiers.map { |c| c[:model_name] }
+ expect(model_names).to include(DiscourseAi::Sentiment::Constants::SENTIMENT_AGENT_MODEL)
+ expect(model_names).not_to include("custom/legacy")
+ end
+
+ it "skips storing when the agent returns no usable classification" do
+ llm_model = Fabricate(:fake_model)
+ ai_agent =
+ agent_for(
+ %w[negative neutral positive].map { |label| { "key" => label, "type" => "number" } },
+ llm_model,
+ )
+ SiteSetting.ai_sentiment_model_configs = ""
+ SiteSetting.ai_sentiment_sentiment_classification_strategy = "agent"
+ SiteSetting.ai_sentiment_sentiment_agent = ai_agent.id
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(["not json"]) do
+ post_classification.classify!(post_1)
+ end
+
+ expect(
+ ClassificationResult.where(
+ model_used: DiscourseAi::Sentiment::Constants::SENTIMENT_AGENT_MODEL,
+ target: post_1,
+ ),
+ ).to be_empty
+ end
+
+ it "falls back to the newest LLM model when the configured agent does not set a default LLM" do
+ Fabricate(:fake_model, name: "older-model", created_at: 1.day.ago)
+ llm_model = Fabricate(:fake_model, name: "newer-model")
+ ai_agent =
+ Fabricate(
+ :ai_agent,
+ default_llm: nil,
+ response_format:
+ %w[negative neutral positive].map { |label| { "key" => label, "type" => "number" } },
+ system_prompt: "Classify this post",
+ )
+ SiteSetting.ai_default_llm_model = ""
+ SiteSetting.ai_sentiment_model_configs = ""
+ SiteSetting.ai_sentiment_sentiment_classification_strategy = "agent"
+ SiteSetting.ai_sentiment_sentiment_agent = ai_agent.id
+
+ classifier =
+ post_classification.send(
+ :agent_classifier,
+ :sentiment,
+ ai_agent.id,
+ DiscourseAi::Sentiment::Constants::SENTIMENT_AGENT_MODEL,
+ )
+
+ expect(classifier[:model]).to eq(llm_model)
+ end
end
describe "#classify_bulk!" do
@@ -172,6 +337,32 @@ RSpec.describe DiscourseAi::Sentiment::PostClassification do
expect(posts).to contain_exactly(classified_post)
end
+ it "does not reclassify agent results when the agent LLM changes" do
+ llm_model = Fabricate(:fake_model)
+ ai_agent =
+ agent_for(
+ %w[negative neutral positive].map { |label| { "key" => label, "type" => "number" } },
+ llm_model,
+ )
+ SiteSetting.ai_sentiment_model_configs = ""
+ SiteSetting.ai_sentiment_sentiment_classification_strategy = "agent"
+ SiteSetting.ai_sentiment_emotion_classification_strategy = "agent"
+ SiteSetting.ai_sentiment_sentiment_agent = ai_agent.id
+ SiteSetting.ai_sentiment_emotion_agent = "0"
+
+ classified_post = Fabricate(:post)
+ Fabricate(
+ :classification_result,
+ target: classified_post,
+ model_used: DiscourseAi::Sentiment::Constants::SENTIMENT_AGENT_MODEL,
+ )
+
+ ai_agent.update!(default_llm: Fabricate(:fake_model, name: "new-model"))
+
+ posts = described_class.backfill_query
+ expect(posts).to be_empty
+ end
+
it "excludes deleted posts" do
Fabricate(:post, deleted_at: 1.hour.ago)