A platform for community discussion. Free, open, simple.
FEATURE: Give Ask AI the endorsed reply of unsolved topics (#45166)
Rafael dos Santos Silvae08ed05b4b1e
diff --git a/plugins/discourse-ai/app/jobs/regular/stream_discover_reply.rb b/plugins/discourse-ai/app/jobs/regular/stream_discover_reply.rbindex 0815907e532..bbd88e7a117 100644--- a/plugins/discourse-ai/app/jobs/regular/stream_discover_reply.rb+++ b/plugins/discourse-ai/app/jobs/regular/stream_discover_reply.rb@@ -94,8 +94,17 @@ module Jobsreturnendreturn if !active_request?(user, args[:request_id])- ask_log.candidate_post_ids =- retrieval_result.synthesis_candidates.pluck("post_id").compact if ask_log+ # passages include the answers added to a candidate, not only its matched post+ if ask_log+ ask_log.candidate_post_ids =+ retrieval_result+ .synthesis_candidates+ .flat_map do |candidate|+ [candidate["post_id"], *Array(candidate["passages"]).pluck("post_id")]+ end+ .compact+ .uniq+ endbase = base.merge(scope_fallback: true) if retrieval_result.scope_fallbackif retrieval_result.synthesis_candidates.empty?ask_log.ask_outcome = :no_answer if ask_logdiff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.ymlindex 4a44a0c2bba..40f7b509de1 100644--- a/plugins/discourse-ai/config/locales/server.en.yml+++ b/plugins/discourse-ai/config/locales/server.en.yml@@ -184,6 +184,7 @@ en:ai_ask_ai_keyword_search_enabled: "Search the forum by keyword for Ask AI answers."ai_ask_ai_semantic_search_enabled: "Search the forum semantically for Ask AI answers. Requires embeddings."ai_ask_ai_search_categories: "Only search these categories, and their subcategories, for Ask AI answers. Personal messages are still searched when asked for. Leave empty to search all categories."+ ai_ask_ai_endorsed_reply_groups: "Also give Ask AI one reply from each topic without an accepted answer. Replies the topic's author liked rank first, then the author's own follow-ups that others liked, then replies members of these groups liked. Staff on communities where they like posts freely may want a narrower group. Leave empty to turn this off."ai_ask_ai_search_max_age_days: "Only search posts created in this many days for Ask AI answers. Set to 0 to search posts of any age."ai_ask_ai_external_sources: "Custom tools Ask AI searches alongside the forum. Each is called with the user's query and must return a list of results, each with a title, an http(s) url, and optionally a blurb, content and published_at. Queries are sent to whatever services the tools call."ai_ask_ai_external_sources_timeout: "Seconds Ask AI waits for its external sources, all searched at once, before answering without the ones still running. Counts towards the 20 seconds an Ask AI request may take, so a longer wait leaves less time to write the answer."diff --git a/plugins/discourse-ai/config/settings.yml b/plugins/discourse-ai/config/settings.ymlindex 8a0fd0d941d..420930c7ed7 100644--- a/plugins/discourse-ai/config/settings.yml+++ b/plugins/discourse-ai/config/settings.yml@@ -601,6 +601,12 @@ discourse_ai:default: 0min: 0area: "ai-features/search"+ ai_ask_ai_endorsed_reply_groups:+ type: group_list+ list_type: compact+ default: "3" # 3: @staff+ allow_any: false+ area: "ai-features/search"ai_ask_ai_external_sources:type: listlist_type: compactdiff --git a/plugins/discourse-ai/lib/agents/ask_ai_synthesis.rb b/plugins/discourse-ai/lib/agents/ask_ai_synthesis.rbindex ba0b156ed08..df990ad2660 100644--- a/plugins/discourse-ai/lib/agents/ask_ai_synthesis.rb+++ b/plugins/discourse-ai/lib/agents/ask_ai_synthesis.rb@@ -48,7 +48,7 @@ module DiscourseAi- Return related_count sources when that many candidates are useful.- Do not include an unrelated source merely to reach the limit.- Prefer authoritative guides and direct answers over discussions that only repeat the question.- - A solved candidate's accepted_answer passage is the answer its community accepted. Prefer it as evidence for how to fix or do something.+ - A passage marked answer is the topic's accepted solution when its candidate is solved, and otherwise its best reply, judged by likes, such as one the topic author liked or the author's own follow-up. Prefer answer passages as evidence for how to fix or do something, accepted solutions most, and only call an answer the solution when its candidate is solved.- Topic authors, staff, and documentation categories are useful signals. Treat likes as a weak signal only.- Consider freshness only for time-sensitive questions.diff --git a/plugins/discourse-ai/lib/discoveries/retrieval.rb b/plugins/discourse-ai/lib/discoveries/retrieval.rbindex 08d5ae00ccf..f07d9a4c29c 100644--- a/plugins/discourse-ai/lib/discoveries/retrieval.rb+++ b/plugins/discourse-ai/lib/discoveries/retrieval.rb@@ -353,7 +353,9 @@ module DiscourseAi.where(id: candidates.pluck("post_id").compact).includes(:user, topic: [{ category: :parent_category }, :tags]).index_by(&:id)- accepted_answers = accepted_answers_by_topic_id(posts.values.map(&:topic_id))+ topic_ids = posts.values.map(&:topic_id).uniq+ accepted_answers = accepted_answers_by_topic_id(topic_ids)+ endorsed_replies = endorsed_replies_by_topic_id(topic_ids - accepted_answers.keys)hidden_tags = DiscourseTagging.hidden_tag_names if SiteSetting.tagging_enabledcandidates.filter_map do |candidate|@@ -382,8 +384,13 @@ module DiscourseAi"is_topic_op" => post.post_number == 1,"topic_bumped_at" => topic.bumped_at&.iso8601,)- answers = accepted_answers[topic.id]- answers.present? ? with_accepted_answers(candidate, answers) : candidate+ if (answers = accepted_answers[topic.id])+ with_answers(candidate, answers).merge("solved" => true)+ elsif (replies = endorsed_replies[topic.id])+ with_answers(candidate, replies)+ else+ candidate+ endendend@@ -406,25 +413,96 @@ module DiscourseAi.group_by(&:topic_id)end+ # The best answer an unsolved topic has is a reply its author liked, the+ # author's own liked follow-up after others replied, or a reply a member of+ # the groups the site trusts liked, in that order. One per topic.+ def endorsed_replies_by_topic_id(topic_ids)+ group_ids = SiteSetting.ai_ask_ai_endorsed_reply_groups_map+ return {} if group_ids.empty? || topic_ids.empty?++ params = {+ topic_ids:,+ group_ids:,+ # every like comes from a logged-in user+ any_liker:+ group_ids.intersect?(+ [Group::AUTO_GROUPS[:everyone], Group::AUTO_GROUPS[:logged_in_users]],+ ),+ like: PostActionType::LIKE_POST_ACTION_ID,+ regular: Post.types[:regular],+ }+ post_ids = DB.query_single(<<~SQL, **params)+ SELECT post_id+ FROM (+ SELECT+ posts.id AS post_id,+ ROW_NUMBER() OVER (+ PARTITION BY posts.topic_id+ ORDER BY+ BOOL_OR(post_actions.user_id = topics.user_id) DESC,+ posts.user_id = topics.user_id DESC,+ posts.like_count DESC,+ posts.post_number+ ) AS rank+ FROM posts+ JOIN topics ON topics.id = posts.topic_id+ JOIN post_actions+ ON post_actions.post_id = posts.id+ AND post_actions.post_action_type_id = :like+ AND post_actions.deleted_at IS NULL+ WHERE posts.topic_id IN (:topic_ids)+ AND posts.post_number > 1+ AND posts.post_type = :regular+ AND NOT posts.hidden+ AND posts.deleted_at IS NULL+ AND (+ post_actions.user_id = topics.user_id+ OR :any_liker+ OR post_actions.user_id IN (+ SELECT user_id FROM group_users WHERE group_id IN (:group_ids)+ )+ OR (+ posts.user_id = topics.user_id+ AND EXISTS (+ SELECT 1+ FROM posts earlier+ WHERE earlier.topic_id = posts.topic_id+ AND earlier.post_number BETWEEN 2 AND posts.post_number - 1+ AND earlier.user_id <> topics.user_id+ AND earlier.post_type = :regular+ AND earlier.deleted_at IS NULL+ )+ )+ )+ GROUP BY posts.id, topics.id+ ) endorsed+ WHERE rank = 1+ SQL+ return {} if post_ids.empty?++ guardian = Guardian.new(@user)+ Post+ .where(id: post_ids)+ .includes(:topic)+ .select { |post| guardian.can_see_post?(post) }+ .group_by(&:topic_id)+ end+# Retrieval mostly matches the question in a topic's first post, so an- # accepted answer would rarely reach the model unless it is added.- def with_accepted_answers(candidate, answers)- answer_ids = answers.map(&:id)- passages =- Array(candidate["passages"]).map do |passage|- next passage if answer_ids.exclude?(passage["post_id"])-- passage.merge("accepted_answer" => true)- end+ # answer would rarely reach the model unless it is added.+ def with_answers(candidate, answers)+ passages = Array(candidate["passages"]).dupanswers.each do |answer|- next if passages.any? { |passage| passage["post_id"] == answer.id }-- passages << passage_from_source(source_from_post(answer, excerpt: answer.cooked)).merge(- "accepted_answer" => true,- )+ if (index = passages.index { |passage| passage["post_id"] == answer.id })+ passages[index] = passages[index].merge("answer" => true)+ else+ passages << passage_from_source(source_from_post(answer, excerpt: answer.cooked)).merge(+ "answer" => true,+ )+ endend- candidate.merge("solved" => true, "passages" => passages)+ candidate.merge("passages" => passages)enddef plain_text(value)diff --git a/plugins/discourse-ai/lib/discoveries/synthesis.rb b/plugins/discourse-ai/lib/discoveries/synthesis.rbindex 1505efb3b39..c1379334f43 100644--- a/plugins/discourse-ai/lib/discoveries/synthesis.rb+++ b/plugins/discourse-ai/lib/discoveries/synthesis.rb@@ -195,7 +195,7 @@ module DiscourseAiinput["solved"] = true if candidate["solved"]if candidate["passages"].present?input["passages"] = candidate["passages"].map do |passage|- passage.slice("post_number", "excerpt", "accepted_answer").compact+ passage.slice("post_number", "excerpt", "answer").compactendelseinput["excerpt"] = candidate["excerpt"] if candidate["excerpt"]diff --git a/plugins/discourse-ai/spec/jobs/regular/stream_discover_reply_spec.rb b/plugins/discourse-ai/spec/jobs/regular/stream_discover_reply_spec.rbindex 30549baa139..c5552db5c4a 100644--- a/plugins/discourse-ai/spec/jobs/regular/stream_discover_reply_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/stream_discover_reply_spec.rb@@ -830,6 +830,30 @@ describe Jobs::StreamDiscoverReply do)end+ it "retains the posts of every passage the model saw" do+ answer_post = Fabricate(:post, topic: source_post.topic)+ allow(retrieval).to receive(:call).and_return(+ DiscourseAi::Discoveries::Retrieval::Result.new(+ candidates: [+ candidate.merge(+ "passages" => [+ { "post_id" => source_post.id },+ { "post_id" => answer_post.id, "answer" => true },+ ],+ ),+ ],+ ),+ )+ allow(retrieval).to receive(:sources_for).and_return([])+ allow(synthesis).to receive(:call).and_return(+ DiscourseAi::Discoveries::Synthesis::Result.new(answerable: false, source_refs: []),+ )++ job.execute(args)++ expect(ask_log.reload.candidate_post_ids).to eq([source_post.id, answer_post.id])+ end+it "records retrieval errors before any answer is available" doallow(retrieval).to receive(:call).and_raise(StandardError)diff --git a/plugins/discourse-ai/spec/lib/discoveries/retrieval_spec.rb b/plugins/discourse-ai/spec/lib/discoveries/retrieval_spec.rbindex e6d15e66cd3..e1ced021b1d 100644--- a/plugins/discourse-ai/spec/lib/discoveries/retrieval_spec.rb+++ b/plugins/discourse-ai/spec/lib/discoveries/retrieval_spec.rb@@ -407,7 +407,7 @@ describe DiscourseAi::Discoveries::Retrieval do"post_id" => answer.id,"post_number" => answer.post_number,"excerpt" => "Restart the robot to fix it",- "accepted_answer" => true,+ "answer" => true,),)expect(candidates.second).not_to include("solved")@@ -417,7 +417,7 @@ describe DiscourseAi::Discoveries::Retrieval docandidates = call([source(answer)]).candidatesexpect(candidates.first.fetch("passages")).to contain_exactly(- include("post_id" => answer.id, "accepted_answer" => true),+ include("post_id" => answer.id, "answer" => true),)end@@ -441,6 +441,122 @@ describe DiscourseAi::Discoveries::Retrieval doendend+ describe "#call with endorsed replies" do+ fab!(:moderator)+ fab!(:liked_reply) { Fabricate(:post, topic: topic_1, raw: "Clear the cache first") }+ fab!(:author_liked_reply) { Fabricate(:post, topic: topic_1, raw: "Restart the robot") }++ def call(lexical)+ described_class.new(+ user:,+ lexical_retriever: ->(_query) { lexical },+ semantic_retriever: ->(_query) { [] },+ ).call("robot")+ end++ def answer_passages(candidate)+ candidate.fetch("passages").select { |passage| passage["answer"] }+ end++ it "adds the reply the topic author liked ahead of more liked ones" do+ PostActionCreator.like(moderator, liked_reply)+ PostActionCreator.like(Fabricate(:user), liked_reply)+ PostActionCreator.like(topic_1.user, author_liked_reply)+ PostActionCreator.like(moderator, author_liked_reply)++ candidates = call([source(post_1), source(post_2)]).candidates++ expect(answer_passages(candidates.first)).to contain_exactly(+ include(+ "post_id" => author_liked_reply.id,+ "excerpt" => "Restart the robot",+ "answer" => true,+ ),+ )+ expect(candidates.first).not_to include("solved")+ expect(answer_passages(candidates.second)).to be_empty+ end++ it "adds a reply liked only by a member of the endorsing groups" do+ PostActionCreator.like(moderator, liked_reply)+ PostActionCreator.like(Fabricate(:user), author_liked_reply)++ candidates = call([source(post_1)]).candidates++ expect(answer_passages(candidates.first)).to contain_exactly(+ include("post_id" => liked_reply.id, "answer" => true),+ )+ end++ it "adds the topic author's liked follow-up ahead of a group member's like" do+ follow_up =+ Fabricate(:post, topic: topic_1, user: topic_1.user, raw: "Fixed it, the cache was stale")+ PostActionCreator.like(Fabricate(:user), follow_up)+ PostActionCreator.like(moderator, liked_reply)+ PostActionCreator.like(Fabricate(:user), liked_reply)++ expect(answer_passages(call([source(post_1)]).candidates.first)).to contain_exactly(+ include("post_id" => follow_up.id, "answer" => true),+ )+ end++ it "ignores the topic author's liked posts before anyone else replied" do+ topic = Fabricate(:topic, category:)+ first_post = Fabricate(:post, topic:, user: topic.user)+ addendum = Fabricate(:post, topic:, user: topic.user)+ PostActionCreator.like(Fabricate(:user), addendum)++ expect(answer_passages(call([source(first_post)]).candidates.first)).to be_empty+ end++ it "marks an endorsed reply that retrieval matched instead of repeating it" do+ PostActionCreator.like(topic_1.user, liked_reply)++ candidates = call([source(liked_reply)]).candidates++ expect(candidates.first.fetch("passages")).to contain_exactly(+ include("post_id" => liked_reply.id, "answer" => true),+ )+ end++ it "leaves out replies that are hidden or only liked by others" do+ PostActionCreator.like(topic_1.user, liked_reply)+ liked_reply.update!(hidden: true)+ PostActionCreator.like(Fabricate(:user), author_liked_reply)++ expect(answer_passages(call([source(post_1)]).candidates.first)).to be_empty+ end++ it "counts any like when the endorsing groups include everyone" do+ SiteSetting.ai_ask_ai_endorsed_reply_groups = Group::AUTO_GROUPS[:everyone].to_s+ PostActionCreator.like(Fabricate(:user), liked_reply)++ expect(answer_passages(call([source(post_1)]).candidates.first)).to contain_exactly(+ include("post_id" => liked_reply.id, "answer" => true),+ )+ end++ it "adds no replies when no endorsing groups are set" do+ SiteSetting.ai_ask_ai_endorsed_reply_groups = ""+ PostActionCreator.like(topic_1.user, liked_reply)++ expect(answer_passages(call([source(post_1)]).candidates.first)).to be_empty+ end++ if defined?(DiscourseSolved)+ it "keeps the accepted answer as a solved topic's only answer" do+ SiteSetting.solved_enabled = true+ Fabricate(:solved_topic, topic: topic_1, answer_post: liked_reply)+ PostActionCreator.like(topic_1.user, author_liked_reply)++ candidate = call([source(post_1)]).candidates.first++ expect(candidate).to include("solved" => true)+ expect(answer_passages(candidate).pluck("post_id")).to eq([liked_reply.id])+ end+ end+ end+describe "#call with external sources" dodef external_source(name: "Help center", &search)Structdiff --git a/plugins/discourse-ai/spec/lib/discoveries/synthesis_spec.rb b/plugins/discourse-ai/spec/lib/discoveries/synthesis_spec.rbindex ba2078ccb19..96521f47382 100644--- a/plugins/discourse-ai/spec/lib/discoveries/synthesis_spec.rb+++ b/plugins/discourse-ai/spec/lib/discoveries/synthesis_spec.rb@@ -77,7 +77,7 @@ describe DiscourseAi::Discoveries::Synthesis do)end- it "flags a solved candidate and its accepted answer passage" do+ it "flags a solved candidate and its answer passage" docandidate = {"source_ref" => "source_1","title" => "Robot stuck",@@ -85,7 +85,7 @@ describe DiscourseAi::Discoveries::Synthesis do"solved" => true,"passages" => [{ "post_number" => 1, "excerpt" => "My robot is stuck." },- { "post_number" => 4, "excerpt" => "Restart it.", "accepted_answer" => true },+ { "post_number" => 4, "excerpt" => "Restart it.", "answer" => true },],}@@ -101,7 +101,7 @@ describe DiscourseAi::Discoveries::Synthesis do"solved" => true,"passages" => [{ "post_number" => 1, "excerpt" => "My robot is stuck." },- { "post_number" => 4, "excerpt" => "Restart it.", "accepted_answer" => true },+ { "post_number" => 4, "excerpt" => "Restart it.", "answer" => true },],)end