A platform for community discussion. Free, open, simple.
FIX: Make Ask AI tolerant of any structured output and drop redundant rechecks (#44960)
Rafael dos Santos Silva9dc06a253955
diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/discover/discoveries_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/discover/discoveries_controller.rbindex 87e3c9cc4c1..3741a07efe8 100644--- a/plugins/discourse-ai/app/controllers/discourse_ai/discover/discoveries_controller.rb+++ b/plugins/discourse-ai/app/controllers/discourse_ai/discover/discoveries_controller.rb@@ -169,6 +169,8 @@ module DiscourseAidef check_permissions!if ask_ai_action?+ # the Ask service checks access itself+ return if action_name == "reply"if !DiscourseAi::Discoveries.enabled_for_user?(current_user)raise Discourse::InvalidAccessenddiff --git a/plugins/discourse-ai/app/jobs/regular/stream_discover_reply.rb b/plugins/discourse-ai/app/jobs/regular/stream_discover_reply.rbindex 2d827edc070..28cda6575a0 100644--- a/plugins/discourse-ai/app/jobs/regular/stream_discover_reply.rb+++ b/plugins/discourse-ai/app/jobs/regular/stream_discover_reply.rb@@ -107,12 +107,21 @@ module Jobssynthesis =DiscourseAi::Discoveries::Synthesis.new(user:, ai_agent:, llm_model:, cancel_manager:)selected_sources = nil- selected_refs = nil- source_selection_invalid = false+ rejected = falsediscovery_title = ""last_published_answer = ""last_streamed_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)+ reject =+ lambda do+ rejected = true+ ask_log.ask_outcome = :no_answer if ask_log+ publish_no_answer(user, base)+ cancel_manager.cancel!+ end+ # Models emit the response keys in any order, so the answer streams as+ # soon as it is meaningful. Once the reply abstains or none of its sources+ # resolve, anything streamed is retracted and generation stops.result =synthesis.call(query:,@@ -122,36 +131,33 @@ module Jobssummary_detail: result_settings[:summary_detail],related_count: result_settings[:related_count],) do |update|- next if cancel_manager.cancelled? || !active_request?(user, args[:request_id])+ # the cancel monitor already stops this once the request is superseded+ next if rejected || cancel_manager.cancelled?+ next reject.call if update[:answerable] == falsediscovery_title = update[:title].to_s.strip if update[:title].present?- if update[:answerable] == true && update[:source_refs].present? &&- DiscourseAi::Discoveries::Synthesis.meaningful_answer?(update[:answer])- streamed_refs = Array(update[:source_refs]).first(result_settings[:related_count])- if selected_refs.present? && selected_refs != streamed_refs- source_selection_invalid = true- elsif selected_refs.nil?- selected_refs = streamed_refs- selected_sources = retrieval.validated_sources(retrieval_result, selected_refs)- source_selection_invalid = true if selected_sources.empty?-- if !source_selection_invalid- publish_update(- user,- base.merge(- done: false,- phase: "sources",- ai_discover_title: discovery_title,- sources: serialize_sources(selected_sources),- candidate_topic_ids:,- ),- )- end- end+ if selected_sources.nil? && update[:source_refs].present?+ selected_sources =+ retrieval.sources_for(+ retrieval_result,+ update[:source_refs],+ limit: result_settings[:related_count],+ )+ next reject.call if selected_sources.empty?++ publish_update(+ user,+ base.merge(+ done: false,+ phase: "sources",+ ai_discover_title: discovery_title,+ sources: serialize_sources(selected_sources),+ candidate_topic_ids:,+ ),+ )end- next if source_selection_invalid || selected_sources.blank?next if !DiscourseAi::Discoveries::Synthesis.meaningful_answer?(update[:answer])next if update[:answer] == last_published_answer@@ -180,17 +186,20 @@ module Jobsendend+ return if rejectedif cancel_manager.cancelled? || Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline ||!active_request?(user, args[:request_id])returnend- final_refs = Array(result.source_refs).first(result_settings[:related_count])final_sources =- retrieval.validated_sources(retrieval_result, final_refs) if final_refs.present?+ retrieval.sources_for(+ retrieval_result,+ result.source_refs,+ limit: result_settings[:related_count],+ )answerable =- result.answerable && !source_selection_invalid && final_sources.present? &&- selected_sources.present? && final_refs == selected_refs &&+ result.answerable && final_sources.present? &&DiscourseAi::Discoveries::Synthesis.meaningful_answer?(result.answer)if answerable@@ -201,13 +210,12 @@ module Jobssuggested_follow_up: result.follow_up,source_post_ids: final_sources.pluck("post_id"),)- selected_sources = final_sourcesDiscourseAi::Discoveries.store_result(user_id: user.id,request_id: args[:request_id],query:,answer: result.answer,- sources: selected_sources,+ sources: final_sources,agent_id: ai_agent.id,follow_up: result.follow_up,)@@ -220,7 +228,7 @@ module Jobsai_discover_title: result.title,ai_discover_reply: result.answer,ai_discover_follow_up: result.follow_up,- sources: serialize_sources(selected_sources),+ sources: serialize_sources(final_sources),candidate_topic_ids:,),)diff --git a/plugins/discourse-ai/lib/agents/ask_ai_synthesis.rb b/plugins/discourse-ai/lib/agents/ask_ai_synthesis.rbindex 327e34f0d29..cefdcae3d93 100644--- a/plugins/discourse-ai/lib/agents/ask_ai_synthesis.rb+++ b/plugins/discourse-ai/lib/agents/ask_ai_synthesis.rb@@ -8,16 +8,18 @@ module DiscourseAienddef response_format+ # Abstaining first avoids streaming an answer that would be retracted;+ # the short title and then the answer follow so both stream early.[{ "key" => "answerable", "type" => "boolean" },+ { "key" => "title", "type" => "string" },+ { "key" => "answer", "type" => "string" },{"key" => "source_refs","type" => "array","array_type" => "string","max_items" => SiteSetting.ai_ask_ai_related_count,},- { "key" => "title", "type" => "string" },- { "key" => "answer", "type" => "string" },{ "key" => "follow_up", "type" => "string" },]end@@ -31,6 +33,8 @@ module DiscourseAi### Answerability+ Decide answerable before writing anything else, by reviewing the candidates first.+Set answerable to true when at least one candidate contains enough relevant information to give a useful, supported answer. One strong source is sufficient.Set answerable to false when the candidates are unrelated, only repeat the question, or do not contain enough information for a supported answer. When false, return no sources and leave the requested title and answer fields empty.@@ -73,7 +77,7 @@ module DiscourseAiDo not include source identifiers, citations, or a sources or references section. The selected discussions are shown separately.Markdown and known relative forum links are allowed when useful.- Before returning, remove any claim not supported by the selected candidates. If no useful answer remains, set answerable to false.+ Include only claims supported by the candidates you select as sources. If you set answerable to true but find no supported answer, leave the answer empty.### Titlediff --git a/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb b/plugins/discourse-ai/lib/completions/json_streaming_tracker.rbindex 0cf98387873..2b81f3ce320 100644--- a/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb+++ b/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb@@ -3,7 +3,7 @@module DiscourseAimodule Completionsclass JsonStreamingTracker- attr_reader :stream_consumer+ attr_reader :stream_consumer, :last_keydef initialize(stream_consumer)@stream_consumer = stream_consumer@@ -42,6 +42,7 @@ module DiscourseAiprivatedef notify_changes(parsed)+ @last_key = parsed.keys.lastparsed.each do |key, value|next if value.nil?next if @last_notified[key] == valuediff --git a/plugins/discourse-ai/lib/completions/structured_output.rb b/plugins/discourse-ai/lib/completions/structured_output.rbindex 07d8820927d..c30f420bb01 100644--- a/plugins/discourse-ai/lib/completions/structured_output.rb+++ b/plugins/discourse-ai/lib/completions/structured_output.rb@@ -107,6 +107,12 @@ module DiscourseAichunkend+ # Until a later key starts or the response finishes, a streamed value may+ # still be growing, e.g. an array whose last item is half written.+ def property_complete?(prop_name)+ @done || @partial_json_tracker.last_key != prop_name.to_s+ end+def notify_progress(key, value)key_sym = key.to_symreturn if !@property_names.include?(key_sym)diff --git a/plugins/discourse-ai/lib/discoveries.rb b/plugins/discourse-ai/lib/discoveries.rbindex b01723499f2..8ec9370a8c2 100644--- a/plugins/discourse-ai/lib/discoveries.rb+++ b/plugins/discourse-ai/lib/discoveries.rb@@ -274,30 +274,14 @@ module DiscourseAidef store_result(user_id:, request_id:, query:, answer:, sources:, agent_id:, follow_up: "")return if !valid_request_id?(request_id) || query.blank? || sources.blank?- posts =- Post- .where(id: sources.map { |source| source.fetch("post_id") }, deleted_at: nil)- .includes(:topic)- .index_by(&:id)- stored_sources =- sources.filter_map do |source|- post = posts[source.fetch("post_id")]- topic = post&.topic- next if topic.nil? || topic.id != source.fetch("topic_id")-- {- post_id: post.id,- topic_id: topic.id,- category_id: topic.category_id,- post_updated_at: post.updated_at.iso8601(6),- }- end- return if stored_sources.length != sources.length-+ stored_sources = sources.map { |source| source.slice("post_id", "topic_id") }payload = { query:, answer:, sources: stored_sources, agent_id:, follow_up: }.to_jsonDiscourse.redis.set(result_key(user_id, request_id), payload, ex: REQUEST_TTL)end+ # The result is read back for follow-ups well after it was answered, so it+ # expires once any source is out of the user's sight: its answer was+ # written from all of them.def cached_result_for(user:, request_id:)return if user.nil? || !valid_request_id?(request_id)@@ -308,31 +292,26 @@ module DiscourseAisources = Array(result["sources"])return if result["query"].blank? || !result.key?("answer") || sources.empty?- posts =- Post- .where(id: sources.pluck("post_id"), deleted_at: nil)- .includes(topic: :category)- .index_by(&:id)+ posts = Post.where(id: sources.pluck("post_id")).includes(:topic).index_by(&:id)guardian = Guardian.new(user)- visible_sources = []- all_sources_visible =- sources.all? do |source|- post = posts[source["post_id"]]- topic = post&.topic- visible =- topic && topic.id == source["topic_id"] &&- [Archetype.default, Archetype.private_message].include?(topic.archetype) &&- topic.deleted_at.nil? && topic.visible? && guardian.can_see?(post) &&- topic.category_id == source["category_id"] &&- post.updated_at.iso8601(6) == source["post_updated_at"]-- if visible- visible_sources << source.merge("title" => topic.title, "url" => post.full_url)- end- visible- end+ if sources.any? { |source|+ post = posts[source["post_id"]]+ post&.topic.nil? || !guardian.can_see?(post)+ }+ return+ end- result.merge("sources" => visible_sources) if all_sources_visible+ result.merge(+ "sources" =>+ sources.map do |source|+ post = posts[source["post_id"]]+ source.merge(+ "topic_id" => post.topic_id,+ "title" => post.topic.title,+ "url" => post.full_url,+ )+ end,+ )rescue JSON::ParserErrornilenddiff --git a/plugins/discourse-ai/lib/discoveries/retrieval.rb b/plugins/discourse-ai/lib/discoveries/retrieval.rbindex c7beb960cc1..6e86ff12a54 100644--- a/plugins/discourse-ai/lib/discoveries/retrieval.rb+++ b/plugins/discourse-ai/lib/discoveries/retrieval.rb@@ -6,12 +6,11 @@ module DiscourseAiRRF_K = 60CANDIDATE_LIMIT = 50SYNTHESIS_LIMIT = 50- SELECTED_SOURCE_LIMIT = 6EXCERPT_LIMIT = 1200SEMANTIC_PRIVATE_MESSAGE_FILTER = /(?:\A|\s)in:(?:messages|personal)(?=\s|\z)/iResult =- Struct.new(:candidates, :private_messages, :scope_fallback, keyword_init: true) do+ Struct.new(:candidates, :scope_fallback, keyword_init: true) dodef synthesis_candidatescandidates.first(Retrieval::SYNTHESIS_LIMIT)end@@ -19,7 +18,6 @@ module DiscourseAidef initialize(user:, lexical_retriever: nil, semantic_retriever: nil)@user = user- @guardian = Guardian.new(user)@lexical_retriever = lexical_retriever || method(:lexical_sources)@semantic_retriever = semantic_retrieverend@@ -68,13 +66,13 @@ module DiscourseAiretrieve_in_parallel(scoped.(keyword_query), scoped.(semantic_query), private_messages:)endcandidates = reciprocal_rank_fusion(rankings)- candidates = revalidate_and_limit(candidates, private_messages:)+ candidates = with_metadata(candidates)candidates =candidates.map.with_index do |candidate, index|candidate.merge("source_ref" => "source_#{index + 1}")end- Result.new(candidates:, private_messages:)+ Result.new(candidates:)enddef self.explicit_filters?(query)@@ -100,21 +98,16 @@ module DiscourseAiexplicit_filters?(query.to_s.gsub(SEMANTIC_PRIVATE_MESSAGE_FILTER, " "))end- def validated_sources(result, source_refs)- source_refs = Array(source_refs)- return [] if source_refs.empty? || source_refs.length > SELECTED_SOURCE_LIMIT- return [] if source_refs.any? { |source_ref| !source_ref.is_a?(String) }- return [] if source_refs.uniq.length != source_refs.length-+ # Candidates were access-checked by the searches seconds earlier, so the+ # model's refs only need resolving; unknown or repeated ones are dropped.+ def sources_for(result, source_refs, limit:)candidates_by_ref =result.synthesis_candidates.index_by { |candidate| candidate["source_ref"] }- selected = source_refs.filter_map { |source_ref| candidates_by_ref[source_ref] }- return [] if selected.length != source_refs.length-- revalidated = revalidate_and_limit(selected, private_messages: result.private_messages)- return [] if revalidated.length != selected.length- revalidated+ Array(source_refs)+ .uniq+ .filter_map { |source_ref| candidates_by_ref[source_ref] }+ .first(limit)endprivate@@ -236,106 +229,41 @@ module DiscourseAisource.slice("post_id", "post_number", "url", "excerpt", "post_updated_at")end- def revalidate_and_limit(candidates, private_messages: false)- post_ids =- candidates.flat_map do |candidate|- candidate.fetch("passages", [candidate]).map { |passage| passage.fetch("post_id") }- end+ # Both retrievers search with the user's guardian, so candidates only need+ # the metadata synthesis and the sources list show.+ def with_metadata(candidates)+ candidates = candidates.first(CANDIDATE_LIMIT)posts =Post- .where(id: post_ids, deleted_at: nil)+ .where(id: candidates.pluck("post_id")).includes(:user, topic: [{ category: :parent_category }, :tags]).index_by(&:id)hidden_tags = DiscourseTagging.hidden_tag_names if SiteSetting.tagging_enabled- private_message_topic_ids = Set.new- if private_messages && @user- private_message_topic_ids =- Topic- .private_messages_for_user(@user)- .where(id: candidates.pluck("topic_id"))- .pluck(:id)- .to_set- end- visible_post_types = Topic.visible_post_types(@user)-- candidates- .filter_map do |candidate|- post = posts[candidate.fetch("post_id")]- topic = post&.topic- next if topic.nil? || topic.id != candidate.fetch("topic_id")- allowed_archetype =- if private_messages- topic.archetype == Archetype.private_message- else- topic.archetype == Archetype.default- end- next if !allowed_archetype- next if topic.deleted_at? || !topic.visible?- if topic.private_message?- next if !private_message_topic_ids.include?(topic.id)- next if post.hidden? || !visible_post_types.include?(post.post_type)- else- next if [email protected]_see?(post)- end- if candidate["post_updated_at"] &&- candidate["post_updated_at"] != post.updated_at.iso8601(6)- next- end- if candidate["topic_updated_at"] &&- candidate["topic_updated_at"] != topic.updated_at.iso8601(6)- next- end- candidate_passages = candidate.fetch("passages", [candidate])- passages =- candidate_passages.filter_map do |passage|- passage_post = posts[passage.fetch("post_id")]- next if passage_post.nil? || passage_post.topic_id != topic.id- if topic.private_message?- if passage_post.hidden? || !visible_post_types.include?(passage_post.post_type)- next- end- elsif [email protected]_see?(passage_post)- next- end- if passage["post_updated_at"] &&- passage["post_updated_at"] != passage_post.updated_at.iso8601(6)- next- end-- passage_from_source(- passage.merge(- "post_number" => passage_post.post_number,- "url" => passage_post.relative_url,- "post_updated_at" => passage_post.updated_at.iso8601(6),- ),- )- end- next if passages.length != candidate_passages.length-- search_metadata =- DiscourseAi::Utils::Search.format_row(- topic:,- post:,- hidden_tags:,- excerpt: plain_text(candidate["excerpt"]),- ).stringify_keys-- candidate.merge(- search_metadata,- "category_id" => topic.category_id,- "username" => post.user&.username,- "name" => SiteSetting.enable_names? ? post.user&.name : nil,- "avatar_template" => post.user&.avatar_template,- "author_is_staff" => post.user&.staff? || false,- "is_topic_op" => post.post_number == 1,- "passages" => passages,- "post_updated_at" => post.updated_at.iso8601(6),- "topic_updated_at" => topic.updated_at.iso8601(6),- "topic_bumped_at" => topic.bumped_at&.iso8601,- )- end- .first(CANDIDATE_LIMIT)+ candidates.filter_map do |candidate|+ post = posts[candidate.fetch("post_id")]+ topic = post&.topic+ next if topic.nil?++ search_metadata =+ DiscourseAi::Utils::Search.format_row(+ topic:,+ post:,+ hidden_tags:,+ excerpt: plain_text(candidate["excerpt"]),+ ).stringify_keys++ candidate.merge(+ search_metadata,+ "category_id" => topic.category_id,+ "username" => post.user&.username,+ "name" => SiteSetting.enable_names? ? post.user&.name : nil,+ "avatar_template" => post.user&.avatar_template,+ "author_is_staff" => post.user&.staff? || false,+ "is_topic_op" => post.post_number == 1,+ "topic_bumped_at" => topic.bumped_at&.iso8601,+ )+ endenddef plain_text(value)diff --git a/plugins/discourse-ai/lib/discoveries/synthesis.rb b/plugins/discourse-ai/lib/discoveries/synthesis.rbindex 5361bce6b68..d378e9a5025 100644--- a/plugins/discourse-ai/lib/discoveries/synthesis.rb+++ b/plugins/discourse-ai/lib/discoveries/synthesis.rb@@ -69,8 +69,11 @@ module DiscourseAianswerable = partial.read_buffered_property(:answerable)values[:answerable] = answerable if !answerable.nil?+ # Locking in a partial array would make later updates look like a changed selection.source_refs = partial.read_buffered_property(:source_refs)- values[:source_refs] = source_refs if !source_refs.nil?+ if !source_refs.nil? && partial.property_complete?(:source_refs)+ values[:source_refs] = source_refs+ endtitle = partial.read_buffered_property(:title)values[:title] << title if title.present?@@ -91,14 +94,17 @@ module DiscourseAiresult =Result.new(answerable: values[:answerable] == true,- source_refs: Array(values[:source_refs]),+ source_refs: known_source_refs(values[:source_refs], candidates:, related_count:),title: values[:title].to_s.strip,answer: answer_without_source_references(values[:answer]),follow_up: values[:follow_up].to_s.strip,)- return empty_result if !result.answerable+ if !result.answerable || result.source_refs.empty? ||+ !self.class.meaningful_answer?(result.answer)+ return empty_result+ end- valid_result?(result, candidates:, related_count:) ? result : empty_result+ resultenddef self.meaningful_answer?(answer)@@ -122,18 +128,12 @@ module DiscourseAiagentend- def valid_result?(result, candidates:, related_count:)- return false if result.source_refs.empty?- return false if result.source_refs.uniq.length != result.source_refs.length- return false if result.source_refs.length > related_count-+ def known_source_refs(source_refs, candidates:, related_count:)candidate_refs = candidates.pluck("source_ref")- if result.source_refs.any? { |source_ref| !candidate_refs.include?(source_ref) }- return false- end- return false if !self.class.meaningful_answer?(result.answer)-- true+ Array(source_refs)+ .uniq+ .select { |source_ref| candidate_refs.include?(source_ref) }+ .first(related_count)enddef empty_resultdiff --git a/plugins/discourse-ai/spec/jobs/regular/stream_discover_reply_spec.rb b/plugins/discourse-ai/spec/jobs/regular/stream_discover_reply_spec.rbindex 29f2fa896dd..afd744aa389 100644--- a/plugins/discourse-ai/spec/jobs/regular/stream_discover_reply_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/stream_discover_reply_spec.rb@@ -65,9 +65,11 @@ describe Jobs::StreamDiscoverReply do)allow(DiscourseAi::Discoveries::Retrieval).to receive(:new).with(user:).and_return(retrieval)allow(retrieval).to receive(:call).and_return(retrieval_result)- allow(retrieval).to receive(:validated_sources).with(retrieval_result, %w[source_1]).and_return(- [candidate],- )+ allow(retrieval).to receive(:sources_for).with(+ retrieval_result,+ %w[source_1],+ limit: an_instance_of(Integer),+ ).and_return([candidate])allow(DiscourseAi::Discoveries::Synthesis).to receive(:new) do |arguments|expect(arguments).to include(user:, ai_agent:, llm_model:)expect(arguments[:cancel_manager]).to be_a(DiscourseAi::Completions::CancelManager)@@ -195,9 +197,11 @@ describe Jobs::StreamDiscoverReply docandidates: [candidate, other_candidate, duplicate_candidate],)allow(retrieval).to receive(:call).and_return(result)- allow(retrieval).to receive(:validated_sources).with(result, %w[source_1]).and_return(- [candidate],- )+ allow(retrieval).to receive(:sources_for).with(+ result,+ %w[source_1],+ limit: an_instance_of(Integer),+ ).and_return([candidate])allow(synthesis).to receive(:call) do |**_, &stream|stream.call(answerable: true,@@ -284,7 +288,7 @@ describe Jobs::StreamDiscoverReply doscope_fallback: true,),)- allow(retrieval).to receive(:validated_sources).and_return([candidate])+ allow(retrieval).to receive(:sources_for).and_return([candidate])messages =MessageBus@@ -409,42 +413,81 @@ describe Jobs::StreamDiscoverReply do)end- it "waits for progressively streamed source references to finish before validating them" do- second_candidate = candidate.merge("source_ref" => "source_2")- progressive_result =- DiscourseAi::Discoveries::Retrieval::Result.new(candidates: [candidate, second_candidate])- allow(retrieval).to receive(:call).and_return(progressive_result)- allow(retrieval).to receive(:validated_sources).with(- progressive_result,- %w[source_1 source_2],- ).and_return([candidate, second_candidate])- allow(synthesis).to receive(- :call,- ) do |query:, candidates:, keyword_query:, original_query_locale:, summary_detail:, related_count:, &stream|- expect(query).to eq(self.query)- expect(keyword_query).to eq(self.query)- expect(candidates).to eq([candidate, second_candidate])- expect(summary_detail).to eq(:balanced)- expect(related_count).to eq(2)+ it "streams the answer before the model selects its sources" do+ allow(synthesis).to receive(:call) do |**, &stream|+ stream.call(answer: "Use the plugin skeleton.", title: "")+ stream.call(answer: "Use the plugin skeleton.", source_refs: %w[source_1], title: "")stream.call(+ answer: "Use the plugin skeleton.",+ source_refs: %w[source_1],+ answerable: true,+ title: "Plugin guide",+ )+ DiscourseAi::Discoveries::Synthesis::Result.new(answerable: true,source_refs: %w[source_1],- title: "Create a Discourse plugin",+ title: "Plugin guide",+ answer: "Use the plugin skeleton.",+ )+ end++ messages =+ MessageBus+ .track_publish("/discourse-ai/discoveries") do+ job.execute(user_id: user.id, query:, request_id:)+ end+ .map(&:data)++ expect(messages.map { |message| message[:phase] }).to eq(+ %w[searching answering sources complete],+ )+ expect(messages.second).to include(ai_discover_reply: "Use the plugin skeleton.")+ expect(messages.last).to include(answerable: true, sources: be_present)+ end++ it "retracts a streamed answer when the model abstains after writing it" do+ allow(synthesis).to receive(:call) do |**, &stream|+ stream.call(answer: "An unsupported guess.", title: "")+ stream.call(answer: "An unsupported guess.", answerable: false, title: "")+ DiscourseAi::Discoveries::Synthesis::Result.new(+ answerable: false,+ source_refs: [],+ title: "",answer: "",)+ end++ messages =+ MessageBus+ .track_publish("/discourse-ai/discoveries") do+ job.execute(user_id: user.id, query:, request_id:)+ end+ .map(&:data)++ expect(messages.map { |message| message[:phase] }).to eq(%w[searching answering complete])+ expect(messages.last).to include(answerable: false, ai_discover_reply: "", sources: [])+ expect(DiscourseAi::Discoveries.cached_result_for(user:, request_id:)).to be_nil+ end++ it "retracts a streamed answer and stops generating when none of its sources resolve" do+ allow(retrieval).to receive(:sources_for).and_return([])+ allow(synthesis).to receive(:call) do |**, &stream|+ stream.call(answer: "Use the plugin skeleton.", title: "")+ stream.call(answer: "Use the plugin skeleton.", source_refs: %w[source_9], title: "")stream.call(+ answer: "Use the plugin skeleton.",+ source_refs: %w[source_9],answerable: true,- source_refs: %w[source_1 source_2],- title: "Create a Discourse plugin",- answer: "Use the two selected discussions.",+ title: "Plugin guide",)DiscourseAi::Discoveries::Synthesis::Result.new(- answerable: true,- source_refs: %w[source_1 source_2],- title: "Create a Discourse plugin",- answer: "Use the two selected discussions.",+ answerable: false,+ source_refs: [],+ title: "",+ answer: "",)end+ expect_any_instance_of(DiscourseAi::Completions::CancelManager).to receive(:cancel!).oncemessages =MessageBus@@ -453,17 +496,12 @@ describe Jobs::StreamDiscoverReply doend.map(&:data)- expect(messages.map { |message| message[:phase] }).to eq(- %w[searching sources answering complete],- )- expect(messages.last).to include(- answerable: true,- ai_discover_reply: "Use the two selected discussions.",- )+ expect(messages.map { |message| message[:phase] }).to eq(%w[searching answering complete])+ expect(messages.last).to include(answerable: false, ai_discover_reply: "", sources: [])endit "does not release an answer when the model selects an invalid source" do- allow(retrieval).to receive(:validated_sources).and_return([])+ allow(retrieval).to receive(:sources_for).and_return([])messages =MessageBus@@ -504,27 +542,6 @@ describe Jobs::StreamDiscoverReply doexpect(messages.last).to include(answerable: false, ai_discover_reply: "", sources: [])end- it "revalidates selected sources before the final answer" do- allow(retrieval).to receive(:validated_sources).and_return([candidate], [])-- messages =- MessageBus- .track_publish("/discourse-ai/discoveries") do- job.execute(user_id: user.id, query:, request_id:)- end- .map(&:data)-- expect(messages.map { |message| message[:phase] }).to include("sources")- expect(messages.last).to include(- phase: "complete",- done: true,- answerable: false,- ai_discover_reply: "",- sources: [],- )- expect(DiscourseAi::Discoveries.cached_result_for(user:, request_id:)).to be_nil- end-it "completes without a model call when retrieval has no candidates" doempty_result = DiscourseAi::Discoveries::Retrieval::Result.new(candidates: [])allow(retrieval).to receive(:call).and_return(empty_result)@@ -744,7 +761,7 @@ describe Jobs::StreamDiscoverReply doallow(retrieval).to receive(:call).and_return(DiscourseAi::Discoveries::Retrieval::Result.new(candidates:),)- allow(retrieval).to receive(:validated_sources).and_return([candidate])+ allow(retrieval).to receive(:sources_for).and_return([candidate])allow(synthesis).to receive(:call) do |**_, &stream|stream.call(answerable: true, source_refs: %w[source_1], answer: "Answer")DiscourseAi::Discoveries::Synthesis::Result.new(@@ -794,7 +811,7 @@ describe Jobs::StreamDiscoverReply doendit "does not save rejected references as final sources" do- allow(retrieval).to receive(:validated_sources).and_return([])+ allow(retrieval).to receive(:sources_for).and_return([])job.execute(args)diff --git a/plugins/discourse-ai/spec/lib/agents/ask_ai_synthesis_spec.rb b/plugins/discourse-ai/spec/lib/agents/ask_ai_synthesis_spec.rbindex efcdc2e0679..0f557de17c5 100644--- a/plugins/discourse-ai/spec/lib/agents/ask_ai_synthesis_spec.rb+++ b/plugins/discourse-ai/spec/lib/agents/ask_ai_synthesis_spec.rb@@ -11,9 +11,9 @@ describe DiscourseAi::Agents::AskAiSynthesis doexpect(agent.response_format).to eq([{ "key" => "answerable", "type" => "boolean" },- { "key" => "source_refs", "type" => "array", "array_type" => "string", "max_items" => 4 },{ "key" => "title", "type" => "string" },{ "key" => "answer", "type" => "string" },+ { "key" => "source_refs", "type" => "array", "array_type" => "string", "max_items" => 4 },{ "key" => "follow_up", "type" => "string" },],)diff --git a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rbindex 7c126bb6fcd..ffb143fabec 100644--- a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb@@ -81,6 +81,30 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput doendend+ describe "#property_complete?" do+ it "treats a property as complete once a later key starts or the response finishes" do+ structured_output << "{\"list\": [\"source_1"+ expect(structured_output.property_complete?(:list)).to eq(false)++ structured_output << "3\"], \"bool\": tr"+ expect(structured_output.property_complete?(:list)).to eq(true)+ expect(structured_output.property_complete?(:bool)).to eq(false)++ structured_output << "ue}"+ structured_output.finish+ expect(structured_output.property_complete?(:bool)).to eq(true)+ end++ it "keeps a property open through a trailing comma and a half-written next key" do+ structured_output << "{\"list\": [\"a\"],"+ expect(structured_output.property_complete?(:list)).to eq(false)++ structured_output << " \"bo"+ expect(structured_output.property_complete?(:list)).to eq(true)+ expect(structured_output.read_buffered_property(:list)).to eq(["a"])+ end+ end+describe "Parsing structured output on the fly" doit "acts as a buffer for an streamed JSON" dochunks = [diff --git a/plugins/discourse-ai/spec/lib/discoveries/retrieval_spec.rb b/plugins/discourse-ai/spec/lib/discoveries/retrieval_spec.rbindex 4294f5ac69c..c5bb0ebcf09 100644--- a/plugins/discourse-ai/spec/lib/discoveries/retrieval_spec.rb+++ b/plugins/discourse-ai/spec/lib/discoveries/retrieval_spec.rb@@ -12,7 +12,6 @@ describe DiscourseAi::Discoveries::Retrieval dofab!(:post_1) { Fabricate(:post, topic: topic_1, raw: "First useful answer") }fab!(:post_2) { Fabricate(:post, topic: topic_2, user: staff_user, raw: "Second useful answer") }fab!(:post_3) { Fabricate(:post, topic: topic_3, raw: "Third useful answer") }- fab!(:private_post) { Fabricate(:post, topic: private_topic, raw: "Private answer") }fab!(:embedding_definition)before do@@ -217,15 +216,15 @@ describe DiscourseAi::Discoveries::Retrieval doexpect(result.candidates.map { |candidate| candidate.fetch("topic_id") }).to eq([topic_2.id])end- it "fuses, permission-checks, and assigns request-owned source references" do+ it "fuses, adds metadata to, and assigns request-owned source references" doSiteSetting.tagging_enabled = trueparent_category = Fabricate(:category)category.update!(parent_category:)tag = Fabricate(:tag, name: "guide")topic_2.tags << tagPostActionCreator.like(user, post_2)- lexical = [source(post_1), source(post_2), source(private_post)]- semantic = [source(post_2), source(post_3), source(private_post)]+ lexical = [source(post_1), source(post_2)]+ semantic = [source(post_2), source(post_3)]result =described_class.new(@@ -293,17 +292,8 @@ describe DiscourseAi::Discoveries::Retrieval doit "uses keyword and semantic retrieval for personal messages the user can see" dopersonal_message = Fabricate(:private_message_post, recipient: user, raw: "Anime plans")- inaccessible_message = Fabricate(:private_message_post, raw: "Hidden anime plans")- lexical_retriever =- instance_spy(- Proc,- call: [source(personal_message), source(inaccessible_message), source(post_1)],- )- semantic_retriever =- instance_spy(- Proc,- call: [source(personal_message), source(inaccessible_message), source(post_1)],- )+ lexical_retriever = instance_spy(Proc, call: [source(personal_message)])+ semantic_retriever = instance_spy(Proc, call: [source(personal_message)])result =described_class.new(user:, lexical_retriever:, semantic_retriever:).call(@@ -358,19 +348,6 @@ describe DiscourseAi::Discoveries::Retrieval do)end- it "excludes personal messages from a regular search" do- personal_message = Fabricate(:private_message_post, recipient: user, raw: "Anime plans")-- result =- described_class.new(- user:,- lexical_retriever: ->(_query) { [source(post_1), source(personal_message)] },- semantic_retriever: ->(_query) { [] },- ).call("anime")-- expect(result.candidates.map { |candidate| candidate.fetch("topic_id") }).to eq([topic_1.id])- end-it "does not broaden explicit search filters through semantic retrieval" dolexical_retriever = instance_spy(Proc, call: [source(post_1)])semantic_retriever = instance_spy(Proc, call: [source(post_2)])@@ -403,8 +380,8 @@ describe DiscourseAi::Discoveries::Retrieval doendend- describe "#validated_sources" do- it "accepts up to six known source references and rechecks access before publication" do+ describe "#sources_for" do+ it "resolves known source references in the model's order, dropping the rest" doretrieval =described_class.new(user:,@@ -413,42 +390,18 @@ describe DiscourseAi::Discoveries::Retrieval do)result = retrieval.call("useful answers")- sources = retrieval.validated_sources(result, %w[source_2 source_1])+ sources =+ retrieval.sources_for(+ result,+ ["source_2", "unknown", "source_2", nil, { "ref" => 1 }, "source_1"],+ limit: 6,+ )+expect(sources.map { |candidate| candidate.fetch("topic_id") }).to eq([topic_2.id, topic_1.id],)-- topic_2.update!(category: private_category)- expect(retrieval.validated_sources(result, %w[source_2 source_1])).to eq([])- end-- it "rejects duplicate, unknown, empty, or excessive source references" do- retrieval =- described_class.new(- user:,- lexical_retriever: ->(_query) { [source(post_1)] },- semantic_retriever: ->(_query) { [] },- )- result = retrieval.call("useful answers")-- expect(retrieval.validated_sources(result, [])).to eq([])- expect(retrieval.validated_sources(result, %w[source_1 source_1])).to eq([])- expect(retrieval.validated_sources(result, %w[source_1 unknown])).to eq([])- expect(retrieval.validated_sources(result, %w[a b c d e f g])).to eq([])- end-- it "rejects a supporting source that changes after retrieval" do- retrieval =- described_class.new(- user:,- lexical_retriever: ->(_query) { [source(post_1)] },- semantic_retriever: ->(_query) { [] },- )- result = retrieval.call("useful answers")-- post_1.update!(raw: "Changed after synthesis began")-- expect(retrieval.validated_sources(result, %w[source_1])).to eq([])+ expect(retrieval.sources_for(result, %w[source_2 source_1], limit: 1).length).to eq(1)+ expect(retrieval.sources_for(result, nil, limit: 6)).to eq([])endend@@ -484,6 +437,20 @@ describe DiscourseAi::Discoveries::Retrieval do)end+ it "excludes personal messages from a regular keyword search" do+ SiteSetting.ai_embeddings_enabled = false+ token = "regularneedle#{SecureRandom.hex(6)}"+ public_post = Fabricate(:post, topic: Fabricate(:topic, category:), raw: token)+ personal_message = Fabricate(:private_message_post, recipient: user, raw: token)+ [public_post, personal_message].each { |post| SearchIndexer.index(post, force: true) }++ result = described_class.new(user:).call(token)++ expect(result.candidates.map { |candidate| candidate.fetch("topic_id") }).to eq(+ [public_post.topic_id],+ )+ end+it "finds only personal messages the user can see without embeddings" doSiteSetting.ai_embeddings_enabled = falseembedding_request = stub_request(:post, embedding_definition.url).to_return(status: 500)diff --git a/plugins/discourse-ai/spec/lib/discoveries/synthesis_spec.rb b/plugins/discourse-ai/spec/lib/discoveries/synthesis_spec.rbindex 0dc903aa96a..f4deb98ed9b 100644--- a/plugins/discourse-ai/spec/lib/discoveries/synthesis_spec.rb+++ b/plugins/discourse-ai/spec/lib/discoveries/synthesis_spec.rb@@ -273,7 +273,7 @@ describe DiscourseAi::Discoveries::Synthesis doexpect(result).to have_attributes(answerable: false, source_refs: [], title: "", answer: "")end- it "rejects source references outside the supplied candidates" do+ it "rejects an answer when none of its source references are supplied candidates" docandidates = [{"source_ref" => "source_1",@@ -297,6 +297,31 @@ describe DiscourseAi::Discoveries::Synthesis doexpect(result).to have_attributes(answerable: false, source_refs: [], title: "", answer: "")end+ it "keeps an answer whose known source references survive alongside invalid ones" do+ candidates =+ %w[source_1 source_2 source_3].map do |source_ref|+ { "source_ref" => source_ref, "title" => "Plugins", "excerpt" => "Use the skeleton." }+ end++ result =+ DiscourseAi::Completions::Llm.with_prepared_responses(+ [+ {+ answerable: true,+ source_refs: %w[source_9 source_2 source_2 source_1 source_3],+ title: "Plugin guide",+ answer: "Use the plugin skeleton.",+ },+ ],+ ) { synthesis.call(query: "how do I create a plugin", candidates:, related_count: 2) }++ expect(result).to have_attributes(+ answerable: true,+ source_refs: %w[source_2 source_1],+ answer: "Use the plugin skeleton.",+ )+ end+it "returns the suggested follow-up question alongside the answer" docandidates = [{@@ -375,6 +400,56 @@ describe DiscourseAi::Discoveries::Synthesis doexpect(result).to have_attributes(answerable: false, source_refs: [], title: "", answer: "")end+ it "streams source references only once the array is complete when the answer comes first" do+ candidates =+ %w[source_4 source_1 source_13].map do |source_ref|+ { "source_ref" => source_ref, "title" => "Python", "excerpt" => "Use the API." }+ end+ chunks = [+ "{\n \"answer\": \"Use the REST API.\",",+ "\n \"answerable\": true,",+ "\n \"source_refs\": [\n \"source_4",+ "\",\n \"source_1",+ "3\",\n \"source_1\"",+ "\n ],\n \"title\": \"Python\"\n}",+ ]+ output =+ DiscourseAi::Completions::StructuredOutput.new(+ answer: {+ type: "string",+ },+ answerable: {+ type: "boolean",+ },+ source_refs: {+ type: "array",+ },+ title: {+ type: "string",+ },+ )+ bot = instance_double(DiscourseAi::Agents::Bot)+ allow(DiscourseAi::Agents::Bot).to receive(:as).and_return(bot)+ allow(bot).to receive(:reply) do |_, &stream|+ chunks.each do |chunk|+ output << chunk+ stream.call(output, nil, :structured_output)+ end+ output.finish+ stream.call(output, nil, :structured_output)+ end+ streamed_refs = []++ result =+ synthesis.call(query: "python", candidates:, related_count: 3) do |update|+ streamed_refs << update[:source_refs] if update[:source_refs]+ end++ expect(streamed_refs.uniq).to eq([%w[source_4 source_13 source_1]])+ expect(result.answerable).to eq(true)+ expect(result.source_refs).to eq(%w[source_4 source_13 source_1])+ end+it "keeps every streamed title fragment" docandidates = [{@@ -398,6 +473,7 @@ describe DiscourseAi::Discoveries::Synthesis dopartials.each do |values|partial = doubleallow(partial).to receive(:read_buffered_property) { |key| values[key] }+ allow(partial).to receive(:property_complete?).and_return(true)stream.call(partial, nil, :structured_output)endend@@ -436,6 +512,7 @@ describe DiscourseAi::Discoveries::Synthesis dopartials.each do |values|partial = doubleallow(partial).to receive(:read_buffered_property) { |key| values[key] }+ allow(partial).to receive(:property_complete?).and_return(true)stream.call(partial, nil, :structured_output)endenddiff --git a/plugins/discourse-ai/spec/lib/discoveries_spec.rb b/plugins/discourse-ai/spec/lib/discoveries_spec.rbindex 67c21544cd4..f550b5b6b1c 100644--- a/plugins/discourse-ai/spec/lib/discoveries_spec.rb+++ b/plugins/discourse-ai/spec/lib/discoveries_spec.rb@@ -270,7 +270,7 @@ describe DiscourseAi::Discoveries doexpect(described_class.cached_result_for(user:, request_id:)).to be_nilend- it "invalidates follow-up context when a supporting post changes" do+ it "keeps follow-up context when a supporting post is edited" dorequest_id = SecureRandom.uuiddescribed_class.store_result(user_id: user.id,@@ -283,6 +283,28 @@ describe DiscourseAi::Discoveries dopost.update!(raw: "Updated supporting content")+ expect(described_class.cached_result_for(user:, request_id:)).to include(+ "answer" => "A useful answer.",+ )+ end++ it "expires the result when any source is out of the user's sight" do+ other_post = Fabricate(:post)+ request_id = SecureRandom.uuid+ described_class.store_result(+ user_id: user.id,+ request_id:,+ query: "plugin setup",+ answer: "A useful answer.",+ sources: [+ { "post_id" => post.id, "topic_id" => post.topic_id },+ { "post_id" => other_post.id, "topic_id" => other_post.topic_id },+ ],+ agent_id: ai_agent.id,+ )++ other_post.topic.update!(category: Fabricate(:private_category, group: Fabricate(:group)))+expect(described_class.cached_result_for(user:, request_id:)).to be_nilend