Skip to content

A platform for community discussion. Free, open, simple.

FIX: Make Ask AI tolerant of any structured output and drop redundant rechecks (#44960)

Rafael dos Santos Silva9dc06a253955
diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/discover/discoveries_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/discover/discoveries_controller.rb
index 87e3c9cc4c1..3741a07efe8 100644
--- a/plugins/discourse-ai/app/controllers/discourse_ai/discover/discoveries_controller.rb
+++ b/plugins/discourse-ai/app/controllers/discourse_ai/discover/discoveries_controller.rb
@@ -169,6 +169,8 @@ module DiscourseAi
def check_permissions!
if ask_ai_action?
+ # the Ask service checks access itself
+ return if action_name == "reply"
if !DiscourseAi::Discoveries.enabled_for_user?(current_user)
raise Discourse::InvalidAccess
end
diff --git a/plugins/discourse-ai/app/jobs/regular/stream_discover_reply.rb b/plugins/discourse-ai/app/jobs/regular/stream_discover_reply.rb
index 2d827edc070..28cda6575a0 100644
--- a/plugins/discourse-ai/app/jobs/regular/stream_discover_reply.rb
+++ b/plugins/discourse-ai/app/jobs/regular/stream_discover_reply.rb
@@ -107,12 +107,21 @@ module Jobs
synthesis =
DiscourseAi::Discoveries::Synthesis.new(user:, ai_agent:, llm_model:, cancel_manager:)
selected_sources = nil
- selected_refs = nil
- source_selection_invalid = false
+ rejected = false
discovery_title = ""
last_published_answer = ""
last_streamed_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ reject =
+ lambda do
+ rejected = true
+ ask_log.ask_outcome = :no_answer if ask_log
+ publish_no_answer(user, base)
+ cancel_manager.cancel!
+ end
+ # Models emit the response keys in any order, so the answer streams as
+ # soon as it is meaningful. Once the reply abstains or none of its sources
+ # resolve, anything streamed is retracted and generation stops.
result =
synthesis.call(
query:,
@@ -122,36 +131,33 @@ module Jobs
summary_detail: result_settings[:summary_detail],
related_count: result_settings[:related_count],
) do |update|
- next if cancel_manager.cancelled? || !active_request?(user, args[:request_id])
+ # the cancel monitor already stops this once the request is superseded
+ next if rejected || cancel_manager.cancelled?
+ next reject.call if update[:answerable] == false
discovery_title = update[:title].to_s.strip if update[:title].present?
- if update[:answerable] == true && update[:source_refs].present? &&
- DiscourseAi::Discoveries::Synthesis.meaningful_answer?(update[:answer])
- streamed_refs = Array(update[:source_refs]).first(result_settings[:related_count])
- if selected_refs.present? && selected_refs != streamed_refs
- source_selection_invalid = true
- elsif selected_refs.nil?
- selected_refs = streamed_refs
- selected_sources = retrieval.validated_sources(retrieval_result, selected_refs)
- source_selection_invalid = true if selected_sources.empty?
-
- if !source_selection_invalid
- publish_update(
- user,
- base.merge(
- done: false,
- phase: "sources",
- ai_discover_title: discovery_title,
- sources: serialize_sources(selected_sources),
- candidate_topic_ids:,
- ),
- )
- end
- end
+ if selected_sources.nil? && update[:source_refs].present?
+ selected_sources =
+ retrieval.sources_for(
+ retrieval_result,
+ update[:source_refs],
+ limit: result_settings[:related_count],
+ )
+ next reject.call if selected_sources.empty?
+
+ publish_update(
+ user,
+ base.merge(
+ done: false,
+ phase: "sources",
+ ai_discover_title: discovery_title,
+ sources: serialize_sources(selected_sources),
+ candidate_topic_ids:,
+ ),
+ )
end
- next if source_selection_invalid || selected_sources.blank?
next if !DiscourseAi::Discoveries::Synthesis.meaningful_answer?(update[:answer])
next if update[:answer] == last_published_answer
@@ -180,17 +186,20 @@ module Jobs
end
end
+ return if rejected
if cancel_manager.cancelled? || Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline ||
!active_request?(user, args[:request_id])
return
end
- final_refs = Array(result.source_refs).first(result_settings[:related_count])
final_sources =
- retrieval.validated_sources(retrieval_result, final_refs) if final_refs.present?
+ retrieval.sources_for(
+ retrieval_result,
+ result.source_refs,
+ limit: result_settings[:related_count],
+ )
answerable =
- result.answerable && !source_selection_invalid && final_sources.present? &&
- selected_sources.present? && final_refs == selected_refs &&
+ result.answerable && final_sources.present? &&
DiscourseAi::Discoveries::Synthesis.meaningful_answer?(result.answer)
if answerable
@@ -201,13 +210,12 @@ module Jobs
suggested_follow_up: result.follow_up,
source_post_ids: final_sources.pluck("post_id"),
)
- selected_sources = final_sources
DiscourseAi::Discoveries.store_result(
user_id: user.id,
request_id: args[:request_id],
query:,
answer: result.answer,
- sources: selected_sources,
+ sources: final_sources,
agent_id: ai_agent.id,
follow_up: result.follow_up,
)
@@ -220,7 +228,7 @@ module Jobs
ai_discover_title: result.title,
ai_discover_reply: result.answer,
ai_discover_follow_up: result.follow_up,
- sources: serialize_sources(selected_sources),
+ sources: serialize_sources(final_sources),
candidate_topic_ids:,
),
)
diff --git a/plugins/discourse-ai/lib/agents/ask_ai_synthesis.rb b/plugins/discourse-ai/lib/agents/ask_ai_synthesis.rb
index 327e34f0d29..cefdcae3d93 100644
--- a/plugins/discourse-ai/lib/agents/ask_ai_synthesis.rb
+++ b/plugins/discourse-ai/lib/agents/ask_ai_synthesis.rb
@@ -8,16 +8,18 @@ module DiscourseAi
end
def response_format
+ # Abstaining first avoids streaming an answer that would be retracted;
+ # the short title and then the answer follow so both stream early.
[
{ "key" => "answerable", "type" => "boolean" },
+ { "key" => "title", "type" => "string" },
+ { "key" => "answer", "type" => "string" },
{
"key" => "source_refs",
"type" => "array",
"array_type" => "string",
"max_items" => SiteSetting.ai_ask_ai_related_count,
},
- { "key" => "title", "type" => "string" },
- { "key" => "answer", "type" => "string" },
{ "key" => "follow_up", "type" => "string" },
]
end
@@ -31,6 +33,8 @@ module DiscourseAi
### Answerability
+ Decide answerable before writing anything else, by reviewing the candidates first.
+
Set answerable to true when at least one candidate contains enough relevant information to give a useful, supported answer. One strong source is sufficient.
Set answerable to false when the candidates are unrelated, only repeat the question, or do not contain enough information for a supported answer. When false, return no sources and leave the requested title and answer fields empty.
@@ -73,7 +77,7 @@ module DiscourseAi
Do not include source identifiers, citations, or a sources or references section. The selected discussions are shown separately.
Markdown and known relative forum links are allowed when useful.
- Before returning, remove any claim not supported by the selected candidates. If no useful answer remains, set answerable to false.
+ Include only claims supported by the candidates you select as sources. If you set answerable to true but find no supported answer, leave the answer empty.
### Title
diff --git a/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb b/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb
index 0cf98387873..2b81f3ce320 100644
--- a/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb
+++ b/plugins/discourse-ai/lib/completions/json_streaming_tracker.rb
@@ -3,7 +3,7 @@
module DiscourseAi
module Completions
class JsonStreamingTracker
- attr_reader :stream_consumer
+ attr_reader :stream_consumer, :last_key
def initialize(stream_consumer)
@stream_consumer = stream_consumer
@@ -42,6 +42,7 @@ module DiscourseAi
private
def notify_changes(parsed)
+ @last_key = parsed.keys.last
parsed.each do |key, value|
next if value.nil?
next if @last_notified[key] == value
diff --git a/plugins/discourse-ai/lib/completions/structured_output.rb b/plugins/discourse-ai/lib/completions/structured_output.rb
index 07d8820927d..c30f420bb01 100644
--- a/plugins/discourse-ai/lib/completions/structured_output.rb
+++ b/plugins/discourse-ai/lib/completions/structured_output.rb
@@ -107,6 +107,12 @@ module DiscourseAi
chunk
end
+ # Until a later key starts or the response finishes, a streamed value may
+ # still be growing, e.g. an array whose last item is half written.
+ def property_complete?(prop_name)
+ @done || @partial_json_tracker.last_key != prop_name.to_s
+ end
+
def notify_progress(key, value)
key_sym = key.to_sym
return if !@property_names.include?(key_sym)
diff --git a/plugins/discourse-ai/lib/discoveries.rb b/plugins/discourse-ai/lib/discoveries.rb
index b01723499f2..8ec9370a8c2 100644
--- a/plugins/discourse-ai/lib/discoveries.rb
+++ b/plugins/discourse-ai/lib/discoveries.rb
@@ -274,30 +274,14 @@ module DiscourseAi
def store_result(user_id:, request_id:, query:, answer:, sources:, agent_id:, follow_up: "")
return if !valid_request_id?(request_id) || query.blank? || sources.blank?
- posts =
- Post
- .where(id: sources.map { |source| source.fetch("post_id") }, deleted_at: nil)
- .includes(:topic)
- .index_by(&:id)
- stored_sources =
- sources.filter_map do |source|
- post = posts[source.fetch("post_id")]
- topic = post&.topic
- next if topic.nil? || topic.id != source.fetch("topic_id")
-
- {
- post_id: post.id,
- topic_id: topic.id,
- category_id: topic.category_id,
- post_updated_at: post.updated_at.iso8601(6),
- }
- end
- return if stored_sources.length != sources.length
-
+ stored_sources = sources.map { |source| source.slice("post_id", "topic_id") }
payload = { query:, answer:, sources: stored_sources, agent_id:, follow_up: }.to_json
Discourse.redis.set(result_key(user_id, request_id), payload, ex: REQUEST_TTL)
end
+ # The result is read back for follow-ups well after it was answered, so it
+ # expires once any source is out of the user's sight: its answer was
+ # written from all of them.
def cached_result_for(user:, request_id:)
return if user.nil? || !valid_request_id?(request_id)
@@ -308,31 +292,26 @@ module DiscourseAi
sources = Array(result["sources"])
return if result["query"].blank? || !result.key?("answer") || sources.empty?
- posts =
- Post
- .where(id: sources.pluck("post_id"), deleted_at: nil)
- .includes(topic: :category)
- .index_by(&:id)
+ posts = Post.where(id: sources.pluck("post_id")).includes(:topic).index_by(&:id)
guardian = Guardian.new(user)
- visible_sources = []
- all_sources_visible =
- sources.all? do |source|
- post = posts[source["post_id"]]
- topic = post&.topic
- visible =
- topic && topic.id == source["topic_id"] &&
- [Archetype.default, Archetype.private_message].include?(topic.archetype) &&
- topic.deleted_at.nil? && topic.visible? && guardian.can_see?(post) &&
- topic.category_id == source["category_id"] &&
- post.updated_at.iso8601(6) == source["post_updated_at"]
-
- if visible
- visible_sources << source.merge("title" => topic.title, "url" => post.full_url)
- end
- visible
- end
+ if sources.any? { |source|
+ post = posts[source["post_id"]]
+ post&.topic.nil? || !guardian.can_see?(post)
+ }
+ return
+ end
- result.merge("sources" => visible_sources) if all_sources_visible
+ result.merge(
+ "sources" =>
+ sources.map do |source|
+ post = posts[source["post_id"]]
+ source.merge(
+ "topic_id" => post.topic_id,
+ "title" => post.topic.title,
+ "url" => post.full_url,
+ )
+ end,
+ )
rescue JSON::ParserError
nil
end
diff --git a/plugins/discourse-ai/lib/discoveries/retrieval.rb b/plugins/discourse-ai/lib/discoveries/retrieval.rb
index c7beb960cc1..6e86ff12a54 100644
--- a/plugins/discourse-ai/lib/discoveries/retrieval.rb
+++ b/plugins/discourse-ai/lib/discoveries/retrieval.rb
@@ -6,12 +6,11 @@ module DiscourseAi
RRF_K = 60
CANDIDATE_LIMIT = 50
SYNTHESIS_LIMIT = 50
- SELECTED_SOURCE_LIMIT = 6
EXCERPT_LIMIT = 1200
SEMANTIC_PRIVATE_MESSAGE_FILTER = /(?:\A|\s)in:(?:messages|personal)(?=\s|\z)/i
Result =
- Struct.new(:candidates, :private_messages, :scope_fallback, keyword_init: true) do
+ Struct.new(:candidates, :scope_fallback, keyword_init: true) do
def synthesis_candidates
candidates.first(Retrieval::SYNTHESIS_LIMIT)
end
@@ -19,7 +18,6 @@ module DiscourseAi
def initialize(user:, lexical_retriever: nil, semantic_retriever: nil)
@user = user
- @guardian = Guardian.new(user)
@lexical_retriever = lexical_retriever || method(:lexical_sources)
@semantic_retriever = semantic_retriever
end
@@ -68,13 +66,13 @@ module DiscourseAi
retrieve_in_parallel(scoped.(keyword_query), scoped.(semantic_query), private_messages:)
end
candidates = reciprocal_rank_fusion(rankings)
- candidates = revalidate_and_limit(candidates, private_messages:)
+ candidates = with_metadata(candidates)
candidates =
candidates.map.with_index do |candidate, index|
candidate.merge("source_ref" => "source_#{index + 1}")
end
- Result.new(candidates:, private_messages:)
+ Result.new(candidates:)
end
def self.explicit_filters?(query)
@@ -100,21 +98,16 @@ module DiscourseAi
explicit_filters?(query.to_s.gsub(SEMANTIC_PRIVATE_MESSAGE_FILTER, " "))
end
- def validated_sources(result, source_refs)
- source_refs = Array(source_refs)
- return [] if source_refs.empty? || source_refs.length > SELECTED_SOURCE_LIMIT
- return [] if source_refs.any? { |source_ref| !source_ref.is_a?(String) }
- return [] if source_refs.uniq.length != source_refs.length
-
+ # Candidates were access-checked by the searches seconds earlier, so the
+ # model's refs only need resolving; unknown or repeated ones are dropped.
+ def sources_for(result, source_refs, limit:)
candidates_by_ref =
result.synthesis_candidates.index_by { |candidate| candidate["source_ref"] }
- selected = source_refs.filter_map { |source_ref| candidates_by_ref[source_ref] }
- return [] if selected.length != source_refs.length
-
- revalidated = revalidate_and_limit(selected, private_messages: result.private_messages)
- return [] if revalidated.length != selected.length
- revalidated
+ Array(source_refs)
+ .uniq
+ .filter_map { |source_ref| candidates_by_ref[source_ref] }
+ .first(limit)
end
private
@@ -236,106 +229,41 @@ module DiscourseAi
source.slice("post_id", "post_number", "url", "excerpt", "post_updated_at")
end
- def revalidate_and_limit(candidates, private_messages: false)
- post_ids =
- candidates.flat_map do |candidate|
- candidate.fetch("passages", [candidate]).map { |passage| passage.fetch("post_id") }
- end
+ # Both retrievers search with the user's guardian, so candidates only need
+ # the metadata synthesis and the sources list show.
+ def with_metadata(candidates)
+ candidates = candidates.first(CANDIDATE_LIMIT)
posts =
Post
- .where(id: post_ids, deleted_at: nil)
+ .where(id: candidates.pluck("post_id"))
.includes(:user, topic: [{ category: :parent_category }, :tags])
.index_by(&:id)
hidden_tags = DiscourseTagging.hidden_tag_names if SiteSetting.tagging_enabled
- private_message_topic_ids = Set.new
- if private_messages && @user
- private_message_topic_ids =
- Topic
- .private_messages_for_user(@user)
- .where(id: candidates.pluck("topic_id"))
- .pluck(:id)
- .to_set
- end
- visible_post_types = Topic.visible_post_types(@user)
-
- candidates
- .filter_map do |candidate|
- post = posts[candidate.fetch("post_id")]
- topic = post&.topic
- next if topic.nil? || topic.id != candidate.fetch("topic_id")
- allowed_archetype =
- if private_messages
- topic.archetype == Archetype.private_message
- else
- topic.archetype == Archetype.default
- end
- next if !allowed_archetype
- next if topic.deleted_at? || !topic.visible?
- if topic.private_message?
- next if !private_message_topic_ids.include?(topic.id)
- next if post.hidden? || !visible_post_types.include?(post.post_type)
- else
- next if [email protected]_see?(post)
- end
- if candidate["post_updated_at"] &&
- candidate["post_updated_at"] != post.updated_at.iso8601(6)
- next
- end
- if candidate["topic_updated_at"] &&
- candidate["topic_updated_at"] != topic.updated_at.iso8601(6)
- next
- end
- candidate_passages = candidate.fetch("passages", [candidate])
- passages =
- candidate_passages.filter_map do |passage|
- passage_post = posts[passage.fetch("post_id")]
- next if passage_post.nil? || passage_post.topic_id != topic.id
- if topic.private_message?
- if passage_post.hidden? || !visible_post_types.include?(passage_post.post_type)
- next
- end
- elsif [email protected]_see?(passage_post)
- next
- end
- if passage["post_updated_at"] &&
- passage["post_updated_at"] != passage_post.updated_at.iso8601(6)
- next
- end
-
- passage_from_source(
- passage.merge(
- "post_number" => passage_post.post_number,
- "url" => passage_post.relative_url,
- "post_updated_at" => passage_post.updated_at.iso8601(6),
- ),
- )
- end
- next if passages.length != candidate_passages.length
-
- search_metadata =
- DiscourseAi::Utils::Search.format_row(
- topic:,
- post:,
- hidden_tags:,
- excerpt: plain_text(candidate["excerpt"]),
- ).stringify_keys
-
- candidate.merge(
- search_metadata,
- "category_id" => topic.category_id,
- "username" => post.user&.username,
- "name" => SiteSetting.enable_names? ? post.user&.name : nil,
- "avatar_template" => post.user&.avatar_template,
- "author_is_staff" => post.user&.staff? || false,
- "is_topic_op" => post.post_number == 1,
- "passages" => passages,
- "post_updated_at" => post.updated_at.iso8601(6),
- "topic_updated_at" => topic.updated_at.iso8601(6),
- "topic_bumped_at" => topic.bumped_at&.iso8601,
- )
- end
- .first(CANDIDATE_LIMIT)
+ candidates.filter_map do |candidate|
+ post = posts[candidate.fetch("post_id")]
+ topic = post&.topic
+ next if topic.nil?
+
+ search_metadata =
+ DiscourseAi::Utils::Search.format_row(
+ topic:,
+ post:,
+ hidden_tags:,
+ excerpt: plain_text(candidate["excerpt"]),
+ ).stringify_keys
+
+ candidate.merge(
+ search_metadata,
+ "category_id" => topic.category_id,
+ "username" => post.user&.username,
+ "name" => SiteSetting.enable_names? ? post.user&.name : nil,
+ "avatar_template" => post.user&.avatar_template,
+ "author_is_staff" => post.user&.staff? || false,
+ "is_topic_op" => post.post_number == 1,
+ "topic_bumped_at" => topic.bumped_at&.iso8601,
+ )
+ end
end
def plain_text(value)
diff --git a/plugins/discourse-ai/lib/discoveries/synthesis.rb b/plugins/discourse-ai/lib/discoveries/synthesis.rb
index 5361bce6b68..d378e9a5025 100644
--- a/plugins/discourse-ai/lib/discoveries/synthesis.rb
+++ b/plugins/discourse-ai/lib/discoveries/synthesis.rb
@@ -69,8 +69,11 @@ module DiscourseAi
answerable = partial.read_buffered_property(:answerable)
values[:answerable] = answerable if !answerable.nil?
+ # Locking in a partial array would make later updates look like a changed selection.
source_refs = partial.read_buffered_property(:source_refs)
- values[:source_refs] = source_refs if !source_refs.nil?
+ if !source_refs.nil? && partial.property_complete?(:source_refs)
+ values[:source_refs] = source_refs
+ end
title = partial.read_buffered_property(:title)
values[:title] << title if title.present?
@@ -91,14 +94,17 @@ module DiscourseAi
result =
Result.new(
answerable: values[:answerable] == true,
- source_refs: Array(values[:source_refs]),
+ source_refs: known_source_refs(values[:source_refs], candidates:, related_count:),
title: values[:title].to_s.strip,
answer: answer_without_source_references(values[:answer]),
follow_up: values[:follow_up].to_s.strip,
)
- return empty_result if !result.answerable
+ if !result.answerable || result.source_refs.empty? ||
+ !self.class.meaningful_answer?(result.answer)
+ return empty_result
+ end
- valid_result?(result, candidates:, related_count:) ? result : empty_result
+ result
end
def self.meaningful_answer?(answer)
@@ -122,18 +128,12 @@ module DiscourseAi
agent
end
- def valid_result?(result, candidates:, related_count:)
- return false if result.source_refs.empty?
- return false if result.source_refs.uniq.length != result.source_refs.length
- return false if result.source_refs.length > related_count
-
+ def known_source_refs(source_refs, candidates:, related_count:)
candidate_refs = candidates.pluck("source_ref")
- if result.source_refs.any? { |source_ref| !candidate_refs.include?(source_ref) }
- return false
- end
- return false if !self.class.meaningful_answer?(result.answer)
-
- true
+ Array(source_refs)
+ .uniq
+ .select { |source_ref| candidate_refs.include?(source_ref) }
+ .first(related_count)
end
def empty_result
diff --git a/plugins/discourse-ai/spec/jobs/regular/stream_discover_reply_spec.rb b/plugins/discourse-ai/spec/jobs/regular/stream_discover_reply_spec.rb
index 29f2fa896dd..afd744aa389 100644
--- a/plugins/discourse-ai/spec/jobs/regular/stream_discover_reply_spec.rb
+++ b/plugins/discourse-ai/spec/jobs/regular/stream_discover_reply_spec.rb
@@ -65,9 +65,11 @@ describe Jobs::StreamDiscoverReply do
)
allow(DiscourseAi::Discoveries::Retrieval).to receive(:new).with(user:).and_return(retrieval)
allow(retrieval).to receive(:call).and_return(retrieval_result)
- allow(retrieval).to receive(:validated_sources).with(retrieval_result, %w[source_1]).and_return(
- [candidate],
- )
+ allow(retrieval).to receive(:sources_for).with(
+ retrieval_result,
+ %w[source_1],
+ limit: an_instance_of(Integer),
+ ).and_return([candidate])
allow(DiscourseAi::Discoveries::Synthesis).to receive(:new) do |arguments|
expect(arguments).to include(user:, ai_agent:, llm_model:)
expect(arguments[:cancel_manager]).to be_a(DiscourseAi::Completions::CancelManager)
@@ -195,9 +197,11 @@ describe Jobs::StreamDiscoverReply do
candidates: [candidate, other_candidate, duplicate_candidate],
)
allow(retrieval).to receive(:call).and_return(result)
- allow(retrieval).to receive(:validated_sources).with(result, %w[source_1]).and_return(
- [candidate],
- )
+ allow(retrieval).to receive(:sources_for).with(
+ result,
+ %w[source_1],
+ limit: an_instance_of(Integer),
+ ).and_return([candidate])
allow(synthesis).to receive(:call) do |**_, &stream|
stream.call(
answerable: true,
@@ -284,7 +288,7 @@ describe Jobs::StreamDiscoverReply do
scope_fallback: true,
),
)
- allow(retrieval).to receive(:validated_sources).and_return([candidate])
+ allow(retrieval).to receive(:sources_for).and_return([candidate])
messages =
MessageBus
@@ -409,42 +413,81 @@ describe Jobs::StreamDiscoverReply do
)
end
- it "waits for progressively streamed source references to finish before validating them" do
- second_candidate = candidate.merge("source_ref" => "source_2")
- progressive_result =
- DiscourseAi::Discoveries::Retrieval::Result.new(candidates: [candidate, second_candidate])
- allow(retrieval).to receive(:call).and_return(progressive_result)
- allow(retrieval).to receive(:validated_sources).with(
- progressive_result,
- %w[source_1 source_2],
- ).and_return([candidate, second_candidate])
- allow(synthesis).to receive(
- :call,
- ) do |query:, candidates:, keyword_query:, original_query_locale:, summary_detail:, related_count:, &stream|
- expect(query).to eq(self.query)
- expect(keyword_query).to eq(self.query)
- expect(candidates).to eq([candidate, second_candidate])
- expect(summary_detail).to eq(:balanced)
- expect(related_count).to eq(2)
+ it "streams the answer before the model selects its sources" do
+ allow(synthesis).to receive(:call) do |**, &stream|
+ stream.call(answer: "Use the plugin skeleton.", title: "")
+ stream.call(answer: "Use the plugin skeleton.", source_refs: %w[source_1], title: "")
stream.call(
+ answer: "Use the plugin skeleton.",
+ source_refs: %w[source_1],
+ answerable: true,
+ title: "Plugin guide",
+ )
+ DiscourseAi::Discoveries::Synthesis::Result.new(
answerable: true,
source_refs: %w[source_1],
- title: "Create a Discourse plugin",
+ title: "Plugin guide",
+ answer: "Use the plugin skeleton.",
+ )
+ end
+
+ messages =
+ MessageBus
+ .track_publish("/discourse-ai/discoveries") do
+ job.execute(user_id: user.id, query:, request_id:)
+ end
+ .map(&:data)
+
+ expect(messages.map { |message| message[:phase] }).to eq(
+ %w[searching answering sources complete],
+ )
+ expect(messages.second).to include(ai_discover_reply: "Use the plugin skeleton.")
+ expect(messages.last).to include(answerable: true, sources: be_present)
+ end
+
+ it "retracts a streamed answer when the model abstains after writing it" do
+ allow(synthesis).to receive(:call) do |**, &stream|
+ stream.call(answer: "An unsupported guess.", title: "")
+ stream.call(answer: "An unsupported guess.", answerable: false, title: "")
+ DiscourseAi::Discoveries::Synthesis::Result.new(
+ answerable: false,
+ source_refs: [],
+ title: "",
answer: "",
)
+ end
+
+ messages =
+ MessageBus
+ .track_publish("/discourse-ai/discoveries") do
+ job.execute(user_id: user.id, query:, request_id:)
+ end
+ .map(&:data)
+
+ expect(messages.map { |message| message[:phase] }).to eq(%w[searching answering complete])
+ expect(messages.last).to include(answerable: false, ai_discover_reply: "", sources: [])
+ expect(DiscourseAi::Discoveries.cached_result_for(user:, request_id:)).to be_nil
+ end
+
+ it "retracts a streamed answer and stops generating when none of its sources resolve" do
+ allow(retrieval).to receive(:sources_for).and_return([])
+ allow(synthesis).to receive(:call) do |**, &stream|
+ stream.call(answer: "Use the plugin skeleton.", title: "")
+ stream.call(answer: "Use the plugin skeleton.", source_refs: %w[source_9], title: "")
stream.call(
+ answer: "Use the plugin skeleton.",
+ source_refs: %w[source_9],
answerable: true,
- source_refs: %w[source_1 source_2],
- title: "Create a Discourse plugin",
- answer: "Use the two selected discussions.",
+ title: "Plugin guide",
)
DiscourseAi::Discoveries::Synthesis::Result.new(
- answerable: true,
- source_refs: %w[source_1 source_2],
- title: "Create a Discourse plugin",
- answer: "Use the two selected discussions.",
+ answerable: false,
+ source_refs: [],
+ title: "",
+ answer: "",
)
end
+ expect_any_instance_of(DiscourseAi::Completions::CancelManager).to receive(:cancel!).once
messages =
MessageBus
@@ -453,17 +496,12 @@ describe Jobs::StreamDiscoverReply do
end
.map(&:data)
- expect(messages.map { |message| message[:phase] }).to eq(
- %w[searching sources answering complete],
- )
- expect(messages.last).to include(
- answerable: true,
- ai_discover_reply: "Use the two selected discussions.",
- )
+ expect(messages.map { |message| message[:phase] }).to eq(%w[searching answering complete])
+ expect(messages.last).to include(answerable: false, ai_discover_reply: "", sources: [])
end
it "does not release an answer when the model selects an invalid source" do
- allow(retrieval).to receive(:validated_sources).and_return([])
+ allow(retrieval).to receive(:sources_for).and_return([])
messages =
MessageBus
@@ -504,27 +542,6 @@ describe Jobs::StreamDiscoverReply do
expect(messages.last).to include(answerable: false, ai_discover_reply: "", sources: [])
end
- it "revalidates selected sources before the final answer" do
- allow(retrieval).to receive(:validated_sources).and_return([candidate], [])
-
- messages =
- MessageBus
- .track_publish("/discourse-ai/discoveries") do
- job.execute(user_id: user.id, query:, request_id:)
- end
- .map(&:data)
-
- expect(messages.map { |message| message[:phase] }).to include("sources")
- expect(messages.last).to include(
- phase: "complete",
- done: true,
- answerable: false,
- ai_discover_reply: "",
- sources: [],
- )
- expect(DiscourseAi::Discoveries.cached_result_for(user:, request_id:)).to be_nil
- end
-
it "completes without a model call when retrieval has no candidates" do
empty_result = DiscourseAi::Discoveries::Retrieval::Result.new(candidates: [])
allow(retrieval).to receive(:call).and_return(empty_result)
@@ -744,7 +761,7 @@ describe Jobs::StreamDiscoverReply do
allow(retrieval).to receive(:call).and_return(
DiscourseAi::Discoveries::Retrieval::Result.new(candidates:),
)
- allow(retrieval).to receive(:validated_sources).and_return([candidate])
+ allow(retrieval).to receive(:sources_for).and_return([candidate])
allow(synthesis).to receive(:call) do |**_, &stream|
stream.call(answerable: true, source_refs: %w[source_1], answer: "Answer")
DiscourseAi::Discoveries::Synthesis::Result.new(
@@ -794,7 +811,7 @@ describe Jobs::StreamDiscoverReply do
end
it "does not save rejected references as final sources" do
- allow(retrieval).to receive(:validated_sources).and_return([])
+ allow(retrieval).to receive(:sources_for).and_return([])
job.execute(args)
diff --git a/plugins/discourse-ai/spec/lib/agents/ask_ai_synthesis_spec.rb b/plugins/discourse-ai/spec/lib/agents/ask_ai_synthesis_spec.rb
index efcdc2e0679..0f557de17c5 100644
--- a/plugins/discourse-ai/spec/lib/agents/ask_ai_synthesis_spec.rb
+++ b/plugins/discourse-ai/spec/lib/agents/ask_ai_synthesis_spec.rb
@@ -11,9 +11,9 @@ describe DiscourseAi::Agents::AskAiSynthesis do
expect(agent.response_format).to eq(
[
{ "key" => "answerable", "type" => "boolean" },
- { "key" => "source_refs", "type" => "array", "array_type" => "string", "max_items" => 4 },
{ "key" => "title", "type" => "string" },
{ "key" => "answer", "type" => "string" },
+ { "key" => "source_refs", "type" => "array", "array_type" => "string", "max_items" => 4 },
{ "key" => "follow_up", "type" => "string" },
],
)
diff --git a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
index 7c126bb6fcd..ffb143fabec 100644
--- a/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/structured_output_spec.rb
@@ -81,6 +81,30 @@ RSpec.describe DiscourseAi::Completions::StructuredOutput do
end
end
+ describe "#property_complete?" do
+ it "treats a property as complete once a later key starts or the response finishes" do
+ structured_output << "{\"list\": [\"source_1"
+ expect(structured_output.property_complete?(:list)).to eq(false)
+
+ structured_output << "3\"], \"bool\": tr"
+ expect(structured_output.property_complete?(:list)).to eq(true)
+ expect(structured_output.property_complete?(:bool)).to eq(false)
+
+ structured_output << "ue}"
+ structured_output.finish
+ expect(structured_output.property_complete?(:bool)).to eq(true)
+ end
+
+ it "keeps a property open through a trailing comma and a half-written next key" do
+ structured_output << "{\"list\": [\"a\"],"
+ expect(structured_output.property_complete?(:list)).to eq(false)
+
+ structured_output << " \"bo"
+ expect(structured_output.property_complete?(:list)).to eq(true)
+ expect(structured_output.read_buffered_property(:list)).to eq(["a"])
+ end
+ end
+
describe "Parsing structured output on the fly" do
it "acts as a buffer for an streamed JSON" do
chunks = [
diff --git a/plugins/discourse-ai/spec/lib/discoveries/retrieval_spec.rb b/plugins/discourse-ai/spec/lib/discoveries/retrieval_spec.rb
index 4294f5ac69c..c5bb0ebcf09 100644
--- a/plugins/discourse-ai/spec/lib/discoveries/retrieval_spec.rb
+++ b/plugins/discourse-ai/spec/lib/discoveries/retrieval_spec.rb
@@ -12,7 +12,6 @@ describe DiscourseAi::Discoveries::Retrieval do
fab!(:post_1) { Fabricate(:post, topic: topic_1, raw: "First useful answer") }
fab!(:post_2) { Fabricate(:post, topic: topic_2, user: staff_user, raw: "Second useful answer") }
fab!(:post_3) { Fabricate(:post, topic: topic_3, raw: "Third useful answer") }
- fab!(:private_post) { Fabricate(:post, topic: private_topic, raw: "Private answer") }
fab!(:embedding_definition)
before do
@@ -217,15 +216,15 @@ describe DiscourseAi::Discoveries::Retrieval do
expect(result.candidates.map { |candidate| candidate.fetch("topic_id") }).to eq([topic_2.id])
end
- it "fuses, permission-checks, and assigns request-owned source references" do
+ it "fuses, adds metadata to, and assigns request-owned source references" do
SiteSetting.tagging_enabled = true
parent_category = Fabricate(:category)
category.update!(parent_category:)
tag = Fabricate(:tag, name: "guide")
topic_2.tags << tag
PostActionCreator.like(user, post_2)
- lexical = [source(post_1), source(post_2), source(private_post)]
- semantic = [source(post_2), source(post_3), source(private_post)]
+ lexical = [source(post_1), source(post_2)]
+ semantic = [source(post_2), source(post_3)]
result =
described_class.new(
@@ -293,17 +292,8 @@ describe DiscourseAi::Discoveries::Retrieval do
it "uses keyword and semantic retrieval for personal messages the user can see" do
personal_message = Fabricate(:private_message_post, recipient: user, raw: "Anime plans")
- inaccessible_message = Fabricate(:private_message_post, raw: "Hidden anime plans")
- lexical_retriever =
- instance_spy(
- Proc,
- call: [source(personal_message), source(inaccessible_message), source(post_1)],
- )
- semantic_retriever =
- instance_spy(
- Proc,
- call: [source(personal_message), source(inaccessible_message), source(post_1)],
- )
+ lexical_retriever = instance_spy(Proc, call: [source(personal_message)])
+ semantic_retriever = instance_spy(Proc, call: [source(personal_message)])
result =
described_class.new(user:, lexical_retriever:, semantic_retriever:).call(
@@ -358,19 +348,6 @@ describe DiscourseAi::Discoveries::Retrieval do
)
end
- it "excludes personal messages from a regular search" do
- personal_message = Fabricate(:private_message_post, recipient: user, raw: "Anime plans")
-
- result =
- described_class.new(
- user:,
- lexical_retriever: ->(_query) { [source(post_1), source(personal_message)] },
- semantic_retriever: ->(_query) { [] },
- ).call("anime")
-
- expect(result.candidates.map { |candidate| candidate.fetch("topic_id") }).to eq([topic_1.id])
- end
-
it "does not broaden explicit search filters through semantic retrieval" do
lexical_retriever = instance_spy(Proc, call: [source(post_1)])
semantic_retriever = instance_spy(Proc, call: [source(post_2)])
@@ -403,8 +380,8 @@ describe DiscourseAi::Discoveries::Retrieval do
end
end
- describe "#validated_sources" do
- it "accepts up to six known source references and rechecks access before publication" do
+ describe "#sources_for" do
+ it "resolves known source references in the model's order, dropping the rest" do
retrieval =
described_class.new(
user:,
@@ -413,42 +390,18 @@ describe DiscourseAi::Discoveries::Retrieval do
)
result = retrieval.call("useful answers")
- sources = retrieval.validated_sources(result, %w[source_2 source_1])
+ sources =
+ retrieval.sources_for(
+ result,
+ ["source_2", "unknown", "source_2", nil, { "ref" => 1 }, "source_1"],
+ limit: 6,
+ )
+
expect(sources.map { |candidate| candidate.fetch("topic_id") }).to eq(
[topic_2.id, topic_1.id],
)
-
- topic_2.update!(category: private_category)
- expect(retrieval.validated_sources(result, %w[source_2 source_1])).to eq([])
- end
-
- it "rejects duplicate, unknown, empty, or excessive source references" do
- retrieval =
- described_class.new(
- user:,
- lexical_retriever: ->(_query) { [source(post_1)] },
- semantic_retriever: ->(_query) { [] },
- )
- result = retrieval.call("useful answers")
-
- expect(retrieval.validated_sources(result, [])).to eq([])
- expect(retrieval.validated_sources(result, %w[source_1 source_1])).to eq([])
- expect(retrieval.validated_sources(result, %w[source_1 unknown])).to eq([])
- expect(retrieval.validated_sources(result, %w[a b c d e f g])).to eq([])
- end
-
- it "rejects a supporting source that changes after retrieval" do
- retrieval =
- described_class.new(
- user:,
- lexical_retriever: ->(_query) { [source(post_1)] },
- semantic_retriever: ->(_query) { [] },
- )
- result = retrieval.call("useful answers")
-
- post_1.update!(raw: "Changed after synthesis began")
-
- expect(retrieval.validated_sources(result, %w[source_1])).to eq([])
+ expect(retrieval.sources_for(result, %w[source_2 source_1], limit: 1).length).to eq(1)
+ expect(retrieval.sources_for(result, nil, limit: 6)).to eq([])
end
end
@@ -484,6 +437,20 @@ describe DiscourseAi::Discoveries::Retrieval do
)
end
+ it "excludes personal messages from a regular keyword search" do
+ SiteSetting.ai_embeddings_enabled = false
+ token = "regularneedle#{SecureRandom.hex(6)}"
+ public_post = Fabricate(:post, topic: Fabricate(:topic, category:), raw: token)
+ personal_message = Fabricate(:private_message_post, recipient: user, raw: token)
+ [public_post, personal_message].each { |post| SearchIndexer.index(post, force: true) }
+
+ result = described_class.new(user:).call(token)
+
+ expect(result.candidates.map { |candidate| candidate.fetch("topic_id") }).to eq(
+ [public_post.topic_id],
+ )
+ end
+
it "finds only personal messages the user can see without embeddings" do
SiteSetting.ai_embeddings_enabled = false
embedding_request = stub_request(:post, embedding_definition.url).to_return(status: 500)
diff --git a/plugins/discourse-ai/spec/lib/discoveries/synthesis_spec.rb b/plugins/discourse-ai/spec/lib/discoveries/synthesis_spec.rb
index 0dc903aa96a..f4deb98ed9b 100644
--- a/plugins/discourse-ai/spec/lib/discoveries/synthesis_spec.rb
+++ b/plugins/discourse-ai/spec/lib/discoveries/synthesis_spec.rb
@@ -273,7 +273,7 @@ describe DiscourseAi::Discoveries::Synthesis do
expect(result).to have_attributes(answerable: false, source_refs: [], title: "", answer: "")
end
- it "rejects source references outside the supplied candidates" do
+ it "rejects an answer when none of its source references are supplied candidates" do
candidates = [
{
"source_ref" => "source_1",
@@ -297,6 +297,31 @@ describe DiscourseAi::Discoveries::Synthesis do
expect(result).to have_attributes(answerable: false, source_refs: [], title: "", answer: "")
end
+ it "keeps an answer whose known source references survive alongside invalid ones" do
+ candidates =
+ %w[source_1 source_2 source_3].map do |source_ref|
+ { "source_ref" => source_ref, "title" => "Plugins", "excerpt" => "Use the skeleton." }
+ end
+
+ result =
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ [
+ {
+ answerable: true,
+ source_refs: %w[source_9 source_2 source_2 source_1 source_3],
+ title: "Plugin guide",
+ answer: "Use the plugin skeleton.",
+ },
+ ],
+ ) { synthesis.call(query: "how do I create a plugin", candidates:, related_count: 2) }
+
+ expect(result).to have_attributes(
+ answerable: true,
+ source_refs: %w[source_2 source_1],
+ answer: "Use the plugin skeleton.",
+ )
+ end
+
it "returns the suggested follow-up question alongside the answer" do
candidates = [
{
@@ -375,6 +400,56 @@ describe DiscourseAi::Discoveries::Synthesis do
expect(result).to have_attributes(answerable: false, source_refs: [], title: "", answer: "")
end
+ it "streams source references only once the array is complete when the answer comes first" do
+ candidates =
+ %w[source_4 source_1 source_13].map do |source_ref|
+ { "source_ref" => source_ref, "title" => "Python", "excerpt" => "Use the API." }
+ end
+ chunks = [
+ "{\n \"answer\": \"Use the REST API.\",",
+ "\n \"answerable\": true,",
+ "\n \"source_refs\": [\n \"source_4",
+ "\",\n \"source_1",
+ "3\",\n \"source_1\"",
+ "\n ],\n \"title\": \"Python\"\n}",
+ ]
+ output =
+ DiscourseAi::Completions::StructuredOutput.new(
+ answer: {
+ type: "string",
+ },
+ answerable: {
+ type: "boolean",
+ },
+ source_refs: {
+ type: "array",
+ },
+ title: {
+ type: "string",
+ },
+ )
+ bot = instance_double(DiscourseAi::Agents::Bot)
+ allow(DiscourseAi::Agents::Bot).to receive(:as).and_return(bot)
+ allow(bot).to receive(:reply) do |_, &stream|
+ chunks.each do |chunk|
+ output << chunk
+ stream.call(output, nil, :structured_output)
+ end
+ output.finish
+ stream.call(output, nil, :structured_output)
+ end
+ streamed_refs = []
+
+ result =
+ synthesis.call(query: "python", candidates:, related_count: 3) do |update|
+ streamed_refs << update[:source_refs] if update[:source_refs]
+ end
+
+ expect(streamed_refs.uniq).to eq([%w[source_4 source_13 source_1]])
+ expect(result.answerable).to eq(true)
+ expect(result.source_refs).to eq(%w[source_4 source_13 source_1])
+ end
+
it "keeps every streamed title fragment" do
candidates = [
{
@@ -398,6 +473,7 @@ describe DiscourseAi::Discoveries::Synthesis do
partials.each do |values|
partial = double
allow(partial).to receive(:read_buffered_property) { |key| values[key] }
+ allow(partial).to receive(:property_complete?).and_return(true)
stream.call(partial, nil, :structured_output)
end
end
@@ -436,6 +512,7 @@ describe DiscourseAi::Discoveries::Synthesis do
partials.each do |values|
partial = double
allow(partial).to receive(:read_buffered_property) { |key| values[key] }
+ allow(partial).to receive(:property_complete?).and_return(true)
stream.call(partial, nil, :structured_output)
end
end
diff --git a/plugins/discourse-ai/spec/lib/discoveries_spec.rb b/plugins/discourse-ai/spec/lib/discoveries_spec.rb
index 67c21544cd4..f550b5b6b1c 100644
--- a/plugins/discourse-ai/spec/lib/discoveries_spec.rb
+++ b/plugins/discourse-ai/spec/lib/discoveries_spec.rb
@@ -270,7 +270,7 @@ describe DiscourseAi::Discoveries do
expect(described_class.cached_result_for(user:, request_id:)).to be_nil
end
- it "invalidates follow-up context when a supporting post changes" do
+ it "keeps follow-up context when a supporting post is edited" do
request_id = SecureRandom.uuid
described_class.store_result(
user_id: user.id,
@@ -283,6 +283,28 @@ describe DiscourseAi::Discoveries do
post.update!(raw: "Updated supporting content")
+ expect(described_class.cached_result_for(user:, request_id:)).to include(
+ "answer" => "A useful answer.",
+ )
+ end
+
+ it "expires the result when any source is out of the user's sight" do
+ other_post = Fabricate(:post)
+ request_id = SecureRandom.uuid
+ described_class.store_result(
+ user_id: user.id,
+ request_id:,
+ query: "plugin setup",
+ answer: "A useful answer.",
+ sources: [
+ { "post_id" => post.id, "topic_id" => post.topic_id },
+ { "post_id" => other_post.id, "topic_id" => other_post.topic_id },
+ ],
+ agent_id: ai_agent.id,
+ )
+
+ other_post.topic.update!(category: Fabricate(:private_category, group: Fabricate(:group)))
+
expect(described_class.cached_result_for(user:, request_id:)).to be_nil
end