Skip to content

A platform for community discussion. Free, open, simple.

DEV: More updates to the highlight dash with more specific conditions (#40994)

Natalie Tay02ebe7327483
diff --git a/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb b/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb
index af746b97adc..1f97b9aff85 100644
--- a/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb
+++ b/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb
@@ -23,6 +23,8 @@ module DiscourseAi
].freeze
MAX_SIGNALS = 6
+ MAX_LANDING_TOPIC_DAYS = 93
+ MAX_STAFF_RATIO_DAYS = 93
METRIC_LABELS = {
new_signups: "new sign-ups",
@@ -41,6 +43,7 @@ module DiscourseAi
@start_date, @end_date = @end_date, @start_date if @start_date > @end_date
length = (@end_date - @start_date).to_i
+ @period_days = length + 1
@prev_end = @start_date - 1
@prev_start = @start_date - (length + 1)
end
@@ -51,6 +54,7 @@ module DiscourseAi
signals =
[
hot_topic,
+ topic_volume,
unanswered_gap,
staff_ratio,
traffic_volume,
@@ -63,7 +67,7 @@ module DiscourseAi
period: {
start_date: @start_date.to_s,
end_date: @end_date.to_s,
- days: (@end_date - @start_date).to_i + 1,
+ days: @period_days,
},
trend: trend(kpis),
metrics: kpis.map { |kpi| metric_for(kpi) },
@@ -80,8 +84,26 @@ module DiscourseAi
end
def metric_for(kpi)
+ key = kpi[:type]&.to_sym
label = METRIC_LABELS[kpi[:type]&.to_sym] || kpi[:type].to_s.tr("_", " ")
- { label: label, value: kpi[:value], delta_pct: kpi[:percent_change] }
+ {
+ key: key,
+ category: metric_category(key),
+ label: label,
+ value: kpi[:value],
+ delta_pct: kpi[:percent_change],
+ }
+ end
+
+ def metric_category(key)
+ case key
+ when :new_signups
+ :acquisition
+ when :dau_mau, :new_contributors
+ :participation
+ when :accepted_solutions
+ :support
+ end
end
def trend(kpis)
@@ -99,12 +121,41 @@ module DiscourseAi
(((current - previous).to_f / previous) * 100).round
end
- def signal(key, headline, score:)
- { key: key, headline: headline, score: score }
+ def signal(key, headline, score:, category:)
+ { key: key, category: category, headline: headline, score: score }
end
# ---- internal signals --------------------------------------------------
+ def new_topics_count(start_date, end_date)
+ DB.query_single(<<~SQL, start_date: start_date, end_date: end_date).first.to_i
+ SELECT COUNT(*)
+ FROM topics t
+ WHERE t.created_at >= :start_date
+ AND t.created_at < (:end_date::date + 1)
+ AND t.deleted_at IS NULL
+ AND t.visible = true
+ AND t.archetype = 'regular'
+ SQL
+ end
+
+ def topic_volume
+ current = new_topics_count(@start_date, @end_date)
+ previous = new_topics_count(@prev_start, @prev_end)
+ delta = delta_pct(current, previous)
+ return if delta.nil? || delta.abs < 30
+ return if delta.positive? && current < 5
+ return if delta.negative? && previous < 5
+
+ direction = delta.positive? ? "up" : "down"
+ signal(
+ :topic_volume,
+ "New topics were #{direction} #{delta.abs}% versus the previous period",
+ score: [delta.abs / 200.0, 0.8].min,
+ category: :participation,
+ )
+ end
+
def hot_topic
row = DB.query(<<~SQL, start_date: @start_date, end_date: @end_date).first
SELECT t.id, t.title, COUNT(p.id) FILTER (WHERE p.post_number > 1) AS replies
@@ -137,6 +188,7 @@ module DiscourseAi
:hot_topic,
"Busiest discussion: \"#{row.title}\" with #{row.replies} replies",
score: 0.7,
+ category: :participation,
)
end
@@ -153,14 +205,22 @@ module DiscourseAi
SQL
return if count < 5
+ total = new_topics_count(@start_date, @end_date)
+ share = total.positive? ? ((count.to_f / total) * 100).round : 0
+ headline = "#{count} new topics received no reply"
+ headline = "#{headline} (#{share}% of new topics)" if share >= 10
+
signal(
:unanswered_gap,
- "#{count} new topics received no reply",
- score: [0.4 + (count / 100.0), 0.9].min,
+ headline,
+ score: [0.4 + (share / 100.0), 0.9].min,
+ category: :support,
)
end
def staff_ratio
+ return if @period_days > MAX_STAFF_RATIO_DAYS
+
row = DB.query(<<~SQL, start_date: @start_date, end_date: @end_date).first
SELECT
COUNT(*) FILTER (WHERE u.admin OR u.moderator) AS staff,
@@ -178,7 +238,12 @@ module DiscourseAi
pct = ((row.staff.to_f / row.total) * 100).round
return if pct < 40
- signal(:staff_ratio, "Staff wrote #{pct}% of posts this period", score: pct / 100.0)
+ signal(
+ :staff_ratio,
+ "Staff wrote #{pct}% of posts this period",
+ score: pct / 100.0,
+ category: :participation,
+ )
end
# ---- external signals --------------------------------------------------
@@ -205,6 +270,7 @@ module DiscourseAi
:traffic_volume,
"Browser pageviews were #{direction} #{delta.abs}% versus the previous period",
score: [delta.abs / 200.0, 0.8].min,
+ category: :acquisition,
)
end
@@ -226,7 +292,7 @@ module DiscourseAi
"Traffic spiked on #{peak.date} (#{multiple}x the typical day)"
end
- signal(:traffic_spike, headline, score: [multiple / 10.0, 0.95].min)
+ signal(:traffic_spike, headline, score: [multiple / 10.0, 0.95].min, category: :acquisition)
end
def dominant_referrer_for(date)
@@ -275,10 +341,13 @@ module DiscourseAi
:geography,
"#{share}% of browser pageviews came from #{top.country_code}",
score: share / 100.0,
+ category: :acquisition,
)
end
def landing_topic
+ return if @period_days > MAX_LANDING_TOPIC_DAYS
+
row = DB.query(<<~SQL, start_date: @start_date, end_date: @end_date).first
SELECT e.topic_id, t.title, COUNT(*) AS visits
FROM browser_pageview_events e
@@ -297,6 +366,7 @@ module DiscourseAi
:landing_topic,
"External visitors mostly landed on \"#{row.title}\" (#{row.visits} visits)",
score: 0.6,
+ category: :acquisition,
)
end
end
diff --git a/plugins/discourse-ai/lib/admin_dashboard/highlight_generator.rb b/plugins/discourse-ai/lib/admin_dashboard/highlight_generator.rb
index 9c87e1aea3e..db7e71e384a 100644
--- a/plugins/discourse-ai/lib/admin_dashboard/highlight_generator.rb
+++ b/plugins/discourse-ai/lib/admin_dashboard/highlight_generator.rb
@@ -7,7 +7,13 @@ module DiscourseAi
# then handed to the admin dashboard highlights agent which explains what changed.
class HighlightGenerator
CACHE_TTL = 6.hours
- CACHE_VERSION = 3
+ CACHE_VERSION = 4
+
+ LENSES = {
+ acquisition: "Acquisition and discovery",
+ participation: "Participation and contribution",
+ support: "Support health",
+ }.freeze
def self.generate(start_date:, end_date:, period: nil)
new(start_date: start_date, end_date: end_date, period: period).generate
@@ -94,22 +100,29 @@ module DiscourseAi
def user_message(facts)
<<~MSG.strip
Community facts for #{facts[:period][:start_date]} to #{facts[:period][:end_date]} — this was a #{facts[:trend]} period.
+ Period length: #{facts[:period][:days]} days. Compare only with the previous #{facts[:period][:days]} days.
Headline metrics (also shown as tiles below the highlight):
#{facts[:metrics].map { |metric| format_metric(metric) }.join("\n")}
+ Community-owner lenses:
+ #{format_lenses(facts)}
+
Notable this period:
#{format_signals(facts[:signals])}
Rules — follow exactly:
- Use ONLY the numbers and facts listed above. Never state a value that is not listed.
+ - Do not mention a metric whose value is "not available".
- Only mention a traffic source, country, or landing topic if it appears in "Notable" above. Do not invent sources, dates, causes, or numbers.
+ - If you mention a traffic source or external referrer, name it exactly as listed or do not mention the source. Never say "a specific external referrer".
- Do not overstate causality. If the facts only show contrast or correlation, phrase it that way.
- - Do not say traffic "translated" or "did not translate" into another metric unless the facts include a conversion metric. Use plain contrast instead.
+ - Do not say traffic "translated", "did not translate", "stemmed", "did not stem", "lifted", or "did not lift" another metric. Do not imply that a traffic spike caused, failed to cause, prevented, or failed to prevent another metric.
+ - If you mention a traffic spike, state only its date, size, and listed referrer/source. Put participation or support concerns in a separate sentence.
- Avoid report phrases like "as evidenced by", "highlighting", "indicating", and "underscoring".
- If little is notable, say it was a steady period rather than inventing drama.
- Write the admin dashboard highlight: 2 or 3 short, scannable sentences for a community owner. Lead with the trend, pick the 2-3 facts most useful for deciding what to inspect next, and prefer relationships between metrics. Warm and plain, no hype, no corporate report phrasing, no emoji.#{language_directive}
+ Write the admin dashboard highlight: 2 or 3 short, scannable sentences for a community owner. Lead with the trend, then choose the 2-3 most useful next inspection areas from the community-owner lenses. For 7-day ranges, focus on immediate follow-up; for 30-day or longer ranges, focus on sustained patterns. Warm and plain, no hype, no corporate report phrasing, no emoji.#{language_directive}
MSG
end
@@ -121,6 +134,7 @@ module DiscourseAi
end
def format_metric(metric)
+ value = metric[:value].presence || "not available"
change =
if metric[:delta_pct].present?
sign = metric[:delta_pct] >= 0 ? "+" : ""
@@ -128,7 +142,32 @@ module DiscourseAi
else
""
end
- "- #{metric[:label]}: #{metric[:value]}#{change}"
+ "- #{metric[:label]}: #{value}#{change}"
+ end
+
+ def format_lenses(facts)
+ LENSES
+ .map do |category, title|
+ entries = lens_entries(facts, category)
+ next if entries.blank?
+
+ "- #{title}: #{entries.join("; ")}"
+ end
+ .compact
+ .join("\n")
+ end
+
+ def lens_entries(facts, category)
+ metrics =
+ facts[:metrics]
+ .select { |metric| metric[:category] == category && metric[:value].present? }
+ .map { |metric| format_metric(metric).delete_prefix("- ") }
+ signals =
+ facts[:signals]
+ .select { |signal| signal[:category] == category }
+ .map { |signal| signal[:headline] }
+
+ metrics + signals
end
def format_signals(signals)
diff --git a/plugins/discourse-ai/lib/agents/admin_dashboard_highlights.rb b/plugins/discourse-ai/lib/agents/admin_dashboard_highlights.rb
index 751c2bb1712..3a08b9ae009 100644
--- a/plugins/discourse-ai/lib/agents/admin_dashboard_highlights.rb
+++ b/plugins/discourse-ai/lib/agents/admin_dashboard_highlights.rb
@@ -15,17 +15,21 @@ module DiscourseAi
<<~PROMPT.strip
You write the brief highlight shown at the top of a Discourse community admin dashboard, for the community owner.
- You are given a set of verified facts for the period: headline metrics (with their change versus the previous period) and a short list of notable signals. This is the only ground truth — everything you write must come from it.
+ You are given a set of verified facts for the period: headline metrics (with their change versus the previous period), community-owner lenses, and a short list of notable signals. This is the only ground truth — everything you write must come from it.
Write 2 or 3 short, scannable sentences that tell the owner what changed and what deserves attention.
- Lead with the overall trend (growing, steady, slipping, or mixed).
- - Pick the 2-3 most useful facts for deciding what to inspect next; do not list everything. The metric tiles are shown right below you, so don't just recite them.
- - Prefer relationships between facts over bare numbers (e.g. "traffic spiked, but sign-ups and contributors still fell" or "new members joined, but 17 topics went unanswered").
+ - Pick the 2-3 most useful inspection areas from the acquisition, participation, and support-health lenses; do not list everything. The metric tiles are shown right below you, so don't just recite them.
+ - For 7-day ranges, focus on immediate follow-up. For 30-day or longer ranges, focus on sustained patterns.
+ - Prefer relationships between facts over bare numbers (e.g. "new members joined, but 17 topics went unanswered"). Do not connect traffic to other metrics unless a conversion metric is given.
- Separate acquisition, participation, and support-health signals when they point in different directions.
+ - Do not mention a metric whose value is "not available".
- Mention a traffic source, country, or specific topic ONLY if it appears in the facts. Never invent sources, dates, causes, or numbers, and never state a metric value that wasn't given.
+ - If you mention a traffic source or external referrer, name it exactly as listed or do not mention the source. Never say "a specific external referrer".
- Do not overstate causality. Use "may", "could", or plain contrast when the facts show correlation, not cause.
- - Do not say traffic "translated" or "did not translate" into another metric unless the facts include a conversion metric. Use plain contrast instead.
+ - Do not say traffic "translated", "did not translate", "stemmed", "did not stem", "lifted", or "did not lift" another metric. Do not imply that a traffic spike caused, failed to cause, prevented, or failed to prevent another metric.
+ - If you mention a traffic spike, state only its date, size, and listed referrer/source. Put participation or support concerns in a separate sentence.
- Avoid report phrases like "as evidenced by", "highlighting", "indicating", and "underscoring".
- If little is notable, say it was a steady period — don't manufacture drama.
- Warm, plain language. No hype, no corporate report phrasing, no emoji.
diff --git a/plugins/discourse-ai/spec/lib/admin_dashboard/admin_dashboard_facts_spec.rb b/plugins/discourse-ai/spec/lib/admin_dashboard/admin_dashboard_facts_spec.rb
index 3402cb0479d..c4f0b41daad 100644
--- a/plugins/discourse-ai/spec/lib/admin_dashboard/admin_dashboard_facts_spec.rb
+++ b/plugins/discourse-ai/spec/lib/admin_dashboard/admin_dashboard_facts_spec.rb
@@ -22,7 +22,14 @@ RSpec.describe DiscourseAi::AdminDashboard::AdminDashboardFacts do
expect(labels).to include("new sign-ups", "new contributors")
expect(facts[:metrics].find { |metric| metric[:label] == "new sign-ups" }).to include(
value: 100,
+ category: :acquisition,
)
+ expect(facts[:metrics].find { |metric| metric[:label] == "new contributors" }).to include(
+ category: :participation,
+ )
+ expect(
+ facts[:metrics].find { |metric| metric[:label] == "questions resolved (accepted solutions)" },
+ ).to include(category: :support)
end
it "classifies the trend from the metric deltas" do
@@ -34,7 +41,97 @@ RSpec.describe DiscourseAi::AdminDashboard::AdminDashboardFacts do
headlines = compute(start_date: 1.day.ago.to_date.to_s).fetch(:signals).map { |s| s[:headline] }
- expect(headlines).to include(match(/new topics received no reply/))
+ expect(headlines).to include(match(/new topics received no reply \(100% of new topics\)/))
+ end
+
+ it "reports when new-topic volume changes sharply" do
+ start_date = Date.parse("2030-01-08")
+ end_date = Date.parse("2030-01-14")
+ Fabricate(:topic, created_at: Date.parse("2030-01-01"))
+ Fabricate.times(6, :topic, created_at: start_date)
+
+ topic_volume =
+ compute(start_date: start_date.to_s, end_date: end_date.to_s)
+ .fetch(:signals)
+ .find { |s| s[:key] == :topic_volume }
+
+ expect(topic_volume).to include(
+ category: :participation,
+ headline: "New topics were up 500% versus the previous period",
+ )
+ end
+
+ it "reports when new-topic volume drops sharply from meaningful previous volume" do
+ start_date = Date.parse("2030-01-08")
+ end_date = Date.parse("2030-01-14")
+ Fabricate.times(20, :topic, created_at: Date.parse("2030-01-01"))
+ Fabricate.times(4, :topic, created_at: start_date)
+
+ topic_volume =
+ compute(start_date: start_date.to_s, end_date: end_date.to_s)
+ .fetch(:signals)
+ .find { |s| s[:key] == :topic_volume }
+
+ expect(topic_volume).to include(
+ category: :participation,
+ headline: "New topics were down 80% versus the previous period",
+ )
+ end
+
+ it "skips the raw landing-topic scan for long date ranges" do
+ queries =
+ track_sql_queries do
+ compute(start_date: 1.year.ago.to_date.to_s, end_date: Date.current.to_s)
+ end
+
+ expect(queries.grep(/FROM browser_pageview_events e/)).to be_empty
+ end
+
+ it "reports the top external landing topic for three-month date ranges" do
+ topic = Fabricate(:topic, title: "Welcome topic for visitors")
+ Fabricate.times(
+ 50,
+ :browser_pageview_event,
+ topic_id: topic.id,
+ normalized_referrer: "example.com",
+ created_at: 1.day.ago,
+ )
+
+ landing_topic =
+ compute(start_date: 3.months.ago.to_date.to_s, end_date: Date.current.to_s)
+ .fetch(:signals)
+ .find { |s| s[:key] == :landing_topic }
+
+ expect(landing_topic).to include(
+ category: :acquisition,
+ headline: 'External visitors mostly landed on "Welcome topic for visitors" (50 visits)',
+ )
+ end
+
+ it "skips the raw staff-ratio post scan for long date ranges" do
+ queries =
+ track_sql_queries do
+ compute(start_date: 1.year.ago.to_date.to_s, end_date: Date.current.to_s)
+ end
+
+ expect(queries.grep(/FROM posts p\s+JOIN users u/m)).to be_empty
+ end
+
+ it "reports staff post ratio for three-month date ranges" do
+ admin = Fabricate(:admin)
+ user = Fabricate(:user)
+ Fabricate.times(12, :post, user: admin, created_at: 1.day.ago)
+ Fabricate.times(8, :post, user: user, created_at: 1.day.ago)
+
+ staff_ratio =
+ compute(start_date: 3.months.ago.to_date.to_s, end_date: Date.current.to_s)
+ .fetch(:signals)
+ .find { |signal| signal[:key] == :staff_ratio }
+
+ expect(staff_ratio).to include(
+ category: :participation,
+ headline: "Staff wrote 60% of posts this period",
+ )
end
it "reports a traffic spike WITHOUT a source when no referrer dominates" do
@@ -51,6 +148,7 @@ RSpec.describe DiscourseAi::AdminDashboard::AdminDashboardFacts do
expect(spike).to be_present
expect(spike[:headline]).to match(/Traffic spiked/)
+ expect(spike[:category]).to eq(:acquisition)
expect(spike[:headline]).not_to match(/driven by/)
end
diff --git a/plugins/discourse-ai/spec/lib/admin_dashboard/highlight_generator_spec.rb b/plugins/discourse-ai/spec/lib/admin_dashboard/highlight_generator_spec.rb
index 31bbb26ed46..0b9d1bb0add 100644
--- a/plugins/discourse-ai/spec/lib/admin_dashboard/highlight_generator_spec.rb
+++ b/plugins/discourse-ai/spec/lib/admin_dashboard/highlight_generator_spec.rb
@@ -95,12 +95,42 @@ RSpec.describe DiscourseAi::AdminDashboard::HighlightGenerator do
message = message_for("2026-05-01", "2026-06-01")
expect(message).to include("new sign-ups: 1100")
+ expect(message).to include("Period length: 32 days")
+ expect(message).to include("Community-owner lenses:")
+ expect(message).to include("Acquisition and discovery: new sign-ups: 1100")
expect(message).to match(/Use ONLY the numbers and facts listed above/)
+ expect(message).to match(/Do not mention a metric whose value is "not available"/)
expect(message).to match(/Do not invent sources, dates, causes, or numbers/)
+ expect(message).to match(/Never say "a specific external referrer"/)
expect(message).to match(/Do not overstate causality/)
expect(message).to match(/Do not say traffic "translated"/)
+ expect(message).to match(/"did not stem"/)
+ expect(message).to match(/state only its date, size, and listed referrer/)
expect(message).to match(/Avoid report phrases/)
- expect(message).to match(/what to inspect next/)
+ expect(message).to match(/next inspection areas/)
+ end
+
+ it "keeps unavailable metrics out of owner lenses" do
+ allow(AdminDashboardHighlights).to receive(:build).and_return(
+ {
+ kpis: [
+ {
+ type: :new_signups,
+ value: nil,
+ previous_value: 10,
+ percent_change: nil,
+ report_type: "signups",
+ report_query: {
+ },
+ },
+ ],
+ },
+ )
+
+ message = message_for("2026-05-01", "2026-06-01")
+
+ expect(message).to include("new sign-ups: not available")
+ expect(message).not_to include("Acquisition and discovery: new sign-ups")
end
it "says nothing stood out when no signal clears its threshold" do