A platform for community discussion. Free, open, simple.
DEV: More updates to the highlight dash with more specific conditions (#40994)
Natalie Tay02ebe7327483
diff --git a/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb b/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rbindex af746b97adc..1f97b9aff85 100644--- a/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb+++ b/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb@@ -23,6 +23,8 @@ module DiscourseAi].freezeMAX_SIGNALS = 6+ MAX_LANDING_TOPIC_DAYS = 93+ MAX_STAFF_RATIO_DAYS = 93METRIC_LABELS = {new_signups: "new sign-ups",@@ -41,6 +43,7 @@ module DiscourseAi@start_date, @end_date = @end_date, @start_date if @start_date > @end_datelength = (@end_date - @start_date).to_i+ @period_days = length + 1@prev_end = @start_date - 1@prev_start = @start_date - (length + 1)end@@ -51,6 +54,7 @@ module DiscourseAisignals =[hot_topic,+ topic_volume,unanswered_gap,staff_ratio,traffic_volume,@@ -63,7 +67,7 @@ module DiscourseAiperiod: {start_date: @start_date.to_s,end_date: @end_date.to_s,- days: (@end_date - @start_date).to_i + 1,+ days: @period_days,},trend: trend(kpis),metrics: kpis.map { |kpi| metric_for(kpi) },@@ -80,8 +84,26 @@ module DiscourseAienddef metric_for(kpi)+ key = kpi[:type]&.to_symlabel = METRIC_LABELS[kpi[:type]&.to_sym] || kpi[:type].to_s.tr("_", " ")- { label: label, value: kpi[:value], delta_pct: kpi[:percent_change] }+ {+ key: key,+ category: metric_category(key),+ label: label,+ value: kpi[:value],+ delta_pct: kpi[:percent_change],+ }+ end++ def metric_category(key)+ case key+ when :new_signups+ :acquisition+ when :dau_mau, :new_contributors+ :participation+ when :accepted_solutions+ :support+ endenddef trend(kpis)@@ -99,12 +121,41 @@ module DiscourseAi(((current - previous).to_f / previous) * 100).roundend- def signal(key, headline, score:)- { key: key, headline: headline, score: score }+ def signal(key, headline, score:, category:)+ { key: key, category: category, headline: headline, score: score }end# ---- internal signals --------------------------------------------------+ def new_topics_count(start_date, end_date)+ DB.query_single(<<~SQL, start_date: start_date, end_date: end_date).first.to_i+ SELECT COUNT(*)+ FROM topics t+ WHERE t.created_at >= :start_date+ AND t.created_at < (:end_date::date + 1)+ AND t.deleted_at IS NULL+ AND t.visible = true+ AND t.archetype = 'regular'+ SQL+ end++ def topic_volume+ current = new_topics_count(@start_date, @end_date)+ previous = new_topics_count(@prev_start, @prev_end)+ delta = delta_pct(current, previous)+ return if delta.nil? || delta.abs < 30+ return if delta.positive? && current < 5+ return if delta.negative? && previous < 5++ direction = delta.positive? ? "up" : "down"+ signal(+ :topic_volume,+ "New topics were #{direction} #{delta.abs}% versus the previous period",+ score: [delta.abs / 200.0, 0.8].min,+ category: :participation,+ )+ end+def hot_topicrow = DB.query(<<~SQL, start_date: @start_date, end_date: @end_date).firstSELECT t.id, t.title, COUNT(p.id) FILTER (WHERE p.post_number > 1) AS replies@@ -137,6 +188,7 @@ module DiscourseAi:hot_topic,"Busiest discussion: \"#{row.title}\" with #{row.replies} replies",score: 0.7,+ category: :participation,)end@@ -153,14 +205,22 @@ module DiscourseAiSQLreturn if count < 5+ total = new_topics_count(@start_date, @end_date)+ share = total.positive? ? ((count.to_f / total) * 100).round : 0+ headline = "#{count} new topics received no reply"+ headline = "#{headline} (#{share}% of new topics)" if share >= 10+signal(:unanswered_gap,- "#{count} new topics received no reply",- score: [0.4 + (count / 100.0), 0.9].min,+ headline,+ score: [0.4 + (share / 100.0), 0.9].min,+ category: :support,)enddef staff_ratio+ return if @period_days > MAX_STAFF_RATIO_DAYS+row = DB.query(<<~SQL, start_date: @start_date, end_date: @end_date).firstSELECTCOUNT(*) FILTER (WHERE u.admin OR u.moderator) AS staff,@@ -178,7 +238,12 @@ module DiscourseAipct = ((row.staff.to_f / row.total) * 100).roundreturn if pct < 40- signal(:staff_ratio, "Staff wrote #{pct}% of posts this period", score: pct / 100.0)+ signal(+ :staff_ratio,+ "Staff wrote #{pct}% of posts this period",+ score: pct / 100.0,+ category: :participation,+ )end# ---- external signals --------------------------------------------------@@ -205,6 +270,7 @@ module DiscourseAi:traffic_volume,"Browser pageviews were #{direction} #{delta.abs}% versus the previous period",score: [delta.abs / 200.0, 0.8].min,+ category: :acquisition,)end@@ -226,7 +292,7 @@ module DiscourseAi"Traffic spiked on #{peak.date} (#{multiple}x the typical day)"end- signal(:traffic_spike, headline, score: [multiple / 10.0, 0.95].min)+ signal(:traffic_spike, headline, score: [multiple / 10.0, 0.95].min, category: :acquisition)enddef dominant_referrer_for(date)@@ -275,10 +341,13 @@ module DiscourseAi:geography,"#{share}% of browser pageviews came from #{top.country_code}",score: share / 100.0,+ category: :acquisition,)enddef landing_topic+ return if @period_days > MAX_LANDING_TOPIC_DAYS+row = DB.query(<<~SQL, start_date: @start_date, end_date: @end_date).firstSELECT e.topic_id, t.title, COUNT(*) AS visitsFROM browser_pageview_events e@@ -297,6 +366,7 @@ module DiscourseAi:landing_topic,"External visitors mostly landed on \"#{row.title}\" (#{row.visits} visits)",score: 0.6,+ category: :acquisition,)endenddiff --git a/plugins/discourse-ai/lib/admin_dashboard/highlight_generator.rb b/plugins/discourse-ai/lib/admin_dashboard/highlight_generator.rbindex 9c87e1aea3e..db7e71e384a 100644--- a/plugins/discourse-ai/lib/admin_dashboard/highlight_generator.rb+++ b/plugins/discourse-ai/lib/admin_dashboard/highlight_generator.rb@@ -7,7 +7,13 @@ module DiscourseAi# then handed to the admin dashboard highlights agent which explains what changed.class HighlightGeneratorCACHE_TTL = 6.hours- CACHE_VERSION = 3+ CACHE_VERSION = 4++ LENSES = {+ acquisition: "Acquisition and discovery",+ participation: "Participation and contribution",+ support: "Support health",+ }.freezedef self.generate(start_date:, end_date:, period: nil)new(start_date: start_date, end_date: end_date, period: period).generate@@ -94,22 +100,29 @@ module DiscourseAidef user_message(facts)<<~MSG.stripCommunity facts for #{facts[:period][:start_date]} to #{facts[:period][:end_date]} — this was a #{facts[:trend]} period.+ Period length: #{facts[:period][:days]} days. Compare only with the previous #{facts[:period][:days]} days.Headline metrics (also shown as tiles below the highlight):#{facts[:metrics].map { |metric| format_metric(metric) }.join("\n")}+ Community-owner lenses:+ #{format_lenses(facts)}+Notable this period:#{format_signals(facts[:signals])}Rules — follow exactly:- Use ONLY the numbers and facts listed above. Never state a value that is not listed.+ - Do not mention a metric whose value is "not available".- Only mention a traffic source, country, or landing topic if it appears in "Notable" above. Do not invent sources, dates, causes, or numbers.+ - If you mention a traffic source or external referrer, name it exactly as listed or do not mention the source. Never say "a specific external referrer".- Do not overstate causality. If the facts only show contrast or correlation, phrase it that way.- - Do not say traffic "translated" or "did not translate" into another metric unless the facts include a conversion metric. Use plain contrast instead.+ - Do not say traffic "translated", "did not translate", "stemmed", "did not stem", "lifted", or "did not lift" another metric. Do not imply that a traffic spike caused, failed to cause, prevented, or failed to prevent another metric.+ - If you mention a traffic spike, state only its date, size, and listed referrer/source. Put participation or support concerns in a separate sentence.- Avoid report phrases like "as evidenced by", "highlighting", "indicating", and "underscoring".- If little is notable, say it was a steady period rather than inventing drama.- Write the admin dashboard highlight: 2 or 3 short, scannable sentences for a community owner. Lead with the trend, pick the 2-3 facts most useful for deciding what to inspect next, and prefer relationships between metrics. Warm and plain, no hype, no corporate report phrasing, no emoji.#{language_directive}+ Write the admin dashboard highlight: 2 or 3 short, scannable sentences for a community owner. Lead with the trend, then choose the 2-3 most useful next inspection areas from the community-owner lenses. For 7-day ranges, focus on immediate follow-up; for 30-day or longer ranges, focus on sustained patterns. Warm and plain, no hype, no corporate report phrasing, no emoji.#{language_directive}MSGend@@ -121,6 +134,7 @@ module DiscourseAienddef format_metric(metric)+ value = metric[:value].presence || "not available"change =if metric[:delta_pct].present?sign = metric[:delta_pct] >= 0 ? "+" : ""@@ -128,7 +142,32 @@ module DiscourseAielse""end- "- #{metric[:label]}: #{metric[:value]}#{change}"+ "- #{metric[:label]}: #{value}#{change}"+ end++ def format_lenses(facts)+ LENSES+ .map do |category, title|+ entries = lens_entries(facts, category)+ next if entries.blank?++ "- #{title}: #{entries.join("; ")}"+ end+ .compact+ .join("\n")+ end++ def lens_entries(facts, category)+ metrics =+ facts[:metrics]+ .select { |metric| metric[:category] == category && metric[:value].present? }+ .map { |metric| format_metric(metric).delete_prefix("- ") }+ signals =+ facts[:signals]+ .select { |signal| signal[:category] == category }+ .map { |signal| signal[:headline] }++ metrics + signalsenddef format_signals(signals)diff --git a/plugins/discourse-ai/lib/agents/admin_dashboard_highlights.rb b/plugins/discourse-ai/lib/agents/admin_dashboard_highlights.rbindex 751c2bb1712..3a08b9ae009 100644--- a/plugins/discourse-ai/lib/agents/admin_dashboard_highlights.rb+++ b/plugins/discourse-ai/lib/agents/admin_dashboard_highlights.rb@@ -15,17 +15,21 @@ module DiscourseAi<<~PROMPT.stripYou write the brief highlight shown at the top of a Discourse community admin dashboard, for the community owner.- You are given a set of verified facts for the period: headline metrics (with their change versus the previous period) and a short list of notable signals. This is the only ground truth — everything you write must come from it.+ You are given a set of verified facts for the period: headline metrics (with their change versus the previous period), community-owner lenses, and a short list of notable signals. This is the only ground truth — everything you write must come from it.Write 2 or 3 short, scannable sentences that tell the owner what changed and what deserves attention.- Lead with the overall trend (growing, steady, slipping, or mixed).- - Pick the 2-3 most useful facts for deciding what to inspect next; do not list everything. The metric tiles are shown right below you, so don't just recite them.- - Prefer relationships between facts over bare numbers (e.g. "traffic spiked, but sign-ups and contributors still fell" or "new members joined, but 17 topics went unanswered").+ - Pick the 2-3 most useful inspection areas from the acquisition, participation, and support-health lenses; do not list everything. The metric tiles are shown right below you, so don't just recite them.+ - For 7-day ranges, focus on immediate follow-up. For 30-day or longer ranges, focus on sustained patterns.+ - Prefer relationships between facts over bare numbers (e.g. "new members joined, but 17 topics went unanswered"). Do not connect traffic to other metrics unless a conversion metric is given.- Separate acquisition, participation, and support-health signals when they point in different directions.+ - Do not mention a metric whose value is "not available".- Mention a traffic source, country, or specific topic ONLY if it appears in the facts. Never invent sources, dates, causes, or numbers, and never state a metric value that wasn't given.+ - If you mention a traffic source or external referrer, name it exactly as listed or do not mention the source. Never say "a specific external referrer".- Do not overstate causality. Use "may", "could", or plain contrast when the facts show correlation, not cause.- - Do not say traffic "translated" or "did not translate" into another metric unless the facts include a conversion metric. Use plain contrast instead.+ - Do not say traffic "translated", "did not translate", "stemmed", "did not stem", "lifted", or "did not lift" another metric. Do not imply that a traffic spike caused, failed to cause, prevented, or failed to prevent another metric.+ - If you mention a traffic spike, state only its date, size, and listed referrer/source. Put participation or support concerns in a separate sentence.- Avoid report phrases like "as evidenced by", "highlighting", "indicating", and "underscoring".- If little is notable, say it was a steady period — don't manufacture drama.- Warm, plain language. No hype, no corporate report phrasing, no emoji.diff --git a/plugins/discourse-ai/spec/lib/admin_dashboard/admin_dashboard_facts_spec.rb b/plugins/discourse-ai/spec/lib/admin_dashboard/admin_dashboard_facts_spec.rbindex 3402cb0479d..c4f0b41daad 100644--- a/plugins/discourse-ai/spec/lib/admin_dashboard/admin_dashboard_facts_spec.rb+++ b/plugins/discourse-ai/spec/lib/admin_dashboard/admin_dashboard_facts_spec.rb@@ -22,7 +22,14 @@ RSpec.describe DiscourseAi::AdminDashboard::AdminDashboardFacts doexpect(labels).to include("new sign-ups", "new contributors")expect(facts[:metrics].find { |metric| metric[:label] == "new sign-ups" }).to include(value: 100,+ category: :acquisition,)+ expect(facts[:metrics].find { |metric| metric[:label] == "new contributors" }).to include(+ category: :participation,+ )+ expect(+ facts[:metrics].find { |metric| metric[:label] == "questions resolved (accepted solutions)" },+ ).to include(category: :support)endit "classifies the trend from the metric deltas" do@@ -34,7 +41,97 @@ RSpec.describe DiscourseAi::AdminDashboard::AdminDashboardFacts doheadlines = compute(start_date: 1.day.ago.to_date.to_s).fetch(:signals).map { |s| s[:headline] }- expect(headlines).to include(match(/new topics received no reply/))+ expect(headlines).to include(match(/new topics received no reply \(100% of new topics\)/))+ end++ it "reports when new-topic volume changes sharply" do+ start_date = Date.parse("2030-01-08")+ end_date = Date.parse("2030-01-14")+ Fabricate(:topic, created_at: Date.parse("2030-01-01"))+ Fabricate.times(6, :topic, created_at: start_date)++ topic_volume =+ compute(start_date: start_date.to_s, end_date: end_date.to_s)+ .fetch(:signals)+ .find { |s| s[:key] == :topic_volume }++ expect(topic_volume).to include(+ category: :participation,+ headline: "New topics were up 500% versus the previous period",+ )+ end++ it "reports when new-topic volume drops sharply from meaningful previous volume" do+ start_date = Date.parse("2030-01-08")+ end_date = Date.parse("2030-01-14")+ Fabricate.times(20, :topic, created_at: Date.parse("2030-01-01"))+ Fabricate.times(4, :topic, created_at: start_date)++ topic_volume =+ compute(start_date: start_date.to_s, end_date: end_date.to_s)+ .fetch(:signals)+ .find { |s| s[:key] == :topic_volume }++ expect(topic_volume).to include(+ category: :participation,+ headline: "New topics were down 80% versus the previous period",+ )+ end++ it "skips the raw landing-topic scan for long date ranges" do+ queries =+ track_sql_queries do+ compute(start_date: 1.year.ago.to_date.to_s, end_date: Date.current.to_s)+ end++ expect(queries.grep(/FROM browser_pageview_events e/)).to be_empty+ end++ it "reports the top external landing topic for three-month date ranges" do+ topic = Fabricate(:topic, title: "Welcome topic for visitors")+ Fabricate.times(+ 50,+ :browser_pageview_event,+ topic_id: topic.id,+ normalized_referrer: "example.com",+ created_at: 1.day.ago,+ )++ landing_topic =+ compute(start_date: 3.months.ago.to_date.to_s, end_date: Date.current.to_s)+ .fetch(:signals)+ .find { |s| s[:key] == :landing_topic }++ expect(landing_topic).to include(+ category: :acquisition,+ headline: 'External visitors mostly landed on "Welcome topic for visitors" (50 visits)',+ )+ end++ it "skips the raw staff-ratio post scan for long date ranges" do+ queries =+ track_sql_queries do+ compute(start_date: 1.year.ago.to_date.to_s, end_date: Date.current.to_s)+ end++ expect(queries.grep(/FROM posts p\s+JOIN users u/m)).to be_empty+ end++ it "reports staff post ratio for three-month date ranges" do+ admin = Fabricate(:admin)+ user = Fabricate(:user)+ Fabricate.times(12, :post, user: admin, created_at: 1.day.ago)+ Fabricate.times(8, :post, user: user, created_at: 1.day.ago)++ staff_ratio =+ compute(start_date: 3.months.ago.to_date.to_s, end_date: Date.current.to_s)+ .fetch(:signals)+ .find { |signal| signal[:key] == :staff_ratio }++ expect(staff_ratio).to include(+ category: :participation,+ headline: "Staff wrote 60% of posts this period",+ )endit "reports a traffic spike WITHOUT a source when no referrer dominates" do@@ -51,6 +148,7 @@ RSpec.describe DiscourseAi::AdminDashboard::AdminDashboardFacts doexpect(spike).to be_presentexpect(spike[:headline]).to match(/Traffic spiked/)+ expect(spike[:category]).to eq(:acquisition)expect(spike[:headline]).not_to match(/driven by/)enddiff --git a/plugins/discourse-ai/spec/lib/admin_dashboard/highlight_generator_spec.rb b/plugins/discourse-ai/spec/lib/admin_dashboard/highlight_generator_spec.rbindex 31bbb26ed46..0b9d1bb0add 100644--- a/plugins/discourse-ai/spec/lib/admin_dashboard/highlight_generator_spec.rb+++ b/plugins/discourse-ai/spec/lib/admin_dashboard/highlight_generator_spec.rb@@ -95,12 +95,42 @@ RSpec.describe DiscourseAi::AdminDashboard::HighlightGenerator domessage = message_for("2026-05-01", "2026-06-01")expect(message).to include("new sign-ups: 1100")+ expect(message).to include("Period length: 32 days")+ expect(message).to include("Community-owner lenses:")+ expect(message).to include("Acquisition and discovery: new sign-ups: 1100")expect(message).to match(/Use ONLY the numbers and facts listed above/)+ expect(message).to match(/Do not mention a metric whose value is "not available"/)expect(message).to match(/Do not invent sources, dates, causes, or numbers/)+ expect(message).to match(/Never say "a specific external referrer"/)expect(message).to match(/Do not overstate causality/)expect(message).to match(/Do not say traffic "translated"/)+ expect(message).to match(/"did not stem"/)+ expect(message).to match(/state only its date, size, and listed referrer/)expect(message).to match(/Avoid report phrases/)- expect(message).to match(/what to inspect next/)+ expect(message).to match(/next inspection areas/)+ end++ it "keeps unavailable metrics out of owner lenses" do+ allow(AdminDashboardHighlights).to receive(:build).and_return(+ {+ kpis: [+ {+ type: :new_signups,+ value: nil,+ previous_value: 10,+ percent_change: nil,+ report_type: "signups",+ report_query: {+ },+ },+ ],+ },+ )++ message = message_for("2026-05-01", "2026-06-01")++ expect(message).to include("new sign-ups: not available")+ expect(message).not_to include("Acquisition and discovery: new sign-ups")endit "says nothing stood out when no signal clears its threshold" do