Skip to content

A platform for community discussion. Free, open, simple.

UX: exclude anonymous endpoint from search terms report (#44218)

Krisacf8e70129f5
diff --git a/app/models/search_log.rb b/app/models/search_log.rb
index e0040597222..e504c24ebbf 100644
--- a/app/models/search_log.rb
+++ b/app/models/search_log.rb
@@ -17,8 +17,18 @@ class SearchLog < ActiveRecord::Base
)
end
scope :excluding_crawlers, -> { where(crawler: false, likely_crawler: false) }
+ # Crawler scoring needs a pageview session, so anonymous searches without one
+ # (direct requests to the search endpoints) can't be vetted and are excluded.
+ scope :from_browser_or_member,
+ -> { where("search_logs.user_id IS NOT NULL OR search_logs.session_id IS NOT NULL") }
scope :human_only,
- -> { CrawlerScorer.enabled? ? non_staff_or_anonymous.excluding_crawlers : non_staff }
+ -> do
+ if CrawlerScorer.enabled?
+ non_staff_or_anonymous.excluding_crawlers.from_browser_or_member
+ else
+ non_staff
+ end
+ end
def ctr
return 0 if click_through == 0 || searches == 0
diff --git a/spec/models/search_log_spec.rb b/spec/models/search_log_spec.rb
index 939edc7f1ae..2dd30fda798 100644
--- a/spec/models/search_log_spec.rb
+++ b/spec/models/search_log_spec.rb
@@ -418,14 +418,15 @@ RSpec.describe SearchLog, type: :model do
).to eq(1)
end
- it "returns non-staff and anonymous searches minus crawlers with the human_only search type" do
+ it "returns non-staff and anonymous browser searches minus crawlers with the human_only search type" do
SiteSetting.improved_crawler_detection = true
member = Fabricate(:user)
Fabricate(:search_log, term: "ruby", user: member)
Fabricate(:search_log, term: "ruby", user: Fabricate(:admin))
Fabricate(:search_log, term: "ruby", user: Fabricate(:moderator))
+ Fabricate(:search_log, term: "ruby", user: nil, session_id: "browser-session")
Fabricate(:search_log, term: "ruby", user: nil)
- Fabricate(:search_log, term: "ruby", user: nil, crawler: true)
+ Fabricate(:search_log, term: "ruby", user: nil, session_id: "browser-session", crawler: true)
expect(
SearchLog.term_details("ruby", :weekly, :human_only)[:data].sum { |point| point[:y] },
@@ -442,6 +443,7 @@ RSpec.describe SearchLog, type: :model do
ip_address: "127.0.0.1",
user_agent:
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
+ session_id: "browser-session",
**opts,
)
end
@@ -487,10 +489,17 @@ RSpec.describe SearchLog, type: :model do
expect(results.map { |trend| [trend.term, trend.searches] }).to eq([["ruby", 1]])
end
- it "returns non-staff and anonymous searches minus crawlers with the human_only search type" do
+ it "returns non-staff and anonymous browser searches minus crawlers with the human_only search type" do
SiteSetting.improved_crawler_detection = true
Fabricate(:search_log, term: "admin-search", user: Fabricate(:admin))
- Fabricate(:search_log, term: "crawler-search", user: nil, crawler: true)
+ Fabricate(:search_log, term: "sessionless-search", user: nil)
+ Fabricate(
+ :search_log,
+ term: "crawler-search",
+ user: nil,
+ session_id: "browser-session",
+ crawler: true,
+ )
results = SearchLog.trending(:all, :human_only).to_a
diff --git a/spec/requests/admin/search_logs_spec.rb b/spec/requests/admin/search_logs_spec.rb
index 581a4fc0aba..802f747c4e3 100644
--- a/spec/requests/admin/search_logs_spec.rb
+++ b/spec/requests/admin/search_logs_spec.rb
@@ -51,17 +51,24 @@ RSpec.describe Admin::SearchLogsController do
)
end
- it "returns non-staff and anonymous searches minus crawlers with the human_only search type" do
+ it "returns non-staff and anonymous browser searches minus crawlers with the human_only search type" do
SiteSetting.improved_crawler_detection = true
Fabricate(:search_log, term: "member-search", user: user)
Fabricate(:search_log, term: "admin-search", user: admin)
- Fabricate(:search_log, term: "anonymous-search", user: nil)
- Fabricate(:search_log, term: "crawler-search", user: nil, crawler: true)
+ Fabricate(:search_log, term: "anonymous-search", user: nil, session_id: "browser-session")
+ Fabricate(:search_log, term: "sessionless-search", user: nil)
+ Fabricate(
+ :search_log,
+ term: "crawler-search",
+ user: nil,
+ session_id: "browser-session",
+ crawler: true,
+ )
get "/admin/logs/search_logs.json", params: { search_type: "human_only" }
expect(response.parsed_body.map { |entry| [entry["term"], entry["searches"]] }).to eq(
- [["anonymous-search", 1], ["member-search", 1], ["ruby", 1]],
+ [["anonymous-search", 1], ["member-search", 1]],
)
end
end
diff --git a/spec/services/admin_dashboard_search_spec.rb b/spec/services/admin_dashboard_search_spec.rb
index cde7fdb9537..17f6a46b696 100644
--- a/spec/services/admin_dashboard_search_spec.rb
+++ b/spec/services/admin_dashboard_search_spec.rb
@@ -58,8 +58,20 @@ RSpec.describe AdminDashboardSearch do
)
Fabricate.times(5, :search_log, term: "ghost", user: user, created_at: "2026-04-26 11:00")
- Fabricate.times(8, :clicked_search_log, term: "onboarding", created_at: "2026-05-05 10:00")
- Fabricate.times(2, :search_log, term: "onboarding", created_at: "2026-05-05 11:00")
+ Fabricate.times(
+ 8,
+ :clicked_search_log,
+ term: "onboarding",
+ session_id: "browser-session",
+ created_at: "2026-05-05 10:00",
+ )
+ Fabricate.times(
+ 2,
+ :search_log,
+ term: "onboarding",
+ session_id: "browser-session",
+ created_at: "2026-05-05 11:00",
+ )
# Searches flagged as crawler traffic must stay out of every metric and out of the
# prior-window deltas. If counted, "crawler-bait" would top trending, inflate the
@@ -78,6 +90,9 @@ RSpec.describe AdminDashboardSearch do
crawler: true,
created_at: "2026-04-26 09:00",
)
+ # Anonymous searches without a pageview session can't be crawler-scored, so they
+ # are excluded the same way.
+ Fabricate.times(30, :search_log, term: "sessionless-bait", created_at: "2026-05-05 12:00")
Fabricate(:search_log, term: "admin-search", user: admin, created_at: "2026-05-05 12:00")
Fabricate(:clicked_search_log, term: "ruby", user: moderator, created_at: "2026-05-05 13:00")
diff --git a/spec/system/admin_dashboard_redesign/search_section_spec.rb b/spec/system/admin_dashboard_redesign/search_section_spec.rb
index 902f09f39a6..ed283a60d20 100644
--- a/spec/system/admin_dashboard_redesign/search_section_spec.rb
+++ b/spec/system/admin_dashboard_redesign/search_section_spec.rb
@@ -185,7 +185,13 @@ describe "Admin Dashboard Redesign | Search section" do
user: user,
created_at: "2026-05-10 11:00",
)
- Fabricate.times(2, :search_log, term: "ghost", created_at: "2026-05-10 12:00")
+ Fabricate.times(
+ 2,
+ :search_log,
+ term: "ghost",
+ session_id: "browser-session",
+ created_at: "2026-05-10 12:00",
+ )
dashboard.visit
search = dashboard.search