Skip to content

A platform for community discussion. Free, open, simple.

DEV: Persist per-heuristic crawler score breakdown (#40022)

Krzysztof Kotlarek52c15cdaf997
diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rb
index 7ae01d2fa5f..30558bda1e9 100644
--- a/app/models/browser_pageview_event.rb
+++ b/app/models/browser_pageview_event.rb
@@ -6,6 +6,8 @@ class BrowserPageviewEvent < ActiveRecord::Base
MAX_REFERRER_LENGTH = 2000
MAX_USER_AGENT_LENGTH = 1000
+ has_one :browser_pageview_event_score, foreign_key: :event_id, dependent: :delete
+
before_save :truncate_fields
private
diff --git a/app/models/browser_pageview_event_score.rb b/app/models/browser_pageview_event_score.rb
new file mode 100644
index 00000000000..7bfcd926b06
--- /dev/null
+++ b/app/models/browser_pageview_event_score.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class BrowserPageviewEventScore < ActiveRecord::Base
+ belongs_to :event, class_name: "BrowserPageviewEvent"
+end
+
+# == Schema Information
+#
+# Table name: browser_pageview_event_scores
+#
+# id :bigint not null, primary key
+# automation_ua_score :integer default(0), not null
+# churn_score :integer default(0), not null
+# known_asn_score :integer default(0), not null
+# rapid_nav_score :integer default(0), not null
+# referrer_score :integer default(0), not null
+# velocity_score :integer default(0), not null
+# event_id :bigint not null
+#
+# Indexes
+#
+# index_browser_pageview_event_scores_on_event_id (event_id) UNIQUE
+#
diff --git a/db/migrate/20260514055648_create_browser_pageview_event_scores.rb b/db/migrate/20260514055648_create_browser_pageview_event_scores.rb
new file mode 100644
index 00000000000..569159906df
--- /dev/null
+++ b/db/migrate/20260514055648_create_browser_pageview_event_scores.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class CreateBrowserPageviewEventScores < ActiveRecord::Migration[8.0]
+ def change
+ create_table :browser_pageview_event_scores do |t|
+ t.bigint :event_id, null: false
+ t.column :automation_ua_score, :smallint, null: false, default: 0
+ t.column :known_asn_score, :smallint, null: false, default: 0
+ t.column :velocity_score, :smallint, null: false, default: 0
+ t.column :churn_score, :smallint, null: false, default: 0
+ t.column :rapid_nav_score, :smallint, null: false, default: 0
+ t.column :referrer_score, :smallint, null: false, default: 0
+ end
+
+ add_index :browser_pageview_event_scores, :event_id, unique: true
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index d9fd35b7c79..9fe0cd9f527 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1800,6 +1800,41 @@ CREATE SEQUENCE public.bookmarks_id_seq
ALTER SEQUENCE public.bookmarks_id_seq OWNED BY public.bookmarks.id;
+--
+-- Name: browser_pageview_event_scores; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.browser_pageview_event_scores (
+ id bigint NOT NULL,
+ event_id bigint NOT NULL,
+ automation_ua_score smallint DEFAULT 0 NOT NULL,
+ known_asn_score smallint DEFAULT 0 NOT NULL,
+ velocity_score smallint DEFAULT 0 NOT NULL,
+ churn_score smallint DEFAULT 0 NOT NULL,
+ rapid_nav_score smallint DEFAULT 0 NOT NULL,
+ referrer_score smallint DEFAULT 0 NOT NULL
+);
+
+
+--
+-- Name: browser_pageview_event_scores_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.browser_pageview_event_scores_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: browser_pageview_event_scores_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.browser_pageview_event_scores_id_seq OWNED BY public.browser_pageview_event_scores.id;
+
+
--
-- Name: browser_pageview_events; Type: TABLE; Schema: public; Owner: -
--
@@ -11515,6 +11550,13 @@ ALTER TABLE ONLY public.badges ALTER COLUMN id SET DEFAULT nextval('public.badge
ALTER TABLE ONLY public.bookmarks ALTER COLUMN id SET DEFAULT nextval('public.bookmarks_id_seq'::regclass);
+--
+-- Name: browser_pageview_event_scores id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.browser_pageview_event_scores ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_event_scores_id_seq'::regclass);
+
+
--
-- Name: browser_pageview_events id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -13579,6 +13621,14 @@ ALTER TABLE ONLY public.bookmarks
ADD CONSTRAINT bookmarks_pkey PRIMARY KEY (id);
+--
+-- Name: browser_pageview_event_scores browser_pageview_event_scores_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.browser_pageview_event_scores
+ ADD CONSTRAINT browser_pageview_event_scores_pkey PRIMARY KEY (id);
+
+
--
-- Name: browser_pageview_events browser_pageview_events_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -16699,6 +16749,13 @@ CREATE INDEX index_bookmarks_on_reminder_set_at ON public.bookmarks USING btree
CREATE INDEX index_bookmarks_on_user_id ON public.bookmarks USING btree (user_id);
+--
+-- Name: index_browser_pageview_event_scores_on_event_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX index_browser_pageview_event_scores_on_event_id ON public.browser_pageview_event_scores USING btree (event_id);
+
+
--
-- Name: index_browser_pageview_events_on_created_at; Type: INDEX; Schema: public; Owner: -
--
@@ -20803,6 +20860,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20260514055648'),
('20260514043815'),
('20260513105516'),
('20260513101242'),
diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rb
index ec77a80072b..6c37e646bac 100644
--- a/lib/crawler_scorer.rb
+++ b/lib/crawler_scorer.rb
@@ -111,55 +111,104 @@ class CrawlerScorer
GROUP BY ip_address, user_agent
),
- totals AS (
+ breakdown AS (
SELECT
e.id,
CASE
WHEN :ua_regex <> '' AND e.user_agent ~* :ua_regex THEN :automation_ua_score
ELSE 0
- END
- + CASE
- WHEN e.asn = ANY(ARRAY[:crawler_asns]::int[]) THEN :known_asn_score
- ELSE 0
- END
- + CASE
- WHEN iu.pageviews >= :velocity_high THEN :velocity_high_score
- WHEN iu.pageviews >= :velocity_medium THEN :velocity_medium_score
- WHEN iu.pageviews >= :velocity_low THEN :velocity_low_score
- ELSE 0
- END
- + CASE
- WHEN iu.distinct_sessions >= :churn_high_min_sessions
- AND iu.pageviews::float / NULLIF(iu.distinct_sessions, 0) <= :churn_max_avg_events
- THEN :churn_high_score
- WHEN iu.distinct_sessions >= :churn_low_min_sessions
- AND iu.pageviews::float / NULLIF(iu.distinct_sessions, 0) <= :churn_max_avg_events
- THEN :churn_low_score
- ELSE 0
- END
- + CASE
- WHEN mg.gap_count >= :rapid_nav_min_gaps
- AND mg.median_gap_seconds < :rapid_nav_max_median_seconds
- THEN :rapid_nav_score
- ELSE 0
- END
- + CASE
- WHEN iu.pageviews >= :referrer_min_events
- AND iu.bad_referrer_ratio >= :referrer_high_ratio THEN :referrer_high_score
- WHEN iu.pageviews >= :referrer_min_events
- AND iu.bad_referrer_ratio >= :referrer_low_ratio THEN :referrer_low_score
- ELSE 0
- END
- AS score
+ END AS automation_ua_score,
+ CASE
+ WHEN e.asn = ANY(ARRAY[:crawler_asns]::int[]) THEN :known_asn_score
+ ELSE 0
+ END AS known_asn_score,
+ CASE
+ WHEN iu.pageviews >= :velocity_high THEN :velocity_high_score
+ WHEN iu.pageviews >= :velocity_medium THEN :velocity_medium_score
+ WHEN iu.pageviews >= :velocity_low THEN :velocity_low_score
+ ELSE 0
+ END AS velocity_score,
+ CASE
+ WHEN iu.distinct_sessions >= :churn_high_min_sessions
+ AND iu.pageviews::float / NULLIF(iu.distinct_sessions, 0) <= :churn_max_avg_events
+ THEN :churn_high_score
+ WHEN iu.distinct_sessions >= :churn_low_min_sessions
+ AND iu.pageviews::float / NULLIF(iu.distinct_sessions, 0) <= :churn_max_avg_events
+ THEN :churn_low_score
+ ELSE 0
+ END AS churn_score,
+ CASE
+ WHEN mg.gap_count >= :rapid_nav_min_gaps
+ AND mg.median_gap_seconds < :rapid_nav_max_median_seconds
+ THEN :rapid_nav_score
+ ELSE 0
+ END AS rapid_nav_score,
+ CASE
+ WHEN iu.pageviews >= :referrer_min_events
+ AND iu.bad_referrer_ratio >= :referrer_high_ratio THEN :referrer_high_score
+ WHEN iu.pageviews >= :referrer_min_events
+ AND iu.bad_referrer_ratio >= :referrer_low_ratio THEN :referrer_low_score
+ ELSE 0
+ END AS referrer_score
FROM events e
LEFT JOIN ipua_stats iu USING (ip_address, user_agent)
LEFT JOIN median_gap mg USING (ip_address, user_agent)
+ ),
+
+ totals AS (
+ SELECT
+ id,
+ automation_ua_score,
+ known_asn_score,
+ velocity_score,
+ churn_score,
+ rapid_nav_score,
+ referrer_score,
+ automation_ua_score + known_asn_score + velocity_score
+ + churn_score + rapid_nav_score + referrer_score AS score
+ FROM breakdown
+ ),
+
+ updated AS (
+ UPDATE browser_pageview_events e
+ SET score = t.score
+ FROM totals t
+ WHERE e.id = t.id
+ AND t.score > 0
+ AND t.score > COALESCE(e.score, 0)
+ RETURNING e.id,
+ t.automation_ua_score,
+ t.known_asn_score,
+ t.velocity_score,
+ t.churn_score,
+ t.rapid_nav_score,
+ t.referrer_score
)
- UPDATE browser_pageview_events e
- SET score = t.score
- FROM totals t
- WHERE e.id = t.id
- AND t.score > COALESCE(e.score, 0);
+ INSERT INTO browser_pageview_event_scores (
+ event_id,
+ automation_ua_score,
+ known_asn_score,
+ velocity_score,
+ churn_score,
+ rapid_nav_score,
+ referrer_score
+ )
+ SELECT
+ id,
+ automation_ua_score,
+ known_asn_score,
+ velocity_score,
+ churn_score,
+ rapid_nav_score,
+ referrer_score
+ FROM updated
+ ON CONFLICT (event_id) DO UPDATE
+ SET automation_ua_score = EXCLUDED.automation_ua_score,
+ known_asn_score = EXCLUDED.known_asn_score,
+ velocity_score = EXCLUDED.velocity_score,
+ churn_score = EXCLUDED.churn_score,
+ rapid_nav_score = EXCLUDED.rapid_nav_score,
+ referrer_score = EXCLUDED.referrer_score;
SQL
end
diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rb
index 04a1cb4cab4..55ba8b04c29 100644
--- a/spec/lib/crawler_scorer_spec.rb
+++ b/spec/lib/crawler_scorer_spec.rb
@@ -25,6 +25,33 @@ RSpec.describe CrawlerScorer do
expect(event.reload.score).to eq(100)
end
+ it "writes the score breakdown per heuristic to the side table" do
+ SiteSetting.crawler_asns = "12345"
+ event =
+ make_event(
+ user_agent: "Mozilla/5.0 (X11; Linux x86_64) HeadlessChrome/120.0.0.0",
+ asn: 12_345,
+ )
+
+ score!
+
+ expect(event.reload.score).to eq(115)
+ breakdown = event.browser_pageview_event_score
+ expect(breakdown.automation_ua_score).to eq(100)
+ expect(breakdown.known_asn_score).to eq(15)
+ expect(breakdown.velocity_score).to eq(0)
+ expect(breakdown.churn_score).to eq(0)
+ expect(breakdown.rapid_nav_score).to eq(0)
+ expect(breakdown.referrer_score).to eq(0)
+ end
+
+ it "does not write a breakdown row for events that score 0" do
+ event = make_event
+ score!
+ expect(event.reload.score).to be_nil
+ expect(event.browser_pageview_event_score).to be_nil
+ end
+
it "scores known crawler ASNs at +15" do
SiteSetting.crawler_asns = "12345"
event = make_event(asn: 12_345)