A platform for community discussion. Free, open, simple.
DEV: Persist per-heuristic crawler score breakdown (#40022)
Krzysztof Kotlarek52c15cdaf997
diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rbindex 7ae01d2fa5f..30558bda1e9 100644--- a/app/models/browser_pageview_event.rb+++ b/app/models/browser_pageview_event.rb@@ -6,6 +6,8 @@ class BrowserPageviewEvent < ActiveRecord::BaseMAX_REFERRER_LENGTH = 2000MAX_USER_AGENT_LENGTH = 1000+ has_one :browser_pageview_event_score, foreign_key: :event_id, dependent: :delete+before_save :truncate_fieldsprivatediff --git a/app/models/browser_pageview_event_score.rb b/app/models/browser_pageview_event_score.rbnew file mode 100644index 00000000000..7bfcd926b06--- /dev/null+++ b/app/models/browser_pageview_event_score.rb@@ -0,0 +1,23 @@+# frozen_string_literal: true++class BrowserPageviewEventScore < ActiveRecord::Base+ belongs_to :event, class_name: "BrowserPageviewEvent"+end++# == Schema Information+#+# Table name: browser_pageview_event_scores+#+# id :bigint not null, primary key+# automation_ua_score :integer default(0), not null+# churn_score :integer default(0), not null+# known_asn_score :integer default(0), not null+# rapid_nav_score :integer default(0), not null+# referrer_score :integer default(0), not null+# velocity_score :integer default(0), not null+# event_id :bigint not null+#+# Indexes+#+# index_browser_pageview_event_scores_on_event_id (event_id) UNIQUE+#diff --git a/db/migrate/20260514055648_create_browser_pageview_event_scores.rb b/db/migrate/20260514055648_create_browser_pageview_event_scores.rbnew file mode 100644index 00000000000..569159906df--- /dev/null+++ b/db/migrate/20260514055648_create_browser_pageview_event_scores.rb@@ -0,0 +1,17 @@+# frozen_string_literal: true++class CreateBrowserPageviewEventScores < ActiveRecord::Migration[8.0]+ def change+ create_table :browser_pageview_event_scores do |t|+ t.bigint :event_id, null: false+ t.column :automation_ua_score, :smallint, null: false, default: 0+ t.column :known_asn_score, :smallint, null: false, default: 0+ t.column :velocity_score, :smallint, null: false, default: 0+ t.column :churn_score, :smallint, null: false, default: 0+ t.column :rapid_nav_score, :smallint, null: false, default: 0+ t.column :referrer_score, :smallint, null: false, default: 0+ end++ add_index :browser_pageview_event_scores, :event_id, unique: true+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex d9fd35b7c79..9fe0cd9f527 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -1800,6 +1800,41 @@ CREATE SEQUENCE public.bookmarks_id_seqALTER SEQUENCE public.bookmarks_id_seq OWNED BY public.bookmarks.id;+--+-- Name: browser_pageview_event_scores; Type: TABLE; Schema: public; Owner: -+--++CREATE TABLE public.browser_pageview_event_scores (+ id bigint NOT NULL,+ event_id bigint NOT NULL,+ automation_ua_score smallint DEFAULT 0 NOT NULL,+ known_asn_score smallint DEFAULT 0 NOT NULL,+ velocity_score smallint DEFAULT 0 NOT NULL,+ churn_score smallint DEFAULT 0 NOT NULL,+ rapid_nav_score smallint DEFAULT 0 NOT NULL,+ referrer_score smallint DEFAULT 0 NOT NULL+);+++--+-- Name: browser_pageview_event_scores_id_seq; Type: SEQUENCE; Schema: public; Owner: -+--++CREATE SEQUENCE public.browser_pageview_event_scores_id_seq+ START WITH 1+ INCREMENT BY 1+ NO MINVALUE+ NO MAXVALUE+ CACHE 1;+++--+-- Name: browser_pageview_event_scores_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -+--++ALTER SEQUENCE public.browser_pageview_event_scores_id_seq OWNED BY public.browser_pageview_event_scores.id;++---- Name: browser_pageview_events; Type: TABLE; Schema: public; Owner: ---@@ -11515,6 +11550,13 @@ ALTER TABLE ONLY public.badges ALTER COLUMN id SET DEFAULT nextval('public.badgeALTER TABLE ONLY public.bookmarks ALTER COLUMN id SET DEFAULT nextval('public.bookmarks_id_seq'::regclass);+--+-- Name: browser_pageview_event_scores id; Type: DEFAULT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.browser_pageview_event_scores ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_event_scores_id_seq'::regclass);++---- Name: browser_pageview_events id; Type: DEFAULT; Schema: public; Owner: ---@@ -13579,6 +13621,14 @@ ALTER TABLE ONLY public.bookmarksADD CONSTRAINT bookmarks_pkey PRIMARY KEY (id);+--+-- Name: browser_pageview_event_scores browser_pageview_event_scores_pkey; Type: CONSTRAINT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.browser_pageview_event_scores+ ADD CONSTRAINT browser_pageview_event_scores_pkey PRIMARY KEY (id);++---- Name: browser_pageview_events browser_pageview_events_pkey; Type: CONSTRAINT; Schema: public; Owner: ---@@ -16699,6 +16749,13 @@ CREATE INDEX index_bookmarks_on_reminder_set_at ON public.bookmarks USING btreeCREATE INDEX index_bookmarks_on_user_id ON public.bookmarks USING btree (user_id);+--+-- Name: index_browser_pageview_event_scores_on_event_id; Type: INDEX; Schema: public; Owner: -+--++CREATE UNIQUE INDEX index_browser_pageview_event_scores_on_event_id ON public.browser_pageview_event_scores USING btree (event_id);++---- Name: index_browser_pageview_events_on_created_at; Type: INDEX; Schema: public; Owner: ---@@ -20803,6 +20860,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groupsSET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES+('20260514055648'),('20260514043815'),('20260513105516'),('20260513101242'),diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rbindex ec77a80072b..6c37e646bac 100644--- a/lib/crawler_scorer.rb+++ b/lib/crawler_scorer.rb@@ -111,55 +111,104 @@ class CrawlerScorerGROUP BY ip_address, user_agent),- totals AS (+ breakdown AS (SELECTe.id,CASEWHEN :ua_regex <> '' AND e.user_agent ~* :ua_regex THEN :automation_ua_scoreELSE 0- END- + CASE- WHEN e.asn = ANY(ARRAY[:crawler_asns]::int[]) THEN :known_asn_score- ELSE 0- END- + CASE- WHEN iu.pageviews >= :velocity_high THEN :velocity_high_score- WHEN iu.pageviews >= :velocity_medium THEN :velocity_medium_score- WHEN iu.pageviews >= :velocity_low THEN :velocity_low_score- ELSE 0- END- + CASE- WHEN iu.distinct_sessions >= :churn_high_min_sessions- AND iu.pageviews::float / NULLIF(iu.distinct_sessions, 0) <= :churn_max_avg_events- THEN :churn_high_score- WHEN iu.distinct_sessions >= :churn_low_min_sessions- AND iu.pageviews::float / NULLIF(iu.distinct_sessions, 0) <= :churn_max_avg_events- THEN :churn_low_score- ELSE 0- END- + CASE- WHEN mg.gap_count >= :rapid_nav_min_gaps- AND mg.median_gap_seconds < :rapid_nav_max_median_seconds- THEN :rapid_nav_score- ELSE 0- END- + CASE- WHEN iu.pageviews >= :referrer_min_events- AND iu.bad_referrer_ratio >= :referrer_high_ratio THEN :referrer_high_score- WHEN iu.pageviews >= :referrer_min_events- AND iu.bad_referrer_ratio >= :referrer_low_ratio THEN :referrer_low_score- ELSE 0- END- AS score+ END AS automation_ua_score,+ CASE+ WHEN e.asn = ANY(ARRAY[:crawler_asns]::int[]) THEN :known_asn_score+ ELSE 0+ END AS known_asn_score,+ CASE+ WHEN iu.pageviews >= :velocity_high THEN :velocity_high_score+ WHEN iu.pageviews >= :velocity_medium THEN :velocity_medium_score+ WHEN iu.pageviews >= :velocity_low THEN :velocity_low_score+ ELSE 0+ END AS velocity_score,+ CASE+ WHEN iu.distinct_sessions >= :churn_high_min_sessions+ AND iu.pageviews::float / NULLIF(iu.distinct_sessions, 0) <= :churn_max_avg_events+ THEN :churn_high_score+ WHEN iu.distinct_sessions >= :churn_low_min_sessions+ AND iu.pageviews::float / NULLIF(iu.distinct_sessions, 0) <= :churn_max_avg_events+ THEN :churn_low_score+ ELSE 0+ END AS churn_score,+ CASE+ WHEN mg.gap_count >= :rapid_nav_min_gaps+ AND mg.median_gap_seconds < :rapid_nav_max_median_seconds+ THEN :rapid_nav_score+ ELSE 0+ END AS rapid_nav_score,+ CASE+ WHEN iu.pageviews >= :referrer_min_events+ AND iu.bad_referrer_ratio >= :referrer_high_ratio THEN :referrer_high_score+ WHEN iu.pageviews >= :referrer_min_events+ AND iu.bad_referrer_ratio >= :referrer_low_ratio THEN :referrer_low_score+ ELSE 0+ END AS referrer_scoreFROM events eLEFT JOIN ipua_stats iu USING (ip_address, user_agent)LEFT JOIN median_gap mg USING (ip_address, user_agent)+ ),++ totals AS (+ SELECT+ id,+ automation_ua_score,+ known_asn_score,+ velocity_score,+ churn_score,+ rapid_nav_score,+ referrer_score,+ automation_ua_score + known_asn_score + velocity_score+ + churn_score + rapid_nav_score + referrer_score AS score+ FROM breakdown+ ),++ updated AS (+ UPDATE browser_pageview_events e+ SET score = t.score+ FROM totals t+ WHERE e.id = t.id+ AND t.score > 0+ AND t.score > COALESCE(e.score, 0)+ RETURNING e.id,+ t.automation_ua_score,+ t.known_asn_score,+ t.velocity_score,+ t.churn_score,+ t.rapid_nav_score,+ t.referrer_score)- UPDATE browser_pageview_events e- SET score = t.score- FROM totals t- WHERE e.id = t.id- AND t.score > COALESCE(e.score, 0);+ INSERT INTO browser_pageview_event_scores (+ event_id,+ automation_ua_score,+ known_asn_score,+ velocity_score,+ churn_score,+ rapid_nav_score,+ referrer_score+ )+ SELECT+ id,+ automation_ua_score,+ known_asn_score,+ velocity_score,+ churn_score,+ rapid_nav_score,+ referrer_score+ FROM updated+ ON CONFLICT (event_id) DO UPDATE+ SET automation_ua_score = EXCLUDED.automation_ua_score,+ known_asn_score = EXCLUDED.known_asn_score,+ velocity_score = EXCLUDED.velocity_score,+ churn_score = EXCLUDED.churn_score,+ rapid_nav_score = EXCLUDED.rapid_nav_score,+ referrer_score = EXCLUDED.referrer_score;SQLenddiff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rbindex 04a1cb4cab4..55ba8b04c29 100644--- a/spec/lib/crawler_scorer_spec.rb+++ b/spec/lib/crawler_scorer_spec.rb@@ -25,6 +25,33 @@ RSpec.describe CrawlerScorer doexpect(event.reload.score).to eq(100)end+ it "writes the score breakdown per heuristic to the side table" do+ SiteSetting.crawler_asns = "12345"+ event =+ make_event(+ user_agent: "Mozilla/5.0 (X11; Linux x86_64) HeadlessChrome/120.0.0.0",+ asn: 12_345,+ )++ score!++ expect(event.reload.score).to eq(115)+ breakdown = event.browser_pageview_event_score+ expect(breakdown.automation_ua_score).to eq(100)+ expect(breakdown.known_asn_score).to eq(15)+ expect(breakdown.velocity_score).to eq(0)+ expect(breakdown.churn_score).to eq(0)+ expect(breakdown.rapid_nav_score).to eq(0)+ expect(breakdown.referrer_score).to eq(0)+ end++ it "does not write a breakdown row for events that score 0" do+ event = make_event+ score!+ expect(event.reload.score).to be_nil+ expect(event.browser_pageview_event_score).to be_nil+ end+it "scores known crawler ASNs at +15" doSiteSetting.crawler_asns = "12345"event = make_event(asn: 12_345)