Skip to content

A platform for community discussion. Free, open, simple.

FIX: score missing engagement per ip+ua ratio, not per session (#42259)

Krzysztof Kotlarek8964e868ba78
diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rb
index c49b8ce7f5c..fffa92c0b6f 100644
--- a/lib/crawler_scorer.rb
+++ b/lib/crawler_scorer.rb
@@ -38,7 +38,10 @@ class CrawlerScorer
REFERRER_LOW_SCORE = 5
REFERRER_HIGH_SCORE = 10
- MISSING_ENGAGEMENT_SCORE = 40
+ MISSING_ENGAGEMENT_LOW_RATIO = 0.5
+ MISSING_ENGAGEMENT_HIGH_RATIO = 0.9
+ MISSING_ENGAGEMENT_LOW_SCORE = 20
+ MISSING_ENGAGEMENT_HIGH_SCORE = 40
def self.score!(window_start:, window_end:)
crawler_asns = SiteSetting.crawler_asns_map.map(&:to_i)
@@ -84,17 +87,35 @@ class CrawlerScorer
referrer_high_ratio: REFERRER_HIGH_RATIO,
referrer_low_score: REFERRER_LOW_SCORE,
referrer_high_score: REFERRER_HIGH_SCORE,
- missing_engagement_score: MISSING_ENGAGEMENT_SCORE,
+ missing_engagement_low_ratio: MISSING_ENGAGEMENT_LOW_RATIO,
+ missing_engagement_high_ratio: MISSING_ENGAGEMENT_HIGH_RATIO,
+ missing_engagement_low_score: MISSING_ENGAGEMENT_LOW_SCORE,
+ missing_engagement_high_score: MISSING_ENGAGEMENT_HIGH_SCORE,
)
end
end
SQL = <<~SQL
WITH events AS (
- SELECT id, session_id, ip_address, user_agent, referrer, asn, url, created_at, source
- FROM browser_pageview_events
- WHERE created_at >= :window_start
- AND created_at < :window_end
+ SELECT
+ e.id,
+ e.session_id,
+ e.ip_address,
+ e.user_agent,
+ e.referrer,
+ e.asn,
+ e.url,
+ e.created_at,
+ e.source,
+ (se.session_id IS NOT NULL) AS engaged
+ FROM browser_pageview_events e
+ LEFT JOIN browser_pageview_session_engagements se
+ ON se.session_id = e.session_id
+ AND (
+ #{BrowserPageviewSessionEngagement::INTERACTION_COLUMNS.map { |column| "se.#{column} > 0" }.join(" OR ")}
+ )
+ WHERE e.created_at >= :window_start
+ AND e.created_at < :window_end
),
-- Per-heuristic stats are partitioned by source as well as ip/ua so that
@@ -113,7 +134,8 @@ class CrawlerScorer
WHEN substring(referrer from '^https?://([^/]+)') = :hostname THEN 0.0
ELSE 1.0
END
- ) AS bad_referrer_ratio
+ ) AS bad_referrer_ratio,
+ AVG(CASE WHEN engaged THEN 0.0 ELSE 1.0 END) AS no_engagement_ratio
FROM events
GROUP BY ip_address, user_agent, source
),
@@ -170,16 +192,16 @@ class CrawlerScorer
CASE
WHEN iu.pageviews = 1
AND e.referrer IS NULL
- AND se.session_id IS NULL
+ AND NOT e.engaged
AND e.url ~ '[?&]#{Discourse::LOCALE_PARAM}(=|&|$)'
THEN :single_request_no_referrer_score + :single_request_locale_param_bonus
WHEN iu.pageviews = 1
AND e.referrer IS NULL
- AND se.session_id IS NULL THEN :single_request_no_referrer_score
+ AND NOT e.engaged THEN :single_request_no_referrer_score
ELSE 0
END AS single_request_no_referrer_score,
CASE
- WHEN se.session_id IS NULL
+ WHEN NOT e.engaged
AND e.user_agent ~ 'Chrome/[0-9]{1,3}'
AND e.user_agent !~* 'HeadlessChrome'
AND (substring(e.user_agent FROM 'Chrome/([0-9]{1,3})'))::int
@@ -223,18 +245,17 @@ class CrawlerScorer
ELSE 0
END AS referrer_score,
CASE
- WHEN se.session_id IS NULL THEN :missing_engagement_score
+ WHEN e.engaged THEN 0
+ WHEN iu.no_engagement_ratio >= :missing_engagement_high_ratio
+ THEN :missing_engagement_high_score
+ WHEN iu.no_engagement_ratio >= :missing_engagement_low_ratio
+ THEN :missing_engagement_low_score
ELSE 0
END AS engagement_score
FROM events e
LEFT JOIN ipua_stats iu USING (ip_address, user_agent, source)
LEFT JOIN median_gap mg USING (ip_address, user_agent, source)
LEFT JOIN session_stats ss USING (session_id, source)
- LEFT JOIN browser_pageview_session_engagements se
- ON se.session_id = e.session_id
- AND (
- #{BrowserPageviewSessionEngagement::INTERACTION_COLUMNS.map { |column| "se.#{column} > 0" }.join(" OR ")}
- )
),
totals AS (
@@ -266,7 +287,7 @@ class CrawlerScorer
SET score = t.score
FROM totals t
WHERE e.id = t.id
- AND (e.score IS NULL OR t.score > e.score)
+ AND e.score IS DISTINCT FROM t.score
RETURNING e.id,
t.automation_ua_score,
t.known_asn_score,
diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rb
index c5319871ca2..e4b331fdb6b 100644
--- a/spec/lib/crawler_scorer_spec.rb
+++ b/spec/lib/crawler_scorer_spec.rb
@@ -332,7 +332,7 @@ RSpec.describe CrawlerScorer do
expect(event.browser_pageview_event_score.engagement_score).to eq(40)
end
- it "keeps the higher score when a later run would lower it" do
+ it "lowers the score when an engagement beacon arrives after an earlier run" do
SiteSetting.crawler_asns = "12345"
event = make_event(asn: 12_345)
@@ -342,7 +342,48 @@ RSpec.describe CrawlerScorer do
Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, key_events: 4)
score!
- expect(event.reload.score).to eq(70)
+ expect(event.reload.score).to eq(30)
+ expect(event.browser_pageview_event_score.engagement_score).to eq(0)
+ end
+
+ it "does not penalise an unengaged session when most of the ip+ua traffic is engaged" do
+ SiteSetting.crawler_asns = "12345"
+ base = 30.minutes.ago
+ engaged_session = "engaged-session"
+ 10.times do |i|
+ make_event(asn: 12_345, session_id: engaged_session, created_at: base + i.minutes)
+ end
+ Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, scroll_events: 8)
+
+ abandoned_tab = make_event(asn: 12_345, session_id: "abandoned-tab", created_at: base)
+
+ score!
+
+ expect(abandoned_tab.reload.score).to eq(30)
+ expect(abandoned_tab.browser_pageview_event_score.engagement_score).to eq(0)
+ end
+
+ it "scores partially engaged ip+ua traffic at the lower band" do
+ SiteSetting.crawler_asns = "12345"
+ base = 30.minutes.ago
+ engaged_session = "partly-engaged"
+ make_event(asn: 12_345, session_id: engaged_session, created_at: base)
+ Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, click_events: 2)
+
+ 3.times do |i|
+ make_event(asn: 12_345, session_id: "unengaged-#{i}", created_at: base + (i + 1).minutes)
+ end
+
+ score!
+
+ expect(BrowserPageviewEvent.where(session_id: "unengaged-0").first.score).to eq(50)
+ expect(
+ BrowserPageviewEvent
+ .where(session_id: "unengaged-0")
+ .first
+ .browser_pageview_event_score
+ .engagement_score,
+ ).to eq(20)
end
it "scores each source but partitions velocity so transports do not inflate each other" do