Skip to content

A platform for community discussion. Free, open, simple.

FIX: Measure crawler engagement ratio over a 6 hour (#42262)

Krzysztof Kotlarek66e03cea1d26
diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rb
index fffa92c0b6f..05aa6507913 100644
--- a/lib/crawler_scorer.rb
+++ b/lib/crawler_scorer.rb
@@ -42,6 +42,7 @@ class CrawlerScorer
MISSING_ENGAGEMENT_HIGH_RATIO = 0.9
MISSING_ENGAGEMENT_LOW_SCORE = 20
MISSING_ENGAGEMENT_HIGH_SCORE = 40
+ ENGAGEMENT_LOOKBACK = 6.hours
def self.score!(window_start:, window_end:)
crawler_asns = SiteSetting.crawler_asns_map.map(&:to_i)
@@ -87,6 +88,7 @@ class CrawlerScorer
referrer_high_ratio: REFERRER_HIGH_RATIO,
referrer_low_score: REFERRER_LOW_SCORE,
referrer_high_score: REFERRER_HIGH_SCORE,
+ engagement_lookback_start: window_end - ENGAGEMENT_LOOKBACK,
missing_engagement_low_ratio: MISSING_ENGAGEMENT_LOW_RATIO,
missing_engagement_high_ratio: MISSING_ENGAGEMENT_HIGH_RATIO,
missing_engagement_low_score: MISSING_ENGAGEMENT_LOW_SCORE,
@@ -111,13 +113,32 @@ class CrawlerScorer
FROM browser_pageview_events e
LEFT JOIN browser_pageview_session_engagements se
ON se.session_id = e.session_id
- AND (
- #{BrowserPageviewSessionEngagement::INTERACTION_COLUMNS.map { |column| "se.#{column} > 0" }.join(" OR ")}
- )
WHERE e.created_at >= :window_start
AND e.created_at < :window_end
),
+ ipua_engagement AS (
+ SELECT
+ e.ip_address,
+ e.user_agent,
+ e.source,
+ AVG(CASE WHEN se.session_id IS NOT NULL THEN 0.0 ELSE 1.0 END)
+ AS no_engagement_ratio
+ FROM browser_pageview_events e
+ LEFT JOIN browser_pageview_session_engagements se
+ ON se.session_id = e.session_id
+ WHERE e.created_at >= :engagement_lookback_start
+ AND e.created_at < :window_end
+ AND EXISTS (
+ SELECT 1
+ FROM events w
+ WHERE w.ip_address = e.ip_address
+ AND w.user_agent = e.user_agent
+ AND w.source = e.source
+ )
+ GROUP BY e.ip_address, e.user_agent, e.source
+ ),
+
-- Per-heuristic stats are partitioned by source as well as ip/ua so that
-- pageviews recorded through different transports (e.g. piggyback vs
-- beacon) never inflate one another's velocity, churn or navigation gaps.
@@ -134,8 +155,7 @@ class CrawlerScorer
WHEN substring(referrer from '^https?://([^/]+)') = :hostname THEN 0.0
ELSE 1.0
END
- ) AS bad_referrer_ratio,
- AVG(CASE WHEN engaged THEN 0.0 ELSE 1.0 END) AS no_engagement_ratio
+ ) AS bad_referrer_ratio
FROM events
GROUP BY ip_address, user_agent, source
),
@@ -246,14 +266,15 @@ class CrawlerScorer
END AS referrer_score,
CASE
WHEN e.engaged THEN 0
- WHEN iu.no_engagement_ratio >= :missing_engagement_high_ratio
+ WHEN ie.no_engagement_ratio >= :missing_engagement_high_ratio
THEN :missing_engagement_high_score
- WHEN iu.no_engagement_ratio >= :missing_engagement_low_ratio
+ WHEN ie.no_engagement_ratio >= :missing_engagement_low_ratio
THEN :missing_engagement_low_score
ELSE 0
END AS engagement_score
FROM events e
LEFT JOIN ipua_stats iu USING (ip_address, user_agent, source)
+ LEFT JOIN ipua_engagement ie USING (ip_address, user_agent, source)
LEFT JOIN median_gap mg USING (ip_address, user_agent, source)
LEFT JOIN session_stats ss USING (session_id, source)
),
diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rb
index e4b331fdb6b..266b7d9fd11 100644
--- a/spec/lib/crawler_scorer_spec.rb
+++ b/spec/lib/crawler_scorer_spec.rb
@@ -318,7 +318,7 @@ RSpec.describe CrawlerScorer do
expect(breakdown.engagement_score).to eq(0)
end
- it "penalises engagement rows crafted without interaction counts" do
+ it "treats any engagement row as engagement, since the client only beacons on activity" do
event = make_event(user_agent: "Mozilla/5.0 HeadlessChrome/120.0.0.0")
Fabricate(
:browser_pageview_session_engagement,
@@ -328,8 +328,8 @@ RSpec.describe CrawlerScorer do
score!
- expect(event.reload.score).to eq(140)
- expect(event.browser_pageview_event_score.engagement_score).to eq(40)
+ expect(event.reload.score).to eq(100)
+ expect(event.browser_pageview_event_score.engagement_score).to eq(0)
end
it "lowers the score when an engagement beacon arrives after an earlier run" do
@@ -363,6 +363,50 @@ RSpec.describe CrawlerScorer do
expect(abandoned_tab.browser_pageview_event_score.engagement_score).to eq(0)
end
+ it "dilutes the ratio with engaged traffic that precedes the scoring window" do
+ SiteSetting.crawler_asns = "12345"
+ engaged_session = "earlier-engaged"
+ 10.times do |i|
+ make_event(asn: 12_345, session_id: engaged_session, created_at: 3.hours.ago + i.minutes)
+ end
+ Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, scroll_events: 8)
+
+ tail = 3.times.map { |i| make_event(asn: 12_345, created_at: 30.minutes.ago + i.minutes) }
+
+ score!
+
+ tail.each do |event|
+ expect(event.reload.score).to eq(30)
+ expect(event.browser_pageview_event_score.engagement_score).to eq(0)
+ end
+ end
+
+ it "still penalises a single unengaged request with no other traffic on the horizon" do
+ SiteSetting.crawler_asns = "12345"
+ event = make_event(asn: 12_345)
+
+ score!
+
+ expect(event.reload.score).to eq(70)
+ expect(event.browser_pageview_event_score.engagement_score).to eq(40)
+ end
+
+ it "ignores engaged traffic older than the lookback horizon" do
+ SiteSetting.crawler_asns = "12345"
+ engaged_session = "long-ago-engaged"
+ 10.times do |i|
+ make_event(asn: 12_345, session_id: engaged_session, created_at: 7.hours.ago + i.minutes)
+ end
+ Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, scroll_events: 8)
+
+ event = make_event(asn: 12_345)
+
+ score!
+
+ expect(event.reload.score).to eq(70)
+ expect(event.browser_pageview_event_score.engagement_score).to eq(40)
+ end
+
it "scores partially engaged ip+ua traffic at the lower band" do
SiteSetting.crawler_asns = "12345"
base = 30.minutes.ago