A platform for community discussion. Free, open, simple.
FIX: Measure crawler engagement ratio over a 6 hour (#42262)
Krzysztof Kotlarek66e03cea1d26
diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rbindex fffa92c0b6f..05aa6507913 100644--- a/lib/crawler_scorer.rb+++ b/lib/crawler_scorer.rb@@ -42,6 +42,7 @@ class CrawlerScorerMISSING_ENGAGEMENT_HIGH_RATIO = 0.9MISSING_ENGAGEMENT_LOW_SCORE = 20MISSING_ENGAGEMENT_HIGH_SCORE = 40+ ENGAGEMENT_LOOKBACK = 6.hoursdef self.score!(window_start:, window_end:)crawler_asns = SiteSetting.crawler_asns_map.map(&:to_i)@@ -87,6 +88,7 @@ class CrawlerScorerreferrer_high_ratio: REFERRER_HIGH_RATIO,referrer_low_score: REFERRER_LOW_SCORE,referrer_high_score: REFERRER_HIGH_SCORE,+ engagement_lookback_start: window_end - ENGAGEMENT_LOOKBACK,missing_engagement_low_ratio: MISSING_ENGAGEMENT_LOW_RATIO,missing_engagement_high_ratio: MISSING_ENGAGEMENT_HIGH_RATIO,missing_engagement_low_score: MISSING_ENGAGEMENT_LOW_SCORE,@@ -111,13 +113,32 @@ class CrawlerScorerFROM browser_pageview_events eLEFT JOIN browser_pageview_session_engagements seON se.session_id = e.session_id- AND (- #{BrowserPageviewSessionEngagement::INTERACTION_COLUMNS.map { |column| "se.#{column} > 0" }.join(" OR ")}- )WHERE e.created_at >= :window_startAND e.created_at < :window_end),+ ipua_engagement AS (+ SELECT+ e.ip_address,+ e.user_agent,+ e.source,+ AVG(CASE WHEN se.session_id IS NOT NULL THEN 0.0 ELSE 1.0 END)+ AS no_engagement_ratio+ FROM browser_pageview_events e+ LEFT JOIN browser_pageview_session_engagements se+ ON se.session_id = e.session_id+ WHERE e.created_at >= :engagement_lookback_start+ AND e.created_at < :window_end+ AND EXISTS (+ SELECT 1+ FROM events w+ WHERE w.ip_address = e.ip_address+ AND w.user_agent = e.user_agent+ AND w.source = e.source+ )+ GROUP BY e.ip_address, e.user_agent, e.source+ ),+-- Per-heuristic stats are partitioned by source as well as ip/ua so that-- pageviews recorded through different transports (e.g. piggyback vs-- beacon) never inflate one another's velocity, churn or navigation gaps.@@ -134,8 +155,7 @@ class CrawlerScorerWHEN substring(referrer from '^https?://([^/]+)') = :hostname THEN 0.0ELSE 1.0END- ) AS bad_referrer_ratio,- AVG(CASE WHEN engaged THEN 0.0 ELSE 1.0 END) AS no_engagement_ratio+ ) AS bad_referrer_ratioFROM eventsGROUP BY ip_address, user_agent, source),@@ -246,14 +266,15 @@ class CrawlerScorerEND AS referrer_score,CASEWHEN e.engaged THEN 0- WHEN iu.no_engagement_ratio >= :missing_engagement_high_ratio+ WHEN ie.no_engagement_ratio >= :missing_engagement_high_ratioTHEN :missing_engagement_high_score- WHEN iu.no_engagement_ratio >= :missing_engagement_low_ratio+ WHEN ie.no_engagement_ratio >= :missing_engagement_low_ratioTHEN :missing_engagement_low_scoreELSE 0END AS engagement_scoreFROM events eLEFT JOIN ipua_stats iu USING (ip_address, user_agent, source)+ LEFT JOIN ipua_engagement ie USING (ip_address, user_agent, source)LEFT JOIN median_gap mg USING (ip_address, user_agent, source)LEFT JOIN session_stats ss USING (session_id, source)),diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rbindex e4b331fdb6b..266b7d9fd11 100644--- a/spec/lib/crawler_scorer_spec.rb+++ b/spec/lib/crawler_scorer_spec.rb@@ -318,7 +318,7 @@ RSpec.describe CrawlerScorer doexpect(breakdown.engagement_score).to eq(0)end- it "penalises engagement rows crafted without interaction counts" do+ it "treats any engagement row as engagement, since the client only beacons on activity" doevent = make_event(user_agent: "Mozilla/5.0 HeadlessChrome/120.0.0.0")Fabricate(:browser_pageview_session_engagement,@@ -328,8 +328,8 @@ RSpec.describe CrawlerScorer doscore!- expect(event.reload.score).to eq(140)- expect(event.browser_pageview_event_score.engagement_score).to eq(40)+ expect(event.reload.score).to eq(100)+ expect(event.browser_pageview_event_score.engagement_score).to eq(0)endit "lowers the score when an engagement beacon arrives after an earlier run" do@@ -363,6 +363,50 @@ RSpec.describe CrawlerScorer doexpect(abandoned_tab.browser_pageview_event_score.engagement_score).to eq(0)end+ it "dilutes the ratio with engaged traffic that precedes the scoring window" do+ SiteSetting.crawler_asns = "12345"+ engaged_session = "earlier-engaged"+ 10.times do |i|+ make_event(asn: 12_345, session_id: engaged_session, created_at: 3.hours.ago + i.minutes)+ end+ Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, scroll_events: 8)++ tail = 3.times.map { |i| make_event(asn: 12_345, created_at: 30.minutes.ago + i.minutes) }++ score!++ tail.each do |event|+ expect(event.reload.score).to eq(30)+ expect(event.browser_pageview_event_score.engagement_score).to eq(0)+ end+ end++ it "still penalises a single unengaged request with no other traffic on the horizon" do+ SiteSetting.crawler_asns = "12345"+ event = make_event(asn: 12_345)++ score!++ expect(event.reload.score).to eq(70)+ expect(event.browser_pageview_event_score.engagement_score).to eq(40)+ end++ it "ignores engaged traffic older than the lookback horizon" do+ SiteSetting.crawler_asns = "12345"+ engaged_session = "long-ago-engaged"+ 10.times do |i|+ make_event(asn: 12_345, session_id: engaged_session, created_at: 7.hours.ago + i.minutes)+ end+ Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, scroll_events: 8)++ event = make_event(asn: 12_345)++ score!++ expect(event.reload.score).to eq(70)+ expect(event.browser_pageview_event_score.engagement_score).to eq(40)+ end+it "scores partially engaged ip+ua traffic at the lower band" doSiteSetting.crawler_asns = "12345"base = 30.minutes.ago