A platform for community discussion. Free, open, simple.
FEATURE: Route full app embed pageviews into a dedicated `page_view_embed` counter (#39535)
Rafael dos Santos Silva647076fea09e
diff --git a/app/models/application_request.rb b/app/models/application_request.rbindex 9fd33152647..52e953e5d6f 100644--- a/app/models/application_request.rb+++ b/app/models/application_request.rb@@ -23,6 +23,7 @@ class ApplicationRequest < ActiveRecord::Basepage_view_anon_browser_mobile_beacon: 18,page_view_logged_in_browser_beacon: 19,page_view_logged_in_browser_mobile_beacon: 20,+ page_view_embed: 21,}include CachedCountingdiff --git a/app/models/concerns/reports/site_traffic.rb b/app/models/concerns/reports/site_traffic.rbindex b3b3c658d9c..9aa63278ffc 100644--- a/app/models/concerns/reports/site_traffic.rb+++ b/app/models/concerns/reports/site_traffic.rb@@ -25,6 +25,7 @@ module Reports::SiteTrafficSUM(CASE WHEN req_type = :page_view_logged_in_browser THEN count ELSE 0 END) AS page_view_logged_in_browser,SUM(CASE WHEN req_type = :page_view_anon_browser THEN count ELSE 0 END) AS page_view_anon_browser,SUM(CASE WHEN req_type = :page_view_crawler THEN count ELSE 0 END) AS page_view_crawler,+ SUM(CASE WHEN req_type = :page_view_embed THEN count ELSE 0 END) AS page_view_embed,SUM(CASE WHEN req_type = :page_view_anon THEN countWHEN req_type = :page_view_logged_in THEN count@@ -46,6 +47,7 @@ module Reports::SiteTrafficpage_view_logged_in: ApplicationRequest.req_types[:page_view_logged_in],page_view_anon_browser: ApplicationRequest.req_types[:page_view_anon_browser],page_view_logged_in_browser: ApplicationRequest.req_types[:page_view_logged_in_browser],+ page_view_embed: ApplicationRequest.req_types[:page_view_embed],first_browser_pageview_date: first_browser_pageview_date,)@@ -68,6 +70,12 @@ module Reports::SiteTrafficcolor: report.colors[:purple],data: data.map { |row| { x: row.date, y: row.page_view_crawler } },},+ {+ req: "page_view_embed",+ label: I18n.t("reports.site_traffic.xaxis.page_view_embed"),+ color: report.colors[:yellow],+ data: data.map { |row| { x: row.date, y: row.page_view_embed } },+ },{req: "page_view_other",label: I18n.t("reports.site_traffic.xaxis.page_view_other"),diff --git a/config/locales/server.en.yml b/config/locales/server.en.ymlindex 53f75437a92..0b8378cbb12 100644--- a/config/locales/server.en.yml+++ b/config/locales/server.en.yml@@ -1425,6 +1425,7 @@ en:page_view_anon_browser: "Pageviews (anonymous)"page_view_logged_in_browser: "Pageviews (logged in)"page_view_crawler: "Known crawlers"+ page_view_embed: "Embedded pageviews"page_view_other: "Other traffic"yaxis: "Day"description: "Pageviews for logged in users, anonymous users, known crawlers and other traffic."@@ -1604,6 +1605,11 @@ en:xaxis: "Day"yaxis: "Logged In Browser Pageviews"description: "Number of pageviews by logged-in visitors using real browsers."+ page_view_embed_reqs:+ title: "Embedded pageviews"+ xaxis: "Day"+ yaxis: "Embedded pageviews"+ description: "Number of pageviews served inside the full app embed iframe."http_background_reqs:title: "Background"xaxis: "Day"diff --git a/frontend/discourse/admin/controllers/admin-reports/show.js b/frontend/discourse/admin/controllers/admin-reports/show.jsindex 8a2dcf8f95d..c126ebcf6a7 100644--- a/frontend/discourse/admin/controllers/admin-reports/show.js+++ b/frontend/discourse/admin/controllers/admin-reports/show.js@@ -25,7 +25,11 @@ export default class AdminReportsShowController extends Controller {if (this.model?.type === "site_traffic") {options.stackedChart = {- hiddenLabels: ["page_view_other", "page_view_crawler"],+ hiddenLabels: [+ "page_view_other",+ "page_view_crawler",+ "page_view_embed",+ ],};}diff --git a/frontend/discourse/admin/controllers/admin/dashboard/general.js b/frontend/discourse/admin/controllers/admin/dashboard/general.jsindex a9b2a4fafd7..ede45856384 100644--- a/frontend/discourse/admin/controllers/admin/dashboard/general.js+++ b/frontend/discourse/admin/controllers/admin/dashboard/general.js@@ -108,7 +108,13 @@ export default class AdminDashboardGeneralController extends AdminDashboardTabCo@computedget siteTrafficOptions() {return {- stackedChart: { hiddenLabels: ["page_view_other", "page_view_crawler"] },+ stackedChart: {+ hiddenLabels: [+ "page_view_other",+ "page_view_crawler",+ "page_view_embed",+ ],+ },};}diff --git a/frontend/discourse/app/lib/ajax.js b/frontend/discourse/app/lib/ajax.jsindex de048e90cd3..55b5c31ece2 100644--- a/frontend/discourse/app/lib/ajax.js+++ b/frontend/discourse/app/lib/ajax.js@@ -1,6 +1,7 @@/* eslint-disable ember/no-jquery */import { run } from "@ember/runloop";import $ from "jquery";+import EmbedMode from "discourse/lib/embed-mode";import { isTesting } from "discourse/lib/environment";import getURL from "discourse/lib/get-url";import DiscourseURL from "discourse/lib/url";@@ -123,6 +124,10 @@ export function ajax() {_trackView = false;args.headers["Discourse-Track-View"] = "true";+ if (EmbedMode.enabled) {+ args.headers["Discourse-Track-View-Embed"] = "true";+ }+if (_trackingSessionId) {args.headers["Discourse-Track-View-Session-Id"] = _trackingSessionId;_trackingSessionId = null;diff --git a/frontend/discourse/app/lib/beacon-pageview.js b/frontend/discourse/app/lib/beacon-pageview.jsindex 92d72965dbc..99f44d16c14 100644--- a/frontend/discourse/app/lib/beacon-pageview.js+++ b/frontend/discourse/app/lib/beacon-pageview.js@@ -1,3 +1,4 @@+import EmbedMode from "discourse/lib/embed-mode";import getURL from "discourse/lib/get-url";let _pendingBeaconRequests = 0;@@ -15,6 +16,9 @@ export function sendBeaconPageview({ sessionId, url, referrer, topicId }) {if (topicId) {body.topic_id = topicId;}+ if (EmbedMode.enabled) {+ body.embed = true;+ }_pendingBeaconRequests++;fetch(getURL("/srv/pv"), {diff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rbindex 3ca470926ae..0f0a4d9dfd2 100644--- a/lib/middleware/request_tracker.rb+++ b/lib/middleware/request_tracker.rb@@ -106,6 +106,9 @@ class Middleware::RequestTrackerif data[:is_crawler]ApplicationRequest.increment!(:page_view_crawler)WebCrawlerRequest.increment!(data[:user_agent])+ elsif data[:is_embed]+ # Embed pageviews are counted in the browser/beacon branches below so+ # community-traffic counters stay unpolluted by iframe traffic.elsif data[:has_auth_cookie]ApplicationRequest.increment!(:page_view_logged_in)ApplicationRequest.increment!(:page_view_logged_in_mobile) if data[:is_mobile]@@ -117,7 +120,9 @@ class Middleware::RequestTrackerif tracks_browser_page_view?(data)if data[:is_beacon]- if data[:has_auth_cookie]+ if data[:is_embed]+ ApplicationRequest.increment!(:page_view_embed)+ elsif data[:has_auth_cookie]ApplicationRequest.increment!(:page_view_logged_in_browser_beacon)if data[:is_mobile]ApplicationRequest.increment!(:page_view_logged_in_browser_mobile_beacon)@@ -128,7 +133,9 @@ class Middleware::RequestTrackerendtrigger_beacon_browser_pageview_event(data)else- if data[:has_auth_cookie]+ if data[:is_embed]+ ApplicationRequest.increment!(:page_view_embed)+ elsif data[:has_auth_cookie]ApplicationRequest.increment!(:page_view_logged_in_browser)ApplicationRequest.increment!(:page_view_logged_in_browser_mobile) if data[:is_mobile]else@@ -138,7 +145,8 @@ class Middleware::RequestTrackertrigger_browser_pageview_event(data)end- if data[:topic_id].present? && (!data[:has_auth_cookie] || data[:current_user_id].present?)+ if data[:topic_id].present? && (!data[:has_auth_cookie] || data[:current_user_id].present?) &&+ !data[:is_embed]TopicsController.defer_topic_view(data[:topic_id],data[:request_remote_ip],@@ -603,12 +611,21 @@ class Middleware::RequestTrackerenv["HTTP_DISCOURSE_TRACK_VIEW_SESSION_ID"]&.slice(0, MAX_SESSION_ID_LENGTH)user_agent = env["HTTP_USER_AGENT"]&.slice(0, MAX_USER_AGENT_LENGTH)+ # An embedded pageview is either an initial HTML load carrying `?embed_mode=true`,+ # or a subsequent XHR from inside the embed iframe which sets the header below.+ # Use `request.GET` (query-string only) rather than `request.params` so a missing+ # or malformed request body cannot raise from here.+ is_embed =+ request.GET["embed_mode"] == "true" ||+ %w[1 true].include?(env["HTTP_DISCOURSE_TRACK_VIEW_EMBED"])+{track_view: track_view,explicit_track_view: explicit_track_view,deferred_track_view: deferred_track_view,implicit_track_view: implicit_track_view,browser_page_view: browser_page_view,+ is_embed: is_embed,topic_id: topic_id,tracking_url: tracking_url,tracking_referrer: tracking_referrer,@@ -650,6 +667,7 @@ class Middleware::RequestTrackertracking_referrer = data["referrer"]&.slice(0, MAX_URL_LENGTH)tracking_session_id = data["session_id"]&.slice(0, MAX_SESSION_ID_LENGTH)user_agent = env["HTTP_USER_AGENT"]&.slice(0, MAX_USER_AGENT_LENGTH)+ is_embed = data["embed"] == true{track_view: false,@@ -658,6 +676,7 @@ class Middleware::RequestTrackerimplicit_track_view: false,browser_page_view: true,is_beacon: true,+ is_embed: is_embed,topic_id: topic_id,tracking_url: tracking_url,tracking_referrer: tracking_referrer,diff --git a/spec/lib/middleware/request_tracker_spec.rb b/spec/lib/middleware/request_tracker_spec.rbindex 5ee7cdf14d3..d4158bb3ec8 100644--- a/spec/lib/middleware/request_tracker_spec.rb+++ b/spec/lib/middleware/request_tracker_spec.rb@@ -267,6 +267,130 @@ RSpec.describe Middleware::RequestTracker doexpect(ApplicationRequest.page_view_anon_browser.first.count).to eq(1)end+ describe "embed mode pageviews" do+ it "does not leak the initial embed HTML load into the legacy page_view_anon counter" do+ data =+ Middleware::RequestTracker.get_data(+ env(path: "/t/topic-slug/1?embed_mode=true"),+ ["200", { "Content-Type" => "text/html" }],+ 0.1,+ )+ Middleware::RequestTracker.log_request(data)+ CachedCounting.flush++ expect(ApplicationRequest.page_view_anon.sum(:count)).to eq(0)+ expect(ApplicationRequest.page_view_embed.sum(:count)).to eq(0)+ end++ it "counts deferred pageview with embed header as page_view_embed" do+ data =+ Middleware::RequestTracker.get_data(+ env(+ :path => "/message-bus/abcde/poll",+ "HTTP_DISCOURSE_TRACK_VIEW_DEFERRED" => "1",+ "HTTP_DISCOURSE_TRACK_VIEW_EMBED" => "true",+ ),+ ["200", { "Content-Type" => "text/html" }],+ 0.1,+ )+ Middleware::RequestTracker.log_request(data)+ CachedCounting.flush++ expect(ApplicationRequest.page_view_embed.first.count).to eq(1)+ expect(ApplicationRequest.page_view_anon_browser.sum(:count)).to eq(0)+ end++ it "counts explicit XHR pageview with embed header only once as page_view_embed" do+ data =+ Middleware::RequestTracker.get_data(+ env("HTTP_DISCOURSE_TRACK_VIEW" => "1", "HTTP_DISCOURSE_TRACK_VIEW_EMBED" => "true"),+ ["200", {}],+ 0.1,+ )+ Middleware::RequestTracker.log_request(data)+ CachedCounting.flush++ expect(ApplicationRequest.page_view_embed.first.count).to eq(1)+ expect(ApplicationRequest.page_view_anon.sum(:count)).to eq(0)+ expect(ApplicationRequest.page_view_anon_browser.sum(:count)).to eq(0)+ end++ it "does not defer a topic view for embed browser pageviews" do+ TopicsController.expects(:defer_topic_view).never+ data =+ Middleware::RequestTracker.get_data(+ env(+ "HTTP_DISCOURSE_TRACK_VIEW" => "1",+ "HTTP_DISCOURSE_TRACK_VIEW_EMBED" => "true",+ "HTTP_DISCOURSE_TRACK_VIEW_TOPIC_ID" => "42",+ ),+ ["200", {}],+ 0.1,+ )+ Middleware::RequestTracker.log_request(data)+ end++ it "counts beacon pageview with embed flag as page_view_embed" do+ SiteSetting.use_beacon_for_browser_page_views = true+ body = {+ session_id: "abc",+ url: "https://example.com/t/slug/1",+ referrer: "https://host.example/page",+ embed: true,+ }.to_json++ data =+ Middleware::RequestTracker.get_data(+ env(+ "REQUEST_METHOD" => "POST",+ :path => Discourse.beacon_pv_tracking_path,+ "rack.input" => StringIO.new(body),+ ),+ ["204", {}],+ 0.1,+ )+ Middleware::RequestTracker.log_request(data)+ CachedCounting.flush++ expect(data[:is_embed]).to eq(true)+ expect(ApplicationRequest.page_view_embed.first.count).to eq(1)+ expect(ApplicationRequest.page_view_anon_browser_beacon.sum(:count)).to eq(0)+ end++ it "survives requests with a missing or unreadable body" do+ # POST without a rack.input must not raise while detecting `is_embed` —+ # we only care about the query string, not the body.+ expect {+ Middleware::RequestTracker.get_data(+ env(+ "REQUEST_METHOD" => "POST",+ :path => "/srv/something?embed_mode=true",+ "rack.input" => nil,+ ),+ ["200", {}],+ 0.1,+ )+ }.not_to raise_error+ end++ it "still counts crawlers as page_view_crawler even on embed URLs" do+ data =+ Middleware::RequestTracker.get_data(+ env(+ :path => "/t/topic-slug/1?embed_mode=true",+ "HTTP_USER_AGENT" => "AdsBot-Google (+http://www.google.com/adsbot.html)",+ ),+ ["200", { "Content-Type" => "text/html" }],+ 0.1,+ )+ Middleware::RequestTracker.log_request(data)+ CachedCounting.flush++ expect(ApplicationRequest.page_view_crawler.first.count).to eq(1)+ expect(ApplicationRequest.page_view_embed.sum(:count)).to eq(0)+ end+ end+it "logs API requests correctly" dodata =Middleware::RequestTracker.get_data(diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rbindex a3284ac0f2c..af967799486 100644--- a/spec/models/report_spec.rb+++ b/spec/models/report_spec.rb@@ -1608,6 +1608,50 @@ RSpec.describe Report doendend+ describe "site_traffic" do+ before do+ freeze_time(Time.now.at_midnight)+ Theme.clear_default!+ end++ let(:reports) { Report.find("site_traffic") }++ context "with no data" do+ it "works" do+ reports.data.each { |report| expect(report[:data]).to be_empty }+ end+ end++ context "with data" do+ before do+ CachedCounting.reset+ CachedCounting.enable+ ApplicationRequest.enable+ end++ after do+ CachedCounting.reset+ ApplicationRequest.disable+ CachedCounting.disable+ end++ it "exposes embedded pageviews as their own series without polluting other series" do+ 2.times { ApplicationRequest.increment!(:page_view_anon) }+ 1.times { ApplicationRequest.increment!(:page_view_anon_browser) }+ 3.times { ApplicationRequest.increment!(:page_view_logged_in) }+ 2.times { ApplicationRequest.increment!(:page_view_logged_in_browser) }+ 4.times { ApplicationRequest.increment!(:page_view_embed) }+ CachedCounting.flush++ embed_series = reports.data.find { |r| r[:req] == "page_view_embed" }+ other_series = reports.data.find { |r| r[:req] == "page_view_other" }++ expect(embed_series[:data][0][:y]).to eq(4)+ expect(other_series[:data][0][:y]).to eq(2)+ end+ end+ end+describe "consolidated_page_views" dobefore dofreeze_time(Time.now.at_midnight)diff --git a/spec/queries/reports/list_query_spec.rb b/spec/queries/reports/list_query_spec.rbindex 7d9f8dbf96b..7b9c3af1675 100644--- a/spec/queries/reports/list_query_spec.rb+++ b/spec/queries/reports/list_query_spec.rb@@ -46,6 +46,7 @@ RSpec.describe Reports::ListQuery dopage_view_anon_reqspage_view_anon_browser_reqspage_view_logged_in_browser_reqs+ page_view_embed_reqs],)end@@ -78,6 +79,7 @@ RSpec.describe Reports::ListQuery dopage_view_anon_browser_reqspage_view_logged_in_browser_reqspage_view_legacy_total_reqs+ page_view_embed_reqs],)end