A platform for community discussion. Free, open, simple.
FIX: Group Site Traffic Explorer locales by language (#43251)
Alan Guo Xiang Tanc41d5ede59f7
diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rbindex a9f32592848..a70c4338da1 100644--- a/app/models/browser_pageview_event.rb+++ b/app/models/browser_pageview_event.rb@@ -6,6 +6,7 @@ class BrowserPageviewEvent < ActiveRecord::BaseMAX_REFERRER_LENGTH = 2000MAX_USER_AGENT_LENGTH = 1000MAX_LANGUAGE_LENGTH = 255+ MAX_NORMALIZED_LANGUAGE_LENGTH = 13MAX_NORMALIZED_REFERRER_LENGTH = 2000MAX_NORMALIZED_URL_LENGTH = 2000RETENTION_PERIOD = 3.months@@ -191,6 +192,7 @@ class BrowserPageviewEvent < ActiveRecord::Basenormalized_referrer = BrowserPageviewEventUrlNormalizer.normalize_referrer(payload[:referrer])normalized_url = BrowserPageviewEventUrlNormalizer.normalize_site_path(payload[:url])user_agent = payload[:user_agent]&.slice(0, MAX_USER_AGENT_LENGTH)+ language = payload[:language]&.slice(0, MAX_LANGUAGE_LENGTH){url: payload[:url]&.slice(0, MAX_URL_LENGTH),@@ -203,7 +205,8 @@ class BrowserPageviewEvent < ActiveRecord::Basenormalized_referrer: normalized_referrer&.slice(0, MAX_NORMALIZED_REFERRER_LENGTH),normalized_referrer_version: BrowserPageviewEventUrlNormalizer::REFERRER_VERSION,user_agent: user_agent,- language: payload[:language]&.slice(0, MAX_LANGUAGE_LENGTH),+ language: language,+ normalized_language: BrowserPageviewEventLanguageNormalizer.normalize(language),browser: BROWSERS.fetch(BrowserDetection.browser(user_agent), BROWSER_UNKNOWN),session_id: payload[:session_id]&.slice(0, MAX_SESSION_ID_LENGTH),user_id: payload[:user_id],@@ -273,6 +276,9 @@ class BrowserPageviewEvent < ActiveRecord::Baseself.referrer = referrer.slice(0, MAX_REFERRER_LENGTH) if referrer.present?self.user_agent = user_agent.slice(0, MAX_USER_AGENT_LENGTH) if user_agent.present?self.language = language.slice(0, MAX_LANGUAGE_LENGTH) if language.present?+ if normalized_language.present?+ self.normalized_language = normalized_language.slice(0, MAX_NORMALIZED_LANGUAGE_LENGTH)+ endself.session_id = session_id.slice(0, MAX_SESSION_ID_LENGTH) if session_id.present?if normalized_referrer.present?self.normalized_referrer = normalized_referrer.slice(0, MAX_NORMALIZED_REFERRER_LENGTH)@@ -293,6 +299,7 @@ end# country_code :string(2)# ip_address :inet not null# language :string(255)+# normalized_language :string# normalized_referrer :string(2000)# normalized_referrer_version :integer# normalized_url :string(2000)diff --git a/app/services/admin_dashboard_site_traffic_explorer.rb b/app/services/admin_dashboard_site_traffic_explorer.rbindex 5c63d5c95cb..afc9f2fc003 100644--- a/app/services/admin_dashboard_site_traffic_explorer.rb+++ b/app/services/admin_dashboard_site_traffic_explorer.rb@@ -104,7 +104,7 @@ class AdminDashboardSiteTrafficExplorerif language.length > BrowserPageviewEvent::MAX_LANGUAGE_LENGTHraise Discourse::InvalidParameters.new(key)end- language+ BrowserPageviewEventLanguageNormalizer.normalize(language)when :ipIPAddr.new(value.to_s).to_swhen :referrer@@ -235,7 +235,7 @@ class AdminDashboardSiteTrafficExplorerbpe.asn,bpe.ip_address,COALESCE(bpe.browser, #{BrowserPageviewEvent::BROWSER_UNKNOWN}) AS browser,- COALESCE(bpe.language, '') AS language,+ COALESCE(bpe.normalized_language, '') AS language,bpe.scoreFROM browser_pageview_events bpeWHERE bpe.created_at >= :start_datediff --git a/db/migrate/20260904063128_add_normalized_language_to_browser_pageview_events.rb b/db/migrate/20260904063128_add_normalized_language_to_browser_pageview_events.rbnew file mode 100644index 00000000000..7e1f0c2226b--- /dev/null+++ b/db/migrate/20260904063128_add_normalized_language_to_browser_pageview_events.rb@@ -0,0 +1,7 @@+# frozen_string_literal: true++class AddNormalizedLanguageToBrowserPageviewEvents < ActiveRecord::Migration[8.0]+ def change+ add_column :browser_pageview_events, :normalized_language, :string+ end+enddiff --git a/db/post_migrate/20260904065041_restart_browser_pageview_language_tracking.rb b/db/post_migrate/20260904065041_restart_browser_pageview_language_tracking.rbnew file mode 100644index 00000000000..f558ab0d966--- /dev/null+++ b/db/post_migrate/20260904065041_restart_browser_pageview_language_tracking.rb@@ -0,0 +1,17 @@+# frozen_string_literal: true++class RestartBrowserPageviewLanguageTracking < ActiveRecord::Migration[8.0]+ def up+ execute <<~SQL+ UPDATE browser_pageview_events+ SET language = NULL,+ normalized_language = NULL+ WHERE language IS NOT NULL+ OR normalized_language IS NOT NULL+ SQL+ end++ def down+ raise ActiveRecord::IrreversibleMigration+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex cbcc0b67800..b3e29002634 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -2112,7 +2112,8 @@ CREATE TABLE public.browser_pageview_events (normalized_url character varying(2000),normalized_url_version integer,browser smallint,- language character varying(255)+ language character varying(255),+ normalized_language character varying);@@ -24407,6 +24408,8 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groupsSET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES+('20260904065041'),+('20260904063128'),('20260903065141'),('20260902150024'),('20260901020329'),diff --git a/lib/browser_pageview_event_language_normalizer.rb b/lib/browser_pageview_event_language_normalizer.rbnew file mode 100644index 00000000000..11a12bd0a5b--- /dev/null+++ b/lib/browser_pageview_event_language_normalizer.rb@@ -0,0 +1,13 @@+# frozen_string_literal: true++class BrowserPageviewEventLanguageNormalizer+ def self.normalize(language)+ return if language.nil?++ tag = I18n::Locale::Tag::Rfc4646.tag(language.tr("_", "-"))+ return "" if tag&.language.nil?++ primary_language = tag.language.split("-", 2).first+ [primary_language, tag.script].compact.join("-")+ end+enddiff --git a/spec/fabricators/browser_pageview_event_fabricator.rb b/spec/fabricators/browser_pageview_event_fabricator.rbindex 87110c2dc3a..3a01181d1e8 100644--- a/spec/fabricators/browser_pageview_event_fabricator.rb+++ b/spec/fabricators/browser_pageview_event_fabricator.rb@@ -12,6 +12,7 @@ Fabricator(:browser_pageview_event) doBrowserPageviewEvent::BROWSER_UNKNOWN,)end+ normalized_language { |attrs| BrowserPageviewEventLanguageNormalizer.normalize(attrs[:language]) }session_id { SecureRandom.hex(16) }enddiff --git a/spec/lib/browser_pageview_event_language_normalizer_spec.rb b/spec/lib/browser_pageview_event_language_normalizer_spec.rbnew file mode 100644index 00000000000..600a76ef860--- /dev/null+++ b/spec/lib/browser_pageview_event_language_normalizer_spec.rb@@ -0,0 +1,53 @@+# frozen_string_literal: true++RSpec.describe BrowserPageviewEventLanguageNormalizer do+ describe ".normalize" do+ it "returns nil when no language was recorded" do+ expect(described_class.normalize(nil)).to be_nil+ end++ it "keeps an empty recorded language empty" do+ expect(described_class.normalize("")).to eq("")+ end++ it "removes the region from Spanish" do+ expect(described_class.normalize("es-MX")).to eq("es")+ end++ it "canonicalizes casing and underscore separators" do+ expect(described_class.normalize("EN_us")).to eq("en")+ end++ it "preserves the Traditional Chinese script" do+ expect(described_class.normalize("zh-Hant-TW")).to eq("zh-Hant")+ end++ it "preserves the Simplified Chinese script and removes an extlang" do+ expect(described_class.normalize("zh-cmn-Hans-CN")).to eq("zh-Hans")+ end++ it "preserves the Serbian Cyrillic script" do+ expect(described_class.normalize("sr-Cyrl-RS")).to eq("sr-Cyrl")+ end++ it "preserves the Serbian Latin script" do+ expect(described_class.normalize("sr-Latn-RS")).to eq("sr-Latn")+ end++ it "preserves the Punjabi Gurmukhi script" do+ expect(described_class.normalize("pa-Guru-IN")).to eq("pa-Guru")+ end++ it "preserves the Punjabi Arabic script" do+ expect(described_class.normalize("pa-Arab-PK")).to eq("pa-Arab")+ end++ it "removes variants from Slovenian" do+ expect(described_class.normalize("sl-rozaj-biske-1994")).to eq("sl")+ end++ it "removes Unicode extensions from English" do+ expect(described_class.normalize("en-US-u-hc-h12")).to eq("en")+ end+ end+enddiff --git a/spec/models/browser_pageview_event_spec.rb b/spec/models/browser_pageview_event_spec.rbindex ac03373b6bc..7e0d07ee0d8 100644--- a/spec/models/browser_pageview_event_spec.rb+++ b/spec/models/browser_pageview_event_spec.rb@@ -189,6 +189,7 @@ RSpec.describe BrowserPageviewEvent doreferrer: "a" * (described_class::MAX_REFERRER_LENGTH + 1),user_agent: "a" * (described_class::MAX_USER_AGENT_LENGTH + 1),language: "a" * (described_class::MAX_LANGUAGE_LENGTH + 1),+ normalized_language: "a" * (described_class::MAX_NORMALIZED_LANGUAGE_LENGTH + 1),ip_address: "1.2.3.4",session_id: "a" * (described_class::MAX_SESSION_ID_LENGTH + 1),normalized_referrer: "a" * (described_class::MAX_NORMALIZED_REFERRER_LENGTH + 1),@@ -198,6 +199,7 @@ RSpec.describe BrowserPageviewEvent doexpect(event.referrer.length).to eq(described_class::MAX_REFERRER_LENGTH)expect(event.user_agent.length).to eq(described_class::MAX_USER_AGENT_LENGTH)expect(event.language.length).to eq(described_class::MAX_LANGUAGE_LENGTH)+ expect(event.normalized_language.length).to eq(described_class::MAX_NORMALIZED_LANGUAGE_LENGTH)expect(event.session_id.length).to eq(described_class::MAX_SESSION_ID_LENGTH)expect(event.normalized_referrer.length).to eq(described_class::MAX_NORMALIZED_REFERRER_LENGTH)end@@ -249,6 +251,7 @@ RSpec.describe BrowserPageviewEvent doexpect(event.source).to eq("beacon")expect(event.browser).to eq("edge")expect(event.language).to eq("en-AU")+ expect(event.normalized_language).to eq("en")expect(described_class.queued_count).to eq(0)enddiff --git a/spec/services/admin_dashboard_site_traffic_explorer_spec.rb b/spec/services/admin_dashboard_site_traffic_explorer_spec.rbindex a361b25df17..3edc2b1a3e9 100644--- a/spec/services/admin_dashboard_site_traffic_explorer_spec.rb+++ b/spec/services/admin_dashboard_site_traffic_explorer_spec.rb@@ -36,7 +36,19 @@ RSpec.describe AdminDashboardSiteTrafficExplorer doendlet(:languages) do- [nil, "", nil, "fr", "en-US", "en-US", "en-US", "fr", "de", "zh-CN", "en-US"]+ [+ nil,+ "",+ nil,+ "fr-FR",+ "en-US",+ "en-GB",+ "en-US",+ "fr-CA",+ "sr-Latn-RS",+ "zh-cmn-Hant-TW",+ "en-US",+ ]endlet!(:pageviews) do@@ -116,13 +128,22 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do{ value: "firefox", label: "Firefox", pageviews: 1 },],"languages" => [- { value: "en-US", label: "en-US", pageviews: 4 },+ { value: "en", label: "en", pageviews: 4 },{ value: "", label: "Unknown", pageviews: 3 },{ value: "fr", label: "fr", pageviews: 2 },- { value: "de", label: "de", pageviews: 1 },- { value: "zh-CN", label: "zh-CN", pageviews: 1 },+ { value: "sr-Latn", label: "sr-Latn", pageviews: 1 },+ { value: "zh-Hant", label: "zh-Hant", pageviews: 1 },],)++ regional_language_traffic =+ described_class.call(params: params.merge(language: "en-US")).traffic+ expect(regional_language_traffic.fetch(:dimensions).fetch("languages")).to eq(+ [{ value: "en", label: "en", pageviews: 4 }],+ )+ expect(regional_language_traffic.fetch(:active_filters)).to eq(+ [{ key: :language, value: "en", label: "en" }],+ )endit "applies all filters to every dimension" do@@ -174,7 +195,7 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do"countries" => [{ value: "US", label: "United States", pageviews: 3 }],"networks" => [{ value: "AS64496", label: "Example Network (AS64496)", pageviews: 3 }],"browsers" => [{ value: "chrome", label: "Google Chrome", pageviews: 3 }],- "languages" => [{ value: "en-US", label: "en-US", pageviews: 3 }],+ "languages" => [{ value: "en", label: "en", pageviews: 3 }],"ip_addresses" => [{ value: "192.0.2.5", label: "192.0.2.5", pageviews: 1 },{ value: "192.0.2.6", label: "192.0.2.6", pageviews: 1 },diff --git a/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb b/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rbindex 99894b1a360..dc8c24e65b7 100644--- a/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb+++ b/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb@@ -333,12 +333,11 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" dotraffic.remove_filter("browser")traffic.select_tab(card: "visitors", tab: "Languages")- expect(traffic).to have_row(card: "visitors", label: "English (United States)", count: "2")- expect(traffic).to have_row(card: "visitors", label: "English (United Kingdom)", count: "1")- traffic.filter_row(card: "visitors", label: "English (United States)")- expect(traffic).to have_filter_pill(dimension: "language", label: "English (United States)")+ expect(traffic).to have_row(card: "visitors", label: "English", count: "3")+ traffic.filter_row(card: "visitors", label: "English")+ expect(traffic).to have_filter_pill(dimension: "language", label: "English")expect(page).to have_current_path(- "/admin/dashboard/site-traffic-explorer?end_date=2026-05-12&language=en-US&range=custom&start_date=2026-05-01",+ "/admin/dashboard/site-traffic-explorer?end_date=2026-05-12&language=en&range=custom&start_date=2026-05-01",)traffic.remove_filter("language")