Skip to content

A platform for community discussion. Free, open, simple.

FIX: Group Site Traffic Explorer locales by language (#43251)

Alan Guo Xiang Tanc41d5ede59f7
diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rb
index a9f32592848..a70c4338da1 100644
--- a/app/models/browser_pageview_event.rb
+++ b/app/models/browser_pageview_event.rb
@@ -6,6 +6,7 @@ class BrowserPageviewEvent < ActiveRecord::Base
MAX_REFERRER_LENGTH = 2000
MAX_USER_AGENT_LENGTH = 1000
MAX_LANGUAGE_LENGTH = 255
+ MAX_NORMALIZED_LANGUAGE_LENGTH = 13
MAX_NORMALIZED_REFERRER_LENGTH = 2000
MAX_NORMALIZED_URL_LENGTH = 2000
RETENTION_PERIOD = 3.months
@@ -191,6 +192,7 @@ class BrowserPageviewEvent < ActiveRecord::Base
normalized_referrer = BrowserPageviewEventUrlNormalizer.normalize_referrer(payload[:referrer])
normalized_url = BrowserPageviewEventUrlNormalizer.normalize_site_path(payload[:url])
user_agent = payload[:user_agent]&.slice(0, MAX_USER_AGENT_LENGTH)
+ language = payload[:language]&.slice(0, MAX_LANGUAGE_LENGTH)
{
url: payload[:url]&.slice(0, MAX_URL_LENGTH),
@@ -203,7 +205,8 @@ class BrowserPageviewEvent < ActiveRecord::Base
normalized_referrer: normalized_referrer&.slice(0, MAX_NORMALIZED_REFERRER_LENGTH),
normalized_referrer_version: BrowserPageviewEventUrlNormalizer::REFERRER_VERSION,
user_agent: user_agent,
- language: payload[:language]&.slice(0, MAX_LANGUAGE_LENGTH),
+ language: language,
+ normalized_language: BrowserPageviewEventLanguageNormalizer.normalize(language),
browser: BROWSERS.fetch(BrowserDetection.browser(user_agent), BROWSER_UNKNOWN),
session_id: payload[:session_id]&.slice(0, MAX_SESSION_ID_LENGTH),
user_id: payload[:user_id],
@@ -273,6 +276,9 @@ class BrowserPageviewEvent < ActiveRecord::Base
self.referrer = referrer.slice(0, MAX_REFERRER_LENGTH) if referrer.present?
self.user_agent = user_agent.slice(0, MAX_USER_AGENT_LENGTH) if user_agent.present?
self.language = language.slice(0, MAX_LANGUAGE_LENGTH) if language.present?
+ if normalized_language.present?
+ self.normalized_language = normalized_language.slice(0, MAX_NORMALIZED_LANGUAGE_LENGTH)
+ end
self.session_id = session_id.slice(0, MAX_SESSION_ID_LENGTH) if session_id.present?
if normalized_referrer.present?
self.normalized_referrer = normalized_referrer.slice(0, MAX_NORMALIZED_REFERRER_LENGTH)
@@ -293,6 +299,7 @@ end
# country_code :string(2)
# ip_address :inet not null
# language :string(255)
+# normalized_language :string
# normalized_referrer :string(2000)
# normalized_referrer_version :integer
# normalized_url :string(2000)
diff --git a/app/services/admin_dashboard_site_traffic_explorer.rb b/app/services/admin_dashboard_site_traffic_explorer.rb
index 5c63d5c95cb..afc9f2fc003 100644
--- a/app/services/admin_dashboard_site_traffic_explorer.rb
+++ b/app/services/admin_dashboard_site_traffic_explorer.rb
@@ -104,7 +104,7 @@ class AdminDashboardSiteTrafficExplorer
if language.length > BrowserPageviewEvent::MAX_LANGUAGE_LENGTH
raise Discourse::InvalidParameters.new(key)
end
- language
+ BrowserPageviewEventLanguageNormalizer.normalize(language)
when :ip
IPAddr.new(value.to_s).to_s
when :referrer
@@ -235,7 +235,7 @@ class AdminDashboardSiteTrafficExplorer
bpe.asn,
bpe.ip_address,
COALESCE(bpe.browser, #{BrowserPageviewEvent::BROWSER_UNKNOWN}) AS browser,
- COALESCE(bpe.language, '') AS language,
+ COALESCE(bpe.normalized_language, '') AS language,
bpe.score
FROM browser_pageview_events bpe
WHERE bpe.created_at >= :start_date
diff --git a/db/migrate/20260904063128_add_normalized_language_to_browser_pageview_events.rb b/db/migrate/20260904063128_add_normalized_language_to_browser_pageview_events.rb
new file mode 100644
index 00000000000..7e1f0c2226b
--- /dev/null
+++ b/db/migrate/20260904063128_add_normalized_language_to_browser_pageview_events.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddNormalizedLanguageToBrowserPageviewEvents < ActiveRecord::Migration[8.0]
+ def change
+ add_column :browser_pageview_events, :normalized_language, :string
+ end
+end
diff --git a/db/post_migrate/20260904065041_restart_browser_pageview_language_tracking.rb b/db/post_migrate/20260904065041_restart_browser_pageview_language_tracking.rb
new file mode 100644
index 00000000000..f558ab0d966
--- /dev/null
+++ b/db/post_migrate/20260904065041_restart_browser_pageview_language_tracking.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class RestartBrowserPageviewLanguageTracking < ActiveRecord::Migration[8.0]
+ def up
+ execute <<~SQL
+ UPDATE browser_pageview_events
+ SET language = NULL,
+ normalized_language = NULL
+ WHERE language IS NOT NULL
+ OR normalized_language IS NOT NULL
+ SQL
+ end
+
+ def down
+ raise ActiveRecord::IrreversibleMigration
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index cbcc0b67800..b3e29002634 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -2112,7 +2112,8 @@ CREATE TABLE public.browser_pageview_events (
normalized_url character varying(2000),
normalized_url_version integer,
browser smallint,
- language character varying(255)
+ language character varying(255),
+ normalized_language character varying
);
@@ -24407,6 +24408,8 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20260904065041'),
+('20260904063128'),
('20260903065141'),
('20260902150024'),
('20260901020329'),
diff --git a/lib/browser_pageview_event_language_normalizer.rb b/lib/browser_pageview_event_language_normalizer.rb
new file mode 100644
index 00000000000..11a12bd0a5b
--- /dev/null
+++ b/lib/browser_pageview_event_language_normalizer.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class BrowserPageviewEventLanguageNormalizer
+ def self.normalize(language)
+ return if language.nil?
+
+ tag = I18n::Locale::Tag::Rfc4646.tag(language.tr("_", "-"))
+ return "" if tag&.language.nil?
+
+ primary_language = tag.language.split("-", 2).first
+ [primary_language, tag.script].compact.join("-")
+ end
+end
diff --git a/spec/fabricators/browser_pageview_event_fabricator.rb b/spec/fabricators/browser_pageview_event_fabricator.rb
index 87110c2dc3a..3a01181d1e8 100644
--- a/spec/fabricators/browser_pageview_event_fabricator.rb
+++ b/spec/fabricators/browser_pageview_event_fabricator.rb
@@ -12,6 +12,7 @@ Fabricator(:browser_pageview_event) do
BrowserPageviewEvent::BROWSER_UNKNOWN,
)
end
+ normalized_language { |attrs| BrowserPageviewEventLanguageNormalizer.normalize(attrs[:language]) }
session_id { SecureRandom.hex(16) }
end
diff --git a/spec/lib/browser_pageview_event_language_normalizer_spec.rb b/spec/lib/browser_pageview_event_language_normalizer_spec.rb
new file mode 100644
index 00000000000..600a76ef860
--- /dev/null
+++ b/spec/lib/browser_pageview_event_language_normalizer_spec.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+RSpec.describe BrowserPageviewEventLanguageNormalizer do
+ describe ".normalize" do
+ it "returns nil when no language was recorded" do
+ expect(described_class.normalize(nil)).to be_nil
+ end
+
+ it "keeps an empty recorded language empty" do
+ expect(described_class.normalize("")).to eq("")
+ end
+
+ it "removes the region from Spanish" do
+ expect(described_class.normalize("es-MX")).to eq("es")
+ end
+
+ it "canonicalizes casing and underscore separators" do
+ expect(described_class.normalize("EN_us")).to eq("en")
+ end
+
+ it "preserves the Traditional Chinese script" do
+ expect(described_class.normalize("zh-Hant-TW")).to eq("zh-Hant")
+ end
+
+ it "preserves the Simplified Chinese script and removes an extlang" do
+ expect(described_class.normalize("zh-cmn-Hans-CN")).to eq("zh-Hans")
+ end
+
+ it "preserves the Serbian Cyrillic script" do
+ expect(described_class.normalize("sr-Cyrl-RS")).to eq("sr-Cyrl")
+ end
+
+ it "preserves the Serbian Latin script" do
+ expect(described_class.normalize("sr-Latn-RS")).to eq("sr-Latn")
+ end
+
+ it "preserves the Punjabi Gurmukhi script" do
+ expect(described_class.normalize("pa-Guru-IN")).to eq("pa-Guru")
+ end
+
+ it "preserves the Punjabi Arabic script" do
+ expect(described_class.normalize("pa-Arab-PK")).to eq("pa-Arab")
+ end
+
+ it "removes variants from Slovenian" do
+ expect(described_class.normalize("sl-rozaj-biske-1994")).to eq("sl")
+ end
+
+ it "removes Unicode extensions from English" do
+ expect(described_class.normalize("en-US-u-hc-h12")).to eq("en")
+ end
+ end
+end
diff --git a/spec/models/browser_pageview_event_spec.rb b/spec/models/browser_pageview_event_spec.rb
index ac03373b6bc..7e0d07ee0d8 100644
--- a/spec/models/browser_pageview_event_spec.rb
+++ b/spec/models/browser_pageview_event_spec.rb
@@ -189,6 +189,7 @@ RSpec.describe BrowserPageviewEvent do
referrer: "a" * (described_class::MAX_REFERRER_LENGTH + 1),
user_agent: "a" * (described_class::MAX_USER_AGENT_LENGTH + 1),
language: "a" * (described_class::MAX_LANGUAGE_LENGTH + 1),
+ normalized_language: "a" * (described_class::MAX_NORMALIZED_LANGUAGE_LENGTH + 1),
ip_address: "1.2.3.4",
session_id: "a" * (described_class::MAX_SESSION_ID_LENGTH + 1),
normalized_referrer: "a" * (described_class::MAX_NORMALIZED_REFERRER_LENGTH + 1),
@@ -198,6 +199,7 @@ RSpec.describe BrowserPageviewEvent do
expect(event.referrer.length).to eq(described_class::MAX_REFERRER_LENGTH)
expect(event.user_agent.length).to eq(described_class::MAX_USER_AGENT_LENGTH)
expect(event.language.length).to eq(described_class::MAX_LANGUAGE_LENGTH)
+ expect(event.normalized_language.length).to eq(described_class::MAX_NORMALIZED_LANGUAGE_LENGTH)
expect(event.session_id.length).to eq(described_class::MAX_SESSION_ID_LENGTH)
expect(event.normalized_referrer.length).to eq(described_class::MAX_NORMALIZED_REFERRER_LENGTH)
end
@@ -249,6 +251,7 @@ RSpec.describe BrowserPageviewEvent do
expect(event.source).to eq("beacon")
expect(event.browser).to eq("edge")
expect(event.language).to eq("en-AU")
+ expect(event.normalized_language).to eq("en")
expect(described_class.queued_count).to eq(0)
end
diff --git a/spec/services/admin_dashboard_site_traffic_explorer_spec.rb b/spec/services/admin_dashboard_site_traffic_explorer_spec.rb
index a361b25df17..3edc2b1a3e9 100644
--- a/spec/services/admin_dashboard_site_traffic_explorer_spec.rb
+++ b/spec/services/admin_dashboard_site_traffic_explorer_spec.rb
@@ -36,7 +36,19 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do
end
let(:languages) do
- [nil, "", nil, "fr", "en-US", "en-US", "en-US", "fr", "de", "zh-CN", "en-US"]
+ [
+ nil,
+ "",
+ nil,
+ "fr-FR",
+ "en-US",
+ "en-GB",
+ "en-US",
+ "fr-CA",
+ "sr-Latn-RS",
+ "zh-cmn-Hant-TW",
+ "en-US",
+ ]
end
let!(:pageviews) do
@@ -116,13 +128,22 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do
{ value: "firefox", label: "Firefox", pageviews: 1 },
],
"languages" => [
- { value: "en-US", label: "en-US", pageviews: 4 },
+ { value: "en", label: "en", pageviews: 4 },
{ value: "", label: "Unknown", pageviews: 3 },
{ value: "fr", label: "fr", pageviews: 2 },
- { value: "de", label: "de", pageviews: 1 },
- { value: "zh-CN", label: "zh-CN", pageviews: 1 },
+ { value: "sr-Latn", label: "sr-Latn", pageviews: 1 },
+ { value: "zh-Hant", label: "zh-Hant", pageviews: 1 },
],
)
+
+ regional_language_traffic =
+ described_class.call(params: params.merge(language: "en-US")).traffic
+ expect(regional_language_traffic.fetch(:dimensions).fetch("languages")).to eq(
+ [{ value: "en", label: "en", pageviews: 4 }],
+ )
+ expect(regional_language_traffic.fetch(:active_filters)).to eq(
+ [{ key: :language, value: "en", label: "en" }],
+ )
end
it "applies all filters to every dimension" do
@@ -174,7 +195,7 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do
"countries" => [{ value: "US", label: "United States", pageviews: 3 }],
"networks" => [{ value: "AS64496", label: "Example Network (AS64496)", pageviews: 3 }],
"browsers" => [{ value: "chrome", label: "Google Chrome", pageviews: 3 }],
- "languages" => [{ value: "en-US", label: "en-US", pageviews: 3 }],
+ "languages" => [{ value: "en", label: "en", pageviews: 3 }],
"ip_addresses" => [
{ value: "192.0.2.5", label: "192.0.2.5", pageviews: 1 },
{ value: "192.0.2.6", label: "192.0.2.6", pageviews: 1 },
diff --git a/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb b/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb
index 99894b1a360..dc8c24e65b7 100644
--- a/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb
+++ b/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb
@@ -333,12 +333,11 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" do
traffic.remove_filter("browser")
traffic.select_tab(card: "visitors", tab: "Languages")
- expect(traffic).to have_row(card: "visitors", label: "English (United States)", count: "2")
- expect(traffic).to have_row(card: "visitors", label: "English (United Kingdom)", count: "1")
- traffic.filter_row(card: "visitors", label: "English (United States)")
- expect(traffic).to have_filter_pill(dimension: "language", label: "English (United States)")
+ expect(traffic).to have_row(card: "visitors", label: "English", count: "3")
+ traffic.filter_row(card: "visitors", label: "English")
+ expect(traffic).to have_filter_pill(dimension: "language", label: "English")
expect(page).to have_current_path(
- "/admin/dashboard/site-traffic-explorer?end_date=2026-05-12&language=en-US&range=custom&start_date=2026-05-01",
+ "/admin/dashboard/site-traffic-explorer?end_date=2026-05-12&language=en&range=custom&start_date=2026-05-01",
)
traffic.remove_filter("language")