Skip to content

A platform for community discussion. Free, open, simple.

PERF: Use daily rollups for DAU/MAU reports (#41958)

Alan Guo Xiang Tan395077a745da
diff --git a/app/jobs/scheduled/maintain_user_visit_daily_rollups.rb b/app/jobs/scheduled/maintain_user_visit_daily_rollups.rb
new file mode 100644
index 00000000000..c6fb48ec96a
--- /dev/null
+++ b/app/jobs/scheduled/maintain_user_visit_daily_rollups.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Jobs
+ class MaintainUserVisitDailyRollups < ::Jobs::Scheduled
+ every 3.hours
+
+ cluster_concurrency 1
+
+ def execute(_args = {})
+ initial_aggregation = UserVisitDailyRollup.none?
+ start_date, end_date = aggregation_window(initial_aggregation: initial_aggregation)
+ return if start_date.nil?
+
+ UserVisitDailyRollup.aggregate(start_date: start_date, end_date: end_date)
+ end
+
+ private
+
+ def aggregation_window(initial_aggregation:)
+ end_date = Time.zone.today
+ return UserVisit.minimum(:visited_at)&.to_date, end_date if initial_aggregation
+
+ start_date = 1.day.ago.to_date
+ latest_rollup_date = UserVisitDailyRollup.maximum(:date)
+ latest_visit_date = UserVisit.maximum(:visited_at)
+
+ if latest_visit_date && latest_visit_date > latest_rollup_date
+ start_date = [start_date, latest_rollup_date + 1.day].min
+ end
+
+ [start_date, end_date]
+ end
+ end
+end
diff --git a/app/models/concerns/reports/dau_by_mau.rb b/app/models/concerns/reports/dau_by_mau.rb
index faf0bcc2bfd..a9e9f393123 100644
--- a/app/models/concerns/reports/dau_by_mau.rb
+++ b/app/models/concerns/reports/dau_by_mau.rb
@@ -14,7 +14,7 @@ module Reports::DauByMau
report.percent = true
data_start = report.facets.include?(:prev_period) ? report.prev_start_date : report.start_date
- data_points = UserVisit.count_by_active_users(data_start, report.end_date)
+ data_points = dau_mau_data_points(start_date: data_start, end_date: report.end_date)
report.data = []
@@ -48,9 +48,15 @@ module Reports::DauByMau
if report.facets.include?(:prev30Days)
prev30_days_data =
- UserVisit.count_by_active_users(report.start_date - 30.days, report.start_date)
+ dau_mau_data_points(start_date: report.start_date - 30.days, end_date: report.start_date)
report.prev30Days = dau_avg.call(prev30_days_data)
end
end
+
+ private
+
+ def dau_mau_data_points(start_date:, end_date:)
+ UserVisitDailyRollup.fetch(start_date: start_date, end_date: end_date)
+ end
end
end
diff --git a/app/models/user_visit_daily_rollup.rb b/app/models/user_visit_daily_rollup.rb
new file mode 100644
index 00000000000..095a7201cb8
--- /dev/null
+++ b/app/models/user_visit_daily_rollup.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class UserVisitDailyRollup < ActiveRecord::Base
+ def self.fetch(start_date:, end_date:)
+ where(date: start_date.to_date..end_date.to_date)
+ .order(:date)
+ .pluck(:date, :dau, :mau)
+ .map { |date, dau, mau| { "date" => date, "dau" => dau, "mau" => mau } }
+ end
+
+ def self.aggregate(start_date:, end_date:)
+ rows = UserVisit.count_by_active_users(start_date, end_date)
+
+ transaction { replace!(start_date: start_date, end_date: end_date, rows: rows) }
+ nil
+ end
+
+ def self.replace!(start_date:, end_date:, rows:)
+ start_date = start_date.to_date
+ end_date = end_date.to_date
+ where(date: start_date..end_date).delete_all
+ return if rows.empty?
+
+ insert_all!(
+ rows.map { |row| { date: row.fetch("date"), dau: row.fetch("dau"), mau: row.fetch("mau") } },
+ )
+ end
+ private_class_method :replace!
+end
+
+# == Schema Information
+#
+# Table name: user_visit_daily_rollups
+#
+# id :bigint not null, primary key
+# date :date not null
+# dau :bigint not null
+# mau :bigint not null
+#
+# Indexes
+#
+# index_user_visit_daily_rollups_on_date (date) UNIQUE
+#
diff --git a/app/services/destroy_task.rb b/app/services/destroy_task.rb
index 1aae65a34b1..e6dca39b698 100644
--- a/app/services/destroy_task.rb
+++ b/app/services/destroy_task.rb
@@ -162,6 +162,7 @@ class DestroyTask
ApplicationRequest.delete_all
IncomingLink.delete_all
UserVisit.delete_all
+ UserVisitDailyRollup.delete_all
UserProfileView.delete_all
UserProfile.update_all(views: 0)
PostAction.unscoped.delete_all
diff --git a/db/migrate/20260723013754_create_user_visit_daily_rollups.rb b/db/migrate/20260723013754_create_user_visit_daily_rollups.rb
new file mode 100644
index 00000000000..8cf39768154
--- /dev/null
+++ b/db/migrate/20260723013754_create_user_visit_daily_rollups.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class CreateUserVisitDailyRollups < ActiveRecord::Migration[8.0]
+ def change
+ create_table :user_visit_daily_rollups do |t|
+ t.date :date, null: false
+ t.bigint :dau, null: false
+ t.bigint :mau, null: false
+ end
+
+ add_index :user_visit_daily_rollups, :date, unique: true
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index 62ab9fdc506..c0514886d9e 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -11970,6 +11970,37 @@ CREATE SEQUENCE public.user_uploads_id_seq
ALTER SEQUENCE public.user_uploads_id_seq OWNED BY public.user_uploads.id;
+--
+-- Name: user_visit_daily_rollups; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.user_visit_daily_rollups (
+ id bigint NOT NULL,
+ date date NOT NULL,
+ dau bigint NOT NULL,
+ mau bigint NOT NULL
+);
+
+
+--
+-- Name: user_visit_daily_rollups_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.user_visit_daily_rollups_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: user_visit_daily_rollups_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.user_visit_daily_rollups_id_seq OWNED BY public.user_visit_daily_rollups.id;
+
+
--
-- Name: user_visits; Type: TABLE; Schema: public; Owner: -
--
@@ -14505,6 +14536,13 @@ ALTER TABLE ONLY public.user_statuses ALTER COLUMN user_id SET DEFAULT nextval('
ALTER TABLE ONLY public.user_uploads ALTER COLUMN id SET DEFAULT nextval('public.user_uploads_id_seq'::regclass);
+--
+-- Name: user_visit_daily_rollups id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.user_visit_daily_rollups ALTER COLUMN id SET DEFAULT nextval('public.user_visit_daily_rollups_id_seq'::regclass);
+
+
--
-- Name: user_visits id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -17103,6 +17141,14 @@ ALTER TABLE ONLY public.user_uploads
ADD CONSTRAINT user_uploads_pkey PRIMARY KEY (id);
+--
+-- Name: user_visit_daily_rollups user_visit_daily_rollups_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.user_visit_daily_rollups
+ ADD CONSTRAINT user_visit_daily_rollups_pkey PRIMARY KEY (id);
+
+
--
-- Name: user_visits user_visits_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -22287,6 +22333,13 @@ CREATE UNIQUE INDEX index_user_uploads_on_upload_id_and_user_id ON public.user_u
CREATE INDEX index_user_uploads_on_user_id_and_upload_id ON public.user_uploads USING btree (user_id, upload_id);
+--
+-- Name: index_user_visit_daily_rollups_on_date; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX index_user_visit_daily_rollups_on_date ON public.user_visit_daily_rollups USING btree (date);
+
+
--
-- Name: index_user_visits_on_user_id_and_visited_at; Type: INDEX; Schema: public; Owner: -
--
@@ -22828,6 +22881,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20260723013754'),
('20260722140539'),
('20260722140536'),
('20260721122254'),
diff --git a/spec/fabricators/user_visit_daily_rollup_fabricator.rb b/spec/fabricators/user_visit_daily_rollup_fabricator.rb
new file mode 100644
index 00000000000..e9226e1bd81
--- /dev/null
+++ b/spec/fabricators/user_visit_daily_rollup_fabricator.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+Fabricator(:user_visit_daily_rollup) do
+ date { Time.zone.today }
+ dau 1
+ mau 1
+end
diff --git a/spec/jobs/export_csv_file_spec.rb b/spec/jobs/export_csv_file_spec.rb
index f41424853fc..802d1f21f79 100644
--- a/spec/jobs/export_csv_file_spec.rb
+++ b/spec/jobs/export_csv_file_spec.rb
@@ -245,8 +245,8 @@ RSpec.describe Jobs::ExportCsvFile do
end
it "works with single-column reports" do
- user.user_visits.create!(visited_at: "2010-01-01", posts_read: 42)
- Fabricate(:user).user_visits.create!(visited_at: "2010-01-03", posts_read: 420)
+ Fabricate(:user_visit_daily_rollup, date: Date.new(2010, 1, 1))
+ Fabricate(:user_visit_daily_rollup, date: Date.new(2010, 1, 3), mau: 2)
exporter.extra["name"] = "dau_by_mau"
report = export_report
diff --git a/spec/jobs/scheduled/maintain_user_visit_daily_rollups_spec.rb b/spec/jobs/scheduled/maintain_user_visit_daily_rollups_spec.rb
new file mode 100644
index 00000000000..138a5d52edf
--- /dev/null
+++ b/spec/jobs/scheduled/maintain_user_visit_daily_rollups_spec.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+RSpec.describe Jobs::MaintainUserVisitDailyRollups do
+ subject(:job) { described_class.new }
+
+ fab!(:recent_visitor, :user)
+ fab!(:historical_visitor, :user)
+
+ before { freeze_time(Time.zone.local(2026, 7, 23, 12)) }
+
+ describe "#execute" do
+ it "aggregates the complete visit history on its first run" do
+ recent_visitor.user_visits.create!(visited_at: Time.zone.today)
+ historical_visitor.user_visits.create!(visited_at: 10.days.ago)
+
+ job.execute({})
+
+ expect(UserVisitDailyRollup.order(:date).pluck(:date, :dau, :mau)).to eq(
+ [[10.days.ago.to_date, 1, 1], [Time.zone.today, 1, 2]],
+ )
+ end
+
+ it "refreshes only yesterday and today after the initial aggregation" do
+ recent_visitor.user_visits.create!(visited_at: Time.zone.today)
+ historical_visitor.user_visits.create!(visited_at: 10.days.ago)
+ job.execute({})
+ late_historical_visitor = Fabricate(:user)
+ late_historical_visitor.user_visits.create!(visited_at: 40.days.ago)
+ another_recent_visitor = Fabricate(:user)
+ another_recent_visitor.user_visits.create!(visited_at: Time.zone.today)
+
+ job.execute({})
+
+ expect(UserVisitDailyRollup.where(date: 40.days.ago).exists?).to eq(false)
+ expect(UserVisitDailyRollup.find_by(date: Time.zone.today)).to have_attributes(dau: 2, mau: 3)
+ end
+
+ it "fills active dates since the most recent rollup" do
+ Fabricate(:user_visit_daily_rollup, date: 3.days.ago)
+ historical_visitor.user_visits.create!(visited_at: 2.days.ago)
+ recent_visitor.user_visits.create!(visited_at: Time.zone.today)
+
+ job.execute({})
+
+ expect(UserVisitDailyRollup.order(:date).pluck(:date)).to eq(
+ [3.days.ago.to_date, 2.days.ago.to_date, Time.zone.today],
+ )
+ end
+ end
+end
diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rb
index 9b507458431..8ad1b81ede4 100644
--- a/spec/models/report_spec.rb
+++ b/spec/models/report_spec.rb
@@ -703,58 +703,6 @@ RSpec.describe Report do
end
end
- describe "DAU/MAU report" do
- let(:report) { Report.find("dau_by_mau") }
-
- include_examples "no data"
-
- context "with different users/visits" do
- before do
- freeze_time_safe
-
- arpit = Fabricate(:user)
- arpit.user_visits.create(visited_at: 1.day.ago)
-
- sam = Fabricate(:user)
- sam.user_visits.create(visited_at: 2.days.ago)
-
- robin = Fabricate(:user)
- robin.user_visits.create(visited_at: 2.days.ago)
-
- michael = Fabricate(:user)
- michael.user_visits.create(visited_at: 35.days.ago)
-
- gerhard = Fabricate(:user)
- gerhard.user_visits.create(visited_at: 45.days.ago)
- end
-
- it "returns a report with data" do
- expect(report.data.first[:y]).to eq(100)
- expect(report.data.last[:y]).to eq(33.34)
- expect(report.prev30Days).to eq(75)
- end
- end
-
- it "returns the current data and previous period average" do
- current_visitor = Fabricate(:user)
- previous_visitor = Fabricate(:user)
-
- current_visitor.user_visits.create!(visited_at: Time.zone.local(2026, 4, 11).to_date)
- previous_visitor.user_visits.create!(visited_at: Time.zone.local(2026, 4, 9).to_date)
-
- report =
- Report.find(
- "dau_by_mau",
- start_date: Time.zone.local(2026, 4, 10).beginning_of_day,
- end_date: Time.zone.local(2026, 4, 11).end_of_day,
- facets: [:prev_period],
- )
-
- expect(report.data.map { |point| point[:x] }).to eq([Date.new(2026, 4, 11)])
- expect(report.prev_period).to eq(100)
- end
- end
-
describe "Daily engaged users" do
let(:report) { Report.find("daily_engaged_users") }
diff --git a/spec/models/reports/dau_by_mau_spec.rb b/spec/models/reports/dau_by_mau_spec.rb
new file mode 100644
index 00000000000..4e9ea958129
--- /dev/null
+++ b/spec/models/reports/dau_by_mau_spec.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+describe Reports::DauByMau do
+ describe ".report_dau_by_mau" do
+ let(:report) { Report.find("dau_by_mau") }
+
+ it "returns an empty report with no data" do
+ expect(report.data).to be_blank
+ end
+
+ it "returns a report with data" do
+ freeze_time_safe
+ Fabricate(:user_visit_daily_rollup, date: 45.days.ago, dau: 1, mau: 1)
+ Fabricate(:user_visit_daily_rollup, date: 35.days.ago, dau: 1, mau: 2)
+ Fabricate(:user_visit_daily_rollup, date: 2.days.ago, dau: 2, mau: 2)
+ Fabricate(:user_visit_daily_rollup, date: 1.day.ago, dau: 1, mau: 3)
+
+ expect(report.data.first[:y]).to eq(100)
+ expect(report.data.last[:y]).to eq(33.34)
+ expect(report.prev30Days).to eq(75)
+ end
+
+ it "returns the current data and previous period average" do
+ Fabricate(:user_visit_daily_rollup, date: Date.new(2026, 4, 9), dau: 1, mau: 1)
+ Fabricate(:user_visit_daily_rollup, date: Date.new(2026, 4, 11), dau: 1, mau: 2)
+
+ report =
+ Report.find(
+ "dau_by_mau",
+ start_date: Time.zone.local(2026, 4, 10).beginning_of_day,
+ end_date: Time.zone.local(2026, 4, 11).end_of_day,
+ facets: [:prev_period],
+ )
+
+ expect(report.data).to eq([{ x: Date.new(2026, 4, 11), y: 50.0 }])
+ expect(report.prev_period).to eq(100)
+ end
+ end
+end
diff --git a/spec/models/user_visit_daily_rollup_spec.rb b/spec/models/user_visit_daily_rollup_spec.rb
new file mode 100644
index 00000000000..0bcb52519b3
--- /dev/null
+++ b/spec/models/user_visit_daily_rollup_spec.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+RSpec.describe UserVisitDailyRollup do
+ fab!(:visitor, :user)
+
+ before { freeze_time(Time.zone.local(2026, 7, 23, 12)) }
+
+ describe ".fetch" do
+ it "returns existing rows in date order and omits dates without rows" do
+ described_class.create!(date: 2.days.ago, dau: 1, mau: 3)
+ described_class.create!(date: Time.zone.today, dau: 2, mau: 4)
+
+ rows = described_class.fetch(start_date: 2.days.ago, end_date: Time.zone.today)
+
+ expect(rows).to eq(
+ [
+ { "date" => 2.days.ago.to_date, "dau" => 1, "mau" => 3 },
+ { "date" => Time.zone.today, "dau" => 2, "mau" => 4 },
+ ],
+ )
+ end
+ end
+
+ describe ".aggregate" do
+ it "stores the same daily values as the established calculation" do
+ visitor.user_visits.create!(visited_at: Time.zone.today)
+ historical_visitor = Fabricate(:user)
+ historical_visitor.user_visits.create!(visited_at: 10.days.ago)
+ expected = UserVisit.count_by_active_users(10.days.ago, Time.zone.today)
+
+ described_class.aggregate(start_date: 10.days.ago, end_date: Time.zone.today)
+
+ expect(described_class.fetch(start_date: 10.days.ago, end_date: Time.zone.today)).to eq(
+ expected,
+ )
+ end
+
+ it "keeps existing report data when a refresh fails" do
+ existing = described_class.create!(date: Time.zone.today, dau: 3, mau: 4)
+ described_class.stubs(:insert_all!).raises(ActiveRecord::StatementInvalid)
+ visitor.user_visits.create!(visited_at: Time.zone.today)
+
+ expect do
+ described_class.aggregate(start_date: Time.zone.today, end_date: Time.zone.today)
+ end.to raise_error(ActiveRecord::StatementInvalid)
+
+ expect(existing.reload).to have_attributes(dau: 3, mau: 4)
+ end
+ end
+end
diff --git a/spec/services/admin_dashboard_highlights_spec.rb b/spec/services/admin_dashboard_highlights_spec.rb
index 895d4565ef6..45169d6e885 100644
--- a/spec/services/admin_dashboard_highlights_spec.rb
+++ b/spec/services/admin_dashboard_highlights_spec.rb
@@ -54,8 +54,8 @@ describe AdminDashboardHighlights do
end
it "averages dau_mau values rather than summing them when there is data" do
- Fabricate(:user_visit, visited_at: Time.zone.local(2026, 4, 10).to_date)
- Fabricate(:user_visit, visited_at: Time.zone.local(2026, 4, 15).to_date)
+ Fabricate(:user_visit_daily_rollup, date: Date.new(2026, 4, 10))
+ Fabricate(:user_visit_daily_rollup, date: Date.new(2026, 4, 15), mau: 2)
result = described_class.build(start_date: "2026-04-01", end_date: "2026-04-28")
dau = result[:kpis].find { |k| k[:type] == :dau_mau }