A platform for community discussion. Free, open, simple.
PERF: Use daily rollups for DAU/MAU reports (#41958)
Alan Guo Xiang Tan395077a745da
diff --git a/app/jobs/scheduled/maintain_user_visit_daily_rollups.rb b/app/jobs/scheduled/maintain_user_visit_daily_rollups.rbnew file mode 100644index 00000000000..c6fb48ec96a--- /dev/null+++ b/app/jobs/scheduled/maintain_user_visit_daily_rollups.rb@@ -0,0 +1,34 @@+# frozen_string_literal: true++module Jobs+ class MaintainUserVisitDailyRollups < ::Jobs::Scheduled+ every 3.hours++ cluster_concurrency 1++ def execute(_args = {})+ initial_aggregation = UserVisitDailyRollup.none?+ start_date, end_date = aggregation_window(initial_aggregation: initial_aggregation)+ return if start_date.nil?++ UserVisitDailyRollup.aggregate(start_date: start_date, end_date: end_date)+ end++ private++ def aggregation_window(initial_aggregation:)+ end_date = Time.zone.today+ return UserVisit.minimum(:visited_at)&.to_date, end_date if initial_aggregation++ start_date = 1.day.ago.to_date+ latest_rollup_date = UserVisitDailyRollup.maximum(:date)+ latest_visit_date = UserVisit.maximum(:visited_at)++ if latest_visit_date && latest_visit_date > latest_rollup_date+ start_date = [start_date, latest_rollup_date + 1.day].min+ end++ [start_date, end_date]+ end+ end+enddiff --git a/app/models/concerns/reports/dau_by_mau.rb b/app/models/concerns/reports/dau_by_mau.rbindex faf0bcc2bfd..a9e9f393123 100644--- a/app/models/concerns/reports/dau_by_mau.rb+++ b/app/models/concerns/reports/dau_by_mau.rb@@ -14,7 +14,7 @@ module Reports::DauByMaureport.percent = truedata_start = report.facets.include?(:prev_period) ? report.prev_start_date : report.start_date- data_points = UserVisit.count_by_active_users(data_start, report.end_date)+ data_points = dau_mau_data_points(start_date: data_start, end_date: report.end_date)report.data = []@@ -48,9 +48,15 @@ module Reports::DauByMauif report.facets.include?(:prev30Days)prev30_days_data =- UserVisit.count_by_active_users(report.start_date - 30.days, report.start_date)+ dau_mau_data_points(start_date: report.start_date - 30.days, end_date: report.start_date)report.prev30Days = dau_avg.call(prev30_days_data)endend++ private++ def dau_mau_data_points(start_date:, end_date:)+ UserVisitDailyRollup.fetch(start_date: start_date, end_date: end_date)+ endendenddiff --git a/app/models/user_visit_daily_rollup.rb b/app/models/user_visit_daily_rollup.rbnew file mode 100644index 00000000000..095a7201cb8--- /dev/null+++ b/app/models/user_visit_daily_rollup.rb@@ -0,0 +1,43 @@+# frozen_string_literal: true++class UserVisitDailyRollup < ActiveRecord::Base+ def self.fetch(start_date:, end_date:)+ where(date: start_date.to_date..end_date.to_date)+ .order(:date)+ .pluck(:date, :dau, :mau)+ .map { |date, dau, mau| { "date" => date, "dau" => dau, "mau" => mau } }+ end++ def self.aggregate(start_date:, end_date:)+ rows = UserVisit.count_by_active_users(start_date, end_date)++ transaction { replace!(start_date: start_date, end_date: end_date, rows: rows) }+ nil+ end++ def self.replace!(start_date:, end_date:, rows:)+ start_date = start_date.to_date+ end_date = end_date.to_date+ where(date: start_date..end_date).delete_all+ return if rows.empty?++ insert_all!(+ rows.map { |row| { date: row.fetch("date"), dau: row.fetch("dau"), mau: row.fetch("mau") } },+ )+ end+ private_class_method :replace!+end++# == Schema Information+#+# Table name: user_visit_daily_rollups+#+# id :bigint not null, primary key+# date :date not null+# dau :bigint not null+# mau :bigint not null+#+# Indexes+#+# index_user_visit_daily_rollups_on_date (date) UNIQUE+#diff --git a/app/services/destroy_task.rb b/app/services/destroy_task.rbindex 1aae65a34b1..e6dca39b698 100644--- a/app/services/destroy_task.rb+++ b/app/services/destroy_task.rb@@ -162,6 +162,7 @@ class DestroyTaskApplicationRequest.delete_allIncomingLink.delete_allUserVisit.delete_all+ UserVisitDailyRollup.delete_allUserProfileView.delete_allUserProfile.update_all(views: 0)PostAction.unscoped.delete_alldiff --git a/db/migrate/20260723013754_create_user_visit_daily_rollups.rb b/db/migrate/20260723013754_create_user_visit_daily_rollups.rbnew file mode 100644index 00000000000..8cf39768154--- /dev/null+++ b/db/migrate/20260723013754_create_user_visit_daily_rollups.rb@@ -0,0 +1,13 @@+# frozen_string_literal: true++class CreateUserVisitDailyRollups < ActiveRecord::Migration[8.0]+ def change+ create_table :user_visit_daily_rollups do |t|+ t.date :date, null: false+ t.bigint :dau, null: false+ t.bigint :mau, null: false+ end++ add_index :user_visit_daily_rollups, :date, unique: true+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex 62ab9fdc506..c0514886d9e 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -11970,6 +11970,37 @@ CREATE SEQUENCE public.user_uploads_id_seqALTER SEQUENCE public.user_uploads_id_seq OWNED BY public.user_uploads.id;+--+-- Name: user_visit_daily_rollups; Type: TABLE; Schema: public; Owner: -+--++CREATE TABLE public.user_visit_daily_rollups (+ id bigint NOT NULL,+ date date NOT NULL,+ dau bigint NOT NULL,+ mau bigint NOT NULL+);+++--+-- Name: user_visit_daily_rollups_id_seq; Type: SEQUENCE; Schema: public; Owner: -+--++CREATE SEQUENCE public.user_visit_daily_rollups_id_seq+ START WITH 1+ INCREMENT BY 1+ NO MINVALUE+ NO MAXVALUE+ CACHE 1;+++--+-- Name: user_visit_daily_rollups_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -+--++ALTER SEQUENCE public.user_visit_daily_rollups_id_seq OWNED BY public.user_visit_daily_rollups.id;++---- Name: user_visits; Type: TABLE; Schema: public; Owner: ---@@ -14505,6 +14536,13 @@ ALTER TABLE ONLY public.user_statuses ALTER COLUMN user_id SET DEFAULT nextval('ALTER TABLE ONLY public.user_uploads ALTER COLUMN id SET DEFAULT nextval('public.user_uploads_id_seq'::regclass);+--+-- Name: user_visit_daily_rollups id; Type: DEFAULT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.user_visit_daily_rollups ALTER COLUMN id SET DEFAULT nextval('public.user_visit_daily_rollups_id_seq'::regclass);++---- Name: user_visits id; Type: DEFAULT; Schema: public; Owner: ---@@ -17103,6 +17141,14 @@ ALTER TABLE ONLY public.user_uploadsADD CONSTRAINT user_uploads_pkey PRIMARY KEY (id);+--+-- Name: user_visit_daily_rollups user_visit_daily_rollups_pkey; Type: CONSTRAINT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.user_visit_daily_rollups+ ADD CONSTRAINT user_visit_daily_rollups_pkey PRIMARY KEY (id);++---- Name: user_visits user_visits_pkey; Type: CONSTRAINT; Schema: public; Owner: ---@@ -22287,6 +22333,13 @@ CREATE UNIQUE INDEX index_user_uploads_on_upload_id_and_user_id ON public.user_uCREATE INDEX index_user_uploads_on_user_id_and_upload_id ON public.user_uploads USING btree (user_id, upload_id);+--+-- Name: index_user_visit_daily_rollups_on_date; Type: INDEX; Schema: public; Owner: -+--++CREATE UNIQUE INDEX index_user_visit_daily_rollups_on_date ON public.user_visit_daily_rollups USING btree (date);++---- Name: index_user_visits_on_user_id_and_visited_at; Type: INDEX; Schema: public; Owner: ---@@ -22828,6 +22881,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groupsSET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES+('20260723013754'),('20260722140539'),('20260722140536'),('20260721122254'),diff --git a/spec/fabricators/user_visit_daily_rollup_fabricator.rb b/spec/fabricators/user_visit_daily_rollup_fabricator.rbnew file mode 100644index 00000000000..e9226e1bd81--- /dev/null+++ b/spec/fabricators/user_visit_daily_rollup_fabricator.rb@@ -0,0 +1,7 @@+# frozen_string_literal: true++Fabricator(:user_visit_daily_rollup) do+ date { Time.zone.today }+ dau 1+ mau 1+enddiff --git a/spec/jobs/export_csv_file_spec.rb b/spec/jobs/export_csv_file_spec.rbindex f41424853fc..802d1f21f79 100644--- a/spec/jobs/export_csv_file_spec.rb+++ b/spec/jobs/export_csv_file_spec.rb@@ -245,8 +245,8 @@ RSpec.describe Jobs::ExportCsvFile doendit "works with single-column reports" do- user.user_visits.create!(visited_at: "2010-01-01", posts_read: 42)- Fabricate(:user).user_visits.create!(visited_at: "2010-01-03", posts_read: 420)+ Fabricate(:user_visit_daily_rollup, date: Date.new(2010, 1, 1))+ Fabricate(:user_visit_daily_rollup, date: Date.new(2010, 1, 3), mau: 2)exporter.extra["name"] = "dau_by_mau"report = export_reportdiff --git a/spec/jobs/scheduled/maintain_user_visit_daily_rollups_spec.rb b/spec/jobs/scheduled/maintain_user_visit_daily_rollups_spec.rbnew file mode 100644index 00000000000..138a5d52edf--- /dev/null+++ b/spec/jobs/scheduled/maintain_user_visit_daily_rollups_spec.rb@@ -0,0 +1,50 @@+# frozen_string_literal: true++RSpec.describe Jobs::MaintainUserVisitDailyRollups do+ subject(:job) { described_class.new }++ fab!(:recent_visitor, :user)+ fab!(:historical_visitor, :user)++ before { freeze_time(Time.zone.local(2026, 7, 23, 12)) }++ describe "#execute" do+ it "aggregates the complete visit history on its first run" do+ recent_visitor.user_visits.create!(visited_at: Time.zone.today)+ historical_visitor.user_visits.create!(visited_at: 10.days.ago)++ job.execute({})++ expect(UserVisitDailyRollup.order(:date).pluck(:date, :dau, :mau)).to eq(+ [[10.days.ago.to_date, 1, 1], [Time.zone.today, 1, 2]],+ )+ end++ it "refreshes only yesterday and today after the initial aggregation" do+ recent_visitor.user_visits.create!(visited_at: Time.zone.today)+ historical_visitor.user_visits.create!(visited_at: 10.days.ago)+ job.execute({})+ late_historical_visitor = Fabricate(:user)+ late_historical_visitor.user_visits.create!(visited_at: 40.days.ago)+ another_recent_visitor = Fabricate(:user)+ another_recent_visitor.user_visits.create!(visited_at: Time.zone.today)++ job.execute({})++ expect(UserVisitDailyRollup.where(date: 40.days.ago).exists?).to eq(false)+ expect(UserVisitDailyRollup.find_by(date: Time.zone.today)).to have_attributes(dau: 2, mau: 3)+ end++ it "fills active dates since the most recent rollup" do+ Fabricate(:user_visit_daily_rollup, date: 3.days.ago)+ historical_visitor.user_visits.create!(visited_at: 2.days.ago)+ recent_visitor.user_visits.create!(visited_at: Time.zone.today)++ job.execute({})++ expect(UserVisitDailyRollup.order(:date).pluck(:date)).to eq(+ [3.days.ago.to_date, 2.days.ago.to_date, Time.zone.today],+ )+ end+ end+enddiff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rbindex 9b507458431..8ad1b81ede4 100644--- a/spec/models/report_spec.rb+++ b/spec/models/report_spec.rb@@ -703,58 +703,6 @@ RSpec.describe Report doendend- describe "DAU/MAU report" do- let(:report) { Report.find("dau_by_mau") }-- include_examples "no data"-- context "with different users/visits" do- before do- freeze_time_safe-- arpit = Fabricate(:user)- arpit.user_visits.create(visited_at: 1.day.ago)-- sam = Fabricate(:user)- sam.user_visits.create(visited_at: 2.days.ago)-- robin = Fabricate(:user)- robin.user_visits.create(visited_at: 2.days.ago)-- michael = Fabricate(:user)- michael.user_visits.create(visited_at: 35.days.ago)-- gerhard = Fabricate(:user)- gerhard.user_visits.create(visited_at: 45.days.ago)- end-- it "returns a report with data" do- expect(report.data.first[:y]).to eq(100)- expect(report.data.last[:y]).to eq(33.34)- expect(report.prev30Days).to eq(75)- end- end-- it "returns the current data and previous period average" do- current_visitor = Fabricate(:user)- previous_visitor = Fabricate(:user)-- current_visitor.user_visits.create!(visited_at: Time.zone.local(2026, 4, 11).to_date)- previous_visitor.user_visits.create!(visited_at: Time.zone.local(2026, 4, 9).to_date)-- report =- Report.find(- "dau_by_mau",- start_date: Time.zone.local(2026, 4, 10).beginning_of_day,- end_date: Time.zone.local(2026, 4, 11).end_of_day,- facets: [:prev_period],- )-- expect(report.data.map { |point| point[:x] }).to eq([Date.new(2026, 4, 11)])- expect(report.prev_period).to eq(100)- end- end-describe "Daily engaged users" dolet(:report) { Report.find("daily_engaged_users") }diff --git a/spec/models/reports/dau_by_mau_spec.rb b/spec/models/reports/dau_by_mau_spec.rbnew file mode 100644index 00000000000..4e9ea958129--- /dev/null+++ b/spec/models/reports/dau_by_mau_spec.rb@@ -0,0 +1,39 @@+# frozen_string_literal: true++describe Reports::DauByMau do+ describe ".report_dau_by_mau" do+ let(:report) { Report.find("dau_by_mau") }++ it "returns an empty report with no data" do+ expect(report.data).to be_blank+ end++ it "returns a report with data" do+ freeze_time_safe+ Fabricate(:user_visit_daily_rollup, date: 45.days.ago, dau: 1, mau: 1)+ Fabricate(:user_visit_daily_rollup, date: 35.days.ago, dau: 1, mau: 2)+ Fabricate(:user_visit_daily_rollup, date: 2.days.ago, dau: 2, mau: 2)+ Fabricate(:user_visit_daily_rollup, date: 1.day.ago, dau: 1, mau: 3)++ expect(report.data.first[:y]).to eq(100)+ expect(report.data.last[:y]).to eq(33.34)+ expect(report.prev30Days).to eq(75)+ end++ it "returns the current data and previous period average" do+ Fabricate(:user_visit_daily_rollup, date: Date.new(2026, 4, 9), dau: 1, mau: 1)+ Fabricate(:user_visit_daily_rollup, date: Date.new(2026, 4, 11), dau: 1, mau: 2)++ report =+ Report.find(+ "dau_by_mau",+ start_date: Time.zone.local(2026, 4, 10).beginning_of_day,+ end_date: Time.zone.local(2026, 4, 11).end_of_day,+ facets: [:prev_period],+ )++ expect(report.data).to eq([{ x: Date.new(2026, 4, 11), y: 50.0 }])+ expect(report.prev_period).to eq(100)+ end+ end+enddiff --git a/spec/models/user_visit_daily_rollup_spec.rb b/spec/models/user_visit_daily_rollup_spec.rbnew file mode 100644index 00000000000..0bcb52519b3--- /dev/null+++ b/spec/models/user_visit_daily_rollup_spec.rb@@ -0,0 +1,50 @@+# frozen_string_literal: true++RSpec.describe UserVisitDailyRollup do+ fab!(:visitor, :user)++ before { freeze_time(Time.zone.local(2026, 7, 23, 12)) }++ describe ".fetch" do+ it "returns existing rows in date order and omits dates without rows" do+ described_class.create!(date: 2.days.ago, dau: 1, mau: 3)+ described_class.create!(date: Time.zone.today, dau: 2, mau: 4)++ rows = described_class.fetch(start_date: 2.days.ago, end_date: Time.zone.today)++ expect(rows).to eq(+ [+ { "date" => 2.days.ago.to_date, "dau" => 1, "mau" => 3 },+ { "date" => Time.zone.today, "dau" => 2, "mau" => 4 },+ ],+ )+ end+ end++ describe ".aggregate" do+ it "stores the same daily values as the established calculation" do+ visitor.user_visits.create!(visited_at: Time.zone.today)+ historical_visitor = Fabricate(:user)+ historical_visitor.user_visits.create!(visited_at: 10.days.ago)+ expected = UserVisit.count_by_active_users(10.days.ago, Time.zone.today)++ described_class.aggregate(start_date: 10.days.ago, end_date: Time.zone.today)++ expect(described_class.fetch(start_date: 10.days.ago, end_date: Time.zone.today)).to eq(+ expected,+ )+ end++ it "keeps existing report data when a refresh fails" do+ existing = described_class.create!(date: Time.zone.today, dau: 3, mau: 4)+ described_class.stubs(:insert_all!).raises(ActiveRecord::StatementInvalid)+ visitor.user_visits.create!(visited_at: Time.zone.today)++ expect do+ described_class.aggregate(start_date: Time.zone.today, end_date: Time.zone.today)+ end.to raise_error(ActiveRecord::StatementInvalid)++ expect(existing.reload).to have_attributes(dau: 3, mau: 4)+ end+ end+enddiff --git a/spec/services/admin_dashboard_highlights_spec.rb b/spec/services/admin_dashboard_highlights_spec.rbindex 895d4565ef6..45169d6e885 100644--- a/spec/services/admin_dashboard_highlights_spec.rb+++ b/spec/services/admin_dashboard_highlights_spec.rb@@ -54,8 +54,8 @@ describe AdminDashboardHighlights doendit "averages dau_mau values rather than summing them when there is data" do- Fabricate(:user_visit, visited_at: Time.zone.local(2026, 4, 10).to_date)- Fabricate(:user_visit, visited_at: Time.zone.local(2026, 4, 15).to_date)+ Fabricate(:user_visit_daily_rollup, date: Date.new(2026, 4, 10))+ Fabricate(:user_visit_daily_rollup, date: Date.new(2026, 4, 15), mau: 2)result = described_class.build(start_date: "2026-04-01", end_date: "2026-04-28")dau = result[:kpis].find { |k| k[:type] == :dau_mau }