High performance, self-hosted, newsletter and mailing list manager with a modern dashboard. Single binary app.
JavaScript campaigncampaign-managementemail-marketingemail-subscriptionlistmonkmailing-listnewsletternewsletter-managementnewsletter-softwareself-hostedsms-gatewaysmtptransactional-emails
Fix incorrect timestamps in dashboard analytics materialized views. (#2952)
Kailash Nadhe35bf878539c
diff --git a/internal/migrations/v6.1.0.go b/internal/migrations/v6.1.0.goindex acdb9728..5681c44a 100644--- a/internal/migrations/v6.1.0.go+++ b/internal/migrations/v6.1.0.go@@ -9,8 +9,67 @@ import ()func V6_1_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf, lo *log.Logger) error {- _, err := db.Exec(`+ if _, err := db.Exec(`INSERT INTO settings (key, value, updated_at) VALUES ('privacy.disable_tracking', 'false', NOW()) ON CONFLICT (key) DO NOTHING- `)- return err+ `); err != nil {+ return err+ }++ // Drop the old UTC-based date indexes and simply use local time with zone consistent+ // with the rest of the schema.+ if _, err := db.Exec(`+ DROP INDEX IF EXISTS idx_views_date; CREATE INDEX IF NOT EXISTS idx_views_date ON campaign_views(created_at);+ DROP INDEX IF EXISTS idx_clicks_date; CREATE INDEX IF NOT EXISTS idx_clicks_date ON link_clicks(created_at);+ DROP INDEX IF EXISTS idx_bounces_date; CREATE INDEX IF NOT EXISTS idx_bounces_date ON bounces(created_at);+ `); err != nil {+ return err+ }++ // Recreate the materialized views to use server local time instead of UTC.+ // Create new views first, let them populate, then drop the old ones and rename the new ones.+ lo.Println("IMPORTANT: recreating analytics materialized views. This might take a while if you have a large database. Please be patient ...")+ if _, err := db.Exec(`+ CREATE MATERIALIZED VIEW IF NOT EXISTS mat_dashboard_charts_v6_1_0 AS+ WITH clicks AS (+ SELECT JSON_AGG(ROW_TO_JSON(row))+ FROM (+ WITH viewDates AS (+ SELECT created_at::DATE AS to_date,+ created_at::DATE - INTERVAL '30 DAY' AS from_date+ FROM link_clicks ORDER BY id DESC LIMIT 1+ )+ SELECT COUNT(*) AS count, created_at::DATE as date FROM link_clicks+ WHERE created_at >= (SELECT from_date FROM viewDates)+ AND created_at < (SELECT to_date FROM viewDates) + INTERVAL '1 day'+ GROUP by date ORDER BY date+ ) row+ ),+ views AS (+ SELECT JSON_AGG(ROW_TO_JSON(row))+ FROM (+ WITH viewDates AS (+ SELECT created_at::DATE AS to_date,+ created_at::DATE - INTERVAL '30 DAY' AS from_date+ FROM campaign_views ORDER BY id DESC LIMIT 1+ )+ SELECT COUNT(*) AS count, created_at::DATE as date FROM campaign_views+ WHERE created_at >= (SELECT from_date FROM viewDates)+ AND created_at < (SELECT to_date FROM viewDates) + INTERVAL '1 day'+ GROUP by date ORDER BY date+ ) row+ )+ SELECT NOW() AS updated_at, JSON_BUILD_OBJECT('link_clicks', COALESCE((SELECT * FROM clicks), '[]'),+ 'campaign_views', COALESCE((SELECT * FROM views), '[]')+ ) AS data;++ DROP INDEX IF EXISTS mat_dashboard_charts_idx;+ DROP MATERIALIZED VIEW IF EXISTS mat_dashboard_charts;++ ALTER MATERIALIZED VIEW mat_dashboard_charts_v6_1_0 RENAME TO mat_dashboard_charts;+ CREATE UNIQUE INDEX IF NOT EXISTS mat_dashboard_charts_idx ON mat_dashboard_charts (updated_at);+ `); err != nil {+ return err+ }++ return nil}diff --git a/schema.sql b/schema.sqlindex e928e2c7..5ea54e16 100644--- a/schema.sql+++ b/schema.sql@@ -163,7 +163,7 @@ CREATE TABLE campaign_views ();DROP INDEX IF EXISTS idx_views_camp_id; CREATE INDEX idx_views_camp_id ON campaign_views(campaign_id);DROP INDEX IF EXISTS idx_views_subscriber_id; CREATE INDEX idx_views_subscriber_id ON campaign_views(subscriber_id);-DROP INDEX IF EXISTS idx_views_date; CREATE INDEX idx_views_date ON campaign_views((TIMEZONE('UTC', created_at)::DATE));+DROP INDEX IF EXISTS idx_views_date; CREATE INDEX idx_views_date ON campaign_views(created_at);-- mediaDROP TABLE IF EXISTS media CASCADE;@@ -216,7 +216,7 @@ CREATE TABLE link_clicks (DROP INDEX IF EXISTS idx_clicks_camp_id; CREATE INDEX idx_clicks_camp_id ON link_clicks(campaign_id);DROP INDEX IF EXISTS idx_clicks_link_id; CREATE INDEX idx_clicks_link_id ON link_clicks(link_id);DROP INDEX IF EXISTS idx_clicks_sub_id; CREATE INDEX idx_clicks_sub_id ON link_clicks(subscriber_id);-DROP INDEX IF EXISTS idx_clicks_date; CREATE INDEX idx_clicks_date ON link_clicks((TIMEZONE('UTC', created_at)::DATE));+DROP INDEX IF EXISTS idx_clicks_date; CREATE INDEX idx_clicks_date ON link_clicks(created_at);-- settingsDROP TABLE IF EXISTS settings CASCADE;@@ -311,7 +311,7 @@ CREATE TABLE bounces (DROP INDEX IF EXISTS idx_bounces_sub_id; CREATE INDEX idx_bounces_sub_id ON bounces(subscriber_id);DROP INDEX IF EXISTS idx_bounces_camp_id; CREATE INDEX idx_bounces_camp_id ON bounces(campaign_id);DROP INDEX IF EXISTS idx_bounces_source; CREATE INDEX idx_bounces_source ON bounces(source);-DROP INDEX IF EXISTS idx_bounces_date; CREATE INDEX idx_bounces_date ON bounces((TIMEZONE('UTC', created_at)::DATE));+DROP INDEX IF EXISTS idx_bounces_date; CREATE INDEX idx_bounces_date ON bounces(created_at);-- rolesDROP TABLE IF EXISTS roles CASCADE;@@ -402,13 +402,13 @@ CREATE MATERIALIZED VIEW mat_dashboard_charts ASSELECT JSON_AGG(ROW_TO_JSON(row))FROM (WITH viewDates AS (- SELECT TIMEZONE('UTC', created_at)::DATE AS to_date,- TIMEZONE('UTC', created_at)::DATE - INTERVAL '30 DAY' AS from_date+ SELECT created_at::DATE AS to_date,+ created_at::DATE - INTERVAL '30 DAY' AS from_dateFROM link_clicks ORDER BY id DESC LIMIT 1)SELECT COUNT(*) AS count, created_at::DATE as date FROM link_clicks- -- use > between < to force the use of the date index.- WHERE TIMEZONE('UTC', created_at)::DATE BETWEEN (SELECT from_date FROM viewDates) AND (SELECT to_date FROM viewDates)+ WHERE created_at >= (SELECT from_date FROM viewDates)+ AND created_at < (SELECT to_date FROM viewDates) + INTERVAL '1 day'GROUP by date ORDER BY date) row),@@ -416,13 +416,13 @@ CREATE MATERIALIZED VIEW mat_dashboard_charts ASSELECT JSON_AGG(ROW_TO_JSON(row))FROM (WITH viewDates AS (- SELECT TIMEZONE('UTC', created_at)::DATE AS to_date,- TIMEZONE('UTC', created_at)::DATE - INTERVAL '30 DAY' AS from_date+ SELECT created_at::DATE AS to_date,+ created_at::DATE - INTERVAL '30 DAY' AS from_dateFROM campaign_views ORDER BY id DESC LIMIT 1)SELECT COUNT(*) AS count, created_at::DATE as date FROM campaign_views- -- use > between < to force the use of the date index.- WHERE TIMEZONE('UTC', created_at)::DATE BETWEEN (SELECT from_date FROM viewDates) AND (SELECT to_date FROM viewDates)+ WHERE created_at >= (SELECT from_date FROM viewDates)+ AND created_at < (SELECT to_date FROM viewDates) + INTERVAL '1 day'GROUP by date ORDER BY date) row)