Skip to content

High performance, self-hosted, newsletter and mailing list manager with a modern dashboard. Single binary app.

Fix incorrect timestamps in dashboard analytics materialized views. (#2952)

Kailash Nadhe35bf878539c
diff --git a/internal/migrations/v6.1.0.go b/internal/migrations/v6.1.0.go
index acdb9728..5681c44a 100644
--- a/internal/migrations/v6.1.0.go
+++ b/internal/migrations/v6.1.0.go
@@ -9,8 +9,67 @@ import (
)
func V6_1_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf, lo *log.Logger) error {
- _, err := db.Exec(`
+ if _, err := db.Exec(`
INSERT INTO settings (key, value, updated_at) VALUES ('privacy.disable_tracking', 'false', NOW()) ON CONFLICT (key) DO NOTHING
- `)
- return err
+ `); err != nil {
+ return err
+ }
+
+ // Drop the old UTC-based date indexes and simply use local time with zone consistent
+ // with the rest of the schema.
+ if _, err := db.Exec(`
+ DROP INDEX IF EXISTS idx_views_date; CREATE INDEX IF NOT EXISTS idx_views_date ON campaign_views(created_at);
+ DROP INDEX IF EXISTS idx_clicks_date; CREATE INDEX IF NOT EXISTS idx_clicks_date ON link_clicks(created_at);
+ DROP INDEX IF EXISTS idx_bounces_date; CREATE INDEX IF NOT EXISTS idx_bounces_date ON bounces(created_at);
+ `); err != nil {
+ return err
+ }
+
+ // Recreate the materialized views to use server local time instead of UTC.
+ // Create new views first, let them populate, then drop the old ones and rename the new ones.
+ lo.Println("IMPORTANT: recreating analytics materialized views. This might take a while if you have a large database. Please be patient ...")
+ if _, err := db.Exec(`
+ CREATE MATERIALIZED VIEW IF NOT EXISTS mat_dashboard_charts_v6_1_0 AS
+ WITH clicks AS (
+ SELECT JSON_AGG(ROW_TO_JSON(row))
+ FROM (
+ WITH viewDates AS (
+ SELECT created_at::DATE AS to_date,
+ created_at::DATE - INTERVAL '30 DAY' AS from_date
+ FROM link_clicks ORDER BY id DESC LIMIT 1
+ )
+ SELECT COUNT(*) AS count, created_at::DATE as date FROM link_clicks
+ WHERE created_at >= (SELECT from_date FROM viewDates)
+ AND created_at < (SELECT to_date FROM viewDates) + INTERVAL '1 day'
+ GROUP by date ORDER BY date
+ ) row
+ ),
+ views AS (
+ SELECT JSON_AGG(ROW_TO_JSON(row))
+ FROM (
+ WITH viewDates AS (
+ SELECT created_at::DATE AS to_date,
+ created_at::DATE - INTERVAL '30 DAY' AS from_date
+ FROM campaign_views ORDER BY id DESC LIMIT 1
+ )
+ SELECT COUNT(*) AS count, created_at::DATE as date FROM campaign_views
+ WHERE created_at >= (SELECT from_date FROM viewDates)
+ AND created_at < (SELECT to_date FROM viewDates) + INTERVAL '1 day'
+ GROUP by date ORDER BY date
+ ) row
+ )
+ SELECT NOW() AS updated_at, JSON_BUILD_OBJECT('link_clicks', COALESCE((SELECT * FROM clicks), '[]'),
+ 'campaign_views', COALESCE((SELECT * FROM views), '[]')
+ ) AS data;
+
+ DROP INDEX IF EXISTS mat_dashboard_charts_idx;
+ DROP MATERIALIZED VIEW IF EXISTS mat_dashboard_charts;
+
+ ALTER MATERIALIZED VIEW mat_dashboard_charts_v6_1_0 RENAME TO mat_dashboard_charts;
+ CREATE UNIQUE INDEX IF NOT EXISTS mat_dashboard_charts_idx ON mat_dashboard_charts (updated_at);
+ `); err != nil {
+ return err
+ }
+
+ return nil
}
diff --git a/schema.sql b/schema.sql
index e928e2c7..5ea54e16 100644
--- a/schema.sql
+++ b/schema.sql
@@ -163,7 +163,7 @@ CREATE TABLE campaign_views (
);
DROP INDEX IF EXISTS idx_views_camp_id; CREATE INDEX idx_views_camp_id ON campaign_views(campaign_id);
DROP INDEX IF EXISTS idx_views_subscriber_id; CREATE INDEX idx_views_subscriber_id ON campaign_views(subscriber_id);
-DROP INDEX IF EXISTS idx_views_date; CREATE INDEX idx_views_date ON campaign_views((TIMEZONE('UTC', created_at)::DATE));
+DROP INDEX IF EXISTS idx_views_date; CREATE INDEX idx_views_date ON campaign_views(created_at);
-- media
DROP TABLE IF EXISTS media CASCADE;
@@ -216,7 +216,7 @@ CREATE TABLE link_clicks (
DROP INDEX IF EXISTS idx_clicks_camp_id; CREATE INDEX idx_clicks_camp_id ON link_clicks(campaign_id);
DROP INDEX IF EXISTS idx_clicks_link_id; CREATE INDEX idx_clicks_link_id ON link_clicks(link_id);
DROP INDEX IF EXISTS idx_clicks_sub_id; CREATE INDEX idx_clicks_sub_id ON link_clicks(subscriber_id);
-DROP INDEX IF EXISTS idx_clicks_date; CREATE INDEX idx_clicks_date ON link_clicks((TIMEZONE('UTC', created_at)::DATE));
+DROP INDEX IF EXISTS idx_clicks_date; CREATE INDEX idx_clicks_date ON link_clicks(created_at);
-- settings
DROP TABLE IF EXISTS settings CASCADE;
@@ -311,7 +311,7 @@ CREATE TABLE bounces (
DROP INDEX IF EXISTS idx_bounces_sub_id; CREATE INDEX idx_bounces_sub_id ON bounces(subscriber_id);
DROP INDEX IF EXISTS idx_bounces_camp_id; CREATE INDEX idx_bounces_camp_id ON bounces(campaign_id);
DROP INDEX IF EXISTS idx_bounces_source; CREATE INDEX idx_bounces_source ON bounces(source);
-DROP INDEX IF EXISTS idx_bounces_date; CREATE INDEX idx_bounces_date ON bounces((TIMEZONE('UTC', created_at)::DATE));
+DROP INDEX IF EXISTS idx_bounces_date; CREATE INDEX idx_bounces_date ON bounces(created_at);
-- roles
DROP TABLE IF EXISTS roles CASCADE;
@@ -402,13 +402,13 @@ CREATE MATERIALIZED VIEW mat_dashboard_charts AS
SELECT JSON_AGG(ROW_TO_JSON(row))
FROM (
WITH viewDates AS (
- SELECT TIMEZONE('UTC', created_at)::DATE AS to_date,
- TIMEZONE('UTC', created_at)::DATE - INTERVAL '30 DAY' AS from_date
+ SELECT created_at::DATE AS to_date,
+ created_at::DATE - INTERVAL '30 DAY' AS from_date
FROM link_clicks ORDER BY id DESC LIMIT 1
)
SELECT COUNT(*) AS count, created_at::DATE as date FROM link_clicks
- -- use > between < to force the use of the date index.
- WHERE TIMEZONE('UTC', created_at)::DATE BETWEEN (SELECT from_date FROM viewDates) AND (SELECT to_date FROM viewDates)
+ WHERE created_at >= (SELECT from_date FROM viewDates)
+ AND created_at < (SELECT to_date FROM viewDates) + INTERVAL '1 day'
GROUP by date ORDER BY date
) row
),
@@ -416,13 +416,13 @@ CREATE MATERIALIZED VIEW mat_dashboard_charts AS
SELECT JSON_AGG(ROW_TO_JSON(row))
FROM (
WITH viewDates AS (
- SELECT TIMEZONE('UTC', created_at)::DATE AS to_date,
- TIMEZONE('UTC', created_at)::DATE - INTERVAL '30 DAY' AS from_date
+ SELECT created_at::DATE AS to_date,
+ created_at::DATE - INTERVAL '30 DAY' AS from_date
FROM campaign_views ORDER BY id DESC LIMIT 1
)
SELECT COUNT(*) AS count, created_at::DATE as date FROM campaign_views
- -- use > between < to force the use of the date index.
- WHERE TIMEZONE('UTC', created_at)::DATE BETWEEN (SELECT from_date FROM viewDates) AND (SELECT to_date FROM viewDates)
+ WHERE created_at >= (SELECT from_date FROM viewDates)
+ AND created_at < (SELECT to_date FROM viewDates) + INTERVAL '1 day'
GROUP by date ORDER BY date
) row
)