Skip to content

High performance, self-hosted, newsletter and mailing list manager with a modern dashboard. Single binary app.

Add missed indexes and performance tweaks to new analytics queries.

Kailash Nadhe71d4b359704
diff --git a/internal/core/lists.go b/internal/core/lists.go
index 8d137e3e..20bf539a 100644
--- a/internal/core/lists.go
+++ b/internal/core/lists.go
@@ -225,17 +225,24 @@ func (c *Core) DeleteLists(ids []int, query, typ, optin, status string, tags []s
return nil
}
-// GetListCharts returns weekly new subscriptions and and campaign view stats.
+// GetListCharts returns weekly subscription, unsubscription, and campaign view stats.
func (c *Core) GetListCharts(id int, campaignViews bool) (types.JSONText, error) {
- if err := c.refreshCache(matListCharts, false); err != nil {
- return nil, err
+ var (
+ out types.JSONText
+ err error
+ )
+
+ if c.consts.CacheSlowQueries {
+ err = c.q.GetListCharts.Get(&out, id, campaignViews)
+ } else {
+ err = c.db.Get(&out, c.q.GetListChartsLive, id, campaignViews)
}
- var out types.JSONText
- if err := c.q.GetListCharts.Get(&out, id, campaignViews); err != nil {
+ if err != nil {
return nil, echo.NewHTTPError(http.StatusInternalServerError,
c.i18n.Ts("globals.messages.errorFetching", "name", "list charts", "error", pqErrMsg(err)))
}
+
return out, nil
}
diff --git a/internal/migrations/v7.0.0.go b/internal/migrations/v7.0.0.go
index 4781b2c8..12607276 100644
--- a/internal/migrations/v7.0.0.go
+++ b/internal/migrations/v7.0.0.go
@@ -46,6 +46,9 @@ func V7_0_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf, lo *log.Logger
// List analytics.
if _, err := db.Exec(`
+CREATE INDEX IF NOT EXISTS idx_sub_lists_created_at ON subscriber_lists(created_at);
+CREATE INDEX IF NOT EXISTS idx_sub_lists_unsubscribed_updated_at ON subscriber_lists(updated_at) WHERE status = 'unsubscribed';
+
DROP MATERIALIZED VIEW IF EXISTS mat_list_charts;
CREATE MATERIALIZED VIEW mat_list_charts AS
WITH views AS (
diff --git a/models/queries.go b/models/queries.go
index f87d96c9..b650d145 100644
--- a/models/queries.go
+++ b/models/queries.go
@@ -58,6 +58,7 @@ type Queries struct {
GetListUnsubscriptionCounts *sqlx.Stmt `query:"get-list-unsubscription-counts"`
GetListLinkCounts *sqlx.Stmt `query:"get-list-link-counts"`
GetListCharts *sqlx.Stmt `query:"get-list-charts"`
+ GetListChartsLive string `query:"get-list-charts-live"`
CreateList *sqlx.Stmt `query:"create-list"`
QueryLists string `query:"query-lists"`
diff --git a/queries/lists.sql b/queries/lists.sql
index 32dd9e61..a71769c9 100644
--- a/queries/lists.sql
+++ b/queries/lists.sql
@@ -116,6 +116,55 @@ SELECT JSON_BUILD_OBJECT(
)
FROM weeks w LEFT JOIN mat_list_charts c ON c.list_id = $1 AND c.date = w.date;
+-- name: get-list-charts-live
+-- raw: true
+-- Compute the requested list live when slow-query caching is disabled.
+-- Executed as a raw unprepared query so that different intervals don't end up using
+-- the same cached query plan which may be suboptimal. This was discovered through
+-- a bunch of profile exercises.
+WITH subscriptions AS (
+ SELECT DATE_TRUNC('week', created_at)::DATE AS date, COUNT(*) AS count
+ FROM subscriber_lists
+ WHERE
+ list_id = $1
+ AND created_at >= CURRENT_DATE - INTERVAL '3 months'
+ AND created_at <= NOW()
+ GROUP BY date
+), unsubscriptions AS (
+ SELECT DATE_TRUNC('week', updated_at)::DATE AS date, COUNT(*) AS count
+ FROM subscriber_lists
+ WHERE
+ list_id = $1
+ AND status = 'unsubscribed'
+ AND updated_at >= CURRENT_DATE - INTERVAL '3 months'
+ AND updated_at <= NOW()
+ GROUP BY date
+), views AS (
+ SELECT DATE_TRUNC('week', v.created_at)::DATE AS date, COUNT(*) AS count
+ FROM campaign_views v JOIN campaign_lists cl ON cl.campaign_id = v.campaign_id
+ WHERE
+ $2::BOOLEAN AND cl.list_id = $1
+ AND v.created_at >= CURRENT_DATE - INTERVAL '3 months'
+ AND v.created_at <= NOW()
+ GROUP BY date
+), weeks AS (
+ SELECT GENERATE_SERIES(DATE_TRUNC('week', CURRENT_DATE - INTERVAL '3 months'),
+ DATE_TRUNC('week', CURRENT_DATE), INTERVAL '1 week')::DATE AS date
+)
+SELECT JSON_BUILD_OBJECT(
+ 'unsubscriptions', JSON_AGG(JSON_BUILD_OBJECT('date', w.date, 'count', COALESCE(u.count, 0)) ORDER BY w.date),
+ 'subscriptions', JSON_AGG(JSON_BUILD_OBJECT('date', w.date, 'count', COALESCE(s.count, 0)) ORDER BY w.date),
+ 'campaign_views',
+ CASE WHEN $2::BOOLEAN
+ THEN JSON_AGG(JSON_BUILD_OBJECT('date', w.date, 'count', COALESCE(v.count, 0)) ORDER BY w.date)
+ ELSE '[]'::JSON
+ END
+)
+FROM weeks w
+LEFT JOIN subscriptions s ON s.date = w.date
+LEFT JOIN unsubscriptions u ON u.date = w.date
+LEFT JOIN views v ON v.date = w.date;
+
-- name: get-list-analytics-counts
-- raw: true
WITH counts AS (
diff --git a/schema.sql b/schema.sql
index 92b3e76e..2414a3e5 100644
--- a/schema.sql
+++ b/schema.sql
@@ -72,6 +72,8 @@ CREATE TABLE subscriber_lists (
DROP INDEX IF EXISTS idx_sub_lists_sub_id; CREATE INDEX idx_sub_lists_sub_id ON subscriber_lists(subscriber_id);
DROP INDEX IF EXISTS idx_sub_lists_list_id; CREATE INDEX idx_sub_lists_list_id ON subscriber_lists(list_id);
DROP INDEX IF EXISTS idx_sub_lists_status; CREATE INDEX idx_sub_lists_status ON subscriber_lists(status);
+CREATE INDEX IF NOT EXISTS idx_sub_lists_created_at ON subscriber_lists(created_at);
+CREATE INDEX IF NOT EXISTS idx_sub_lists_unsubscribed_updated_at ON subscriber_lists(updated_at) WHERE status = 'unsubscribed';
-- templates
DROP TABLE IF EXISTS templates CASCADE;