Skip to content

High performance, self-hosted, newsletter and mailing list manager with a modern dashboard. Single binary app.

Add cron-based `VACUUM ANALYZE` support for DB maintenance.

Kailash Nadh570bb46d7580
diff --git a/cmd/handlers.go b/cmd/handlers.go
index 68604bc3..2efdc795 100644
--- a/cmd/handlers.go
+++ b/cmd/handlers.go
@@ -107,6 +107,7 @@ func initHTTPHandlers(e *echo.Echo, a *App) {
g.GET("/api/settings", pm(a.GetSettings, "settings:get"))
g.PUT("/api/settings", pm(a.UpdateSettings, "settings:manage"))
+ g.PUT("/api/settings/:key", pm(a.UpdateSettingsByKey, "settings:manage"))
g.POST("/api/settings/smtp/test", pm(a.TestSMTPSettings, "settings:manage"))
g.POST("/api/admin/reload", pm(a.ReloadApp, "settings:manage"))
g.GET("/api/logs", pm(a.GetLogs, "settings:get"))
diff --git a/cmd/init.go b/cmd/init.go
index 24a1af37..613164ba 100644
--- a/cmd/init.go
+++ b/cmd/init.go
@@ -8,7 +8,9 @@ import (
"errors"
"fmt"
"html/template"
+ "io"
"log"
+ "log/slog"
"maps"
"net/http"
"os"
@@ -941,27 +943,49 @@ func initCaptcha() *captcha.Captcha {
return captcha.New(opt)
}
-// initCron initializes the cron job for refreshing slow query cache.
-func initCron(co *core.Core) {
- intval := ko.String("app.cache_slow_queries_interval")
- if intval == "" {
- lo.Println("error: invalid cron interval string")
- return
+// initCron initializes cron jobs for slow query cache refresh and database vacuum.
+func initCron(co *core.Core, db *sqlx.DB) {
+ c := cron.New(cron.WithLogger(slog.New(slog.NewTextHandler(io.Discard, nil))))
+
+ // Slow query cache cron job.
+ if ko.Bool("app.cache_slow_queries") {
+ intval := ko.String("app.cache_slow_queries_interval")
+ if intval == "" {
+ lo.Println("error: invalid cron interval string for slow query cache")
+ } else {
+ _, err := c.Add(intval, func() {
+ lo.Println("refreshing slow query cache")
+ _ = co.RefreshMatViews(true)
+ lo.Println("done refreshing slow query cache")
+ })
+ if err != nil {
+ lo.Printf("error initializing slow cache query cron: %v", err)
+ } else {
+ lo.Printf("IMPORTANT: database slow query caching is enabled. Aggregate numbers and stats will not be realtime. Next refresh at: %v", c.Entries()[len(c.Entries())-1].Next)
+ }
+ }
}
- c := cron.New()
- _, err := c.Add(intval, func() {
- lo.Println("refreshing slow query cache")
- _ = co.RefreshMatViews(true)
- lo.Println("done refreshing slow query cache")
- })
- if err != nil {
- lo.Printf("error initializing slow cache query cron: %v", err)
- return
+ // Database vacuum cron job.
+ if ko.Bool("maintenance.db.vacuum") {
+ intval := ko.String("maintenance.db.vacuum_cron_interval")
+ if intval == "" {
+ lo.Println("error: invalid cron interval string for database vacuum")
+ } else {
+ _, err := c.Add(intval, func() {
+ RunDBVacuum(db, lo)
+ })
+ if err != nil {
+ lo.Printf("error initializing database vacuum cron: %v", err)
+ } else {
+ lo.Printf("database VACUUM cron enabled at interval: %s", intval)
+ }
+ }
}
- c.Start()
- lo.Printf("IMPORTANT: database slow query caching is enabled. Aggregate numbers and stats will not be realtime. Next refresh at: %v", c.Entries()[0].Next)
+ if len(c.Entries()) > 0 {
+ c.Start()
+ }
}
// awaitReload waits for a SIGHUP signal to reload the app. Every setting change on the UI causes a reload.
diff --git a/cmd/main.go b/cmd/main.go
index c10f658d..082303a2 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -232,9 +232,7 @@ func main() {
}
// Start cronjobs.
- if ko.Bool("app.cache_slow_queries") {
- initCron(core)
- }
+ initCron(core, db)
// Start the campaign manager workers. The campaign batches (fetch from DB, push out
// messages) get processed at the specified interval.
diff --git a/cmd/maintenance.go b/cmd/maintenance.go
index 5bdac8c0..2a8d8395 100644
--- a/cmd/maintenance.go
+++ b/cmd/maintenance.go
@@ -1,9 +1,11 @@
package main
import (
+ "log"
"net/http"
"time"
+ "github.com/jmoiron/sqlx"
"github.com/labstack/echo/v4"
)
@@ -81,3 +83,14 @@ func (a *App) GCCampaignAnalytics(c echo.Context) error {
return c.JSON(http.StatusOK, okResp{true})
}
+
+// RunDBVacuum runs a full VACUUM on the PostgreSQL database.
+// VACUUM reclaims storage occupied by dead tuples and updates planner statistics.
+func RunDBVacuum(db *sqlx.DB, lo *log.Logger) {
+ lo.Println("running database VACUUM ANALYZE")
+ if _, err := db.Exec("VACUUM ANALYZE"); err != nil {
+ lo.Printf("error running VACUUM ANALYZE: %v", err)
+ return
+ }
+ lo.Println("finished database VACUUM ANALYZE")
+}
diff --git a/cmd/settings.go b/cmd/settings.go
index ccec8bc9..12ecf2f7 100644
--- a/cmd/settings.go
+++ b/cmd/settings.go
@@ -2,6 +2,7 @@ package main
import (
"bytes"
+ "encoding/json"
"io"
"net/http"
"net/url"
@@ -15,7 +16,7 @@ import (
"github.com/gdgvda/cron"
"github.com/gofrs/uuid/v5"
"github.com/jmoiron/sqlx/types"
- "github.com/knadh/koanf/parsers/json"
+ koanfjson "github.com/knadh/koanf/parsers/json"
"github.com/knadh/koanf/providers/rawbytes"
"github.com/knadh/koanf/v2"
"github.com/knadh/listmonk/internal/auth"
@@ -290,6 +291,33 @@ func (a *App) UpdateSettings(c echo.Context) error {
return err
}
+ return a.handleSettingsRestart(c)
+}
+
+// UpdateSettingsByKey updates a single setting key-value in the DB.
+func (a *App) UpdateSettingsByKey(c echo.Context) error {
+ key := c.Param("key")
+ if key == "" {
+ return echo.NewHTTPError(http.StatusBadRequest, a.i18n.T("globals.messages.invalidData"))
+ }
+
+ // Read the raw JSON body as the value.
+ var b json.RawMessage
+ if err := c.Bind(&b); err != nil {
+ return err
+ }
+
+ // Update the value in the DB.
+ if err := a.core.UpdateSettingsByKey(key, b); err != nil {
+ return err
+ }
+
+ return a.handleSettingsRestart(c)
+}
+
+// handleSettingsRestart checks for running campaigns and either triggers an
+// immediate app restart or marks the app as needing a restart.
+func (a *App) handleSettingsRestart(c echo.Context) error {
// If there are any active campaigns, don't do an auto reload and
// warn the user on the frontend.
if a.manager.HasRunningCampaigns() {
@@ -302,7 +330,7 @@ func (a *App) UpdateSettings(c echo.Context) error {
}{true}})
}
- // No running campaigns. Reload the a.
+ // No running campaigns. Reload the app.
go func() {
<-time.After(time.Millisecond * 500)
a.chReload <- syscall.SIGHUP
@@ -327,7 +355,7 @@ func (a *App) TestSMTPSettings(c echo.Context) error {
// Load the JSON into koanf to parse SMTP settings properly including timestrings.
ko := koanf.New(".")
- if err := ko.Load(rawbytes.Provider(reqBody), json.Parser()); err != nil {
+ if err := ko.Load(rawbytes.Provider(reqBody), koanfjson.Parser()); err != nil {
a.log.Printf("error unmarshalling SMTP test request: %v", err)
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.Ts("globals.messages.internalError"))
}
diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js
index 842e4e78..cac1af8f 100644
--- a/frontend/src/api/index.js
+++ b/frontend/src/api/index.js
@@ -431,6 +431,12 @@ export const updateSettings = async (data) => http.put(
{ loading: models.settings },
);
+export const updateSettingsByKey = async (key, data) => http.put(
+ `/api/settings/${key}`,
+ data,
+ { loading: models.settings },
+);
+
export const testSMTP = async (data) => http.post(
'/api/settings/smtp/test',
data,
diff --git a/frontend/src/main.js b/frontend/src/main.js
index 0b914c5c..e36c137f 100644
--- a/frontend/src/main.js
+++ b/frontend/src/main.js
@@ -101,6 +101,31 @@ const v = new Vue({
loadConfig() {
initConfig();
},
+
+ // awaitRestart handles app restart polling after settings changes.
+ // Shows a toast and polls until the backend is back up.
+ // Returns a promise that resolves with { needsRestart: boolean }.
+ awaitRestart(response) {
+ return new Promise((resolve) => {
+ // If there are running campaigns, app won't auto restart.
+ if (response && typeof response === 'object' && response.needsRestart) {
+ this.loadConfig();
+ resolve({ needsRestart: true });
+ return;
+ }
+
+ Vue.prototype.$utils.toast(i18n.t('settings.messengers.messageSaved'));
+
+ // Poll until backend is back up.
+ const pollId = setInterval(() => {
+ api.getHealth().then(() => {
+ clearInterval(pollId);
+ this.loadConfig();
+ resolve({ needsRestart: false });
+ });
+ }, 1000);
+ });
+ },
},
mounted() {
diff --git a/frontend/src/views/Maintenance.vue b/frontend/src/views/Maintenance.vue
index 0363f126..fdeed2a2 100644
--- a/frontend/src/views/Maintenance.vue
+++ b/frontend/src/views/Maintenance.vue
@@ -107,6 +107,39 @@
</div>
</div>
</div><!-- analytics -->
+
+ <form @submit.prevent="onUpdateDBSettings" class="box mt-6">
+ <h4 class="is-size-4">
+ {{ $t('maintenance.database.title') }}
+ </h4><br />
+ <h5 class="is-size-5">Vacuum</h5>
+ <p class="has-text-grey is-size-7">
+ {{ $t('maintenance.database.vacuumHelp') }}
+ </p>
+ <br />
+ <div class="columns">
+ <div class="column is-2">
+ <b-field :label="$t('globals.buttons.enabled')">
+ <b-switch v-model="dbSettings.vacuum" />
+ </b-field>
+ </div>
+ <div class="column is-4" :class="{ disabled: !dbSettings.vacuum }">
+ <b-field :label="$t('settings.maintenance.cron')">
+ <b-input v-model="dbSettings.vacuum_cron_interval" placeholder="0 2 * * *" :disabled="!dbSettings.vacuum"
+ pattern="((\*|[0-9,\-\/]+)\s+){4}(\*|[0-9,\-\/]+)" />
+ </b-field>
+ </div>
+ <div class="column is-3" />
+ <div class="column is-3">
+ <br />
+ <b-button type="is-primary" native-type="submit" :loading="loading.settings" expanded>
+ {{ $t('globals.buttons.save') }}
+ </b-button>
+ </div>
+ </div>
+ </form><!-- database -->
+
+ <b-loading :is-full-page="true" v-if="isLoading" active />
</section>
</template>
@@ -121,14 +154,23 @@ export default Vue.extend({
data() {
return {
+ isLoading: false,
subscriberType: 'orphan',
analyticsType: 'all',
subscriptionType: 'optin',
analyticsDate: dayjs().subtract(7, 'day').toDate(),
subscriptionDate: dayjs().subtract(7, 'day').toDate(),
+ dbSettings: {
+ vacuum: false,
+ vacuum_cron_interval: '0 2 * * *',
+ },
};
},
+ mounted() {
+ this.loadDBSettings();
+ },
+
methods: {
formatDateTime(s) {
return dayjs(s).format('YYYY-MM-DD');
@@ -173,6 +215,21 @@ export default Vue.extend({
},
);
},
+
+ loadDBSettings() {
+ this.$api.getSettings().then((data) => {
+ if (data['maintenance.db'] !== undefined) {
+ this.dbSettings = { ...data['maintenance.db'] };
+ }
+ });
+ },
+
+ async onUpdateDBSettings() {
+ this.isLoading = true;
+ const data = await this.$api.updateSettingsByKey('maintenance.db', this.dbSettings);
+ await this.$root.awaitRestart(data);
+ this.isLoading = false;
+ },
},
computed: {
diff --git a/frontend/src/views/Settings.vue b/frontend/src/views/Settings.vue
index 5a33de55..d476055e 100644
--- a/frontend/src/views/Settings.vue
+++ b/frontend/src/views/Settings.vue
@@ -106,7 +106,7 @@ export default Vue.extend({
},
methods: {
- onSubmit() {
+ async onSubmit() {
const form = JSON.parse(JSON.stringify(this.form));
// SMTP boxes.
@@ -197,30 +197,13 @@ export default Vue.extend({
form['privacy.domain_allowlist'] = form['privacy.domain_allowlist'].split('\n').map((v) => v.trim().toLowerCase()).filter((v) => v !== '');
this.isLoading = true;
- this.$api.updateSettings(form).then((data) => {
- if (typeof data === 'object' && data !== null && data.needsRestart) {
- // There are running campaigns and the app didn't auto restart.
- // The UI will show a warning.
- this.$root.loadConfig();
- this.getSettings();
- this.isLoading = false;
- return;
- }
-
- this.$utils.toast(this.$t('settings.messengers.messageSaved'));
-
- // Poll until there's a 200 response, waiting for the app
- // to restart and come back up.
- const pollId = setInterval(() => {
- this.$api.getHealth().then(() => {
- clearInterval(pollId);
- this.$root.loadConfig();
- this.getSettings();
- });
- }, 1000);
- }, () => {
+ try {
+ const data = await this.$api.updateSettings(form);
+ await this.$root.awaitRestart(data);
+ this.getSettings();
+ } finally {
this.isLoading = false;
- });
+ }
return false;
},
diff --git a/i18n/en.json b/i18n/en.json
index e6c4bd50..c6ed4aef 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -686,5 +686,7 @@
"users.totpSecret": "Secret key",
"users.invalidPassword": "Invalid password",
"lists.archived": "Archived",
- "lists.archivedHelp": "Archiving hides the lists from lists page, campaigns, and public forms. It can be unarchived anytime. It is useful for hiding old and rarely used lists."
+ "lists.archivedHelp": "Archiving hides the lists from lists page, campaigns, and public forms. It can be unarchived anytime. It is useful for hiding old and rarely used lists.",
+ "maintenance.database.title": "Database",
+ "maintenance.database.vacuumHelp": "PostgreSQL VACUUM ANALYZE reclaims storage used by deleted rows and significantly speeds up database performance on large databases. IMPORTANT: For large databases, this is a slow, blocking operation. Schedule to run this during off-peak hours."
}
diff --git a/internal/core/settings.go b/internal/core/settings.go
index c3dec64c..2d277e46 100644
--- a/internal/core/settings.go
+++ b/internal/core/settings.go
@@ -48,3 +48,13 @@ func (c *Core) UpdateSettings(s models.Settings) error {
return nil
}
+
+// UpdateSettingsByKey updates a single setting by key.
+func (c *Core) UpdateSettingsByKey(key string, value json.RawMessage) error {
+ if _, err := c.q.UpdateSettingsByKey.Exec(key, value); err != nil {
+ return echo.NewHTTPError(http.StatusInternalServerError,
+ c.i18n.Ts("globals.messages.errorUpdating", "name", "{globals.terms.settings}", "error", pqErrMsg(err)))
+ }
+
+ return nil
+}
diff --git a/internal/migrations/v5.1.0.go b/internal/migrations/v5.1.0.go
index 5b93e9e0..033622c1 100644
--- a/internal/migrations/v5.1.0.go
+++ b/internal/migrations/v5.1.0.go
@@ -50,5 +50,15 @@ func V5_1_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf, lo *log.Logger
return err
}
+ // Add maintenance.db setting if not present.
+ _, err = db.Exec(`
+ INSERT INTO settings (key, value, updated_at)
+ VALUES ('maintenance.db', '{"vacuum": false, "vacuum_cron_interval": "0 2 * * *"}', NOW())
+ ON CONFLICT (key) DO NOTHING
+ `)
+ if err != nil {
+ return err
+ }
+
return nil
}
diff --git a/models/queries.go b/models/queries.go
index 6922b689..0703b6c3 100644
--- a/models/queries.go
+++ b/models/queries.go
@@ -104,8 +104,9 @@ type Queries struct {
CreateLink *sqlx.Stmt `query:"create-link"`
RegisterLinkClick *sqlx.Stmt `query:"register-link-click"`
- GetSettings *sqlx.Stmt `query:"get-settings"`
- UpdateSettings *sqlx.Stmt `query:"update-settings"`
+ GetSettings *sqlx.Stmt `query:"get-settings"`
+ UpdateSettings *sqlx.Stmt `query:"update-settings"`
+ UpdateSettingsByKey *sqlx.Stmt `query:"update-settings-by-key"`
// GetStats *sqlx.Stmt `query:"get-stats"`
RecordBounce *sqlx.Stmt `query:"record-bounce"`
diff --git a/models/settings.go b/models/settings.go
index 503b5fef..0051cb17 100644
--- a/models/settings.go
+++ b/models/settings.go
@@ -143,6 +143,11 @@ type Settings struct {
ScanInterval string `json:"scan_interval"`
} `json:"bounce.mailboxes"`
+ MaintenanceDB struct {
+ Vacuum bool `json:"vacuum"`
+ VacuumInterval string `json:"vacuum_cron_interval"`
+ } `json:"maintenance.db"`
+
AdminCustomCSS string `json:"appearance.admin.custom_css"`
AdminCustomJS string `json:"appearance.admin.custom_js"`
PublicCustomCSS string `json:"appearance.public.custom_css"`
diff --git a/queries/misc.sql b/queries/misc.sql
index 72f18f62..5b2e0bbe 100644
--- a/queries/misc.sql
+++ b/queries/misc.sql
@@ -12,6 +12,9 @@ UPDATE settings AS s SET value = c.value
-- For each key in the incoming JSON map, update the row with the key and its value.
FROM(SELECT * FROM JSONB_EACH($1)) AS c(key, value) WHERE s.key = c.key;
+-- name: update-settings-by-key
+UPDATE settings SET value = $2, updated_at = NOW() WHERE key = $1;
+
-- name: get-db-info
SELECT JSON_BUILD_OBJECT('version', (SELECT VERSION()),
'size_mb', (SELECT ROUND(pg_database_size((SELECT CURRENT_DATABASE()))/(1024^2)))) AS info;
diff --git a/schema.sql b/schema.sql
index 09735051..80a0cd81 100644
--- a/schema.sql
+++ b/schema.sql
@@ -292,7 +292,8 @@ INSERT INTO settings (key, value) VALUES
('appearance.admin.custom_css', '""'),
('appearance.admin.custom_js', '""'),
('appearance.public.custom_css', '""'),
- ('appearance.public.custom_js', '""');
+ ('appearance.public.custom_js', '""'),
+ ('maintenance.db', '{"vacuum": false, "vacuum_cron_interval": "0 2 * * *"}');
-- bounces
DROP TABLE IF EXISTS bounces CASCADE;