High performance, self-hosted, newsletter and mailing list manager with a modern dashboard. Single binary app.
JavaScript campaigncampaign-managementemail-marketingemail-subscriptionlistmonkmailing-listnewsletternewsletter-managementnewsletter-softwareself-hostedsms-gatewaysmtptransactional-emails
Add cron-based `VACUUM ANALYZE` support for DB maintenance.
Kailash Nadh570bb46d7580
diff --git a/cmd/handlers.go b/cmd/handlers.goindex 68604bc3..2efdc795 100644--- a/cmd/handlers.go+++ b/cmd/handlers.go@@ -107,6 +107,7 @@ func initHTTPHandlers(e *echo.Echo, a *App) {g.GET("/api/settings", pm(a.GetSettings, "settings:get"))g.PUT("/api/settings", pm(a.UpdateSettings, "settings:manage"))+ g.PUT("/api/settings/:key", pm(a.UpdateSettingsByKey, "settings:manage"))g.POST("/api/settings/smtp/test", pm(a.TestSMTPSettings, "settings:manage"))g.POST("/api/admin/reload", pm(a.ReloadApp, "settings:manage"))g.GET("/api/logs", pm(a.GetLogs, "settings:get"))diff --git a/cmd/init.go b/cmd/init.goindex 24a1af37..613164ba 100644--- a/cmd/init.go+++ b/cmd/init.go@@ -8,7 +8,9 @@ import ("errors""fmt""html/template"+ "io""log"+ "log/slog""maps""net/http""os"@@ -941,27 +943,49 @@ func initCaptcha() *captcha.Captcha {return captcha.New(opt)}-// initCron initializes the cron job for refreshing slow query cache.-func initCron(co *core.Core) {- intval := ko.String("app.cache_slow_queries_interval")- if intval == "" {- lo.Println("error: invalid cron interval string")- return+// initCron initializes cron jobs for slow query cache refresh and database vacuum.+func initCron(co *core.Core, db *sqlx.DB) {+ c := cron.New(cron.WithLogger(slog.New(slog.NewTextHandler(io.Discard, nil))))++ // Slow query cache cron job.+ if ko.Bool("app.cache_slow_queries") {+ intval := ko.String("app.cache_slow_queries_interval")+ if intval == "" {+ lo.Println("error: invalid cron interval string for slow query cache")+ } else {+ _, err := c.Add(intval, func() {+ lo.Println("refreshing slow query cache")+ _ = co.RefreshMatViews(true)+ lo.Println("done refreshing slow query cache")+ })+ if err != nil {+ lo.Printf("error initializing slow cache query cron: %v", err)+ } else {+ lo.Printf("IMPORTANT: database slow query caching is enabled. Aggregate numbers and stats will not be realtime. Next refresh at: %v", c.Entries()[len(c.Entries())-1].Next)+ }+ }}- c := cron.New()- _, err := c.Add(intval, func() {- lo.Println("refreshing slow query cache")- _ = co.RefreshMatViews(true)- lo.Println("done refreshing slow query cache")- })- if err != nil {- lo.Printf("error initializing slow cache query cron: %v", err)- return+ // Database vacuum cron job.+ if ko.Bool("maintenance.db.vacuum") {+ intval := ko.String("maintenance.db.vacuum_cron_interval")+ if intval == "" {+ lo.Println("error: invalid cron interval string for database vacuum")+ } else {+ _, err := c.Add(intval, func() {+ RunDBVacuum(db, lo)+ })+ if err != nil {+ lo.Printf("error initializing database vacuum cron: %v", err)+ } else {+ lo.Printf("database VACUUM cron enabled at interval: %s", intval)+ }+ }}- c.Start()- lo.Printf("IMPORTANT: database slow query caching is enabled. Aggregate numbers and stats will not be realtime. Next refresh at: %v", c.Entries()[0].Next)+ if len(c.Entries()) > 0 {+ c.Start()+ }}// awaitReload waits for a SIGHUP signal to reload the app. Every setting change on the UI causes a reload.diff --git a/cmd/main.go b/cmd/main.goindex c10f658d..082303a2 100644--- a/cmd/main.go+++ b/cmd/main.go@@ -232,9 +232,7 @@ func main() {}// Start cronjobs.- if ko.Bool("app.cache_slow_queries") {- initCron(core)- }+ initCron(core, db)// Start the campaign manager workers. The campaign batches (fetch from DB, push out// messages) get processed at the specified interval.diff --git a/cmd/maintenance.go b/cmd/maintenance.goindex 5bdac8c0..2a8d8395 100644--- a/cmd/maintenance.go+++ b/cmd/maintenance.go@@ -1,9 +1,11 @@package mainimport (+ "log""net/http""time"+ "github.com/jmoiron/sqlx""github.com/labstack/echo/v4")@@ -81,3 +83,14 @@ func (a *App) GCCampaignAnalytics(c echo.Context) error {return c.JSON(http.StatusOK, okResp{true})}++// RunDBVacuum runs a full VACUUM on the PostgreSQL database.+// VACUUM reclaims storage occupied by dead tuples and updates planner statistics.+func RunDBVacuum(db *sqlx.DB, lo *log.Logger) {+ lo.Println("running database VACUUM ANALYZE")+ if _, err := db.Exec("VACUUM ANALYZE"); err != nil {+ lo.Printf("error running VACUUM ANALYZE: %v", err)+ return+ }+ lo.Println("finished database VACUUM ANALYZE")+}diff --git a/cmd/settings.go b/cmd/settings.goindex ccec8bc9..12ecf2f7 100644--- a/cmd/settings.go+++ b/cmd/settings.go@@ -2,6 +2,7 @@ package mainimport ("bytes"+ "encoding/json""io""net/http""net/url"@@ -15,7 +16,7 @@ import ("github.com/gdgvda/cron""github.com/gofrs/uuid/v5""github.com/jmoiron/sqlx/types"- "github.com/knadh/koanf/parsers/json"+ koanfjson "github.com/knadh/koanf/parsers/json""github.com/knadh/koanf/providers/rawbytes""github.com/knadh/koanf/v2""github.com/knadh/listmonk/internal/auth"@@ -290,6 +291,33 @@ func (a *App) UpdateSettings(c echo.Context) error {return err}+ return a.handleSettingsRestart(c)+}++// UpdateSettingsByKey updates a single setting key-value in the DB.+func (a *App) UpdateSettingsByKey(c echo.Context) error {+ key := c.Param("key")+ if key == "" {+ return echo.NewHTTPError(http.StatusBadRequest, a.i18n.T("globals.messages.invalidData"))+ }++ // Read the raw JSON body as the value.+ var b json.RawMessage+ if err := c.Bind(&b); err != nil {+ return err+ }++ // Update the value in the DB.+ if err := a.core.UpdateSettingsByKey(key, b); err != nil {+ return err+ }++ return a.handleSettingsRestart(c)+}++// handleSettingsRestart checks for running campaigns and either triggers an+// immediate app restart or marks the app as needing a restart.+func (a *App) handleSettingsRestart(c echo.Context) error {// If there are any active campaigns, don't do an auto reload and// warn the user on the frontend.if a.manager.HasRunningCampaigns() {@@ -302,7 +330,7 @@ func (a *App) UpdateSettings(c echo.Context) error {}{true}})}- // No running campaigns. Reload the a.+ // No running campaigns. Reload the app.go func() {<-time.After(time.Millisecond * 500)a.chReload <- syscall.SIGHUP@@ -327,7 +355,7 @@ func (a *App) TestSMTPSettings(c echo.Context) error {// Load the JSON into koanf to parse SMTP settings properly including timestrings.ko := koanf.New(".")- if err := ko.Load(rawbytes.Provider(reqBody), json.Parser()); err != nil {+ if err := ko.Load(rawbytes.Provider(reqBody), koanfjson.Parser()); err != nil {a.log.Printf("error unmarshalling SMTP test request: %v", err)return echo.NewHTTPError(http.StatusBadRequest, a.i18n.Ts("globals.messages.internalError"))}diff --git a/frontend/src/api/index.js b/frontend/src/api/index.jsindex 842e4e78..cac1af8f 100644--- a/frontend/src/api/index.js+++ b/frontend/src/api/index.js@@ -431,6 +431,12 @@ export const updateSettings = async (data) => http.put({ loading: models.settings },);+export const updateSettingsByKey = async (key, data) => http.put(+ `/api/settings/${key}`,+ data,+ { loading: models.settings },+);+export const testSMTP = async (data) => http.post('/api/settings/smtp/test',data,diff --git a/frontend/src/main.js b/frontend/src/main.jsindex 0b914c5c..e36c137f 100644--- a/frontend/src/main.js+++ b/frontend/src/main.js@@ -101,6 +101,31 @@ const v = new Vue({loadConfig() {initConfig();},++ // awaitRestart handles app restart polling after settings changes.+ // Shows a toast and polls until the backend is back up.+ // Returns a promise that resolves with { needsRestart: boolean }.+ awaitRestart(response) {+ return new Promise((resolve) => {+ // If there are running campaigns, app won't auto restart.+ if (response && typeof response === 'object' && response.needsRestart) {+ this.loadConfig();+ resolve({ needsRestart: true });+ return;+ }++ Vue.prototype.$utils.toast(i18n.t('settings.messengers.messageSaved'));++ // Poll until backend is back up.+ const pollId = setInterval(() => {+ api.getHealth().then(() => {+ clearInterval(pollId);+ this.loadConfig();+ resolve({ needsRestart: false });+ });+ }, 1000);+ });+ },},mounted() {diff --git a/frontend/src/views/Maintenance.vue b/frontend/src/views/Maintenance.vueindex 0363f126..fdeed2a2 100644--- a/frontend/src/views/Maintenance.vue+++ b/frontend/src/views/Maintenance.vue@@ -107,6 +107,39 @@</div></div></div><!-- analytics -->++ <form @submit.prevent="onUpdateDBSettings" class="box mt-6">+ <h4 class="is-size-4">+ {{ $t('maintenance.database.title') }}+ </h4><br />+ <h5 class="is-size-5">Vacuum</h5>+ <p class="has-text-grey is-size-7">+ {{ $t('maintenance.database.vacuumHelp') }}+ </p>+ <br />+ <div class="columns">+ <div class="column is-2">+ <b-field :label="$t('globals.buttons.enabled')">+ <b-switch v-model="dbSettings.vacuum" />+ </b-field>+ </div>+ <div class="column is-4" :class="{ disabled: !dbSettings.vacuum }">+ <b-field :label="$t('settings.maintenance.cron')">+ <b-input v-model="dbSettings.vacuum_cron_interval" placeholder="0 2 * * *" :disabled="!dbSettings.vacuum"+ pattern="((\*|[0-9,\-\/]+)\s+){4}(\*|[0-9,\-\/]+)" />+ </b-field>+ </div>+ <div class="column is-3" />+ <div class="column is-3">+ <br />+ <b-button type="is-primary" native-type="submit" :loading="loading.settings" expanded>+ {{ $t('globals.buttons.save') }}+ </b-button>+ </div>+ </div>+ </form><!-- database -->++ <b-loading :is-full-page="true" v-if="isLoading" active /></section></template>@@ -121,14 +154,23 @@ export default Vue.extend({data() {return {+ isLoading: false,subscriberType: 'orphan',analyticsType: 'all',subscriptionType: 'optin',analyticsDate: dayjs().subtract(7, 'day').toDate(),subscriptionDate: dayjs().subtract(7, 'day').toDate(),+ dbSettings: {+ vacuum: false,+ vacuum_cron_interval: '0 2 * * *',+ },};},+ mounted() {+ this.loadDBSettings();+ },+methods: {formatDateTime(s) {return dayjs(s).format('YYYY-MM-DD');@@ -173,6 +215,21 @@ export default Vue.extend({},);},++ loadDBSettings() {+ this.$api.getSettings().then((data) => {+ if (data['maintenance.db'] !== undefined) {+ this.dbSettings = { ...data['maintenance.db'] };+ }+ });+ },++ async onUpdateDBSettings() {+ this.isLoading = true;+ const data = await this.$api.updateSettingsByKey('maintenance.db', this.dbSettings);+ await this.$root.awaitRestart(data);+ this.isLoading = false;+ },},computed: {diff --git a/frontend/src/views/Settings.vue b/frontend/src/views/Settings.vueindex 5a33de55..d476055e 100644--- a/frontend/src/views/Settings.vue+++ b/frontend/src/views/Settings.vue@@ -106,7 +106,7 @@ export default Vue.extend({},methods: {- onSubmit() {+ async onSubmit() {const form = JSON.parse(JSON.stringify(this.form));// SMTP boxes.@@ -197,30 +197,13 @@ export default Vue.extend({form['privacy.domain_allowlist'] = form['privacy.domain_allowlist'].split('\n').map((v) => v.trim().toLowerCase()).filter((v) => v !== '');this.isLoading = true;- this.$api.updateSettings(form).then((data) => {- if (typeof data === 'object' && data !== null && data.needsRestart) {- // There are running campaigns and the app didn't auto restart.- // The UI will show a warning.- this.$root.loadConfig();- this.getSettings();- this.isLoading = false;- return;- }-- this.$utils.toast(this.$t('settings.messengers.messageSaved'));-- // Poll until there's a 200 response, waiting for the app- // to restart and come back up.- const pollId = setInterval(() => {- this.$api.getHealth().then(() => {- clearInterval(pollId);- this.$root.loadConfig();- this.getSettings();- });- }, 1000);- }, () => {+ try {+ const data = await this.$api.updateSettings(form);+ await this.$root.awaitRestart(data);+ this.getSettings();+ } finally {this.isLoading = false;- });+ }return false;},diff --git a/i18n/en.json b/i18n/en.jsonindex e6c4bd50..c6ed4aef 100644--- a/i18n/en.json+++ b/i18n/en.json@@ -686,5 +686,7 @@"users.totpSecret": "Secret key","users.invalidPassword": "Invalid password","lists.archived": "Archived",- "lists.archivedHelp": "Archiving hides the lists from lists page, campaigns, and public forms. It can be unarchived anytime. It is useful for hiding old and rarely used lists."+ "lists.archivedHelp": "Archiving hides the lists from lists page, campaigns, and public forms. It can be unarchived anytime. It is useful for hiding old and rarely used lists.",+ "maintenance.database.title": "Database",+ "maintenance.database.vacuumHelp": "PostgreSQL VACUUM ANALYZE reclaims storage used by deleted rows and significantly speeds up database performance on large databases. IMPORTANT: For large databases, this is a slow, blocking operation. Schedule to run this during off-peak hours."}diff --git a/internal/core/settings.go b/internal/core/settings.goindex c3dec64c..2d277e46 100644--- a/internal/core/settings.go+++ b/internal/core/settings.go@@ -48,3 +48,13 @@ func (c *Core) UpdateSettings(s models.Settings) error {return nil}++// UpdateSettingsByKey updates a single setting by key.+func (c *Core) UpdateSettingsByKey(key string, value json.RawMessage) error {+ if _, err := c.q.UpdateSettingsByKey.Exec(key, value); err != nil {+ return echo.NewHTTPError(http.StatusInternalServerError,+ c.i18n.Ts("globals.messages.errorUpdating", "name", "{globals.terms.settings}", "error", pqErrMsg(err)))+ }++ return nil+}diff --git a/internal/migrations/v5.1.0.go b/internal/migrations/v5.1.0.goindex 5b93e9e0..033622c1 100644--- a/internal/migrations/v5.1.0.go+++ b/internal/migrations/v5.1.0.go@@ -50,5 +50,15 @@ func V5_1_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf, lo *log.Loggerreturn err}+ // Add maintenance.db setting if not present.+ _, err = db.Exec(`+ INSERT INTO settings (key, value, updated_at)+ VALUES ('maintenance.db', '{"vacuum": false, "vacuum_cron_interval": "0 2 * * *"}', NOW())+ ON CONFLICT (key) DO NOTHING+ `)+ if err != nil {+ return err+ }+return nil}diff --git a/models/queries.go b/models/queries.goindex 6922b689..0703b6c3 100644--- a/models/queries.go+++ b/models/queries.go@@ -104,8 +104,9 @@ type Queries struct {CreateLink *sqlx.Stmt `query:"create-link"`RegisterLinkClick *sqlx.Stmt `query:"register-link-click"`- GetSettings *sqlx.Stmt `query:"get-settings"`- UpdateSettings *sqlx.Stmt `query:"update-settings"`+ GetSettings *sqlx.Stmt `query:"get-settings"`+ UpdateSettings *sqlx.Stmt `query:"update-settings"`+ UpdateSettingsByKey *sqlx.Stmt `query:"update-settings-by-key"`// GetStats *sqlx.Stmt `query:"get-stats"`RecordBounce *sqlx.Stmt `query:"record-bounce"`diff --git a/models/settings.go b/models/settings.goindex 503b5fef..0051cb17 100644--- a/models/settings.go+++ b/models/settings.go@@ -143,6 +143,11 @@ type Settings struct {ScanInterval string `json:"scan_interval"`} `json:"bounce.mailboxes"`+ MaintenanceDB struct {+ Vacuum bool `json:"vacuum"`+ VacuumInterval string `json:"vacuum_cron_interval"`+ } `json:"maintenance.db"`+AdminCustomCSS string `json:"appearance.admin.custom_css"`AdminCustomJS string `json:"appearance.admin.custom_js"`PublicCustomCSS string `json:"appearance.public.custom_css"`diff --git a/queries/misc.sql b/queries/misc.sqlindex 72f18f62..5b2e0bbe 100644--- a/queries/misc.sql+++ b/queries/misc.sql@@ -12,6 +12,9 @@ UPDATE settings AS s SET value = c.value-- For each key in the incoming JSON map, update the row with the key and its value.FROM(SELECT * FROM JSONB_EACH($1)) AS c(key, value) WHERE s.key = c.key;+-- name: update-settings-by-key+UPDATE settings SET value = $2, updated_at = NOW() WHERE key = $1;+-- name: get-db-infoSELECT JSON_BUILD_OBJECT('version', (SELECT VERSION()),'size_mb', (SELECT ROUND(pg_database_size((SELECT CURRENT_DATABASE()))/(1024^2)))) AS info;diff --git a/schema.sql b/schema.sqlindex 09735051..80a0cd81 100644--- a/schema.sql+++ b/schema.sql@@ -292,7 +292,8 @@ INSERT INTO settings (key, value) VALUES('appearance.admin.custom_css', '""'),('appearance.admin.custom_js', '""'),('appearance.public.custom_css', '""'),- ('appearance.public.custom_js', '""');+ ('appearance.public.custom_js', '""'),+ ('maintenance.db', '{"vacuum": false, "vacuum_cron_interval": "0 2 * * *"}');-- bouncesDROP TABLE IF EXISTS bounces CASCADE;