High performance, self-hosted, newsletter and mailing list manager with a modern dashboard. Single binary app.
JavaScript campaigncampaign-managementemail-marketingemail-subscriptionlistmonkmailing-listnewsletternewsletter-managementnewsletter-softwareself-hostedsms-gatewaysmtptransactional-emails
package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/mail"
"net/url"
"regexp"
"runtime"
"strings"
"syscall"
"time"
"unicode/utf8"
"github.com/gofrs/uuid/v5"
"github.com/jmoiron/sqlx/types"
koanfjson "github.com/knadh/koanf/parsers/json"
"github.com/knadh/koanf/providers/rawbytes"
"github.com/knadh/koanf/v2"
"github.com/knadh/listmonk/internal/auth"
"github.com/knadh/listmonk/internal/messenger/email"
"github.com/knadh/listmonk/internal/notifs"
"github.com/knadh/listmonk/models"
"github.com/labstack/echo/v5"
)
const pwdMask = "•"
type aboutHost struct {
OS string `json:"os"`
Machine string `json:"arch"`
Hostname string `json:"hostname"`
}
type aboutSystem struct {
NumCPU int `json:"num_cpu"`
AllocMB uint64 `json:"memory_alloc_mb"`
OSMB uint64 `json:"memory_from_os_mb"`
}
type about struct {
Version string `json:"version"`
Build string `json:"build"`
GoVersion string `json:"go_version"`
GoArch string `json:"go_arch"`
Database types.JSONText `json:"database"`
System aboutSystem `json:"system"`
Host aboutHost `json:"host"`
}
var (
reAlphaNum = regexp.MustCompile(`[^a-z0-9\-]`)
)
// GetSettings returns settings from the DB.
func (a *App) GetSettings(c *echo.Context) error {
s, err := a.core.GetSettings()
if err != nil {
return err
}
maskSettings(&s)
return c.JSON(http.StatusOK, okResp{s})
}
// maskSettings replaces sensitive values (passwords, secrets, keys) in the
// settings with masked placeholders of the same length.
func maskSettings(s *models.Settings) {
for i := range s.SMTP {
s.SMTP[i].Password = strings.Repeat(pwdMask, utf8.RuneCountInString(s.SMTP[i].Password))
}
for i := range s.BounceBoxes {
s.BounceBoxes[i].Password = strings.Repeat(pwdMask, utf8.RuneCountInString(s.BounceBoxes[i].Password))
}
for i := range s.Messengers {
s.Messengers[i].Password = strings.Repeat(pwdMask, utf8.RuneCountInString(s.Messengers[i].Password))
}
s.UploadS3AwsSecretAccessKey = strings.Repeat(pwdMask, utf8.RuneCountInString(s.UploadS3AwsSecretAccessKey))
s.BounceSendgrid.Key = strings.Repeat(pwdMask, utf8.RuneCountInString(s.BounceSendgrid.Key))
s.BounceAzure.SharedSecret = strings.Repeat(pwdMask, utf8.RuneCountInString(s.BounceAzure.SharedSecret))
s.BouncePostmark.Password = strings.Repeat(pwdMask, utf8.RuneCountInString(s.BouncePostmark.Password))
s.BounceForwardEmail.Key = strings.Repeat(pwdMask, utf8.RuneCountInString(s.BounceForwardEmail.Key))
s.BounceLettermint.Key = strings.Repeat(pwdMask, utf8.RuneCountInString(s.BounceLettermint.Key))
s.SecurityCaptcha.HCaptcha.Secret = strings.Repeat(pwdMask, utf8.RuneCountInString(s.SecurityCaptcha.HCaptcha.Secret))
s.OIDC.ClientSecret = strings.Repeat(pwdMask, utf8.RuneCountInString(s.OIDC.ClientSecret))
}
// roleOption is a minimal role representation for the settings OIDC role dropdowns.
type roleOption struct {
ID int `json:"id"`
Name string `json:"name"`
}
// settingsView is the admin page view for the settings form.
type settingsView struct {
adminView
Settings models.Settings
UserRoles []roleOption
ListRoles []roleOption
Version string
}
// ViewSettings renders the HTML view for the settings form.
func (a *App) ViewSettings(c *echo.Context) error {
s, err := a.core.GetSettings()
if err != nil {
return err
}
maskSettings(&s)
// Roles for the OIDC default-role dropdowns (security tab).
var userRoles, listRoles []roleOption
if can(auth.GetUser(c), auth.PermRolesGet) {
ur, err := a.core.GetRoles("", "", "")
if err != nil {
return err
}
for _, r := range ur {
userRoles = append(userRoles, roleOption{ID: r.ID, Name: r.Name.String})
}
lr, err := a.core.GetListRoles("", "", "")
if err != nil {
return err
}
for _, r := range lr {
listRoles = append(listRoles, roleOption{ID: r.ID, Name: r.Name.String})
}
}
data := settingsView{
adminView: newAdminView(c, a.i18n.T("globals.terms.settings"), "", "settings.all"),
Settings: s,
UserRoles: userRoles,
ListRoles: listRoles,
Version: versionString,
}
return c.Render(http.StatusOK, "admin-settings", data)
}
// UpdateSettings returns settings from the DB.
func (a *App) UpdateSettings(c *echo.Context) error {
// Unmarshal and marshal the fields once to sanitize the settings blob.
var set models.Settings
if err := c.Bind(&set); err != nil {
return err
}
// Sanitize `from` addresses.
addr, err := sanitizeFromAddresses(set.AppFromEmail)
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest, err.Error())
}
set.AppFromEmail = addr
// Get the existing settings.
cur, err := a.core.GetSettings()
if err != nil {
return err
}
// Preserve the OIDC role mapping when older clients do not send the field.
// An explicit empty array still clears the mapping.
if set.OIDC.Roles == nil {
set.OIDC.Roles = cur.OIDC.Roles
}
// Validate and sanitize postback Messenger names along with SMTP names
// (where each SMTP is also considered as a standalone messenger).
// Duplicates are disallowed and "email" is a reserved name.
names := map[string]bool{email.MessengerName: true}
// There should be at least one enabled SMTP server for the default messenger.
has := false
for i, s := range set.SMTP {
if s.Enabled && !s.ExcludeFromDefault {
has = true
}
// Sanitize and normalize the SMTP server name.
name := reAlphaNum.ReplaceAllString(strings.ToLower(strings.TrimSpace(s.Name)), "-")
if name != "" {
if !strings.HasPrefix(name, "email-") {
name = "email-" + name
}
if _, ok := names[name]; ok {
return echo.NewHTTPError(http.StatusBadRequest,
a.i18n.Ts("settings.duplicateMessengerName", "name", name))
}
names[name] = true
}
if s.ExcludeFromDefault && name == "" {
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.T("settings.smtp.excludeFromDefaultNameRequired"))
}
set.SMTP[i].Name = name
// Assign a UUID. The frontend only sends a password when the user explicitly
// changes the password. In other cases, the existing password in the DB
// is copied while updating the settings and the UUID is used to match
// the incoming array of SMTP blocks with the array in the DB.
if s.UUID == "" {
set.SMTP[i].UUID = uuid.Must(uuid.NewV4()).String()
}
// Ensure the HOST is trimmed of any whitespace.
// This is a common mistake when copy-pasting SMTP settings.
set.SMTP[i].Host = strings.TrimSpace(s.Host)
// If there's no password coming in from the frontend, copy the existing
// password by matching the UUID.
if s.Password == "" {
for _, c := range cur.SMTP {
if s.UUID == c.UUID {
set.SMTP[i].Password = c.Password
}
}
}
}
if !has {
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.T("settings.smtp.defaultRequired"))
}
// Normalize `from_addresses``. Values are either an e-mail address
// or an FQDN. Duplicate domains across server blocks are allowed
// (they get round-robin'd while sending).
for i, s := range set.SMTP {
if !s.Enabled {
continue
}
addrs := make([]string, 0, len(s.FromAddresses))
for _, addr := range s.FromAddresses {
if k := email.NormalizeAddr(addr); k != "" {
addrs = append(addrs, k)
}
}
set.SMTP[i].FromAddresses = addrs
}
// Always remove the trailing slash from the app root URL.
set.AppRootURL = strings.TrimRight(set.AppRootURL, "/")
// Bounce boxes.
for i, s := range set.BounceBoxes {
// Assign a UUID. The frontend only sends a password when the user explicitly
// changes the password. In other cases, the existing password in the DB
// is copied while updating the settings and the UUID is used to match
// the incoming array of blocks with the array in the DB.
if s.UUID == "" {
set.BounceBoxes[i].UUID = uuid.Must(uuid.NewV4()).String()
}
// Ensure the HOST is trimmed of any whitespace.
// This is a common mistake when copy-pasting SMTP settings.
set.BounceBoxes[i].Host = strings.TrimSpace(s.Host)
if d, _ := time.ParseDuration(s.ScanInterval); d.Minutes() < 1 {
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.T("settings.bounces.invalidScanInterval"))
}
// If there's no password coming in from the frontend, copy the existing
// password by matching the UUID.
if s.Password == "" {
for _, c := range cur.BounceBoxes {
if s.UUID == c.UUID {
set.BounceBoxes[i].Password = c.Password
}
}
}
}
for i, m := range set.Messengers {
// UUID to keep track of password changes similar to the SMTP logic above.
if m.UUID == "" {
set.Messengers[i].UUID = uuid.Must(uuid.NewV4()).String()
}
if m.Password == "" {
for _, c := range cur.Messengers {
if m.UUID == c.UUID {
set.Messengers[i].Password = c.Password
}
}
}
name := reAlphaNum.ReplaceAllString(strings.ToLower(m.Name), "")
if _, ok := names[name]; ok {
return echo.NewHTTPError(http.StatusBadRequest,
a.i18n.Ts("settings.duplicateMessengerName", "name", name))
}
if len(name) == 0 {
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.T("settings.invalidMessengerName"))
}
set.Messengers[i].Name = name
names[name] = true
}
// S3 password?
if set.UploadS3AwsSecretAccessKey == "" {
set.UploadS3AwsSecretAccessKey = cur.UploadS3AwsSecretAccessKey
}
if set.BounceSendgrid.Key == "" {
set.BounceSendgrid.Key = cur.BounceSendgrid.Key
}
if set.BounceAzure.SharedSecret == "" {
set.BounceAzure.SharedSecret = cur.BounceAzure.SharedSecret
}
if set.BouncePostmark.Password == "" {
set.BouncePostmark.Password = cur.BouncePostmark.Password
}
if set.BounceForwardEmail.Key == "" {
set.BounceForwardEmail.Key = cur.BounceForwardEmail.Key
}
if set.BounceLettermint.Key == "" {
set.BounceLettermint.Key = cur.BounceLettermint.Key
}
if set.SecurityCaptcha.HCaptcha.Secret == "" {
set.SecurityCaptcha.HCaptcha.Secret = cur.SecurityCaptcha.HCaptcha.Secret
}
if set.OIDC.ClientSecret == "" {
set.OIDC.ClientSecret = cur.OIDC.ClientSecret
}
if err := a.validateOIDCSettings(set); err != nil {
return err
}
for n, v := range set.UploadExtensions {
set.UploadExtensions[n] = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(v), "."))
}
// Domain blocklist / allowlist.
doms := make([]string, 0, len(set.DomainBlocklist))
for _, d := range set.DomainBlocklist {
if d = strings.TrimSpace(strings.ToLower(d)); d != "" {
doms = append(doms, d)
}
}
set.DomainBlocklist = doms
doms = make([]string, 0, len(set.DomainAllowlist))
for _, d := range set.DomainAllowlist {
if d = strings.TrimSpace(strings.ToLower(d)); d != "" {
doms = append(doms, d)
}
}
set.DomainAllowlist = doms
// Validate and clean trusted URLs.
urls := make([]string, 0, len(set.SecurityTrustedURLs))
for _, d := range set.SecurityTrustedURLs {
if d = strings.TrimSpace(d); d != "" {
if d == "*" {
urls = append(urls, d)
continue
}
// Parse and validate the URL.
u, err := url.Parse(d)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return echo.NewHTTPError(http.StatusBadRequest,
a.i18n.Ts("globals.messages.invalidData")+": invalid trusted URL: "+d)
}
urls = append(urls, d)
}
}
set.SecurityTrustedURLs = urls
// Validate slow query caching cron.
if set.CacheSlowQueries {
if _, err := parseCronString(set.CacheSlowQueriesInterval); err != nil {
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.Ts("globals.messages.invalidData")+": slow query cron: "+err.Error())
}
}
// Update the settings in the DB.
if err := a.core.UpdateSettings(set); err != nil {
return err
}
return a.handleSettingsRestart(c)
}
// UpdateSettingsByKey updates a single setting key-value in the DB.
func (a *App) UpdateSettingsByKey(c *echo.Context) error {
key := c.Param("key")
if key == "" {
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.T("globals.messages.invalidData"))
}
// Read the raw JSON body as the value.
var b json.RawMessage
if err := c.Bind(&b); err != nil {
return err
}
// Validate OIDC defaults and mappings on both settings endpoints.
if key == "security.oidc" {
var set models.Settings
if err := json.Unmarshal(b, &set.OIDC); err != nil {
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.T("globals.messages.invalidData"))
}
if err := a.validateOIDCSettings(set); err != nil {
return err
}
}
// Update the value in the DB.
if err := a.core.UpdateSettingsByKey(key, b); err != nil {
return err
}
return a.handleSettingsRestart(c)
}
// validateOIDCSettings validates default roles and claim mapping.
func (a *App) validateOIDCSettings(set models.Settings) error {
userRoles, err := a.core.GetRoles("", "", "")
if err != nil {
return err
}
listRoles, err := a.core.GetListRoles("", "", "")
if err != nil {
return err
}
if err := validateOIDCSettings(set, userRoles, listRoles); err != nil {
return echo.NewHTTPError(http.StatusBadRequest, err.Error())
}
return nil
}
// handleSettingsRestart checks for running campaigns and either triggers an
// immediate app restart or marks the app as needing a restart.
func (a *App) handleSettingsRestart(c *echo.Context) error {
// If there are any active campaigns, don't do an auto reload and
// warn the user on the frontend.
if a.manager.HasRunningCampaigns() {
a.Lock()
a.needsRestart = true
a.Unlock()
return c.JSON(http.StatusOK, okResp{struct {
NeedsRestart bool `json:"needs_restart"`
}{true}})
}
// No running campaigns. Reload the app.
go func() {
<-time.After(time.Millisecond * 500)
a.chReload <- syscall.SIGHUP
}()
return c.JSON(http.StatusOK, okResp{true})
}
// logsView is the admin page view for the log viewer.
type logsView struct {
adminView
Lines []string
}
// ViewLogs renders the HTML view for the application log viewer.
func (a *App) ViewLogs(c *echo.Context) error {
data := logsView{
adminView: newAdminView(c, a.i18n.T("globals.terms.logs"), "", "settings.logs"),
Lines: a.bufLog.Lines(),
}
return c.Render(http.StatusOK, "admin-logs", data)
}
// GetLogs returns the log entries stored in the log buffer.
func (a *App) GetLogs(c *echo.Context) error {
return c.JSON(http.StatusOK, okResp{a.bufLog.Lines()})
}
// TestSMTPSettings returns the log entries stored in the log buffer.
func (a *App) TestSMTPSettings(c *echo.Context) error {
// Copy the raw JSON post body.
reqBody, err := io.ReadAll(c.Request().Body)
if err != nil {
a.log.Printf("error reading SMTP test: %v", err)
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.Ts("globals.messages.internalError"))
}
// Load the JSON into koanf to parse SMTP settings properly including timestrings.
ko := koanf.New(".")
if err := ko.Load(rawbytes.Provider(reqBody), koanfjson.Parser()); err != nil {
a.log.Printf("error unmarshalling SMTP test request: %v", err)
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.Ts("globals.messages.internalError"))
}
req := email.Server{}
if err := ko.UnmarshalWithConf("", &req, koanf.UnmarshalConf{Tag: "json"}); err != nil {
a.log.Printf("error scanning SMTP test request: %v", err)
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.Ts("globals.messages.internalError"))
}
to := ko.String("email")
if to == "" {
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.Ts("globals.messages.missingFields", "name", "email"))
}
// Initialize a new SMTP pool.
req.MaxConns = 1
req.IdleTimeout = time.Second * 2
req.PoolWaitTimeout = time.Second * 2
msgr, err := email.New("", req)
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest,
a.i18n.Ts("globals.messages.errorCreating", "name", "SMTP", "error", err.Error()))
}
// Render the test email template body.
var b bytes.Buffer
if err := notifs.Tpls.ExecuteTemplate(&b, "smtp-test", nil); err != nil {
a.log.Printf("error compiling notification template '%s': %v", "smtp-test", err)
return err
}
m := models.Message{}
m.From = a.cfg.FromEmail
m.To = []string{to}
m.Subject = a.i18n.T("globals.buttons.testConnection")
m.Body = b.Bytes()
if err := msgr.Push(m); err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
return c.JSON(http.StatusOK, okResp{a.bufLog.Lines()})
}
func (a *App) GetAboutInfo(c *echo.Context) error {
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
out := a.about
out.System.AllocMB = mem.Alloc / 1024 / 1024
out.System.OSMB = mem.Sys / 1024 / 1024
return c.JSON(http.StatusOK, out)
}
func sanitizeFromAddresses(addresses []string) ([]string, error) {
if len(addresses) > 250 {
return nil, fmt.Errorf("omax 250 `from` addresses are allowed")
}
out := make([]string, 0, len(addresses))
seen := make(map[string]struct{}, len(addresses))
for _, a := range addresses {
a = strings.TrimSpace(a)
if a == "" {
continue
}
if utf8.RuneCountInString(a) > stdInputMaxLen {
return nil, fmt.Errorf("`from` address cannot exceed %d characters", stdInputMaxLen)
}
// The first from email address must not contain template expressions as that's
// used in non-campaign emails also.
if models.HasTplExpr(a) {
if len(out) == 0 {
return nil, fmt.Errorf("default (first) `from` address cannot contain template expressions")
}
} else {
if _, err := mail.ParseAddress(a); err != nil {
return nil, fmt.Errorf("invalid from email address: %s", a)
}
}
if _, ok := seen[a]; ok {
continue
}
seen[a] = struct{}{}
out = append(out, a)
}
if len(out) == 0 {
return nil, fmt.Errorf("at least one `from` address is required")
}
return out, nil
}