Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

14.8 KB
Raw Blame
package api

import (
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"sync/atomic"

	"github.com/ente/museum/ente"
	"github.com/ente/museum/pkg/external/listmonk"
	"github.com/ente/museum/pkg/utils/auth"
	"github.com/ente/museum/pkg/utils/crypto"
	emailUtil "github.com/ente/museum/pkg/utils/email"
	"github.com/ente/museum/pkg/utils/handler"
	"github.com/ente/museum/pkg/utils/time"
	"github.com/ente/stacktrace"
	"github.com/gin-contrib/requestid"
	"github.com/gin-gonic/gin"
	"github.com/sirupsen/logrus"
	"github.com/spf13/viper"
)

const listmonkMissingSubscribersDefaultPageSize = 10000
const listmonkMissingSubscribersMaxLoggedEmails = 10
const listmonkMissingSubscribersMaxUnsubscribeAttempts = 500

var listmonkMissingSubscribersInFlight atomic.Bool

type listmonkMissingSubscribersJobConfig struct {
	Endpoint           string
	Username           string
	Password           string
	PageSize           int
	HasUpperLimit      bool
	UpperLimit         int
	LogMissingEmails   bool
	UnsubscribeMissing bool
	TargetListIDSet    map[int]struct{}
}

type listmonkMissingSubscribersJobSummary struct {
	StartedAt             int64
	CompletedAt           int64
	MissingCount          int
	MatchedCount          int
	ScannedCount          int
	FetchedCount          int
	PagesFetched          int
	ListmonkTotal         int
	StopReason            string
	Capped                bool
	UnsubscribeMissing    bool
	UnsubscribeAttempted  int
	UnsubscribeSucceeded  int
	UnsubscribeFailed     int
	UnsubscribeCapReached bool
}

func (h *AdminHandler) GetListmonkMissingSubscribersCount(c *gin.Context) {
	baseURL := strings.TrimRight(viper.GetString("listmonk.server-url"), "/")
	username := viper.GetString("listmonk.username")
	password := viper.GetString("listmonk.password")
	listIDs := viper.GetIntSlice("listmonk.list-ids")
	pageSize := listmonkMissingSubscribersDefaultPageSize
	if pageSizeParam := strings.TrimSpace(c.Query("pageSize")); pageSizeParam != "" {
		parsedPageSize, err := strconv.Atoi(pageSizeParam)
		if err != nil || parsedPageSize <= 0 {
			handler.Error(c, stacktrace.Propagate(
				ente.NewBadRequestWithMessage("invalid pageSize query param, expected positive integer"),
				"",
			))
			return
		}
		pageSize = parsedPageSize
	}

	hasUpperLimit := false
	upperLimit := 0
	if upperLimitParam := strings.TrimSpace(c.Query("upperLimit")); upperLimitParam != "" {
		parsedUpperLimit, err := strconv.Atoi(upperLimitParam)
		if err != nil || parsedUpperLimit <= 0 {
			handler.Error(c, stacktrace.Propagate(
				ente.NewBadRequestWithMessage("invalid upperLimit query param, expected positive integer"),
				"",
			))
			return
		}
		hasUpperLimit = true
		upperLimit = parsedUpperLimit
	}

	logMissingEmails := false
	if logMissingEmailsParam := strings.TrimSpace(c.Query("logMissingEmails")); logMissingEmailsParam != "" {
		parsed, err := strconv.ParseBool(logMissingEmailsParam)
		if err != nil {
			handler.Error(c, stacktrace.Propagate(
				ente.NewBadRequestWithMessage("invalid logMissingEmails query param, expected boolean"),
				"",
			))
			return
		}
		logMissingEmails = parsed
	}

	unsubscribeMissing := false
	if unsubscribeMissingParam := strings.TrimSpace(c.Query("unsubscribeMissing")); unsubscribeMissingParam != "" {
		parsed, err := strconv.ParseBool(unsubscribeMissingParam)
		if err != nil {
			handler.Error(c, stacktrace.Propagate(
				ente.NewBadRequestWithMessage("invalid unsubscribeMissing query param, expected boolean"),
				"",
			))
			return
		}
		unsubscribeMissing = parsed
	}

	upperLimitLogValue := interface{}("none")
	if hasUpperLimit {
		upperLimitLogValue = upperLimit
	}
	upperLimitResponseValue := interface{}(nil)
	if hasUpperLimit {
		upperLimitResponseValue = upperLimit
	}

	adminID := auth.GetUserID(c.Request.Header)
	reqID := requestid.Get(c)
	logger := logrus.WithFields(logrus.Fields{
		"admin_id":      adminID,
		"req_id":        reqID,
		"req_ctx":       "listmonk_missing_subscribers_count",
		"page_size":     pageSize,
		"upper_limit":   upperLimitLogValue,
		"log_missing":   logMissingEmails,
		"unsubscribe":   unsubscribeMissing,
		"listmonk_user": username,
	})

	if baseURL == "" || username == "" || password == "" || len(listIDs) == 0 {
		logger.Warn("listmonk missing-subscribers count requested without required listmonk config")
		handler.Error(c, stacktrace.Propagate(
			ente.NewBadRequestWithMessage("listmonk is not configured (server-url, username, password, list-ids are required)"),
			"",
		))
		return
	}

	if !listmonkMissingSubscribersInFlight.CompareAndSwap(false, true) {
		logger.Warn("listmonk missing-subscribers job already in progress on this host")
		c.JSON(http.StatusConflict, gin.H{
			"error": "listmonk missing-subscribers processing is already in progress on this host",
		})
		return
	}

	targetListSet := make(map[int]struct{}, len(listIDs))
	for _, listID := range listIDs {
		targetListSet[listID] = struct{}{}
	}

	jobConfig := listmonkMissingSubscribersJobConfig{
		Endpoint:           baseURL + "/api/subscribers",
		Username:           username,
		Password:           password,
		PageSize:           pageSize,
		HasUpperLimit:      hasUpperLimit,
		UpperLimit:         upperLimit,
		LogMissingEmails:   logMissingEmails,
		UnsubscribeMissing: unsubscribeMissing,
		TargetListIDSet:    targetListSet,
	}

	go h.runListmonkMissingSubscribersJobAsync(logger, jobConfig)

	responsePayload := gin.H{
		"status":             "started",
		"requestID":          reqID,
		"pageSize":           pageSize,
		"upperLimit":         upperLimitResponseValue,
		"logMissingEmails":   logMissingEmails,
		"unsubscribeMissing": unsubscribeMissing,
		"acceptedAt":         time.Microseconds(),
	}
	logger.WithField("response_payload", responsePayload).Info("accepted listmonk missing-subscribers async job")
	c.JSON(http.StatusAccepted, responsePayload)
}

func (h *AdminHandler) runListmonkMissingSubscribersJobAsync(logger *logrus.Entry, cfg listmonkMissingSubscribersJobConfig) {
	defer func() {
		if r := recover(); r != nil {
			logger.WithField("panic", r).Error("listmonk missing-subscribers async job panicked")
			if h.DiscordController != nil {
				h.DiscordController.Notify(fmt.Sprintf("Listmonk missing-subscribers job panicked on `%s`", h.DiscordController.HostName))
			}
		}
		listmonkMissingSubscribersInFlight.Store(false)
	}()

	logger.Info("starting listmonk missing-subscribers async job")
	summary, err := h.runListmonkMissingSubscribersJob(cfg, logger)
	if err != nil {
		logger.WithError(err).Error("listmonk missing-subscribers async job failed")
		if h.DiscordController != nil {
			h.DiscordController.Notify(
				fmt.Sprintf("Listmonk missing-subscribers job failed on `%s`", h.DiscordController.HostName),
			)
		}
		return
	}

	durationMs := (summary.CompletedAt - summary.StartedAt) / 1000
	logger.WithFields(logrus.Fields{
		"pages_fetched":           summary.PagesFetched,
		"fetched_count":           summary.FetchedCount,
		"scanned_count":           summary.ScannedCount,
		"matched_count":           summary.MatchedCount,
		"missing_count":           summary.MissingCount,
		"listmonk_total":          summary.ListmonkTotal,
		"stop_reason":             summary.StopReason,
		"capped":                  summary.Capped,
		"unsubscribe_missing":     summary.UnsubscribeMissing,
		"unsubscribe_attempted":   summary.UnsubscribeAttempted,
		"unsubscribe_succeeded":   summary.UnsubscribeSucceeded,
		"unsubscribe_failed":      summary.UnsubscribeFailed,
		"unsubscribe_cap_reached": summary.UnsubscribeCapReached,
		"duration_ms":             durationMs,
	}).Info("completed listmonk missing-subscribers async job")

	if h.DiscordController != nil {
		// Summary intentionally contains no PII.
		h.DiscordController.Notify(fmt.Sprintf(
			"Listmonk missing-subscribers summary on `%s`: active_missing=%d, active_found=%d, eligible=%d, fetched=%d, pages=%d, listmonk_total=%d, unsubscribe_missing=%t, unsub_attempted=%d, unsub_succeeded=%d, unsub_failed=%d, unsub_cap=%t, capped=%t, stop_reason=%s, duration_ms=%d",
			h.DiscordController.HostName,
			summary.MissingCount,
			summary.MatchedCount,
			summary.ScannedCount,
			summary.FetchedCount,
			summary.PagesFetched,
			summary.ListmonkTotal,
			summary.UnsubscribeMissing,
			summary.UnsubscribeAttempted,
			summary.UnsubscribeSucceeded,
			summary.UnsubscribeFailed,
			summary.UnsubscribeCapReached,
			summary.Capped,
			summary.StopReason,
			durationMs,
		))
	}
}

func (h *AdminHandler) runListmonkMissingSubscribersJob(cfg listmonkMissingSubscribersJobConfig, logger *logrus.Entry) (listmonkMissingSubscribersJobSummary, error) {
	summary := listmonkMissingSubscribersJobSummary{
		StartedAt:          time.Microseconds(),
		StopReason:         "exhausted_pages",
		UnsubscribeMissing: cfg.UnsubscribeMissing,
	}

	pageNumber := 1
	seenHashes := make(map[string]struct{})
	emailsToUnsubscribe := make([]string, 0, listmonkMissingSubscribersMaxUnsubscribeAttempts)

	for {
		if cfg.HasUpperLimit && summary.FetchedCount >= cfg.UpperLimit {
			summary.StopReason = "upper_limit_reached"
			summary.Capped = true
			logger.WithFields(logrus.Fields{
				"fetched_count": summary.FetchedCount,
				"stop_reason":   summary.StopReason,
			}).Info("stopping listmonk scan after upper limit")
			break
		}

		page, err := listmonk.ListSubscribers(cfg.Endpoint, cfg.Username, cfg.Password, pageNumber, cfg.PageSize)
		if err != nil {
			return summary, stacktrace.Propagate(err, "failed to fetch listmonk subscribers page %d", pageNumber)
		}

		summary.PagesFetched++
		if pageNumber == 1 {
			summary.ListmonkTotal = page.Total
		}
		originalPageResultsCount := len(page.Results)
		if originalPageResultsCount == 0 {
			summary.StopReason = "empty_page"
			break
		}

		pageResults := page.Results
		if cfg.HasUpperLimit {
			remaining := cfg.UpperLimit - summary.FetchedCount
			if remaining < len(pageResults) {
				pageResults = pageResults[:remaining]
				summary.Capped = true
				summary.StopReason = "upper_limit_reached"
			}
		}
		summary.FetchedCount += len(pageResults)

		pageHashes := make([]string, 0, len(pageResults))
		pageHashToEmail := make(map[string]string, len(pageResults))
		for _, subscriber := range pageResults {
			if !subscriberInTargetLists(subscriber.ListIDs, cfg.TargetListIDSet) {
				continue
			}
			email := emailUtil.NormalizeEmail(subscriber.Email)
			if email == "" {
				continue
			}
			emailHash, err := crypto.GetHash(email, h.HashingKey)
			if err != nil {
				return summary, stacktrace.Propagate(err, "failed to hash subscriber email")
			}
			if _, exists := seenHashes[emailHash]; exists {
				continue
			}
			seenHashes[emailHash] = struct{}{}
			pageHashes = append(pageHashes, emailHash)
			pageHashToEmail[emailHash] = email
		}

		summary.ScannedCount += len(pageHashes)
		matchedHashes, err := h.UserRepo.GetActiveUserEmailHashes(pageHashes)
		if err != nil {
			return summary, stacktrace.Propagate(err, "failed to count matched users")
		}
		pageMatchedCount := len(matchedHashes)
		summary.MatchedCount += pageMatchedCount
		matchedHashSet := make(map[string]struct{}, len(matchedHashes))
		for _, hash := range matchedHashes {
			matchedHashSet[hash] = struct{}{}
		}

		pageMissingCount := len(pageHashes) - pageMatchedCount
		if cfg.UnsubscribeMissing && pageMissingCount > 0 {
			for _, hash := range pageHashes {
				if _, exists := matchedHashSet[hash]; exists {
					continue
				}
				if len(emailsToUnsubscribe) >= listmonkMissingSubscribersMaxUnsubscribeAttempts {
					summary.UnsubscribeCapReached = true
					break
				}
				emailsToUnsubscribe = append(emailsToUnsubscribe, pageHashToEmail[hash])
			}
		}

		if cfg.LogMissingEmails && pageMissingCount > 0 {
			missingEmailSamples := make([]string, 0, listmonkMissingSubscribersMaxLoggedEmails)
			for _, hash := range pageHashes {
				if _, exists := matchedHashSet[hash]; exists {
					continue
				}
				missingEmailSamples = append(missingEmailSamples, pageHashToEmail[hash])
				if len(missingEmailSamples) >= listmonkMissingSubscribersMaxLoggedEmails {
					break
				}
			}
			logger.WithFields(logrus.Fields{
				"page":                 pageNumber,
				"page_missing_count":   pageMissingCount,
				"missing_email_sample": missingEmailSamples,
				"sample_count":         len(missingEmailSamples),
			}).Info("missing emails sample for listmonk page")
		}

		logger.WithFields(logrus.Fields{
			"page":                    pageNumber,
			"page_results":            originalPageResultsCount,
			"processed_results":       len(pageResults),
			"page_unique_hashes":      len(pageHashes),
			"page_matched":            pageMatchedCount,
			"page_missing":            pageMissingCount,
			"fetched_count":           summary.FetchedCount,
			"scanned_count":           summary.ScannedCount,
			"matched_count":           summary.MatchedCount,
			"unsubscribe_queued":      len(emailsToUnsubscribe),
			"unsubscribe_cap_reached": summary.UnsubscribeCapReached,
		}).Info("processed listmonk subscribers page")

		if summary.StopReason == "upper_limit_reached" {
			logger.WithFields(logrus.Fields{
				"fetched_count": summary.FetchedCount,
				"stop_reason":   summary.StopReason,
			}).Info("stopping listmonk scan after upper limit")
			break
		}

		effectivePerPage := cfg.PageSize
		if page.PerPage > 0 {
			effectivePerPage = page.PerPage
		}
		if originalPageResultsCount < effectivePerPage {
			summary.StopReason = "last_partial_page"
			break
		}
		pageNumber++
	}

	if cfg.UnsubscribeMissing && len(emailsToUnsubscribe) > 0 {
		logger.WithField("unsubscribe_queued", len(emailsToUnsubscribe)).Info("starting listmonk unsubscribe for missing subscribers")
		for _, email := range emailsToUnsubscribe {
			summary.UnsubscribeAttempted++
			if err := listmonkUnsubscribeByEmail(cfg.Endpoint, cfg.Username, cfg.Password, email); err != nil {
				summary.UnsubscribeFailed++
				logger.WithFields(logrus.Fields{
					"unsubscribe_attempted": summary.UnsubscribeAttempted,
					"unsubscribe_failed":    summary.UnsubscribeFailed,
				}).WithError(err).Warn("failed to unsubscribe missing listmonk subscriber")
				continue
			}
			summary.UnsubscribeSucceeded++
		}
	}

	summary.CompletedAt = time.Microseconds()
	summary.MissingCount = summary.ScannedCount - summary.MatchedCount
	logger.WithField("missing_count", summary.MissingCount).Info("final missing subscribers count for async job")
	return summary, nil
}

func subscriberInTargetLists(subscriberListIDs []int, targetListSet map[int]struct{}) bool {
	for _, subscriberListID := range subscriberListIDs {
		if _, ok := targetListSet[subscriberListID]; ok {
			return true
		}
	}
	return false
}

func listmonkUnsubscribeByEmail(endpoint string, username string, password string, email string) error {
	subscriberID, err := listmonk.GetSubscriberID(endpoint, username, password, email)
	if err != nil {
		return stacktrace.Propagate(err, "failed to find listmonk subscriber by email")
	}
	return stacktrace.Propagate(
		listmonk.SendRequest(
			http.MethodDelete,
			fmt.Sprintf("%s/%d", endpoint, subscriberID),
			map[string]interface{}{},
			username,
			password,
		),
		"failed to unsubscribe listmonk subscriber",
	)
}