Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
package notification
import (
"bytes"
"cmp"
"context"
"encoding/json/v2"
"errors"
"fmt"
"io"
"log/slog"
"maps"
"net/http"
"slices"
"strings"
"time"
"github.com/getarcaneapp/arcane/types/v2/imageupdate"
"github.com/getarcaneapp/arcane/types/v2/notification"
"github.com/getarcaneapp/arcane/types/v2/system"
"go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/crypto"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
"golang.org/x/sync/errgroup"
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/apns"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/notification/children/templates"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation"
)
var notificationCredentialFieldsByProvider = map[notifications.NotificationProvider][]string{
notifications.NotificationProviderDiscord: {"token"},
notifications.NotificationProviderEmail: {"smtpPassword"},
notifications.NotificationProviderTelegram: {"botToken"},
notifications.NotificationProviderSignal: {"password", "token"},
notifications.NotificationProviderSlack: {"token"},
notifications.NotificationProviderNtfy: {"password"},
notifications.NotificationProviderPushover: {"token"},
notifications.NotificationProviderGotify: {"token"},
notifications.NotificationProviderMatrix: {"password"},
// The Google Chat webhook URL embeds its key and token, so the whole URL is a credential.
notifications.NotificationProviderGoogleChat: {"webhookUrl"},
}
var notificationTargetFieldByProvider = map[notifications.NotificationProvider]string{
notifications.NotificationProviderEmail: "smtpHost",
notifications.NotificationProviderSignal: "host",
notifications.NotificationProviderNtfy: "host",
notifications.NotificationProviderGotify: "host",
notifications.NotificationProviderMatrix: "host",
}
const (
notificationDispatchConcurrency = 4
notificationTestTypeSimple = "simple"
notificationTestTypeImageUpdate = "image-update"
notificationTestTypeBatchImageUpdate = "batch-image-update"
notificationTestTypeVulnerability = "vulnerability-found"
notificationTestTypePruneReport = "prune-report"
notificationTestTypeAutoHeal = "auto-heal"
)
// notificationTestEventTypes lists supported test types and the event a real notification of that kind is gated on ("" = none).
var notificationTestEventTypes = map[string]notifications.NotificationEventType{
notificationTestTypeSimple: "",
notificationTestTypeImageUpdate: notifications.NotificationEventImageUpdate,
notificationTestTypeBatchImageUpdate: notifications.NotificationEventImageUpdate,
notificationTestTypeVulnerability: notifications.NotificationEventVulnerabilityFound,
notificationTestTypePruneReport: notifications.NotificationEventPruneReport,
notificationTestTypeAutoHeal: notifications.NotificationEventAutoHeal,
}
var (
ErrUnauthorizedNotificationDispatch = errors.New("unauthorized notification dispatch")
ErrUnsupportedDispatchKind = errors.New("unsupported notification dispatch kind")
)
var notificationsSent = func() metric.Int64Counter {
counter, err := otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.notifications.sent",
metric.WithDescription("Notification provider sends by outcome"), metric.WithUnit("{notification}"))
if err != nil {
otel.Handle(err)
}
return counter
}()
type NotificationService struct {
db *database.DB
config *config.Config
environmentSvc *environment.EnvironmentService
eventSvc *event.EventService
apnsSvc *apns.ApnsService
templates *templates.Service
httpClient *http.Client
}
type NotificationTarget struct {
EnvironmentID string
EnvironmentName string
}
func logManagerDispatchNotification(ctx context.Context, target NotificationTarget, kind notification.DispatchKind) {
slog.InfoContext(ctx,
"Manager dispatching notification on behalf of agent",
"environmentId", target.EnvironmentID,
"environmentName", target.EnvironmentName,
"kind", string(kind),
)
}
func NewNotificationService(db *database.DB, cfg *config.Config, environmentSvc *environment.EnvironmentService, eventSvc *event.EventService, apnsSvc *apns.ApnsService) *NotificationService {
return &NotificationService{
db: db,
config: cfg,
environmentSvc: environmentSvc,
eventSvc: eventSvc,
apnsSvc: apnsSvc,
templates: templates.NewService(cfg.GetAppURL),
httpClient: &http.Client{Timeout: 15 * time.Second},
}
}
func (s *NotificationService) ResolveNotificationTarget(ctx context.Context, environmentID string) (NotificationTarget, error) {
trimmedEnvironmentID := cmp.Or(strings.TrimSpace(environmentID), "0")
if s.environmentSvc != nil {
env, err := s.environmentSvc.GetEnvironmentByID(ctx, trimmedEnvironmentID)
if err == nil && env != nil {
return NotificationTarget{
EnvironmentID: env.ID,
EnvironmentName: cmp.Or(strings.TrimSpace(env.Name), kit.Ternary(env.ID == "0", "Local Docker", "")),
}, nil
}
if trimmedEnvironmentID != "0" {
if err != nil {
return NotificationTarget{}, fmt.Errorf("failed to resolve notification environment: %w", err)
}
return NotificationTarget{}, nil
}
if err != nil {
slog.WarnContext(ctx, "Failed to resolve local environment, falling back to 'Local Docker'", "error", err)
}
}
return NotificationTarget{
EnvironmentID: "0",
EnvironmentName: "Local Docker",
}, nil
}
func (s *NotificationService) dispatchNotificationToManager(ctx context.Context, payload notification.DispatchRequest) (notification.DispatchResponse, error) {
body, err := json.Marshal(payload)
if err != nil {
return notification.DispatchResponse{}, fmt.Errorf("failed to marshal notification dispatch payload: %w", err)
}
publishViaTunnel := func() error {
return edge.PublishEventToManager(ctx, &edge.TunnelEvent{
Type: edge.TunnelEventTypeNotificationDispatch,
Title: string(payload.Kind),
MetadataJSON: body,
})
}
// Tunnel-connected edge agents often lack MANAGER_API_URL/AGENT_TOKEN, so they dispatch over the tunnel (#3002).
if s.config == nil || strings.TrimSpace(httpx.ManagerBaseURL(s.config.ManagerApiUrl)) == "" || strings.TrimSpace(s.config.AgentToken) == "" {
if publishViaTunnelErr := publishViaTunnel(); publishViaTunnelErr != nil {
return notification.DispatchResponse{}, fmt.Errorf("notification dispatch needs either MANAGER_API_URL + AGENT_TOKEN or a connected edge tunnel: %w", publishViaTunnelErr)
}
return notification.DispatchResponse{Message: "notification dispatched via edge tunnel", Delivered: 1}, nil
}
dispatchURL := strings.TrimRight(httpx.ManagerBaseURL(s.config.ManagerApiUrl), "/") + "/api/notifications/dispatch"
req, err := http.NewRequestWithContext(ctx, http.MethodPost, dispatchURL, bytes.NewReader(body))
if err != nil {
return notification.DispatchResponse{}, fmt.Errorf("failed to create notification dispatch request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set(middleware.HeaderApiKey, s.config.AgentToken)
resp, err := s.httpClient.Do(req)
if err != nil {
// An agent configured for HTTP may still hold a healthy edge tunnel, so try it before giving up.
if tunnelErr := publishViaTunnel(); tunnelErr == nil {
return notification.DispatchResponse{Message: "notification dispatched via edge tunnel", Delivered: 1}, nil
}
return notification.DispatchResponse{}, fmt.Errorf("failed to dispatch notification to manager: %w", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices {
var apiResponse struct {
Data notification.DispatchResponse `json:"data"`
}
if unmarshalReadErr := json.UnmarshalRead(resp.Body, &apiResponse); unmarshalReadErr != nil {
return notification.DispatchResponse{}, fmt.Errorf("failed to decode manager notification dispatch response: %w", unmarshalReadErr)
}
return apiResponse.Data, nil
}
responseBody, _ := io.ReadAll(io.LimitReader(resp.Body, 8*1024))
return notification.DispatchResponse{}, fmt.Errorf("manager notification dispatch failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(responseBody)))
}
func (s *NotificationService) DispatchNotification(ctx context.Context, accessToken string, payload notification.DispatchRequest) (notification.DispatchResponse, error) {
if s.config != nil && s.config.AgentMode {
return notification.DispatchResponse{}, errors.New("notification dispatch is manager-only")
}
if s.environmentSvc == nil {
return notification.DispatchResponse{}, errors.New("environment service not initialized")
}
env, err := s.environmentSvc.ResolveEnvironmentByAccessToken(ctx, accessToken)
if err != nil {
if errors.Is(err, environment.ErrEnvironmentAccessTokenRequired) || errors.Is(err, environment.ErrInvalidEnvironmentAccessToken) {
return notification.DispatchResponse{}, ErrUnauthorizedNotificationDispatch
}
return notification.DispatchResponse{}, err
}
target := NotificationTarget{
EnvironmentID: env.ID,
EnvironmentName: cmp.Or(strings.TrimSpace(env.Name), kit.Ternary(env.ID == "0", "Local Docker", "")),
}
return s.dispatchForTarget(ctx, target, payload)
}
// DispatchNotificationForEnvironment dispatches for an agent already identified by its edge tunnel session (#3002).
func (s *NotificationService) DispatchNotificationForEnvironment(ctx context.Context, environmentID string, payload notification.DispatchRequest) (notification.DispatchResponse, error) {
if s.config != nil && s.config.AgentMode {
return notification.DispatchResponse{}, errors.New("notification dispatch is manager-only")
}
target, err := s.ResolveNotificationTarget(ctx, environmentID)
if err != nil {
return notification.DispatchResponse{}, err
}
return s.dispatchForTarget(ctx, target, payload)
}
func (s *NotificationService) dispatchForTarget(ctx context.Context, target NotificationTarget, payload notification.DispatchRequest) (notification.DispatchResponse, error) {
delivered, err := s.deliverForTarget(ctx, target, payload)
if err != nil && delivered == 0 {
return notification.DispatchResponse{}, err
}
if err != nil {
slog.WarnContext(ctx, "Notification dispatch partially failed", "kind", payload.Kind, "delivered", delivered, "error", err)
return notification.DispatchResponse{
Message: "Notification dispatched with errors: " + err.Error(),
Delivered: delivered,
}, nil
}
return notification.DispatchResponse{
Message: "Notification dispatched successfully",
Delivered: delivered,
}, nil
}
func (s *NotificationService) deliverForTarget(ctx context.Context, target NotificationTarget, payload notification.DispatchRequest) (int, error) {
switch payload.Kind {
case notification.DispatchKindImageUpdate:
if payload.ImageUpdate == nil {
return 0, common.Classify(common.ErrValidation, errors.New("image update payload is required"))
}
logManagerDispatchNotification(ctx, target, payload.Kind)
imageUpdate := payload.ImageUpdate
return s.sendImageUpdateNotificationForTarget(ctx, target, imageUpdate.ImageRef, &imageUpdate.UpdateInfo, notifications.NotificationEventImageUpdate)
case notification.DispatchKindBatchImageUpdate:
if payload.BatchImageUpdate == nil {
return 0, common.Classify(common.ErrValidation, errors.New("batch image update payload is required"))
}
logManagerDispatchNotification(ctx, target, payload.Kind)
return s.sendBatchImageUpdateNotificationForTarget(ctx, target, payload.BatchImageUpdate.Updates)
case notification.DispatchKindContainerUpdate:
if payload.ContainerUpdate == nil {
return 0, common.Classify(common.ErrValidation, errors.New("container update payload is required"))
}
logManagerDispatchNotification(ctx, target, payload.Kind)
update := payload.ContainerUpdate
return s.sendContainerUpdateNotificationForTarget(ctx, target, update.ContainerName, update.ImageRef, update.OldDigest, update.NewDigest)
case notification.DispatchKindBatchContainerUpdate:
if payload.BatchContainerUpdate == nil {
return 0, common.Classify(common.ErrValidation, errors.New("batch container update payload is required"))
}
logManagerDispatchNotification(ctx, target, payload.Kind)
entries := make([]notifications.ContainerUpdateBatchEntry, 0, len(payload.BatchContainerUpdate.Updates))
for _, update := range payload.BatchContainerUpdate.Updates {
entries = append(entries, notifications.ContainerUpdateBatchEntry{
ContainerName: update.ContainerName,
ImageRef: update.ImageRef,
OldDigest: update.OldDigest,
NewDigest: update.NewDigest,
})
}
return s.sendBatchContainerUpdateNotificationForTarget(ctx, target, entries)
case notification.DispatchKindVulnerabilityFound:
if payload.VulnerabilityFound == nil {
return 0, common.Classify(common.ErrValidation, errors.New("vulnerability payload is required"))
}
logManagerDispatchNotification(ctx, target, payload.Kind)
return s.sendVulnerabilityNotificationForTarget(ctx, target, VulnerabilityNotificationPayload(*payload.VulnerabilityFound))
case notification.DispatchKindPruneReport:
if payload.PruneReport == nil {
return 0, common.Classify(common.ErrValidation, errors.New("prune report payload is required"))
}
logManagerDispatchNotification(ctx, target, payload.Kind)
return s.sendPruneReportNotificationForTarget(ctx, target, &payload.PruneReport.Result)
case notification.DispatchKindAutoHeal:
if payload.AutoHeal == nil {
return 0, common.Classify(common.ErrValidation, errors.New("auto-heal payload is required"))
}
logManagerDispatchNotification(ctx, target, payload.Kind)
return s.sendAutoHealNotificationForTarget(ctx, target, payload.AutoHeal.ContainerName, payload.AutoHeal.ContainerID)
default:
return 0, fmt.Errorf("%s: %w", payload.Kind, ErrUnsupportedDispatchKind)
}
}
func (s *NotificationService) GetAllSettings(ctx context.Context) ([]NotificationSettings, error) {
var settings []NotificationSettings
if err := s.db.WithContext(ctx).Find(&settings).Error; err != nil {
return nil, fmt.Errorf("failed to get notification settings: %w", err)
}
return settings, nil
}
func (s *NotificationService) GetSettingsByProvider(ctx context.Context, provider notifications.NotificationProvider) (*NotificationSettings, error) {
var setting NotificationSettings
if err := s.db.WithContext(ctx).Where("provider = ?", provider).First(&setting).Error; err != nil {
return nil, err
}
return &setting, nil
}
func (s *NotificationService) CreateOrUpdateSettings(ctx context.Context, provider notifications.NotificationProvider, enabled bool, localConfig database.JSON) (*NotificationSettings, error) {
if provider == notifications.NotificationProviderGeneric {
genericConfig, decodeErr := notifications.DecodeConfig[notifications.GenericConfig](localConfig, "Generic")
if decodeErr != nil {
return nil, decodeErr
}
// Shoutrrr renders the payload template internally at send time, so a
// broken template is validated here where the error surfaces against
// the user's own configuration instead of as an opaque send failure.
if validateErr := notifications.ValidateGenericPayloadTemplate(genericConfig); validateErr != nil {
return nil, common.Classify(common.ErrInvalidNotificationPayloadTemplate, validateErr)
}
}
var setting NotificationSettings
err := s.db.WithContext(ctx).Where("provider = ?", provider).First(&setting).Error
existingConfig := kit.Ternary(err == nil, setting.Config, database.JSON(nil))
encryptedConfig, encryptErr := encryptNotificationConfigCredentials(provider, localConfig, existingConfig)
if encryptErr != nil {
return nil, encryptErr
}
localConfig = encryptedConfig
if _, ok := localConfig["events"]; !ok {
if existingEvents, localOk := existingConfig["events"]; localOk {
localConfig["events"] = existingEvents
}
}
if err != nil {
setting = NotificationSettings{
Provider: provider,
Enabled: enabled,
Config: localConfig,
}
if createSettingsErr := s.db.WithContext(ctx).Create(&setting).Error; createSettingsErr != nil {
return nil, fmt.Errorf("failed to create notification settings: %w", createSettingsErr)
}
} else {
setting.Enabled = enabled
setting.Config = localConfig
if updateSettingsErr := s.db.WithContext(ctx).Save(&setting).Error; updateSettingsErr != nil {
return nil, fmt.Errorf("failed to update notification settings: %w", updateSettingsErr)
}
}
return &setting, nil
}
// RedactNotificationConfigCredentials returns a copy of config with provider credential fields blanked for API responses.
func RedactNotificationConfigCredentials(provider notifications.NotificationProvider, localConfig database.JSON) database.JSON {
redacted := cloneNotificationConfig(localConfig)
for _, field := range notificationCredentialFieldsByProvider[provider] {
value, ok := redacted[field]
if !ok {
continue
}
if value == "" {
delete(redacted, field)
continue
}
redacted[field] = ""
}
return redacted
}
// encryptNotificationConfigCredentials encrypts credential fields, preserving stored ones left blank.
// Kept separate from CreateOrUpdateSettings to stay under the gocognit limit.
func encryptNotificationConfigCredentials(provider notifications.NotificationProvider, localConfig, existingConfig database.JSON) (database.JSON, error) {
encryptedConfig := cloneNotificationConfig(localConfig)
// Switching auth mode drops the stored credential of the other mode.
preserveConfig := cloneNotificationConfig(existingConfig)
switch provider {
case notifications.NotificationProviderSignal:
user, _ := localConfig["user"].(string)
password, _ := localConfig["password"].(string)
token, _ := localConfig["token"].(string)
if strings.TrimSpace(token) != "" {
delete(preserveConfig, "password")
}
if strings.TrimSpace(user) != "" || strings.TrimSpace(password) != "" {
delete(preserveConfig, "token")
}
case notifications.NotificationProviderEmail:
if authMode, _ := localConfig["authMode"].(string); authMode == string(notifications.EmailAuthModeNone) {
delete(preserveConfig, "smtpPassword")
}
}
if targetField := notificationTargetFieldByProvider[provider]; targetField != "" {
currentTarget, _ := existingConfig[targetField].(string)
nextTarget, _ := encryptedConfig[targetField].(string)
if strings.TrimSpace(nextTarget) == "" && strings.TrimSpace(currentTarget) != "" {
nextTarget = currentTarget
encryptedConfig[targetField] = currentTarget
}
storedCredentials := make(map[string]bool, len(notificationCredentialFieldsByProvider[provider]))
updatedCredentials := make(map[string]bool, len(notificationCredentialFieldsByProvider[provider]))
for _, field := range notificationCredentialFieldsByProvider[provider] {
preservedValue, _ := preserveConfig[field].(string)
updatedValue, _ := encryptedConfig[field].(string)
storedCredentials[field] = preservedValue != ""
updatedCredentials[field] = updatedValue != ""
}
if err := validation.ValidateCredentialTargetChange(
targetField,
currentTarget,
new(nextTarget),
func(value string) string {
return strings.ToLower(strings.TrimSuffix(strings.TrimSpace(value), "."))
},
storedCredentials,
updatedCredentials,
); err != nil {
return nil, err
}
}
for _, field := range notificationCredentialFieldsByProvider[provider] {
value, _ := encryptedConfig[field].(string)
if value == "" {
if existingValue, ok := preserveConfig[field].(string); ok && existingValue != "" {
encryptedConfig[field] = existingValue
}
continue
}
// Already-encrypted values are kept as is.
if _, decryptErr := crypto.Decrypt(value); decryptErr == nil {
continue
}
encrypted, err := crypto.Encrypt(value)
if err != nil {
return nil, fmt.Errorf("failed to encrypt notification credential %q: %w", field, err)
}
encryptedConfig[field] = encrypted
}
return encryptedConfig, nil
}
// cloneNotificationConfig copies config, returning an empty non-nil map for nil.
func cloneNotificationConfig(localConfig database.JSON) database.JSON {
if localConfig == nil {
return database.JSON{}
}
cloned := make(database.JSON, len(localConfig))
maps.Copy(cloned, localConfig)
return cloned
}
func (s *NotificationService) DeleteSettings(ctx context.Context, provider notifications.NotificationProvider) error {
if err := s.db.WithContext(ctx).Where("provider = ?", provider).Delete(&NotificationSettings{}).Error; err != nil {
return fmt.Errorf("failed to delete notification settings: %w", err)
}
return nil
}
// isEventEnabled defaults to enabled when the config has no entry for eventType.
func (s *NotificationService) isEventEnabled(localConfig database.JSON, eventType notifications.NotificationEventType) bool {
events, ok := localConfig["events"].(map[string]any)
if !ok {
return true
}
enabled, ok := events[string(eventType)].(bool)
return !ok || enabled
}
// send delivers content to one provider inside a notification.send span and counts the outcome.
func (s *NotificationService) send(
ctx context.Context,
target NotificationTarget,
eventType notifications.NotificationEventType,
setting *NotificationSettings,
content notifications.Content,
) (handled bool, err error) {
providerAttr := attribute.String("arcane.notification.provider", string(setting.Provider))
ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "notification.send", trace.WithAttributes(
providerAttr,
attribute.String("arcane.notification.kind", string(eventType)),
attribute.String("arcane.environment.id", target.EnvironmentID),
attribute.Int64("arcane.notification.setting_id", int64(setting.ID)),
))
defer func() {
// Delivery errors can embed webhook URLs whose paths hold secrets, so spans record only the error type.
if err != nil {
span.SetStatus(codes.Error, "notification delivery failed")
span.SetAttributes(attribute.String("error.type", fmt.Sprintf("%T", err)))
}
span.End()
}()
handled, err = notifications.Deliver(ctx, setting.Provider, setting.Config, content)
if handled {
outcomeAttr := attribute.String("arcane.notification.outcome", kit.Ternary(err == nil, "success", "failure"))
notificationsSent.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes(providerAttr, outcomeAttr))
}
return handled, err
}
// notifyEnabledProviders sends content to every enabled provider subscribed to eventType, logs each attempt,
// and returns how many providers it was delivered to along with the aggregated send errors.
func (s *NotificationService) notifyEnabledProviders(
ctx context.Context,
target NotificationTarget,
eventType notifications.NotificationEventType,
logRef string,
metadata database.JSON,
content notifications.Content,
) (int, error) {
settings, err := s.GetAllSettings(ctx)
if err != nil {
return 0, fmt.Errorf("failed to get notification settings: %w", err)
}
eligible := make([]NotificationSettings, 0, len(settings))
for _, setting := range settings {
if setting.Enabled && s.isEventEnabled(setting.Config, eventType) {
eligible = append(eligible, setting)
}
}
handled := make([]bool, len(eligible))
sendErrs := make([]error, len(eligible))
var g errgroup.Group
g.SetLimit(notificationDispatchConcurrency)
for i := range eligible {
g.Go(func() error {
defer utils.RecoverToError(&sendErrs[i], "notification dispatch", "provider", eligible[i].Provider)
handled[i] = true
handled[i], sendErrs[i] = s.send(ctx, target, eventType, &eligible[i], content)
return nil
})
}
var errs []string
if waitErr := g.Wait(); waitErr != nil {
errs = append(errs, fmt.Sprintf("notification dispatch: %v", waitErr))
}
delivered := 0
for i, setting := range eligible {
if !handled[i] {
slog.WarnContext(ctx, "Unknown notification provider", "provider", setting.Provider)
continue
}
providerName := string(setting.Provider)
req := event.CreateEventRequest{
Type: event.EventTypeNotificationSend,
Severity: event.EventSeveritySuccess,
Title: "Notification sent via " + providerName,
Description: logRef,
ResourceType: new("notification"),
ResourceName: &providerName,
EnvironmentID: &target.EnvironmentID,
Metadata: cloneNotificationConfig(metadata),
}
req.Metadata["provider"] = providerName
req.Metadata["status"] = "success"
if sendErrs[i] == nil {
delivered++
} else {
msg := sendErrs[i].Error()
errs = append(errs, fmt.Sprintf("%s: %s", providerName, msg))
req.Severity = event.EventSeverityError
req.Title = "Notification failed via " + providerName
req.Description = fmt.Sprintf("%s: %s", logRef, msg)
req.Metadata["status"] = "failed"
}
if s.eventSvc == nil {
continue
}
done := make(chan struct{})
go func() {
defer close(done)
defer utils.RecoverToError(nil, "notification event logging", "provider", providerName)
logCtx, cancelLog := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancelLog()
if _, createErr := s.eventSvc.CreateEvent(logCtx, req); createErr != nil {
slog.WarnContext(logCtx, "Failed to log notification event", "provider", providerName, "error", createErr.Error())
}
}()
// Cancellation ends the caller's wait, not persistence of a completed attempt.
select {
case <-done:
case <-ctx.Done():
}
}
if s.apnsSvc != nil {
// A queued push counts as a delivery, so push-only setups still mark
// updates notified instead of re-sending them on every poll.
enqueued, enqueueErr := s.apnsSvc.Enqueue(ctx, target.EnvironmentID, target.EnvironmentName, eventType, logRef, metadata)
if enqueueErr != nil {
slog.WarnContext(ctx, "Failed to enqueue mobile push notification", "error", enqueueErr)
}
if enqueued {
delivered++
}
}
if len(errs) > 0 {
return delivered, fmt.Errorf("notification errors: %s", strings.Join(errs, "; "))
}
return delivered, nil
}
// VulnerabilityNotificationPayload is the data sent to all providers for vulnerability_found events.
// Only vulnerabilities with a fixed version should trigger this notification.
type VulnerabilityNotificationPayload struct {
CVEID string // e.g. CVE-2024-1234
CVELink string // e.g. https://nvd.nist.gov/vuln/detail/CVE-2024-1234
Severity string // CRITICAL, HIGH, MEDIUM, LOW, UNKNOWN
ImageName string // e.g. nginx:latest
FixedVersion string
PkgName string // optional
InstalledVersion string // optional
}
// --- Event entry points ---
// SendBatchContainerUpdateNotification dispatches one notification covering all containers updated in an auto-update run.
func (s *NotificationService) SendBatchContainerUpdateNotification(ctx context.Context, entries []notifications.ContainerUpdateBatchEntry) error {
if len(entries) == 0 {
return nil
}
if s.config != nil && s.config.AgentMode {
updates := make([]notification.DispatchContainerUpdate, 0, len(entries))
for _, entry := range entries {
updates = append(updates, notification.DispatchContainerUpdate{
ContainerName: entry.ContainerName,
ImageRef: entry.ImageRef,
OldDigest: entry.OldDigest,
NewDigest: entry.NewDigest,
})
}
_, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindBatchContainerUpdate,
BatchContainerUpdate: ¬ification.DispatchBatchContainerUpdate{Updates: updates},
})
return err
}
target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return err
}
_, err = s.sendBatchContainerUpdateNotificationForTarget(ctx, target, entries)
return err
}
func (s *NotificationService) sendBatchContainerUpdateNotificationForTarget(ctx context.Context, target NotificationTarget, entries []notifications.ContainerUpdateBatchEntry) (int, error) {
containerNames := make([]string, 0, len(entries))
for _, entry := range entries {
containerNames = append(containerNames, entry.ContainerName)
}
metadata := database.JSON{
"containerNames": containerNames,
"updateCount": len(entries),
"eventType": string(notifications.NotificationEventContainerUpdate),
"batch": true,
}
content := s.templates.BatchContainerUpdate(target.EnvironmentName, entries)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventContainerUpdate)
logRef := strings.Join(containerNames, ", ")
return s.notifyEnabledProviders(ctx, target, notifications.NotificationEventContainerUpdate, logRef, metadata, content)
}
// SendImageUpdateNotification returns how many providers the notification was delivered to.
// Zero means no provider has this event enabled, so callers must not mark the update notified.
func (s *NotificationService) SendImageUpdateNotification(ctx context.Context, imageRef string, updateInfo *imageupdate.Response, eventType notifications.NotificationEventType) (int, error) {
if updateInfo == nil {
return 0, errors.New("updateInfo is required")
}
if s.config != nil && s.config.AgentMode {
dispatchResponse, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindImageUpdate,
ImageUpdate: ¬ification.DispatchImageUpdate{
ImageRef: imageRef,
UpdateInfo: *updateInfo,
},
})
if err != nil {
return 0, err
}
return dispatchResponse.Delivered, nil
}
target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return 0, err
}
return s.sendImageUpdateNotificationForTarget(ctx, target, imageRef, updateInfo, eventType)
}
func (s *NotificationService) sendImageUpdateNotificationForTarget(
ctx context.Context,
target NotificationTarget,
imageRef string,
updateInfo *imageupdate.Response,
eventType notifications.NotificationEventType,
) (int, error) {
metadata := database.JSON{
"hasUpdate": updateInfo.HasUpdate,
"currentDigest": updateInfo.CurrentDigest,
"latestDigest": updateInfo.LatestDigest,
"updateType": updateInfo.UpdateType,
"eventType": string(eventType),
}
content := s.templates.ImageUpdate(target.EnvironmentName, imageRef, updateInfo)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, eventType)
return s.notifyEnabledProviders(ctx, target, eventType, imageRef, metadata, content)
}
func (s *NotificationService) SendContainerUpdateNotification(ctx context.Context, containerName, imageRef, oldDigest, newDigest string) error {
if s.config != nil && s.config.AgentMode {
_, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindContainerUpdate,
ContainerUpdate: ¬ification.DispatchContainerUpdate{
ContainerName: containerName,
ImageRef: imageRef,
OldDigest: oldDigest,
NewDigest: newDigest,
},
})
return err
}
target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return err
}
_, err = s.sendContainerUpdateNotificationForTarget(ctx, target, containerName, imageRef, oldDigest, newDigest)
return err
}
func (s *NotificationService) sendContainerUpdateNotificationForTarget(ctx context.Context, target NotificationTarget, containerName, imageRef, oldDigest, newDigest string) (int, error) {
metadata := database.JSON{
"containerName": containerName,
"oldDigest": oldDigest,
"newDigest": newDigest,
"eventType": string(notifications.NotificationEventContainerUpdate),
}
content := s.templates.ContainerUpdate(target.EnvironmentName, containerName, imageRef, oldDigest, newDigest)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventContainerUpdate)
return s.notifyEnabledProviders(ctx, target, notifications.NotificationEventContainerUpdate, imageRef, metadata, content)
}
// SendVulnerabilityNotification notifies providers subscribed to vulnerability_found events.
// Only daily summary payloads are sent; legacy per-CVE payloads are ignored.
func (s *NotificationService) SendVulnerabilityNotification(ctx context.Context, payload VulnerabilityNotificationPayload) error {
if !strings.HasPrefix(strings.ToUpper(strings.TrimSpace(payload.CVEID)), "DAILY SUMMARY") {
slog.InfoContext(ctx, "skipping legacy individual vulnerability notification payload", "cve", payload.CVEID)
return nil
}
if s.config != nil && s.config.AgentMode {
_, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindVulnerabilityFound,
VulnerabilityFound: ¬ification.DispatchVulnerabilityFound{
CVEID: payload.CVEID,
CVELink: payload.CVELink,
Severity: payload.Severity,
ImageName: payload.ImageName,
FixedVersion: payload.FixedVersion,
PkgName: payload.PkgName,
InstalledVersion: payload.InstalledVersion,
},
})
return err
}
target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return err
}
_, err = s.sendVulnerabilityNotificationForTarget(ctx, target, payload)
return err
}
func (s *NotificationService) sendVulnerabilityNotificationForTarget(ctx context.Context, target NotificationTarget, payload VulnerabilityNotificationPayload) (int, error) {
metadata := database.JSON{
"cveId": payload.CVEID,
"severity": payload.Severity,
"fixedVersion": payload.FixedVersion,
"eventType": string(notifications.NotificationEventVulnerabilityFound),
}
content := s.templates.Vulnerability(target.EnvironmentName, notification.DispatchVulnerabilityFound(payload))
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventVulnerabilityFound)
return s.notifyEnabledProviders(ctx, target, notifications.NotificationEventVulnerabilityFound, payload.ImageName, metadata, content)
}
// SendBatchImageUpdateNotification returns how many providers the notification was delivered to.
// Zero means no provider has this event enabled, so callers must not mark the updates notified.
func (s *NotificationService) SendBatchImageUpdateNotification(ctx context.Context, updates map[string]*imageupdate.Response) (int, error) {
updatesWithChanges := maps.Clone(updates)
maps.DeleteFunc(updatesWithChanges, func(_ string, update *imageupdate.Response) bool { return update == nil || !update.HasUpdate })
if len(updatesWithChanges) == 0 {
return 0, nil
}
if s.config != nil && s.config.AgentMode {
dispatchResponse, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindBatchImageUpdate,
BatchImageUpdate: ¬ification.DispatchBatchImageUpdate{
Updates: updatesWithChanges,
},
})
if err != nil {
return 0, err
}
return dispatchResponse.Delivered, nil
}
target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return 0, err
}
return s.sendBatchImageUpdateNotificationForTarget(ctx, target, updatesWithChanges)
}
func (s *NotificationService) sendBatchImageUpdateNotificationForTarget(ctx context.Context, target NotificationTarget, updates map[string]*imageupdate.Response) (int, error) {
updatesWithChanges := maps.Clone(updates)
maps.DeleteFunc(updatesWithChanges, func(_ string, update *imageupdate.Response) bool { return update == nil || !update.HasUpdate })
if len(updatesWithChanges) == 0 {
return 0, nil
}
metadata := database.JSON{
"updateCount": len(updatesWithChanges),
"eventType": string(notifications.NotificationEventImageUpdate),
"batch": true,
}
content := s.templates.BatchImageUpdate(target.EnvironmentName, updatesWithChanges)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventImageUpdate)
return s.notifyEnabledProviders(ctx, target, notifications.NotificationEventImageUpdate, batchImageUpdateLogRefInternal(updatesWithChanges), metadata, content)
}
// batchImageUpdateLogRefInternal lists the image refs in sorted order. The push body is built
// from it, and the relay dedupes on that body, so a repeat of the same set must read the same.
func batchImageUpdateLogRefInternal(updates map[string]*imageupdate.Response) string {
return strings.Join(slices.Sorted(maps.Keys(updates)), ", ")
}
func (s *NotificationService) SendPruneReportNotification(ctx context.Context, result *system.PruneAllResult) error {
if result == nil {
slog.InfoContext(ctx, "skipping prune report notification because no prune result was reported")
return nil
}
hasChanges := result.SpaceReclaimed > 0 ||
len(result.ContainersPruned) > 0 ||
len(result.ImagesDeleted) > 0 ||
len(result.VolumesDeleted) > 0 ||
len(result.NetworksDeleted) > 0
hasErrors := len(result.Errors) > 0
if !hasChanges && !hasErrors {
slog.InfoContext(ctx, "skipping prune report notification because no resources were pruned and no errors were reported")
return nil
}
if !hasChanges {
slog.WarnContext(ctx, "sending prune report notification with errors but no resources were pruned", "errorCount", len(result.Errors))
}
if s.config != nil && s.config.AgentMode {
_, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindPruneReport,
PruneReport: ¬ification.DispatchPruneReport{
Result: *result,
},
})
return err
}
target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return err
}
_, err = s.sendPruneReportNotificationForTarget(ctx, target, result)
return err
}
func (s *NotificationService) sendPruneReportNotificationForTarget(ctx context.Context, target NotificationTarget, result *system.PruneAllResult) (int, error) {
metadata := database.JSON{
"spaceReclaimed": result.SpaceReclaimed,
"eventType": string(notifications.NotificationEventPruneReport),
}
content := s.templates.PruneReport(target.EnvironmentName, result)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventPruneReport)
return s.notifyEnabledProviders(ctx, target, notifications.NotificationEventPruneReport, "System Prune Report", metadata, content)
}
// SendAutoHealNotification sends a notification when a container is auto-healed.
func (s *NotificationService) SendAutoHealNotification(ctx context.Context, containerName, containerID string) error {
if s.config != nil && s.config.AgentMode {
_, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindAutoHeal,
AutoHeal: ¬ification.DispatchAutoHeal{
ContainerName: containerName,
ContainerID: containerID,
},
})
return err
}
target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return err
}
_, err = s.sendAutoHealNotificationForTarget(ctx, target, containerName, containerID)
return err
}
func (s *NotificationService) sendAutoHealNotificationForTarget(ctx context.Context, target NotificationTarget, containerName, containerID string) (int, error) {
metadata := database.JSON{
"containerID": containerID,
"eventType": string(notifications.NotificationEventAutoHeal),
}
content := s.templates.AutoHeal(target.EnvironmentName, containerName)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventAutoHeal)
return s.notifyEnabledProviders(ctx, target, notifications.NotificationEventAutoHeal, containerName, metadata, content)
}
// --- Test notifications ---
// TestNotification sends a test message even to a disabled provider (testing before enabling is legitimate)
// and returns a warning when a real notification of the tested kind would not send.
func (s *NotificationService) TestNotification(ctx context.Context, environmentID string, provider notifications.NotificationProvider, testType string) (string, error) {
setting, err := s.GetSettingsByProvider(ctx, provider)
if errors.Is(err, gorm.ErrRecordNotFound) {
return "", common.Classify(common.ErrBadRequest, fmt.Errorf("please save your %s settings before testing", provider))
}
if err != nil {
return "", fmt.Errorf("please save your %s settings before testing", provider)
}
testType = cmp.Or(strings.TrimSpace(testType), notificationTestTypeSimple)
eventType, ok := notificationTestEventTypes[testType]
if !ok {
return "", fmt.Errorf("unsupported notification test type: %s", testType)
}
var warning string
switch {
case !setting.Enabled:
warning = fmt.Sprintf("%s is disabled, so real notifications will not send", setting.Provider)
case eventType != "" && !s.isEventEnabled(setting.Config, eventType):
warning = fmt.Sprintf("%s events are disabled for %s, so real notifications will not send", eventType, setting.Provider)
case eventType == "" && !slices.ContainsFunc(notifications.AllNotificationEventTypes, func(candidate notifications.NotificationEventType) bool {
return s.isEventEnabled(setting.Config, candidate)
}):
warning = fmt.Sprintf("no events are subscribed for %s, so real notifications will not send", setting.Provider)
}
target, err := s.ResolveNotificationTarget(ctx, environmentID)
if err != nil {
return "", err
}
var content notifications.Content
switch {
case provider == notifications.NotificationProviderEmail && testType == notificationTestTypeSimple:
content = s.templates.TestEmail(target.EnvironmentName)
case testType == notificationTestTypeVulnerability:
content = s.templates.Vulnerability(target.EnvironmentName, notification.DispatchVulnerabilityFound{
CVEID: "Daily Summary - " + time.Now().UTC().Format("2006-01-02"),
Severity: "Critical:1 High:3 Medium:2 Low:1 Unknown:0",
ImageName: "5 image(s) scanned, 2 with fixable vulnerabilities",
FixedVersion: "7 fixable vulnerability record(s)",
PkgName: "CVE-2025-1234, CVE-2025-5678, CVE-2026-0001",
})
case testType == notificationTestTypeAutoHeal:
content = s.templates.AutoHeal(target.EnvironmentName, "test-container")
case testType == notificationTestTypePruneReport:
content = s.templates.PruneReport(target.EnvironmentName, &system.PruneAllResult{
Success: true,
ContainersPruned: []string{"a1b2c3d4e5f6", "f6e5d4c3b2a1"},
ImagesDeleted: []string{"sha256:1111111111111111111111111111111111111111111111111111111111111111"},
VolumesDeleted: []string{"arcane_test_volume"},
NetworksDeleted: []string{"arcane_test_network"},
SpaceReclaimed: 3825205248,
ContainerSpaceReclaimed: 503316480,
ImageSpaceReclaimed: 2449473536,
VolumeSpaceReclaimed: 641728512,
BuildCacheSpaceReclaimed: 230162432,
Errors: []string{},
})
case testType == notificationTestTypeBatchImageUpdate:
content = s.templates.BatchImageUpdate(target.EnvironmentName, map[string]*imageupdate.Response{
"nginx:latest": {
HasUpdate: true,
UpdateType: "digest",
CurrentDigest: "sha256:abc123def456789012345678901234567890",
LatestDigest: "sha256:xyz789ghi012345678901234567890123456",
CheckTime: time.Now(),
ResponseTimeMs: 100,
},
"postgres:16-alpine": {
HasUpdate: true,
UpdateType: "digest",
CurrentDigest: "sha256:def456abc123789012345678901234567890",
LatestDigest: "sha256:ghi789xyz012345678901234567890123456",
CheckTime: time.Now(),
ResponseTimeMs: 120,
},
"redis:7.2-alpine": {
HasUpdate: true,
UpdateType: "digest",
CurrentDigest: "sha256:123456789abc012345678901234567890def",
LatestDigest: "sha256:456789012def345678901234567890123abc",
CheckTime: time.Now(),
ResponseTimeMs: 95,
},
})
default: // simple and image-update
imageRef := kit.Ternary(testType == notificationTestTypeSimple, "test/image:latest", "nginx:latest")
content = s.templates.ImageUpdate(target.EnvironmentName, imageRef, &imageupdate.Response{
HasUpdate: true,
UpdateType: "digest",
CurrentDigest: "sha256:abc123def456789012345678901234567890",
LatestDigest: "sha256:xyz789ghi012345678901234567890123456",
CheckTime: time.Now(),
ResponseTimeMs: 100,
})
}
// Event vars let the test exercise a generic payload template; the simple test renders image-update content.
testEventType := cmp.Or(eventType, notifications.NotificationEventImageUpdate)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, testEventType)
handled, sendErr := s.send(ctx, target, testEventType, setting, content)
if !handled {
return "", fmt.Errorf("unknown provider: %s", provider)
}
return warning, sendErr
}