Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: dispatch agent notifications over the edge tunnel so remote environments send notifications (#3567)
Kyle Mendellebc900d8c329
diff --git a/backend/internal/bootstrap/edge_bootstrap.go b/backend/internal/bootstrap/edge_bootstrap.goindex c03ecba90..39b983371 100644--- a/backend/internal/bootstrap/edge_bootstrap.go+++ b/backend/internal/bootstrap/edge_bootstrap.go@@ -14,7 +14,9 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/event""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/models"+ "github.com/getarcaneapp/arcane/backend/v2/internal/notification""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"+ notificationdto "github.com/getarcaneapp/arcane/types/v2/notification""github.com/labstack/echo/v5""go.uber.org/fx")@@ -30,6 +32,7 @@ func registerEdgeTunnelRoutes(apiGroup *echo.Group,environmentService *environment.EnvironmentService,eventService *event.EventService,+ notificationService *notification.NotificationService,registry *edge.TunnelRegistry,) *edge.TunnelServer {// Resolver that validates API key and returns the environment ID@@ -47,6 +50,18 @@ func registerEdgeTunnelRoutes(return errors.New("event payload is required")}+ if evt.Type == edge.TunnelEventTypeNotificationDispatch {+ if notificationService == nil {+ return errors.New("notification service is not available for edge dispatch")+ }+ var payload notificationdto.DispatchRequest+ if err := json.Unmarshal(evt.MetadataJSON, &payload); err != nil {+ return errors.WrapIf(err, "failed to decode edge notification dispatch payload")+ }+ _, err := notificationService.DispatchNotificationForEnvironment(ctx, envID, payload)+ return errors.WrapIf(err, "failed to dispatch edge notification")+ }+var metadata models.JSONif len(evt.MetadataJSON) > 0 {metadata = models.JSON{}diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.goindex 06ce77c51..9bee21641 100644--- a/backend/internal/bootstrap/router_bootstrap.go+++ b/backend/internal/bootstrap/router_bootstrap.go@@ -279,7 +279,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {// This is only registered when NOT in agent mode (i.e., running as manager)var tunnelServer *edge.TunnelServerif !cfg.AgentMode {- tunnelServer = registerEdgeTunnelRoutes(ctx, p.Lifecycle, p.ActorRuntime, cfg, apiGroup, deps.Environment.Service(), deps.Event.Service(), p.TunnelRegistry)+ tunnelServer = registerEdgeTunnelRoutes(ctx, p.Lifecycle, p.ActorRuntime, cfg, apiGroup, deps.Environment.Service(), deps.Event.Service(), deps.Notification.Service(), p.TunnelRegistry)}if cfg.Environment != "production" {diff --git a/backend/internal/notification/notification.go b/backend/internal/notification/notification.goindex 2e828b719..819004962 100644--- a/backend/internal/notification/notification.go+++ b/backend/internal/notification/notification.go@@ -23,6 +23,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/event""github.com/getarcaneapp/arcane/backend/v2/internal/models"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation""github.com/getarcaneapp/arcane/backend/v2/resources"@@ -151,18 +152,29 @@ func (s *NotificationService) resolveNotificationTargetForAccessTokenInternal(ct}func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.Context, payload notificationdto.DispatchRequest) (notificationdto.DispatchResponse, error) {- if s.config == nil || strings.TrimSpace(s.config.GetManagerBaseURL()) == "" {- return notificationdto.DispatchResponse{}, errors.New("manager API URL is required for notification dispatch")- }- if strings.TrimSpace(s.config.AgentToken) == "" {- return notificationdto.DispatchResponse{}, errors.New("agent token is required for notification dispatch")- }-body, err := json.Marshal(payload)if err != nil {return notificationdto.DispatchResponse{}, errors.WrapIf(err, "failed to marshal notification dispatch payload")}+ publishViaTunnel := func() error {+ return edge.PublishEventToManager(&edge.TunnelEvent{+ Type: edge.TunnelEventTypeNotificationDispatch,+ Title: string(payload.Kind),+ MetadataJSON: body,+ })+ }++ // Tunnel-connected edge agents typically have neither MANAGER_API_URL nor+ // AGENT_TOKEN configured for HTTP; ride the agent-to-manager event channel+ // instead so their notifications are not silently lost (#3002).+ if s.config == nil || strings.TrimSpace(s.config.GetManagerBaseURL()) == "" || strings.TrimSpace(s.config.AgentToken) == "" {+ if err := publishViaTunnel(); err != nil {+ return notificationdto.DispatchResponse{}, errors.WrapIf(err, "notification dispatch needs either MANAGER_API_URL + AGENT_TOKEN or a connected edge tunnel")+ }+ return notificationdto.DispatchResponse{Message: "notification dispatched via edge tunnel"}, nil+ }+dispatchURL := strings.TrimRight(s.config.GetManagerBaseURL(), "/") + "/api/notifications/dispatch"req, err := http.NewRequestWithContext(ctx, http.MethodPost, dispatchURL, bytes.NewReader(body))if err != nil {@@ -173,6 +185,11 @@ func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.resp, err := s.httpClient.Do(req)if err != nil {+ // HTTP unreachable does not mean the manager is: an agent configured for+ // HTTP may still hold a healthy edge tunnel, so ride that before giving up.+ if tunnelErr := publishViaTunnel(); tunnelErr == nil {+ return notificationdto.DispatchResponse{Message: "notification dispatched via edge tunnel"}, nil+ }return notificationdto.DispatchResponse{}, errors.WrapIf(err, "failed to dispatch notification to manager")}defer func() { _ = resp.Body.Close() }()@@ -187,7 +204,14 @@ func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.return apiResponse.Data, nil}+ // A non-2xx also means the notification was not dispatched, and the HTTP+ // path can fail independently of the tunnel (stale AGENT_TOKEN, wrong+ // MANAGER_API_URL, intermediate proxy), so the tunnel fallback applies here+ // too before surfacing the status error.responseBody, _ := io.ReadAll(resp.Body)+ if tunnelErr := publishViaTunnel(); tunnelErr == nil {+ return notificationdto.DispatchResponse{Message: "notification dispatched via edge tunnel"}, nil+ }return notificationdto.DispatchResponse{}, errors.Errorf("manager notification dispatch failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(responseBody)))}@@ -201,6 +225,27 @@ func (s *NotificationService) DispatchNotification(ctx context.Context, accessToreturn notificationdto.DispatchResponse{}, err}+ return s.dispatchForTargetInternal(ctx, target, payload)+}++// DispatchNotificationForEnvironment dispatches on behalf of an agent whose+// identity was already established by its edge tunnel session, so no access+// token is exchanged (#3002).+func (s *NotificationService) DispatchNotificationForEnvironment(ctx context.Context, environmentID string, payload notificationdto.DispatchRequest) (notificationdto.DispatchResponse, error) {+ if s.config != nil && s.config.AgentMode {+ return notificationdto.DispatchResponse{}, errors.New("notification dispatch is manager-only")+ }++ target, err := s.resolveNotificationTargetInternal(ctx, environmentID)+ if err != nil {+ return notificationdto.DispatchResponse{}, err+ }++ return s.dispatchForTargetInternal(ctx, target, payload)+}++func (s *NotificationService) dispatchForTargetInternal(ctx context.Context, target NotificationTarget, payload notificationdto.DispatchRequest) (notificationdto.DispatchResponse, error) {+ var err errordispatchResponse := notificationdto.DispatchResponse{Message: "Notification dispatched successfully"}switch payload.Kind {case notificationdto.DispatchKindImageUpdate:diff --git a/backend/internal/notification/service_test.go b/backend/internal/notification/service_test.goindex 79f8cd5f9..9b5b70b8d 100644--- a/backend/internal/notification/service_test.go+++ b/backend/internal/notification/service_test.go@@ -791,3 +791,67 @@ func TestSupportedNotificationTestTypes_IncludesAutoHeal(t *testing.T) {require.Len(t, supportedNotificationTestTypes, len(expected),"supportedNotificationTestTypes has unexpected entries")}++func TestNotificationService_DispatchNotificationForEnvironment_ResolvesTunnelSessionEnvironment(t *testing.T) {+ ctx := context.Background()+ db, _, svc := setupNotificationTestServiceInternal(t)++ now := time.Now()+ require.NoError(t, db.WithContext(ctx).Create(&models.Environment{+ BaseModel: models.BaseModel{ID: "env-edge", CreatedAt: now, UpdatedAt: &now},+ Name: "edge-env",+ ApiUrl: "http://edge:3553",+ Enabled: true,+ }).Error)++ // No access token involved: the environment comes from the tunnel session (#3002).+ resp, err := svc.DispatchNotificationForEnvironment(ctx, "env-edge", notificationdto.DispatchRequest{+ Kind: notificationdto.DispatchKindImageUpdate,+ ImageUpdate: ¬ificationdto.DispatchImageUpdate{+ ImageRef: "nginx:latest",+ UpdateInfo: *newNotificationTestUpdateInfoInternal(),+ },+ })+ require.NoError(t, err)+ require.Equal(t, "Notification dispatched successfully", resp.Message)++ _, err = svc.DispatchNotificationForEnvironment(ctx, "env-edge", notificationdto.DispatchRequest{Kind: "bogus"})+ require.ErrorIs(t, err, ErrUnsupportedDispatchKind)+}++func TestNotificationService_AgentDispatchWithoutHTTPConfigFallsBackToTunnel(t *testing.T) {+ ctx := context.Background()+ db := setupNotificationTestDB(t)+ svc := NewNotificationService(db, &config.Config{AgentMode: true}, nil, event.NewEventService(db, &config.Config{AgentMode: true}, nil))++ // No MANAGER_API_URL/AGENT_TOKEN and no active tunnel: the error must point+ // at both options instead of only the HTTP env vars (#3002).+ _, err := svc.dispatchNotificationToManagerInternal(ctx, notificationdto.DispatchRequest{+ Kind: notificationdto.DispatchKindImageUpdate,+ ImageUpdate: ¬ificationdto.DispatchImageUpdate{+ ImageRef: "nginx:latest",+ UpdateInfo: *newNotificationTestUpdateInfoInternal(),+ },+ })+ require.Error(t, err)+ require.Contains(t, err.Error(), "connected edge tunnel")+}++func TestNotificationService_AgentDispatchHTTPFailureFallsBackToTunnel(t *testing.T) {+ ctx := context.Background()+ db := setupNotificationTestDB(t)+ cfg := &config.Config{AgentMode: true, ManagerApiUrl: "http://127.0.0.1:0", AgentToken: "token"}+ svc := NewNotificationService(db, cfg, nil, event.NewEventService(db, cfg, nil))++ // HTTP dispatch fails and no tunnel is connected either: the fallback must+ // not mask the HTTP transport error.+ _, err := svc.dispatchNotificationToManagerInternal(ctx, notificationdto.DispatchRequest{+ Kind: notificationdto.DispatchKindImageUpdate,+ ImageUpdate: ¬ificationdto.DispatchImageUpdate{+ ImageRef: "nginx:latest",+ UpdateInfo: *newNotificationTestUpdateInfoInternal(),+ },+ })+ require.Error(t, err)+ require.Contains(t, err.Error(), "failed to dispatch notification to manager")+}diff --git a/backend/pkg/libarcane/edge/event_sync.go b/backend/pkg/libarcane/edge/event_sync.goindex 562b843a3..cd47c3328 100644--- a/backend/pkg/libarcane/edge/event_sync.go+++ b/backend/pkg/libarcane/edge/event_sync.go@@ -9,8 +9,16 @@ import ("github.com/google/uuid")-// ErrNoActiveAgentTunnel is returned when no active agent tunnel exists for outbound event sync.-const ErrNoActiveAgentTunnel = errors.Sentinel("no active edge agent tunnel")+const (+ // ErrNoActiveAgentTunnel is returned when no active agent tunnel exists for outbound event sync.+ ErrNoActiveAgentTunnel = errors.Sentinel("no active edge agent tunnel")++ // TunnelEventTypeNotificationDispatch marks a tunnel event that carries a+ // notification dispatch request in MetadataJSON instead of an event-log entry.+ // Tunnel-connected agents have no manager HTTP config, so notification+ // dispatch rides the existing agent-to-manager event channel (#3002).+ TunnelEventTypeNotificationDispatch = "notification.dispatch"+)var agentTunnelState struct {mu sync.RWMutex