Skip to content

Modern Docker Management, Designed for Everyone

fix: dispatch agent notifications over the edge tunnel so remote environments send notifications (#3567)

Kyle Mendellebc900d8c329
diff --git a/backend/internal/bootstrap/edge_bootstrap.go b/backend/internal/bootstrap/edge_bootstrap.go
index c03ecba90..39b983371 100644
--- a/backend/internal/bootstrap/edge_bootstrap.go
+++ b/backend/internal/bootstrap/edge_bootstrap.go
@@ -14,7 +14,9 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/notification"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
+ notificationdto "github.com/getarcaneapp/arcane/types/v2/notification"
"github.com/labstack/echo/v5"
"go.uber.org/fx"
)
@@ -30,6 +32,7 @@ func registerEdgeTunnelRoutes(
apiGroup *echo.Group,
environmentService *environment.EnvironmentService,
eventService *event.EventService,
+ notificationService *notification.NotificationService,
registry *edge.TunnelRegistry,
) *edge.TunnelServer {
// Resolver that validates API key and returns the environment ID
@@ -47,6 +50,18 @@ func registerEdgeTunnelRoutes(
return errors.New("event payload is required")
}
+ if evt.Type == edge.TunnelEventTypeNotificationDispatch {
+ if notificationService == nil {
+ return errors.New("notification service is not available for edge dispatch")
+ }
+ var payload notificationdto.DispatchRequest
+ if err := json.Unmarshal(evt.MetadataJSON, &payload); err != nil {
+ return errors.WrapIf(err, "failed to decode edge notification dispatch payload")
+ }
+ _, err := notificationService.DispatchNotificationForEnvironment(ctx, envID, payload)
+ return errors.WrapIf(err, "failed to dispatch edge notification")
+ }
+
var metadata models.JSON
if len(evt.MetadataJSON) > 0 {
metadata = models.JSON{}
diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.go
index 06ce77c51..9bee21641 100644
--- a/backend/internal/bootstrap/router_bootstrap.go
+++ b/backend/internal/bootstrap/router_bootstrap.go
@@ -279,7 +279,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
// This is only registered when NOT in agent mode (i.e., running as manager)
var tunnelServer *edge.TunnelServer
if !cfg.AgentMode {
- tunnelServer = registerEdgeTunnelRoutes(ctx, p.Lifecycle, p.ActorRuntime, cfg, apiGroup, deps.Environment.Service(), deps.Event.Service(), p.TunnelRegistry)
+ tunnelServer = registerEdgeTunnelRoutes(ctx, p.Lifecycle, p.ActorRuntime, cfg, apiGroup, deps.Environment.Service(), deps.Event.Service(), deps.Notification.Service(), p.TunnelRegistry)
}
if cfg.Environment != "production" {
diff --git a/backend/internal/notification/notification.go b/backend/internal/notification/notification.go
index 2e828b719..819004962 100644
--- a/backend/internal/notification/notification.go
+++ b/backend/internal/notification/notification.go
@@ -23,6 +23,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation"
"github.com/getarcaneapp/arcane/backend/v2/resources"
@@ -151,18 +152,29 @@ func (s *NotificationService) resolveNotificationTargetForAccessTokenInternal(ct
}
func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.Context, payload notificationdto.DispatchRequest) (notificationdto.DispatchResponse, error) {
- if s.config == nil || strings.TrimSpace(s.config.GetManagerBaseURL()) == "" {
- return notificationdto.DispatchResponse{}, errors.New("manager API URL is required for notification dispatch")
- }
- if strings.TrimSpace(s.config.AgentToken) == "" {
- return notificationdto.DispatchResponse{}, errors.New("agent token is required for notification dispatch")
- }
-
body, err := json.Marshal(payload)
if err != nil {
return notificationdto.DispatchResponse{}, errors.WrapIf(err, "failed to marshal notification dispatch payload")
}
+ publishViaTunnel := func() error {
+ return edge.PublishEventToManager(&edge.TunnelEvent{
+ Type: edge.TunnelEventTypeNotificationDispatch,
+ Title: string(payload.Kind),
+ MetadataJSON: body,
+ })
+ }
+
+ // Tunnel-connected edge agents typically have neither MANAGER_API_URL nor
+ // AGENT_TOKEN configured for HTTP; ride the agent-to-manager event channel
+ // instead so their notifications are not silently lost (#3002).
+ if s.config == nil || strings.TrimSpace(s.config.GetManagerBaseURL()) == "" || strings.TrimSpace(s.config.AgentToken) == "" {
+ if err := publishViaTunnel(); err != nil {
+ return notificationdto.DispatchResponse{}, errors.WrapIf(err, "notification dispatch needs either MANAGER_API_URL + AGENT_TOKEN or a connected edge tunnel")
+ }
+ return notificationdto.DispatchResponse{Message: "notification dispatched via edge tunnel"}, nil
+ }
+
dispatchURL := strings.TrimRight(s.config.GetManagerBaseURL(), "/") + "/api/notifications/dispatch"
req, err := http.NewRequestWithContext(ctx, http.MethodPost, dispatchURL, bytes.NewReader(body))
if err != nil {
@@ -173,6 +185,11 @@ func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.
resp, err := s.httpClient.Do(req)
if err != nil {
+ // HTTP unreachable does not mean the manager is: an agent configured for
+ // HTTP may still hold a healthy edge tunnel, so ride that before giving up.
+ if tunnelErr := publishViaTunnel(); tunnelErr == nil {
+ return notificationdto.DispatchResponse{Message: "notification dispatched via edge tunnel"}, nil
+ }
return notificationdto.DispatchResponse{}, errors.WrapIf(err, "failed to dispatch notification to manager")
}
defer func() { _ = resp.Body.Close() }()
@@ -187,7 +204,14 @@ func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.
return apiResponse.Data, nil
}
+ // A non-2xx also means the notification was not dispatched, and the HTTP
+ // path can fail independently of the tunnel (stale AGENT_TOKEN, wrong
+ // MANAGER_API_URL, intermediate proxy), so the tunnel fallback applies here
+ // too before surfacing the status error.
responseBody, _ := io.ReadAll(resp.Body)
+ if tunnelErr := publishViaTunnel(); tunnelErr == nil {
+ return notificationdto.DispatchResponse{Message: "notification dispatched via edge tunnel"}, nil
+ }
return notificationdto.DispatchResponse{}, errors.Errorf("manager notification dispatch failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(responseBody)))
}
@@ -201,6 +225,27 @@ func (s *NotificationService) DispatchNotification(ctx context.Context, accessTo
return notificationdto.DispatchResponse{}, err
}
+ return s.dispatchForTargetInternal(ctx, target, payload)
+}
+
+// DispatchNotificationForEnvironment dispatches on behalf of an agent whose
+// identity was already established by its edge tunnel session, so no access
+// token is exchanged (#3002).
+func (s *NotificationService) DispatchNotificationForEnvironment(ctx context.Context, environmentID string, payload notificationdto.DispatchRequest) (notificationdto.DispatchResponse, error) {
+ if s.config != nil && s.config.AgentMode {
+ return notificationdto.DispatchResponse{}, errors.New("notification dispatch is manager-only")
+ }
+
+ target, err := s.resolveNotificationTargetInternal(ctx, environmentID)
+ if err != nil {
+ return notificationdto.DispatchResponse{}, err
+ }
+
+ return s.dispatchForTargetInternal(ctx, target, payload)
+}
+
+func (s *NotificationService) dispatchForTargetInternal(ctx context.Context, target NotificationTarget, payload notificationdto.DispatchRequest) (notificationdto.DispatchResponse, error) {
+ var err error
dispatchResponse := notificationdto.DispatchResponse{Message: "Notification dispatched successfully"}
switch payload.Kind {
case notificationdto.DispatchKindImageUpdate:
diff --git a/backend/internal/notification/service_test.go b/backend/internal/notification/service_test.go
index 79f8cd5f9..9b5b70b8d 100644
--- a/backend/internal/notification/service_test.go
+++ b/backend/internal/notification/service_test.go
@@ -791,3 +791,67 @@ func TestSupportedNotificationTestTypes_IncludesAutoHeal(t *testing.T) {
require.Len(t, supportedNotificationTestTypes, len(expected),
"supportedNotificationTestTypes has unexpected entries")
}
+
+func TestNotificationService_DispatchNotificationForEnvironment_ResolvesTunnelSessionEnvironment(t *testing.T) {
+ ctx := context.Background()
+ db, _, svc := setupNotificationTestServiceInternal(t)
+
+ now := time.Now()
+ require.NoError(t, db.WithContext(ctx).Create(&models.Environment{
+ BaseModel: models.BaseModel{ID: "env-edge", CreatedAt: now, UpdatedAt: &now},
+ Name: "edge-env",
+ ApiUrl: "http://edge:3553",
+ Enabled: true,
+ }).Error)
+
+ // No access token involved: the environment comes from the tunnel session (#3002).
+ resp, err := svc.DispatchNotificationForEnvironment(ctx, "env-edge", notificationdto.DispatchRequest{
+ Kind: notificationdto.DispatchKindImageUpdate,
+ ImageUpdate: &notificationdto.DispatchImageUpdate{
+ ImageRef: "nginx:latest",
+ UpdateInfo: *newNotificationTestUpdateInfoInternal(),
+ },
+ })
+ require.NoError(t, err)
+ require.Equal(t, "Notification dispatched successfully", resp.Message)
+
+ _, err = svc.DispatchNotificationForEnvironment(ctx, "env-edge", notificationdto.DispatchRequest{Kind: "bogus"})
+ require.ErrorIs(t, err, ErrUnsupportedDispatchKind)
+}
+
+func TestNotificationService_AgentDispatchWithoutHTTPConfigFallsBackToTunnel(t *testing.T) {
+ ctx := context.Background()
+ db := setupNotificationTestDB(t)
+ svc := NewNotificationService(db, &config.Config{AgentMode: true}, nil, event.NewEventService(db, &config.Config{AgentMode: true}, nil))
+
+ // No MANAGER_API_URL/AGENT_TOKEN and no active tunnel: the error must point
+ // at both options instead of only the HTTP env vars (#3002).
+ _, err := svc.dispatchNotificationToManagerInternal(ctx, notificationdto.DispatchRequest{
+ Kind: notificationdto.DispatchKindImageUpdate,
+ ImageUpdate: &notificationdto.DispatchImageUpdate{
+ ImageRef: "nginx:latest",
+ UpdateInfo: *newNotificationTestUpdateInfoInternal(),
+ },
+ })
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "connected edge tunnel")
+}
+
+func TestNotificationService_AgentDispatchHTTPFailureFallsBackToTunnel(t *testing.T) {
+ ctx := context.Background()
+ db := setupNotificationTestDB(t)
+ cfg := &config.Config{AgentMode: true, ManagerApiUrl: "http://127.0.0.1:0", AgentToken: "token"}
+ svc := NewNotificationService(db, cfg, nil, event.NewEventService(db, cfg, nil))
+
+ // HTTP dispatch fails and no tunnel is connected either: the fallback must
+ // not mask the HTTP transport error.
+ _, err := svc.dispatchNotificationToManagerInternal(ctx, notificationdto.DispatchRequest{
+ Kind: notificationdto.DispatchKindImageUpdate,
+ ImageUpdate: &notificationdto.DispatchImageUpdate{
+ ImageRef: "nginx:latest",
+ UpdateInfo: *newNotificationTestUpdateInfoInternal(),
+ },
+ })
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "failed to dispatch notification to manager")
+}
diff --git a/backend/pkg/libarcane/edge/event_sync.go b/backend/pkg/libarcane/edge/event_sync.go
index 562b843a3..cd47c3328 100644
--- a/backend/pkg/libarcane/edge/event_sync.go
+++ b/backend/pkg/libarcane/edge/event_sync.go
@@ -9,8 +9,16 @@ import (
"github.com/google/uuid"
)
-// ErrNoActiveAgentTunnel is returned when no active agent tunnel exists for outbound event sync.
-const ErrNoActiveAgentTunnel = errors.Sentinel("no active edge agent tunnel")
+const (
+ // ErrNoActiveAgentTunnel is returned when no active agent tunnel exists for outbound event sync.
+ ErrNoActiveAgentTunnel = errors.Sentinel("no active edge agent tunnel")
+
+ // TunnelEventTypeNotificationDispatch marks a tunnel event that carries a
+ // notification dispatch request in MetadataJSON instead of an event-log entry.
+ // Tunnel-connected agents have no manager HTTP config, so notification
+ // dispatch rides the existing agent-to-manager event channel (#3002).
+ TunnelEventTypeNotificationDispatch = "notification.dispatch"
+)
var agentTunnelState struct {
mu sync.RWMutex