Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: keep edge agents on gRPC after a manager restart
Kyle Mendell4febf6d0dcf6
diff --git a/backend/pkg/libarcane/edge/client.go b/backend/pkg/libarcane/edge/client.goindex f17226525..0b86d134b 100644--- a/backend/pkg/libarcane/edge/client.go+++ b/backend/pkg/libarcane/edge/client.go@@ -57,6 +57,9 @@ const (// errTunnelRegistrationTimeout marks a registration attempt where the manager// accepted the connection but never answered the register message.errTunnelRegistrationTimeout = errors.Sentinel("timed out waiting for tunnel registration response")+ // errEstablishedTunnelSessionEnded marks a session that the manager accepted+ // and later dropped, as opposed to a transport that never connected.+ errEstablishedTunnelSessionEnded = errors.Sentinel("established edge tunnel session ended"))func (t *commandRequestTransfer) stopInternal() {@@ -97,6 +100,7 @@ func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {heartbeatInterval: DefaultHeartbeatInterval,registrationTimeout: DefaultGRPCRegistrationTimeout,websocketPreferenceTTL: DefaultWebSocketPreferenceTTL,+ healthySessionDuration: healthyTunnelSessionDuration,managerURL: managerURL,managerGRPCAddr: managerGRPCAddr,localPort: localPort,@@ -216,10 +220,17 @@ func (c *TunnelClient) connectAndServeManagedTunnelInternal(ctx context.Context)return c.connectAndServeWebSocket(ctx)}+ sessionStart := time.Now()if err := c.connectAndServeGRPC(ctx); err != nil {if ctx.Err() != nil {return ctx.Err()}+ // A healthy gRPC session that the manager dropped (restart, redeploy)+ // is not a broken gRPC path: let the outer loop back off and retry+ // gRPC first instead of racing a websocket dial against the restart.+ if errors.Is(err, errEstablishedTunnelSessionEnded) && time.Since(sessionStart) >= c.healthySessionDurationInternal() {+ return err+ }c.noteGRPCTunnelFailureInternal()if transports.websocket {managerWSURL := c.managerWebSocketURLInternal()@@ -308,6 +319,13 @@ func (c *TunnelClient) requestTimeoutInternal() time.Duration {return c.requestTimeout}+func (c *TunnelClient) healthySessionDurationInternal() time.Duration {+ if c == nil || c.healthySessionDuration <= 0 {+ return healthyTunnelSessionDuration+ }+ return c.healthySessionDuration+}+func (c *TunnelClient) websocketPreferenceTTLInternal() time.Duration {if c == nil || c.websocketPreferenceTTL <= 0 {return DefaultWebSocketPreferenceTTL@@ -483,7 +501,10 @@ func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn Tunnworkers.Go(func() { c.heartbeatLoop(connCtx, conn) })- return c.messageLoop(connCtx, conn, &workers)+ if err := c.messageLoop(connCtx, conn, &workers); err != nil {+ return fmt.Errorf("%w: %w", errEstablishedTunnelSessionEnded, err)+ }+ return nil}// heartbeatLoop sends periodic heartbeatsdiff --git a/backend/pkg/libarcane/edge/client_test.go b/backend/pkg/libarcane/edge/client_test.goindex d11beb1ae..55fcde838 100644--- a/backend/pkg/libarcane/edge/client_test.go+++ b/backend/pkg/libarcane/edge/client_test.go@@ -1702,6 +1702,76 @@ func startTestTunnelServiceOnAPIPathInternal(t *testing.T, ctx context.Context,return "http://" + lis.Addr().String(), cleanup}+func TestTunnelClient_connectAndServe_AutoDoesNotFallbackWhenEstablishedGRPCSessionDrops(t *testing.T) {+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()++ envID := "env-auto-grpc-drop"+ GetRegistry().Unregister(envID)+ defer GetRegistry().Unregister(envID)++ resolver := func(ctx context.Context, token string) (string, error) {+ if token != "valid-token" {+ return "", errors.New("invalid token")+ }+ return envID, nil+ }++ tunnelServer := NewTunnelServerWithRegistry(GetRegistry(), resolver, nil)+ go tunnelServer.StartCleanupLoop(ctx)+ defer tunnelServer.WaitForCleanupDone()++ grpcManagerURL, stopGRPCManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)+ defer stopGRPCManager()++ wsManagerURL, wsConnectedCh, stopWSManager := startTestWebSocketTunnelManagerInternal(t, ctx)+ defer stopWSManager()++ client := NewTunnelClient(&Config{+ EdgeTransport: EdgeTransportAuto,+ ManagerApiUrl: wsManagerURL,+ AgentToken: "valid-token",+ }, http.NotFoundHandler())+ grpcAddr, err := url.Parse(grpcManagerURL)+ require.NoError(t, err)+ client.managerGRPCAddr = grpcAddr.Host+ client.healthySessionDuration = time.Millisecond++ errCh := make(chan error, 1)+ go func() {+ errCh <- client.connectAndServe(ctx)+ }()++ var tunnel *AgentTunnel+ require.Eventually(t, func() bool {+ registered, ok := GetRegistry().Get(envID).Get()+ if !ok || registered == nil || registered.Conn == nil || registered.Conn.IsClosed() {+ return false+ }+ tunnel = registered+ return true+ }, 3*time.Second, 20*time.Millisecond)+ require.Equal(t, EdgeTransportGRPC, tunnel.Conn.Transport())++ time.Sleep(5 * time.Millisecond)+ require.NoError(t, tunnel.CloseWithReason("manager restart"))++ select {+ case err := <-errCh:+ require.ErrorIs(t, err, errEstablishedTunnelSessionEnded)+ case <-time.After(3 * time.Second):+ require.FailNow(t, "expected dropped gRPC session to return instead of falling back to websocket")+ }++ select {+ case <-wsConnectedCh:+ require.FailNow(t, "dropped gRPC session must not fall back to websocket")+ default:+ }+ assert.Zero(t, client.grpcFailureStreakInternal())+ cancel()+}+func TestTunnelClient_connectAndServePoll_OpensGRPCWhenRequired(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)defer cancel()diff --git a/backend/pkg/libarcane/edge/types.go b/backend/pkg/libarcane/edge/types.goindex c5a9b5ccc..322d1a8f5 100644--- a/backend/pkg/libarcane/edge/types.go+++ b/backend/pkg/libarcane/edge/types.go@@ -45,6 +45,7 @@ type TunnelClient struct {heartbeatInterval time.DurationregistrationTimeout time.DurationwebsocketPreferenceTTL time.Duration+ healthySessionDuration time.DurationmanagerURL stringmanagerGRPCAddr stringlocalPort string // Port the agent is running on locally