Skip to content

Modern Docker Management, Designed for Everyone

fix: keep edge agents on gRPC after a manager restart

Kyle Mendell4febf6d0dcf6
diff --git a/backend/pkg/libarcane/edge/client.go b/backend/pkg/libarcane/edge/client.go
index f17226525..0b86d134b 100644
--- a/backend/pkg/libarcane/edge/client.go
+++ b/backend/pkg/libarcane/edge/client.go
@@ -57,6 +57,9 @@ const (
// errTunnelRegistrationTimeout marks a registration attempt where the manager
// accepted the connection but never answered the register message.
errTunnelRegistrationTimeout = errors.Sentinel("timed out waiting for tunnel registration response")
+ // errEstablishedTunnelSessionEnded marks a session that the manager accepted
+ // and later dropped, as opposed to a transport that never connected.
+ errEstablishedTunnelSessionEnded = errors.Sentinel("established edge tunnel session ended")
)
func (t *commandRequestTransfer) stopInternal() {
@@ -97,6 +100,7 @@ func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {
heartbeatInterval: DefaultHeartbeatInterval,
registrationTimeout: DefaultGRPCRegistrationTimeout,
websocketPreferenceTTL: DefaultWebSocketPreferenceTTL,
+ healthySessionDuration: healthyTunnelSessionDuration,
managerURL: managerURL,
managerGRPCAddr: managerGRPCAddr,
localPort: localPort,
@@ -216,10 +220,17 @@ func (c *TunnelClient) connectAndServeManagedTunnelInternal(ctx context.Context)
return c.connectAndServeWebSocket(ctx)
}
+ sessionStart := time.Now()
if err := c.connectAndServeGRPC(ctx); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
+ // A healthy gRPC session that the manager dropped (restart, redeploy)
+ // is not a broken gRPC path: let the outer loop back off and retry
+ // gRPC first instead of racing a websocket dial against the restart.
+ if errors.Is(err, errEstablishedTunnelSessionEnded) && time.Since(sessionStart) >= c.healthySessionDurationInternal() {
+ return err
+ }
c.noteGRPCTunnelFailureInternal()
if transports.websocket {
managerWSURL := c.managerWebSocketURLInternal()
@@ -308,6 +319,13 @@ func (c *TunnelClient) requestTimeoutInternal() time.Duration {
return c.requestTimeout
}
+func (c *TunnelClient) healthySessionDurationInternal() time.Duration {
+ if c == nil || c.healthySessionDuration <= 0 {
+ return healthyTunnelSessionDuration
+ }
+ return c.healthySessionDuration
+}
+
func (c *TunnelClient) websocketPreferenceTTLInternal() time.Duration {
if c == nil || c.websocketPreferenceTTL <= 0 {
return DefaultWebSocketPreferenceTTL
@@ -483,7 +501,10 @@ func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn Tunn
workers.Go(func() { c.heartbeatLoop(connCtx, conn) })
- return c.messageLoop(connCtx, conn, &workers)
+ if err := c.messageLoop(connCtx, conn, &workers); err != nil {
+ return fmt.Errorf("%w: %w", errEstablishedTunnelSessionEnded, err)
+ }
+ return nil
}
// heartbeatLoop sends periodic heartbeats
diff --git a/backend/pkg/libarcane/edge/client_test.go b/backend/pkg/libarcane/edge/client_test.go
index d11beb1ae..55fcde838 100644
--- a/backend/pkg/libarcane/edge/client_test.go
+++ b/backend/pkg/libarcane/edge/client_test.go
@@ -1702,6 +1702,76 @@ func startTestTunnelServiceOnAPIPathInternal(t *testing.T, ctx context.Context,
return "http://" + lis.Addr().String(), cleanup
}
+func TestTunnelClient_connectAndServe_AutoDoesNotFallbackWhenEstablishedGRPCSessionDrops(t *testing.T) {
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+
+ envID := "env-auto-grpc-drop"
+ GetRegistry().Unregister(envID)
+ defer GetRegistry().Unregister(envID)
+
+ resolver := func(ctx context.Context, token string) (string, error) {
+ if token != "valid-token" {
+ return "", errors.New("invalid token")
+ }
+ return envID, nil
+ }
+
+ tunnelServer := NewTunnelServerWithRegistry(GetRegistry(), resolver, nil)
+ go tunnelServer.StartCleanupLoop(ctx)
+ defer tunnelServer.WaitForCleanupDone()
+
+ grpcManagerURL, stopGRPCManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)
+ defer stopGRPCManager()
+
+ wsManagerURL, wsConnectedCh, stopWSManager := startTestWebSocketTunnelManagerInternal(t, ctx)
+ defer stopWSManager()
+
+ client := NewTunnelClient(&Config{
+ EdgeTransport: EdgeTransportAuto,
+ ManagerApiUrl: wsManagerURL,
+ AgentToken: "valid-token",
+ }, http.NotFoundHandler())
+ grpcAddr, err := url.Parse(grpcManagerURL)
+ require.NoError(t, err)
+ client.managerGRPCAddr = grpcAddr.Host
+ client.healthySessionDuration = time.Millisecond
+
+ errCh := make(chan error, 1)
+ go func() {
+ errCh <- client.connectAndServe(ctx)
+ }()
+
+ var tunnel *AgentTunnel
+ require.Eventually(t, func() bool {
+ registered, ok := GetRegistry().Get(envID).Get()
+ if !ok || registered == nil || registered.Conn == nil || registered.Conn.IsClosed() {
+ return false
+ }
+ tunnel = registered
+ return true
+ }, 3*time.Second, 20*time.Millisecond)
+ require.Equal(t, EdgeTransportGRPC, tunnel.Conn.Transport())
+
+ time.Sleep(5 * time.Millisecond)
+ require.NoError(t, tunnel.CloseWithReason("manager restart"))
+
+ select {
+ case err := <-errCh:
+ require.ErrorIs(t, err, errEstablishedTunnelSessionEnded)
+ case <-time.After(3 * time.Second):
+ require.FailNow(t, "expected dropped gRPC session to return instead of falling back to websocket")
+ }
+
+ select {
+ case <-wsConnectedCh:
+ require.FailNow(t, "dropped gRPC session must not fall back to websocket")
+ default:
+ }
+ assert.Zero(t, client.grpcFailureStreakInternal())
+ cancel()
+}
+
func TestTunnelClient_connectAndServePoll_OpensGRPCWhenRequired(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
diff --git a/backend/pkg/libarcane/edge/types.go b/backend/pkg/libarcane/edge/types.go
index c5a9b5ccc..322d1a8f5 100644
--- a/backend/pkg/libarcane/edge/types.go
+++ b/backend/pkg/libarcane/edge/types.go
@@ -45,6 +45,7 @@ type TunnelClient struct {
heartbeatInterval time.Duration
registrationTimeout time.Duration
websocketPreferenceTTL time.Duration
+ healthySessionDuration time.Duration
managerURL string
managerGRPCAddr string
localPort string // Port the agent is running on locally