Skip to content

Modern Docker Management, Designed for Everyone

refactor: unify edge tunnel transport lifecycle and relocate proto to backend/proto (#3416)

Kyle Mendell5ea6675e5778
diff --git a/Justfile b/Justfile
index 2abb9558e..5994914e7 100644
--- a/Justfile
+++ b/Justfile
@@ -6,7 +6,7 @@ set working-directory := './'
export GOEXPERIMENT := "jsonv2"
-edge_proto_dir := 'backend/pkg/libarcane/edge/proto'
+edge_proto_dir := 'backend/proto'
_default:
@just --list
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index b9ad8da34..61025c28c 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -16,7 +16,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
- tunnelpb "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/labstack/echo/v5"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
diff --git a/backend/internal/bootstrap/server_bootstrap.go b/backend/internal/bootstrap/server_bootstrap.go
index 347605ed9..806c1abd4 100644
--- a/backend/internal/bootstrap/server_bootstrap.go
+++ b/backend/internal/bootstrap/server_bootstrap.go
@@ -13,8 +13,8 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
- tunnelpb "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/labstack/echo/v5"
"go.uber.org/fx"
"google.golang.org/grpc"
diff --git a/backend/pkg/libarcane/edge/client.go b/backend/pkg/libarcane/edge/client.go
index a93b53844..e59764a9b 100644
--- a/backend/pkg/libarcane/edge/client.go
+++ b/backend/pkg/libarcane/edge/client.go
@@ -26,15 +26,23 @@ const (
// DefaultRequestTimeout is the timeout for executing local requests
DefaultRequestTimeout = 5 * time.Minute
// DefaultGRPCRegistrationTimeout bounds how long the agent waits for the
- // manager to acknowledge gRPC tunnel registration before treating it as a
- // failed transport attempt.
+ // manager to acknowledge tunnel registration on any transport before
+ // treating it as a failed transport attempt.
DefaultGRPCRegistrationTimeout = 10 * time.Second
// DefaultWebSocketPreferenceTTL keeps websocket as the preferred transport
// for a short period after a successful auto-mode fallback.
DefaultWebSocketPreferenceTTL = 2 * time.Minute
- defaultCommandChunkSize = 256 * 1024
+ // maxWebSocketPreferenceTTL caps the exponential backoff applied to the
+ // websocket preference window while gRPC keeps failing, so a persistently
+ // broken gRPC path is retried at most this often but never disabled.
+ maxWebSocketPreferenceTTL = 30 * time.Minute
+ defaultCommandChunkSize = 256 * 1024
)
+// errTunnelRegistrationTimeout marks a registration attempt where the manager
+// accepted the connection but never answered the register message.
+const errTunnelRegistrationTimeout = errors.Sentinel("timed out waiting for tunnel registration response")
+
func (t *commandRequestTransfer) stopInternal() {
if t == nil {
return
@@ -82,19 +90,19 @@ func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {
}
return &TunnelClient{
- cfg: cfg,
- handler: handler,
- reconnectInterval: reconnectInterval,
- heartbeatInterval: DefaultHeartbeatInterval,
- grpcRegistrationTimeout: DefaultGRPCRegistrationTimeout,
- websocketPreferenceTTL: DefaultWebSocketPreferenceTTL,
- managerURL: managerURL,
- managerGRPCAddr: managerGRPCAddr,
- localPort: localPort,
- httpClient: &http.Client{},
- stopCh: make(chan struct{}),
- requestTimeout: DefaultRequestTimeout,
- agentInstanceID: uuid.NewString(),
+ cfg: cfg,
+ handler: handler,
+ reconnectInterval: reconnectInterval,
+ heartbeatInterval: DefaultHeartbeatInterval,
+ registrationTimeout: DefaultGRPCRegistrationTimeout,
+ websocketPreferenceTTL: DefaultWebSocketPreferenceTTL,
+ managerURL: managerURL,
+ managerGRPCAddr: managerGRPCAddr,
+ localPort: localPort,
+ httpClient: &http.Client{},
+ stopCh: make(chan struct{}),
+ requestTimeout: DefaultRequestTimeout,
+ agentInstanceID: uuid.NewString(),
}
}
@@ -192,6 +200,7 @@ func (c *TunnelClient) connectAndServeManagedTunnelInternal(ctx context.Context)
if preferredUntil, ok := c.preferredWebSocketUntilInternal(time.Now()).Get(); ok {
slog.InfoContext(ctx, "Temporarily preferring websocket edge tunnel transport after recent websocket success",
"preferred_until", preferredUntil,
+ "grpc_failure_streak", c.grpcFailureStreakInternal(),
"manager_ws_url", c.managerWebSocketURLInternal(),
)
return c.connectAndServeWebSocket(ctx)
@@ -201,6 +210,7 @@ func (c *TunnelClient) connectAndServeManagedTunnelInternal(ctx context.Context)
if ctx.Err() != nil {
return ctx.Err()
}
+ c.noteGRPCTunnelFailureInternal()
if c.shouldFallbackToWebSocketInternal() {
managerWSURL := c.managerWebSocketURLInternal()
slog.WarnContext(ctx, "gRPC edge tunnel connection failed, falling back to websocket transport",
@@ -288,11 +298,11 @@ func (c *TunnelClient) managerWebSocketURLInternal() string {
return HTTPToWebSocketURL(managerBaseURL) + "/api/tunnel/connect"
}
-func (c *TunnelClient) grpcRegistrationTimeoutInternal() time.Duration {
- if c == nil || c.grpcRegistrationTimeout <= 0 {
+func (c *TunnelClient) registrationTimeoutInternal() time.Duration {
+ if c == nil || c.registrationTimeout <= 0 {
return DefaultGRPCRegistrationTimeout
}
- return c.grpcRegistrationTimeout
+ return c.registrationTimeout
}
func (c *TunnelClient) requestTimeoutInternal() time.Duration {
@@ -335,16 +345,42 @@ func (c *TunnelClient) markTransportConnectedInternal(transport string) {
switch transport {
case EdgeTransportGRPC:
c.preferWebSocketUntil = time.Time{}
+ c.grpcFailureStreak = 0
case EdgeTransportWebSocket:
if c.shouldAttemptGRPCTunnelInternal() && c.shouldAttemptWebSocketTunnelInternal() {
- c.preferWebSocketUntil = time.Now().Add(c.websocketPreferenceTTLInternal())
+ // Back the preference window off exponentially while gRPC keeps
+ // failing so each reconnect does not re-pay the registration
+ // timeout against a persistently broken gRPC path.
+ ttl := c.websocketPreferenceTTLInternal()
+ if streak := min(c.grpcFailureStreak, 4); streak > 0 {
+ ttl = min(ttl<<streak, maxWebSocketPreferenceTTL)
+ }
+ c.preferWebSocketUntil = time.Now().Add(ttl)
}
}
}
+func (c *TunnelClient) noteGRPCTunnelFailureInternal() {
+ if c == nil {
+ return
+ }
+ c.transportPreferenceMu.Lock()
+ defer c.transportPreferenceMu.Unlock()
+ c.grpcFailureStreak++
+}
+
+func (c *TunnelClient) grpcFailureStreakInternal() int {
+ if c == nil {
+ return 0
+ }
+ c.transportPreferenceMu.RLock()
+ defer c.transportPreferenceMu.RUnlock()
+ return c.grpcFailureStreak
+}
+
func (c *TunnelClient) registerMessageInternal() *TunnelMessage {
capabilities := AdvertisedEdgeCommands()
- capabilities = append(capabilities, tunnelCapabilityChunkedRequest)
+ capabilities = append(capabilities, tunnelCapabilityChunkedRequest, tunnelCapabilityProtoParity)
return &TunnelMessage{
Type: MessageTypeRegister,
AgentToken: c.cfg.AgentToken,
@@ -368,7 +404,7 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context) (*TunnelMe
err error
}
- timeout := c.grpcRegistrationTimeoutInternal()
+ timeout := c.registrationTimeoutInternal()
recvCh := make(chan registrationResult, 1)
go func() {
msg, err := conn.Receive()
@@ -384,7 +420,7 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context) (*TunnelMe
return nil, ctx.Err()
case <-timer.C:
_ = conn.Close()
- return nil, errors.Errorf("timed out waiting for tunnel registration response after %s", timeout)
+ return nil, errors.WrapIff(errTunnelRegistrationTimeout, "after %s", timeout)
case result := <-recvCh:
if result.err != nil {
return nil, errors.WrapIf(result.err, "failed to receive tunnel registration response")
@@ -399,10 +435,52 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context) (*TunnelMe
return nil, errors.Errorf("manager rejected tunnel registration: %s", result.msg.Error)
}
c.sessionID = result.msg.SessionID
+ c.managerCapabilities = append([]string(nil), result.msg.Capabilities...)
return result.msg, nil
}
}
+// serveTunnelSessionInternal runs the shared register/heartbeat/message
+// lifecycle on an established tunnel connection, for every transport.
+func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn TunnelConnection, managerAddr string) error {
+ c.setConn(conn)
+ setActiveAgentTunnelConn(conn)
+ defer clearActiveAgentTunnelConn(conn)
+ // Always tear down through the wrapper so the peer sees a clean close
+ // (websocket close frame / gRPC half-close) and later Sends hit the
+ // closed guard instead of the dead socket.
+ defer func() { _ = conn.Close() }()
+
+ if err := conn.Send(c.registerMessageInternal()); err != nil {
+ return errors.WrapIff(err, "failed to send %s tunnel register message", conn.Transport())
+ }
+
+ registerMsg, err := c.awaitRegistrationInternal(ctx)
+ if err != nil {
+ return err
+ }
+
+ slog.InfoContext(ctx, "Edge tunnel connected to manager",
+ "transport", conn.Transport(),
+ "manager_addr", managerAddr,
+ "environment_id", registerMsg.EnvironmentID,
+ "session_id", registerMsg.SessionID,
+ )
+ c.markTransportConnectedInternal(conn.Transport())
+
+ connCtx, connCancel := context.WithCancel(ctx)
+ defer connCancel()
+ // A blocked websocket Receive is not ctx-aware; closing the connection is
+ // what unblocks messageLoop when the session context is cancelled.
+ go func() {
+ <-connCtx.Done()
+ _ = conn.Close()
+ }()
+ go c.heartbeatLoop(connCtx)
+
+ return c.messageLoop(connCtx)
+}
+
// heartbeatLoop sends periodic heartbeats
func (c *TunnelClient) heartbeatLoop(ctx context.Context) {
ticker := time.NewTicker(c.heartbeatInterval)
@@ -487,8 +565,8 @@ func (c *TunnelClient) messageLoop(ctx context.Context) error {
if !msg.Accepted {
return errors.Errorf("manager rejected tunnel registration: %s", msg.Error)
}
- slog.InfoContext(ctx, "Edge gRPC tunnel connected to manager",
- "manager_addr", c.managerGRPCAddr,
+ slog.InfoContext(ctx, "Edge tunnel re-registered",
+ "transport", conn.Transport(),
"environment_id", msg.EnvironmentID,
)
case MessageTypeRegister:
@@ -635,6 +713,8 @@ func (c *TunnelClient) buildLocalHTTPRequest(ctx context.Context, msg *TunnelMes
}
}
+ // Set headers. Note: Go's net/http does not populate req.Host from
+ // Header.Set("Host", ...) — it must be set explicitly on the field.
for k, v := range msg.Headers {
if http.CanonicalHeaderKey(k) == "Host" {
req.Host = v
@@ -646,6 +726,16 @@ func (c *TunnelClient) buildLocalHTTPRequest(ctx context.Context, msg *TunnelMes
return req, nil
}
+// agentAuthCredentialsInternal returns the canonical header/metadata credential
+// set every agent transport presents, so the three transports cannot drift.
+func agentAuthCredentialsInternal(token string) map[string]string {
+ return map[string]string{
+ HeaderAgentToken: token,
+ HeaderAPIKey: token,
+ HeaderAuthorization: "Bearer " + token,
+ }
+}
+
// handleRequest processes an incoming request and sends back a response
func (c *TunnelClient) handleRequest(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {
if isGRPCConnection(conn) {
@@ -659,41 +749,12 @@ func (c *TunnelClient) handleRequest(ctx context.Context, conn TunnelConnection,
slog.DebugContext(reqCtx, "Processing tunneled request", "id", msg.ID, "method", msg.Method, "path", msg.Path, "bodyLength", len(msg.Body))
- // Build the request
- var body io.Reader
- var bodyBytes []byte
- if len(msg.Body) > 0 {
- bodyBytes = msg.Body
- body = bytes.NewReader(bodyBytes)
- }
-
- path := msg.Path
- if msg.Query != "" {
- path = path + "?" + msg.Query
- }
-
- req, err := http.NewRequestWithContext(reqCtx, msg.Method, path, body)
+ req, err := c.buildLocalHTTPRequest(reqCtx, msg)
if err != nil {
c.sendErrorResponse(conn, msg.ID, http.StatusInternalServerError, fmt.Sprintf("failed to create request: %v", err))
return
}
- if bodyBytes != nil {
- req.GetBody = func() (io.ReadCloser, error) {
- return io.NopCloser(bytes.NewReader(bodyBytes)), nil
- }
- }
-
- // Set headers. Note: Go's net/http does not populate req.Host from
- // Header.Set("Host", ...) — it must be set explicitly on the field.
- for k, v := range msg.Headers {
- if http.CanonicalHeaderKey(k) == "Host" {
- req.Host = v
- continue
- }
- req.Header.Set(k, v)
- }
-
// Use a response recorder to capture the response
rw := &responseRecorder{
headers: make(http.Header),
@@ -730,8 +791,7 @@ func isGRPCConnection(conn TunnelConnection) bool {
if conn == nil {
return false
}
- _, isGRPC := conn.(*GRPCAgentTunnelConn)
- return isGRPC
+ return conn.Transport() == EdgeTransportGRPC
}
func (c *TunnelClient) handleRequestStreaming(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {
@@ -741,40 +801,12 @@ func (c *TunnelClient) handleRequestStreaming(ctx context.Context, conn TunnelCo
slog.DebugContext(reqCtx, "Processing tunneled request (streaming)", "id", msg.ID, "method", msg.Method, "path", msg.Path, "bodyLength", len(msg.Body))
- var body io.Reader
- var bodyBytes []byte
- if len(msg.Body) > 0 {
- bodyBytes = msg.Body
- body = bytes.NewReader(bodyBytes)
- }
-
- path := msg.Path
- if msg.Query != "" {
- path = path + "?" + msg.Query
- }
-
- req, err := http.NewRequestWithContext(reqCtx, msg.Method, path, body)
+ req, err := c.buildLocalHTTPRequest(reqCtx, msg)
if err != nil {
c.sendErrorResponse(conn, msg.ID, http.StatusInternalServerError, fmt.Sprintf("failed to create request: %v", err))
return
}
- if bodyBytes != nil {
- req.GetBody = func() (io.ReadCloser, error) {
- return io.NopCloser(bytes.NewReader(bodyBytes)), nil
- }
- }
-
- // Set headers. Note: Go's net/http does not populate req.Host from
- // Header.Set("Host", ...) — it must be set explicitly on the field.
- for k, v := range msg.Headers {
- if http.CanonicalHeaderKey(k) == "Host" {
- req.Host = v
- continue
- }
- req.Header.Set(k, v)
- }
-
recorder := newStreamingResponseRecorder(msg.ID, conn)
c.handler.ServeHTTP(recorder, req)
diff --git a/backend/pkg/libarcane/edge/client_test.go b/backend/pkg/libarcane/edge/client_test.go
index a3e3765a9..c979f2fab 100644
--- a/backend/pkg/libarcane/edge/client_test.go
+++ b/backend/pkg/libarcane/edge/client_test.go
@@ -19,8 +19,8 @@ import (
"testing"
"time"
- tunnelpb "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1"
httputil "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/gorilla/websocket"
"github.com/labstack/echo/v5"
slogecho "github.com/samber/slog-echo/v2"
@@ -720,6 +720,8 @@ func (f *fakeTunnelConnForTransportCheck) IsClosed() bool {
return false
}
+func (f *fakeTunnelConnForTransportCheck) Transport() string { return EdgeTransportWebSocket }
+
type capturingTunnelConnForHandleRequest struct {
sent []*TunnelMessage
}
@@ -753,6 +755,8 @@ func (c *capturingTunnelConnForHandleRequest) IsClosed() bool {
return false
}
+func (c *capturingTunnelConnForHandleRequest) Transport() string { return EdgeTransportWebSocket }
+
type failingHeartbeatConn struct {
closeCalled bool
}
@@ -778,6 +782,8 @@ func (f *failingHeartbeatConn) IsClosed() bool {
return false
}
+func (f *failingHeartbeatConn) Transport() string { return EdgeTransportWebSocket }
+
type stallingTunnelService struct {
tunnelpb.UnimplementedTunnelServiceServer
connectCount atomic.Int32
@@ -825,6 +831,8 @@ func (c *blockingRegistrationConn) IsClosed() bool {
}
}
+func (c *blockingRegistrationConn) Transport() string { return EdgeTransportWebSocket }
+
func (s *stallingTunnelService) Connect(stream grpc.BidiStreamingServer[tunnelpb.AgentMessage, tunnelpb.ManagerMessage]) error {
s.connectCount.Add(1)
if _, err := stream.Recv(); err != nil {
@@ -1097,33 +1105,35 @@ func TestTunnelClient_connectAndServeGRPC_TimesOutWithoutRegisterResponse(t *tes
ManagerApiUrl: managerURL,
AgentToken: "valid-token",
}, http.NotFoundHandler())
- client.grpcRegistrationTimeout = 100 * time.Millisecond
+ client.registrationTimeout = 100 * time.Millisecond
err := client.connectAndServeGRPC(ctx)
require.Error(t, err)
+ assert.ErrorIs(t, err, errTunnelRegistrationTimeout)
assert.Contains(t, err.Error(), "timed out waiting for tunnel registration response")
+ assert.Contains(t, err.Error(), "not forwarding gRPC")
assert.EqualValues(t, 1, service.connectCount.Load())
assert.True(t, client.getConn() == nil || client.getConn().IsClosed())
}
-func TestTunnelClient_awaitGRPCRegistrationInternal_ClosesConnOnContextDone(t *testing.T) {
+func TestTunnelClient_awaitRegistrationInternal_ClosesConnOnContextDone(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
conn := newBlockingRegistrationConnInternal()
client := &TunnelClient{
- grpcRegistrationTimeout: time.Second,
+ registrationTimeout: time.Second,
}
client.setConn(conn)
- msg, err := client.awaitGRPCRegistrationInternal(ctx)
+ msg, err := client.awaitRegistrationInternal(ctx)
require.Nil(t, msg)
require.ErrorIs(t, err, context.DeadlineExceeded)
assert.True(t, conn.IsClosed())
assert.EqualValues(t, 1, conn.closeCount.Load())
}
-func TestTunnelClient_awaitGRPCRegistrationInternal_ClosesAttemptConnWhenClientConnChanges(t *testing.T) {
+func TestTunnelClient_awaitRegistrationInternal_ClosesAttemptConnWhenClientConnChanges(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -1135,7 +1145,7 @@ func TestTunnelClient_awaitGRPCRegistrationInternal_ClosesAttemptConnWhenClientC
})
client := &TunnelClient{
- grpcRegistrationTimeout: time.Second,
+ registrationTimeout: time.Second,
}
client.setConn(firstConn)
@@ -1145,7 +1155,7 @@ func TestTunnelClient_awaitGRPCRegistrationInternal_ClosesAttemptConnWhenClientC
}
resultCh := make(chan registrationResult, 1)
go func() {
- msg, err := client.awaitGRPCRegistrationInternal(ctx)
+ msg, err := client.awaitRegistrationInternal(ctx)
resultCh <- registrationResult{msg: msg, err: err}
}()
@@ -1407,7 +1417,7 @@ func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCUnavailabl
grpcAddr, releaseGRPCAddr := reserveTCPAddressInternal(t)
defer releaseGRPCAddr()
client.managerGRPCAddr = grpcAddr
- client.grpcRegistrationTimeout = 100 * time.Millisecond
+ client.registrationTimeout = 100 * time.Millisecond
errCh := make(chan error, 1)
go func() {
@@ -1445,7 +1455,7 @@ func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCSetupHangs
AgentToken: "valid-token",
}, http.NotFoundHandler())
client.managerGRPCAddr = grpcAddr
- client.grpcRegistrationTimeout = 100 * time.Millisecond
+ client.registrationTimeout = 100 * time.Millisecond
errCh := make(chan error, 1)
go func() {
@@ -1482,7 +1492,7 @@ func TestTunnelClient_connectAndServe_GRPCDoesNotFallbackToWebSocket(t *testing.
grpcAddr, releaseGRPCAddr := reserveTCPAddressInternal(t)
defer releaseGRPCAddr()
client.managerGRPCAddr = grpcAddr
- client.grpcRegistrationTimeout = 100 * time.Millisecond
+ client.registrationTimeout = 100 * time.Millisecond
err := client.connectAndServe(ctx)
require.Error(t, err)
@@ -1807,7 +1817,7 @@ func TestTunnelClient_connectAndServePoll_OpensWebSocketWhenRequired(t *testing.
ManagerApiUrl: managerServer.URL,
AgentToken: "valid-token",
}, http.NotFoundHandler())
- client.grpcRegistrationTimeout = 100 * time.Millisecond
+ client.registrationTimeout = 100 * time.Millisecond
err := client.connectAndServe(ctx)
require.Error(t, err)
@@ -1964,7 +1974,7 @@ func TestTunnelClient_connectAndServePoll_RetriesAfterTransientPollError(t *test
ManagerApiUrl: managerServer.URL,
AgentToken: "valid-token",
}, http.NotFoundHandler())
- client.grpcRegistrationTimeout = 100 * time.Millisecond
+ client.registrationTimeout = 100 * time.Millisecond
err := client.connectAndServe(ctx)
require.Error(t, err)
@@ -2208,6 +2218,8 @@ func (f *fakeTunnelConn) IsClosed() bool {
return f.closed
}
+func (f *fakeTunnelConn) Transport() string { return EdgeTransportWebSocket }
+
func TestStreamingResponseRecorder_Sequence(t *testing.T) {
conn := &fakeTunnelConn{}
r := newStreamingResponseRecorder("req-1", conn)
@@ -2251,3 +2263,61 @@ func TestStreamingResponseRecorder_WriteHeaderAndClose(t *testing.T) {
assert.Equal(t, http.StatusCreated, conn.msgs[0].Status)
assert.Equal(t, MessageTypeStreamEnd, conn.msgs[1].Type)
}
+
+func TestConnectAndServeWebSocket_CancelUnblocksMessageLoop(t *testing.T) {
+ registered := make(chan struct{})
+ managerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ upgrader := websocket.Upgrader{}
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ defer func() { _ = conn.Close() }()
+
+ _, _, err = conn.ReadMessage()
+ if err != nil {
+ return
+ }
+ registerResp, _ := json.Marshal(&TunnelMessage{
+ Type: MessageTypeRegisterResponse,
+ Accepted: true,
+ SessionID: "session-cancel",
+ })
+ _ = conn.WriteMessage(websocket.TextMessage, registerResp)
+ close(registered)
+ // Go silent: keep reading so the socket stays open.
+ for {
+ if _, _, err := conn.ReadMessage(); err != nil {
+ return
+ }
+ }
+ }))
+ defer managerServer.Close()
+
+ cfg := &Config{
+ EdgeTransport: EdgeTransportWebSocket,
+ ManagerApiUrl: managerServer.URL,
+ AgentToken: "test-token",
+ }
+ client := NewTunnelClient(cfg, http.NotFoundHandler())
+ client.managerURL = "ws" + strings.TrimPrefix(managerServer.URL, "http")
+
+ ctx, cancel := context.WithCancel(t.Context())
+ done := make(chan error, 1)
+ go func() { done <- client.connectAndServeWebSocket(ctx) }()
+
+ select {
+ case <-registered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("timed out waiting for registration")
+ }
+ cancel()
+
+ select {
+ case <-done:
+ // Cancellation must close the socket and unblock the message loop well
+ // under the poll-managed teardown timeout.
+ case <-time.After(defaultPollManagedSessionStopTimeout):
+ t.Fatal("messageLoop did not unblock after context cancellation")
+ }
+}
diff --git a/backend/pkg/libarcane/edge/client_transport_grpc.go b/backend/pkg/libarcane/edge/client_transport_grpc.go
index aab0e4444..32b1b0713 100644
--- a/backend/pkg/libarcane/edge/client_transport_grpc.go
+++ b/backend/pkg/libarcane/edge/client_transport_grpc.go
@@ -10,7 +10,7 @@ import (
"emperror.dev/errors"
- tunnelpb "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"google.golang.org/grpc"
"google.golang.org/grpc/backoff"
"google.golang.org/grpc/connectivity"
@@ -72,11 +72,9 @@ func (c *TunnelClient) connectAndServeGRPC(ctx context.Context) error {
return errors.WrapIf(err, "manager gRPC endpoint is not ready")
}
- streamCtx, streamCancel := context.WithCancel(metadata.NewOutgoingContext(ctx, metadata.Pairs(
- strings.ToLower(HeaderAgentToken), c.cfg.AgentToken,
- strings.ToLower(HeaderAPIKey), c.cfg.AgentToken,
- strings.ToLower(HeaderAuthorization), "Bearer "+c.cfg.AgentToken,
- )))
+ // metadata.New lowercases the keys itself.
+ streamCtx, streamCancel := context.WithCancel(metadata.NewOutgoingContext(ctx,
+ metadata.New(agentAuthCredentialsInternal(c.cfg.AgentToken))))
defer streamCancel()
method := c.grpcConnectMethodInternal()
@@ -85,30 +83,17 @@ func (c *TunnelClient) connectAndServeGRPC(ctx context.Context) error {
return errors.WrapIf(err, "failed to open tunnel stream")
}
- tunnelConn := NewGRPCAgentTunnelConn(stream, streamCancel)
- c.setConn(tunnelConn)
- setActiveAgentTunnelConn(tunnelConn)
- defer clearActiveAgentTunnelConn(tunnelConn)
- if err := tunnelConn.Send(c.registerMessageInternal()); err != nil {
- return errors.WrapIf(err, "failed to send register message")
- }
-
- registerMsg, err := c.awaitGRPCRegistrationInternal(ctx)
- if err != nil {
+ if err := c.serveTunnelSessionInternal(ctx, NewGRPCAgentTunnelConn(stream, streamCancel), managerAddr); err != nil {
+ if errors.Is(err, errTunnelRegistrationTimeout) {
+ // The channel already reached Ready, so TCP/TLS works but gRPC
+ // framing was never answered end to end.
+ return errors.WrapIf(err,
+ "manager accepted the TCP/TLS connection but never answered gRPC tunnel registration; "+
+ "if a reverse proxy (Traefik/Pangolin/Nginx) fronts the manager, it is likely not forwarding gRPC (HTTP/2 with trailers) on /api/tunnel/connect")
+ }
return err
}
-
- slog.InfoContext(ctx, "Edge gRPC tunnel connected to manager",
- "manager_addr", c.managerGRPCAddr,
- "environment_id", registerMsg.EnvironmentID,
- )
- c.markTransportConnectedInternal(EdgeTransportGRPC)
-
- connCtx, connCancel := context.WithCancel(ctx)
- defer connCancel()
- go c.heartbeatLoop(connCtx)
-
- return c.messageLoop(connCtx)
+ return nil
}
func (c *TunnelClient) waitForGRPCReadyInternal(ctx context.Context, conn *grpc.ClientConn) error {
@@ -116,7 +101,7 @@ func (c *TunnelClient) waitForGRPCReadyInternal(ctx context.Context, conn *grpc.
return errors.New("manager gRPC connection is not initialized")
}
- timeout := c.grpcRegistrationTimeoutInternal()
+ timeout := c.registrationTimeoutInternal()
readyCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
@@ -155,10 +140,6 @@ func (c *TunnelClient) grpcConnectMethodInternal() string {
return "/api/tunnel/connect"
}
-func (c *TunnelClient) awaitGRPCRegistrationInternal(ctx context.Context) (*TunnelMessage, error) {
- return c.awaitRegistrationInternal(ctx)
-}
-
func (c *TunnelClient) useTLSForManagerGRPC() bool {
baseURL := strings.TrimSpace(c.cfg.GetManagerBaseURL())
if baseURL == "" {
diff --git a/backend/pkg/libarcane/edge/client_transport_grpc_test.go b/backend/pkg/libarcane/edge/client_transport_grpc_test.go
index 0b7deef7d..102137e1c 100644
--- a/backend/pkg/libarcane/edge/client_transport_grpc_test.go
+++ b/backend/pkg/libarcane/edge/client_transport_grpc_test.go
@@ -8,7 +8,7 @@ import (
"testing"
"time"
- tunnelpb "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
diff --git a/backend/pkg/libarcane/edge/client_transport_poll.go b/backend/pkg/libarcane/edge/client_transport_poll.go
index 9505356e6..4c16846a3 100644
--- a/backend/pkg/libarcane/edge/client_transport_poll.go
+++ b/backend/pkg/libarcane/edge/client_transport_poll.go
@@ -174,9 +174,9 @@ func (c *TunnelClient) pollTunnelControlInternal(ctx context.Context, pollURL st
return nil, errors.WrapIf(err, "failed to create poll request")
}
req.Header.Set("Content-Type", "application/json")
- req.Header.Set(HeaderAgentToken, c.cfg.AgentToken)
- req.Header.Set(HeaderAPIKey, c.cfg.AgentToken)
- req.Header.Set(HeaderAuthorization, "Bearer "+c.cfg.AgentToken)
+ for header, value := range agentAuthCredentialsInternal(c.cfg.AgentToken) {
+ req.Header.Set(header, value)
+ }
resp, err := c.httpClient.Do(req)
if err != nil {
diff --git a/backend/pkg/libarcane/edge/client_transport_websocket.go b/backend/pkg/libarcane/edge/client_transport_websocket.go
index aa7450dfd..60b367ee0 100644
--- a/backend/pkg/libarcane/edge/client_transport_websocket.go
+++ b/backend/pkg/libarcane/edge/client_transport_websocket.go
@@ -2,6 +2,7 @@ package edge
import (
"context"
+ "crypto/tls"
"io"
"log/slog"
"net/http"
@@ -29,13 +30,16 @@ func (c *TunnelClient) connectAndServeWebSocket(ctx context.Context) error {
if err != nil {
return errors.WrapIf(err, "failed to configure edge websocket TLS")
}
+ if tlsConfig == nil {
+ tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12}
+ }
dialer.TLSClientConfig = tlsConfig
}
headers := http.Header{}
- headers.Set(HeaderAgentToken, c.cfg.AgentToken)
- headers.Set(HeaderAPIKey, c.cfg.AgentToken)
- headers.Set(HeaderAuthorization, "Bearer "+c.cfg.AgentToken)
+ for header, value := range agentAuthCredentialsInternal(c.cfg.AgentToken) {
+ headers.Set(header, value)
+ }
slog.DebugContext(ctx, "Dialing manager for websocket edge tunnel", "url", managerWSURL)
@@ -48,29 +52,6 @@ func (c *TunnelClient) connectAndServeWebSocket(ctx context.Context) error {
}
return errors.WrapIf(err, "failed to connect to manager websocket endpoint")
}
- defer func() { _ = conn.Close() }()
-
- tunnelConn := NewTunnelConn(conn)
- c.setConn(tunnelConn)
- setActiveAgentTunnelConn(tunnelConn)
- defer clearActiveAgentTunnelConn(tunnelConn)
- if err := tunnelConn.Send(c.registerMessageInternal()); err != nil {
- return errors.WrapIf(err, "failed to send websocket register message")
- }
- registerMsg, err := c.awaitRegistrationInternal(ctx)
- if err != nil {
- return err
- }
- slog.InfoContext(ctx, "WebSocket edge tunnel connected to manager", "manager_url", managerWSURL)
- slog.InfoContext(ctx, "Edge websocket tunnel registered",
- "environment_id", registerMsg.EnvironmentID,
- "session_id", registerMsg.SessionID,
- )
- c.markTransportConnectedInternal(EdgeTransportWebSocket)
-
- connCtx, connCancel := context.WithCancel(ctx)
- defer connCancel()
- go c.heartbeatLoop(connCtx)
- return c.messageLoop(connCtx)
+ return c.serveTunnelSessionInternal(ctx, NewTunnelConn(conn), managerWSURL)
}
diff --git a/backend/pkg/libarcane/edge/command_client.go b/backend/pkg/libarcane/edge/command_client.go
index f36e4d3ea..4276c14dd 100644
--- a/backend/pkg/libarcane/edge/command_client.go
+++ b/backend/pkg/libarcane/edge/command_client.go
@@ -13,7 +13,12 @@ import (
const (
tunnelCapabilityChunkedRequest = "chunked-request"
- bodyTransferMetadataKey = "body_transfer_id"
+ // tunnelCapabilityProtoParity signals the peer can decode the full
+ // TunnelMessage vocabulary natively over gRPC (stream_data/stream_end
+ // manager->agent, cancel_request agent->manager) instead of the legacy
+ // re-encoded forms.
+ tunnelCapabilityProtoParity = "proto-parity-v1"
+ bodyTransferMetadataKey = "body_transfer_id"
)
func NewCommandClient() *CommandClient {
diff --git a/backend/pkg/libarcane/edge/event_sync_test.go b/backend/pkg/libarcane/edge/event_sync_test.go
index c571690d3..0114a5edf 100644
--- a/backend/pkg/libarcane/edge/event_sync_test.go
+++ b/backend/pkg/libarcane/edge/event_sync_test.go
@@ -38,6 +38,8 @@ func (f *fakeEventTunnelConn) Close() error { f.closed = true; return nil }
func (f *fakeEventTunnelConn) IsClosed() bool { return f.closed }
+func (f *fakeEventTunnelConn) Transport() string { return EdgeTransportWebSocket }
+
func TestPublishEventToManager_NoActiveTunnel(t *testing.T) {
clearActiveAgentTunnelConn(getActiveAgentTunnelConn())
diff --git a/backend/pkg/libarcane/edge/poll_control.go b/backend/pkg/libarcane/edge/poll_control.go
index 836649683..73630bf12 100644
--- a/backend/pkg/libarcane/edge/poll_control.go
+++ b/backend/pkg/libarcane/edge/poll_control.go
@@ -216,12 +216,7 @@ func (s *TunnelServer) pollStatusInternal(envID string) TunnelPollResponse {
if tunnel, ok := s.registry.Get(envID).Get(); ok && tunnel != nil && tunnel.Conn != nil && !tunnel.Conn.IsClosed() {
hasActiveTunnel = true
- switch tunnel.Conn.(type) {
- case *GRPCManagerTunnelConn:
- activeTransport = EdgeTransportGRPC
- case *TunnelConn:
- activeTransport = EdgeTransportWebSocket
- }
+ activeTransport = tunnel.Conn.Transport()
}
status := GetDemandRegistry().DesiredStatus(envID, hasActiveTunnel, time.Now())
diff --git a/backend/pkg/libarcane/edge/proxy_test.go b/backend/pkg/libarcane/edge/proxy_test.go
index 556e171dd..988f4bc96 100644
--- a/backend/pkg/libarcane/edge/proxy_test.go
+++ b/backend/pkg/libarcane/edge/proxy_test.go
@@ -12,7 +12,7 @@ import (
"testing"
"time"
- tunnelpb "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/gorilla/websocket"
"github.com/labstack/echo/v5"
"github.com/stretchr/testify/assert"
diff --git a/backend/pkg/libarcane/edge/remenv_test.go b/backend/pkg/libarcane/edge/remenv_test.go
index bdd16d15e..3b438ea1c 100644
--- a/backend/pkg/libarcane/edge/remenv_test.go
+++ b/backend/pkg/libarcane/edge/remenv_test.go
@@ -12,8 +12,8 @@ import (
"testing"
"time"
- tunnelpb "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1"
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/labstack/echo/v5"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
diff --git a/backend/pkg/libarcane/edge/server.go b/backend/pkg/libarcane/edge/server.go
index 07ad5d882..9c23b3c42 100644
--- a/backend/pkg/libarcane/edge/server.go
+++ b/backend/pkg/libarcane/edge/server.go
@@ -6,14 +6,15 @@ import (
"io"
"log/slog"
"net/http"
+ "slices"
"strings"
"time"
"emperror.dev/emperror"
"emperror.dev/errors"
- tunnelpb "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1"
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/google/uuid"
"github.com/gorilla/websocket"
"github.com/labstack/echo/v5"
@@ -29,6 +30,10 @@ import (
const (
// TunnelStaleTimeout is how long before a tunnel is considered stale.
TunnelStaleTimeout = 2 * time.Minute
+ // tunnelStaleSweepInterval is how often stale tunnels are reaped. The sweep
+ // is a backstop behind read-deadline liveness, so it runs well under
+ // TunnelStaleTimeout to keep the dead-but-selectable window short.
+ tunnelStaleSweepInterval = time.Minute
// streamDeliveryTimeout bounds per-message delivery wait to a pending consumer.
// This prevents silent data loss while avoiding indefinite blocking.
streamDeliveryTimeout = 5 * time.Second
@@ -289,7 +294,9 @@ func (s *TunnelServer) Connect(stream grpc.BidiStreamingServer[tunnelpb.AgentMes
return status.Error(codes.Unauthenticated, err.Error())
}
- tunnel := NewAgentTunnelWithConn(envID, NewGRPCManagerTunnelConn(stream))
+ managerConn := NewGRPCManagerTunnelConn(stream)
+ managerConn.SetParityEncoding(slices.Contains(register.GetCapabilities(), tunnelCapabilityProtoParity))
+ tunnel := NewAgentTunnelWithConn(envID, managerConn)
s.populateSessionMetadata(tunnel, &TunnelMessage{
AgentInstance: register.GetAgentInstanceId(),
Capabilities: append([]string(nil), register.GetCapabilities()...),
@@ -314,11 +321,8 @@ func (s *TunnelServer) populateSessionMetadata(tunnel *AgentTunnel, registerMsg
tunnel.Capabilities = append([]string(nil), registerMsg.Capabilities...)
}
- switch tunnel.Conn.(type) {
- case *GRPCManagerTunnelConn:
- tunnel.Transport = EdgeTransportGRPC
- default:
- tunnel.Transport = EdgeTransportWebSocket
+ if tunnel.Conn != nil {
+ tunnel.Transport = tunnel.Conn.Transport()
}
}
@@ -333,16 +337,17 @@ func (s *TunnelServer) resolveEnvironment(ctx context.Context, token string) (st
return s.resolver(ctx, token)
}
+// agentTokenHeaderPriority is the shared credential fallback order for both
+// header- and metadata-based token extraction, so the transports cannot drift.
+var agentTokenHeaderPriority = []string{HeaderAgentToken, HeaderAPIKey}
+
func tokenFromMetadataInternal(ctx context.Context) string {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return ""
}
- for _, key := range []string{
- strings.ToLower(HeaderAgentToken),
- strings.ToLower(HeaderAPIKey),
- } {
- values := md.Get(key)
+ for _, header := range agentTokenHeaderPriority {
+ values := md.Get(strings.ToLower(header))
for _, value := range values {
trimmed := strings.TrimSpace(value)
if trimmed != "" {
@@ -371,7 +376,7 @@ func tokenFromHeadersWithSourceInternal(req *http.Request) (string, string) {
if req == nil {
return "", ""
}
- for _, header := range []string{HeaderAgentToken, HeaderAPIKey} {
+ for _, header := range agentTokenHeaderPriority {
if token := strings.TrimSpace(req.Header.Get(header)); token != "" {
return token, header
}
@@ -406,13 +411,22 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, callbackCtx co
"security_mode", tunnel.SecurityMode,
)
+ // Echo the agent's capabilities and append the manager's own so the agent
+ // learns what this manager supports (older agents ignore extra strings).
+ capabilities := append([]string(nil), tunnel.Capabilities...)
+ for _, capability := range []string{tunnelCapabilityProtoParity, tunnelCapabilityChunkedRequest} {
+ if !slices.Contains(capabilities, capability) {
+ capabilities = append(capabilities, capability)
+ }
+ }
+
if err := tunnel.Conn.Send(&TunnelMessage{
Type: MessageTypeRegisterResponse,
Accepted: true,
EnvironmentID: tunnel.EnvironmentID,
SessionID: tunnel.SessionID,
SecurityMode: tunnel.SecurityMode,
- Capabilities: append([]string(nil), tunnel.Capabilities...),
+ Capabilities: capabilities,
DrainPrevious: drainPrevious,
}); err != nil {
slog.WarnContext(ctx, "Failed to send register response", "environment_id", tunnel.EnvironmentID, "error", err)
@@ -561,7 +575,7 @@ func (s *TunnelServer) handleEvent(ctx context.Context, tunnel *AgentTunnel, msg
// StartCleanupLoop periodically cleans up stale tunnels.
func (s *TunnelServer) StartCleanupLoop(ctx context.Context) {
defer close(s.cleanupDone)
- ticker := time.NewTicker(5 * time.Minute)
+ ticker := time.NewTicker(tunnelStaleSweepInterval)
defer ticker.Stop()
for {
diff --git a/backend/pkg/libarcane/edge/server_test.go b/backend/pkg/libarcane/edge/server_test.go
index b8da880cb..64350756a 100644
--- a/backend/pkg/libarcane/edge/server_test.go
+++ b/backend/pkg/libarcane/edge/server_test.go
@@ -601,6 +601,8 @@ func (f *fakeServerTunnelConn) Close() error { return nil }
func (f *fakeServerTunnelConn) IsClosed() bool { return false }
+func (f *fakeServerTunnelConn) Transport() string { return EdgeTransportWebSocket }
+
type registerResponseOrderConn struct {
sendHook func(*TunnelMessage) error
recvErr error
@@ -626,6 +628,8 @@ func (f *registerResponseOrderConn) Close() error { return nil }
func (f *registerResponseOrderConn) IsClosed() bool { return false }
+func (f *registerResponseOrderConn) Transport() string { return EdgeTransportGRPC }
+
func TestTunnelServer_ManageConnectedTunnel_RegistersBeforeSendingGRPCRegisterResponse(t *testing.T) {
server := NewTunnelServer(nil, nil)
envID := "env-grpc-register-order"
diff --git a/backend/pkg/libarcane/edge/transport.go b/backend/pkg/libarcane/edge/transport.go
index 3dd57c80a..67d234dbc 100644
--- a/backend/pkg/libarcane/edge/transport.go
+++ b/backend/pkg/libarcane/edge/transport.go
@@ -129,14 +129,10 @@ func GetActiveTunnelTransport(envID string) mo.Option[string] {
return mo.None[string]()
}
- switch tunnel.Conn.(type) {
- case *GRPCManagerTunnelConn, *GRPCAgentTunnelConn:
- return mo.Some(EdgeTransportGRPC)
- case *TunnelConn:
- return mo.Some(EdgeTransportWebSocket)
- default:
- return mo.None[string]()
+ if transport := tunnel.Conn.Transport(); transport != "" {
+ return mo.Some(transport)
}
+ return mo.None[string]()
}
// GetTunnelRuntimeState returns live metadata for an active tunnel.
@@ -147,13 +143,7 @@ func GetTunnelRuntimeState(envID string) mo.Option[*TunnelRuntimeState] {
}
state := &TunnelRuntimeState{}
-
- switch tunnel.Conn.(type) {
- case *GRPCManagerTunnelConn, *GRPCAgentTunnelConn:
- state.Transport = EdgeTransportGRPC
- case *TunnelConn:
- state.Transport = EdgeTransportWebSocket
- }
+ state.Transport = tunnel.Conn.Transport()
state.ConnectedAt = new(tunnel.ConnectedAt)
state.LastHeartbeat = new(tunnel.GetLastHeartbeat())
diff --git a/backend/pkg/libarcane/edge/transport_test.go b/backend/pkg/libarcane/edge/transport_test.go
index 18978146c..55f84c482 100644
--- a/backend/pkg/libarcane/edge/transport_test.go
+++ b/backend/pkg/libarcane/edge/transport_test.go
@@ -12,7 +12,7 @@ import (
"testing"
"time"
- tunnelpb "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/gorilla/websocket"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
@@ -185,6 +185,8 @@ func (u *unknownTunnelConn) Close() error { return nil }
func (u *unknownTunnelConn) IsClosed() bool { return false }
+func (u *unknownTunnelConn) Transport() string { return "" }
+
func BenchmarkEdgeTunnelProxyRequest(b *testing.B) {
previousLogger := slog.Default()
slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil)))
diff --git a/backend/pkg/libarcane/edge/tunnel.go b/backend/pkg/libarcane/edge/tunnel.go
index 2f2654cf7..8e524b2df 100644
--- a/backend/pkg/libarcane/edge/tunnel.go
+++ b/backend/pkg/libarcane/edge/tunnel.go
@@ -4,10 +4,13 @@ import (
"context"
"encoding/json/v2"
"io"
+ "log/slog"
+ "net"
+ "time"
"emperror.dev/errors"
- tunnelpb "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/gorilla/websocket"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -18,6 +21,20 @@ type TunnelMessageType string
const maxGRPCTunnelMessageSize = 16 * 1024 * 1024
+// maxWebSocketTunnelMessageSize caps inbound websocket tunnel frames at 2x the
+// gRPC limit, since JSON base64-encodes binary bodies a gRPC peer would accept.
+const maxWebSocketTunnelMessageSize = 2 * maxGRPCTunnelMessageSize
+
+// ErrTunnelConnectionClosed is returned by Send and Receive on every transport
+// once the tunnel connection is closed.
+const ErrTunnelConnectionClosed = errors.Sentinel("edge tunnel connection is closed")
+
+// tunnelReadWait bounds how long a websocket tunnel read may sit idle. Both
+// peers see traffic at least every DefaultHeartbeatInterval (agent heartbeat,
+// manager heartbeat_ack), so 3x tolerates transient stalls while still
+// detecting a silently dead peer. Variable so tests can shorten it.
+var tunnelReadWait = 3 * DefaultHeartbeatInterval
+
const (
// MessageTypeRequest is sent from manager to agent to initiate a request.
MessageTypeRequest TunnelMessageType = "request"
@@ -68,10 +85,12 @@ type TunnelConnection interface {
IsExpectedReceiveError(err error) bool
Close() error
IsClosed() bool
+ Transport() string
}
// NewTunnelConn creates a new WebSocket tunnel connection wrapper.
func NewTunnelConn(conn *websocket.Conn) *TunnelConn {
+ conn.SetReadLimit(maxWebSocketTunnelMessageSize)
return &TunnelConn{conn: conn}
}
@@ -81,20 +100,33 @@ func (t *TunnelConn) Send(msg *TunnelMessage) error {
defer t.mu.Unlock()
if t.closed.Load() {
- return websocket.ErrCloseSent
+ return ErrTunnelConnectionClosed
}
data, err := json.Marshal(msg)
if err != nil {
return err
}
- return t.conn.WriteMessage(websocket.TextMessage, data)
+ _ = t.conn.SetWriteDeadline(time.Now().Add(DefaultWriteTimeout))
+ if err := t.conn.WriteMessage(websocket.TextMessage, data); err != nil {
+ // A gorilla connection is unusable after a write error.
+ t.closed.Store(true)
+ return err
+ }
+ return nil
}
// Receive receives a tunnel message from the WebSocket connection.
func (t *TunnelConn) Receive() (*TunnelMessage, error) {
+ if t.closed.Load() {
+ return nil, ErrTunnelConnectionClosed
+ }
+
+ _ = t.conn.SetReadDeadline(time.Now().Add(tunnelReadWait))
_, data, err := t.conn.ReadMessage()
if err != nil {
+ // Read-deadline expiry and network errors are terminal for gorilla conns.
+ t.closed.Store(true)
return nil, err
}
@@ -114,6 +146,9 @@ func (t *TunnelConn) IsExpectedReceiveError(err error) bool {
if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return true
}
+ if errors.Is(err, net.ErrClosed) || errors.Is(err, ErrTunnelConnectionClosed) {
+ return true
+ }
return websocket.IsCloseError(err,
websocket.CloseNormalClosure,
@@ -122,12 +157,18 @@ func (t *TunnelConn) IsExpectedReceiveError(err error) bool {
)
}
-// Close closes the WebSocket tunnel connection.
+// Close closes the WebSocket tunnel connection, sending a close frame on the
+// first call so the peer observes a normal closure instead of 1006.
func (t *TunnelConn) Close() error {
- t.closed.Store(true)
+ if t.closed.Swap(true) {
+ return t.conn.Close()
+ }
- t.mu.Lock()
- defer t.mu.Unlock()
+ // WriteControl is safe concurrently with a data writer, so a stuck Send
+ // cannot block teardown.
+ _ = t.conn.WriteControl(websocket.CloseMessage,
+ websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""),
+ time.Now().Add(DefaultWriteTimeout))
return t.conn.Close()
}
@@ -136,6 +177,11 @@ func (t *TunnelConn) IsClosed() bool {
return t.closed.Load()
}
+// Transport identifies the underlying tunnel transport.
+func (t *TunnelConn) Transport() string {
+ return EdgeTransportWebSocket
+}
+
type grpcManagerStream interface {
Send(msg *tunnelpb.ManagerMessage) error
Recv() (*tunnelpb.AgentMessage, error)
@@ -149,7 +195,9 @@ type grpcAgentStream interface {
CloseSend() error
}
-// NewGRPCManagerTunnelConn creates a manager-side gRPC tunnel wrapper.
+// NewGRPCManagerTunnelConn creates a manager-side gRPC tunnel wrapper. A nil
+// stream yields a connection whose Send/Receive report the closed sentinel
+// instead of dereferencing nil.
func NewGRPCManagerTunnelConn(stream grpcManagerStream) *GRPCManagerTunnelConn {
if stream == nil {
return &GRPCManagerTunnelConn{}
@@ -170,11 +218,11 @@ func (t *GRPCManagerTunnelConn) Send(msg *TunnelMessage) error {
t.mu.Lock()
defer t.mu.Unlock()
- if t.IsClosed() {
- return io.EOF
+ if t.IsClosed() || t.stream == nil {
+ return ErrTunnelConnectionClosed
}
- protoMsg, err := tunnelMessageToManagerProto(msg)
+ protoMsg, err := tunnelMessageToManagerProto(msg, t.parityEncoding.Load())
if err != nil {
return err
}
@@ -186,21 +234,39 @@ func (t *GRPCManagerTunnelConn) Send(msg *TunnelMessage) error {
return nil
}
+// SetParityEncoding enables native proto encodings for message types that are
+// otherwise re-encoded for legacy agents. Set when the agent advertises the
+// proto-parity-v1 capability during registration.
+func (t *GRPCManagerTunnelConn) SetParityEncoding(enabled bool) {
+ t.parityEncoding.Store(enabled)
+}
+
// Receive receives an agent->manager tunnel message from gRPC.
func (t *GRPCManagerTunnelConn) Receive() (*TunnelMessage, error) {
- protoMsg, err := t.stream.Recv()
- if err != nil {
- if errors.Is(err, io.EOF) {
+ if t.stream == nil {
+ return nil, ErrTunnelConnectionClosed
+ }
+
+ for {
+ protoMsg, err := t.stream.Recv()
+ if err != nil {
+ // A gRPC stream is dead after any Recv error, not just io.EOF.
t.markClosed()
+ return nil, err
}
- return nil, err
- }
- msg, err := agentProtoToTunnelMessage(protoMsg)
- if err != nil {
- return nil, err
+ msg, err := agentProtoToTunnelMessage(protoMsg)
+ if err != nil {
+ if errors.Is(err, errUnknownTunnelPayload) {
+ // A newer peer sent a payload this build cannot decode yet;
+ // skip it like unknown websocket message types are skipped.
+ slog.Debug("Ignoring unknown edge tunnel payload", "error", err)
+ continue
+ }
+ return nil, err
+ }
+ return msg, nil
}
- return msg, nil
}
// IsExpectedReceiveError returns true for expected gRPC stream shutdown errors.
@@ -226,6 +292,11 @@ func (t *GRPCManagerTunnelConn) markClosed() {
t.closed.Store(true)
}
+// Transport identifies the underlying tunnel transport.
+func (t *GRPCManagerTunnelConn) Transport() string {
+ return EdgeTransportGRPC
+}
+
// NewGRPCAgentTunnelConn creates an agent-side gRPC tunnel wrapper.
func NewGRPCAgentTunnelConn(stream grpcAgentStream, cancelFns ...context.CancelFunc) *GRPCAgentTunnelConn {
var cancel context.CancelFunc
@@ -241,8 +312,8 @@ func (t *GRPCAgentTunnelConn) Send(msg *TunnelMessage) error {
t.mu.Lock()
defer t.mu.Unlock()
- if t.IsClosed() {
- return io.EOF
+ if t.IsClosed() || t.stream == nil {
+ return ErrTunnelConnectionClosed
}
protoMsg, err := tunnelMessageToAgentProto(msg)
@@ -259,19 +330,30 @@ func (t *GRPCAgentTunnelConn) Send(msg *TunnelMessage) error {
// Receive receives a manager->agent tunnel message from gRPC.
func (t *GRPCAgentTunnelConn) Receive() (*TunnelMessage, error) {
- protoMsg, err := t.stream.Recv()
- if err != nil {
- if errors.Is(err, io.EOF) {
+ if t.stream == nil {
+ return nil, ErrTunnelConnectionClosed
+ }
+
+ for {
+ protoMsg, err := t.stream.Recv()
+ if err != nil {
+ // A gRPC stream is dead after any Recv error, not just io.EOF.
t.markClosed()
+ return nil, err
}
- return nil, err
- }
- msg, err := managerProtoToTunnelMessage(protoMsg)
- if err != nil {
- return nil, err
+ msg, err := managerProtoToTunnelMessage(protoMsg)
+ if err != nil {
+ if errors.Is(err, errUnknownTunnelPayload) {
+ // A newer peer sent a payload this build cannot decode yet;
+ // skip it like unknown websocket message types are skipped.
+ slog.Debug("Ignoring unknown edge tunnel payload", "error", err)
+ continue
+ }
+ return nil, err
+ }
+ return msg, nil
}
- return msg, nil
}
// IsExpectedReceiveError returns true for expected gRPC stream shutdown errors.
@@ -279,16 +361,22 @@ func (t *GRPCAgentTunnelConn) IsExpectedReceiveError(err error) bool {
return isExpectedGRPCReceiveErrorInternal(err)
}
-// Close closes the client send stream.
+// Close closes the client send stream. The half-close happens before cancel so
+// the manager observes a clean io.EOF (the gRPC analog of a websocket close
+// frame) instead of codes.Canceled.
func (t *GRPCAgentTunnelConn) Close() error {
- t.markClosed()
+ if t.closed.Swap(true) {
+ return nil
+ }
+
+ var err error
+ if t.stream != nil {
+ err = t.stream.CloseSend()
+ }
if t.cancel != nil {
t.cancel()
}
- if t.stream == nil {
- return nil
- }
- return t.stream.CloseSend()
+ return err
}
// IsClosed returns whether the stream is closed.
@@ -300,6 +388,11 @@ func (t *GRPCAgentTunnelConn) markClosed() {
t.closed.Store(true)
}
+// Transport identifies the underlying tunnel transport.
+func (t *GRPCAgentTunnelConn) Transport() string {
+ return EdgeTransportGRPC
+}
+
func (s *cancelableGRPCManagerStream) Send(msg *tunnelpb.ManagerMessage) error {
return s.stream.Send(msg)
}
@@ -336,6 +429,9 @@ func isExpectedGRPCReceiveErrorInternal(err error) bool {
if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return true
}
+ if errors.Is(err, ErrTunnelConnectionClosed) {
+ return true
+ }
code := status.Code(err)
return code == codes.Canceled || code == codes.DeadlineExceeded
diff --git a/backend/pkg/libarcane/edge/tunnel_proto_adapter.go b/backend/pkg/libarcane/edge/tunnel_proto_adapter.go
index aa3b8616b..b310e9c46 100644
--- a/backend/pkg/libarcane/edge/tunnel_proto_adapter.go
+++ b/backend/pkg/libarcane/edge/tunnel_proto_adapter.go
@@ -6,10 +6,19 @@ import (
"emperror.dev/errors"
- tunnelpb "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
)
-func tunnelMessageToManagerProto(msg *TunnelMessage) (*tunnelpb.ManagerMessage, error) {
+// errUnknownTunnelPayload marks a proto payload this peer cannot decode, e.g.
+// a oneof case added by a newer peer. Receivers skip such messages instead of
+// killing the stream, mirroring how unknown JSON message types are ignored on
+// the websocket transport.
+const errUnknownTunnelPayload = errors.Sentinel("unknown tunnel payload type")
+
+// tunnelMessageToManagerProto encodes a manager->agent message. parity is true
+// once the agent advertised tunnelCapabilityProtoParity; without it, legacy
+// agents receive stream_data re-encoded as ws_data and cannot receive stream_end.
+func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.ManagerMessage, error) {
if msg == nil {
return nil, errors.New("message is nil")
}
@@ -25,7 +34,7 @@ func tunnelMessageToManagerProto(msg *TunnelMessage) (*tunnelpb.ManagerMessage,
Body: msg.Body,
}}}, nil
case MessageTypeHeartbeatAck:
- return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_HeartbeatPong{HeartbeatPong: &tunnelpb.HeartbeatPong{}}}, nil
+ return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_HeartbeatPong{HeartbeatPong: &tunnelpb.HeartbeatPong{Id: msg.ID}}}, nil
case MessageTypeWebSocketStart:
return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_WsStart{WsStart: &tunnelpb.WebSocketStart{
StreamId: msg.ID,
@@ -48,15 +57,28 @@ func tunnelMessageToManagerProto(msg *TunnelMessage) (*tunnelpb.ManagerMessage,
if err != nil {
return nil, err
}
+ if parity {
+ return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_StreamData{StreamData: &tunnelpb.StreamData{
+ RequestId: msg.ID,
+ Data: msg.Body,
+ MessageType: messageType,
+ }}}, nil
+ }
return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_WsData{WsData: &tunnelpb.WebSocketData{
StreamId: msg.ID,
Data: msg.Body,
MessageType: messageType,
}}}, nil
+ case MessageTypeStreamEnd:
+ if parity {
+ return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_StreamEnd{StreamEnd: &tunnelpb.StreamEnd{RequestId: msg.ID}}}, nil
+ }
+ return nil, errors.Errorf("unsupported manager message type: %s", msg.Type)
case MessageTypeWebSocketClose:
return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_WsClose{WsClose: &tunnelpb.WebSocketClose{StreamId: msg.ID}}}, nil
case MessageTypeRegisterResponse:
return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_RegisterResponse{RegisterResponse: &tunnelpb.RegisterResponse{
+ Id: msg.ID,
Accepted: msg.Accepted,
EnvironmentId: msg.EnvironmentID,
Error: msg.Error,
@@ -103,10 +125,10 @@ func tunnelMessageToManagerProto(msg *TunnelMessage) (*tunnelpb.ManagerMessage,
Data: msg.Body,
Sequence: msg.Sequence,
Eof: msg.EOF,
+ Metadata: cloneHeaderMap(msg.Metadata),
}}}, nil
case MessageTypeResponse,
MessageTypeHeartbeat,
- MessageTypeStreamEnd,
MessageTypeRegister,
MessageTypeEvent,
MessageTypeCommandAck,
@@ -135,7 +157,7 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,
Body: payload.HttpRequest.GetBody(),
}, nil
case *tunnelpb.ManagerMessage_HeartbeatPong:
- return &TunnelMessage{Type: MessageTypeHeartbeatAck}, nil
+ return &TunnelMessage{ID: payload.HeartbeatPong.GetId(), Type: MessageTypeHeartbeatAck}, nil
case *tunnelpb.ManagerMessage_WsStart:
return &TunnelMessage{
ID: payload.WsStart.GetStreamId(),
@@ -155,6 +177,7 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,
return &TunnelMessage{ID: payload.WsClose.GetStreamId(), Type: MessageTypeWebSocketClose}, nil
case *tunnelpb.ManagerMessage_RegisterResponse:
return &TunnelMessage{
+ ID: payload.RegisterResponse.GetId(),
Type: MessageTypeRegisterResponse,
Accepted: payload.RegisterResponse.GetAccepted(),
EnvironmentID: payload.RegisterResponse.GetEnvironmentId(),
@@ -207,9 +230,19 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,
Body: payload.FileChunk.GetData(),
Sequence: payload.FileChunk.GetSequence(),
EOF: payload.FileChunk.GetEof(),
+ Metadata: cloneHeaderMap(payload.FileChunk.GetMetadata()),
}, nil
+ case *tunnelpb.ManagerMessage_StreamData:
+ return &TunnelMessage{
+ ID: payload.StreamData.GetRequestId(),
+ Type: MessageTypeStreamData,
+ Body: payload.StreamData.GetData(),
+ WSMessageType: int(payload.StreamData.GetMessageType()),
+ }, nil
+ case *tunnelpb.ManagerMessage_StreamEnd:
+ return &TunnelMessage{ID: payload.StreamEnd.GetRequestId(), Type: MessageTypeStreamEnd}, nil
default:
- return nil, errors.Errorf("unsupported manager payload type %T", payload)
+ return nil, errors.WrapIff(errUnknownTunnelPayload, "manager payload %T", payload)
}
}
@@ -231,7 +264,7 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erro
Body: msg.Body,
}}}, nil
case MessageTypeHeartbeat:
- return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_HeartbeatPing{HeartbeatPing: &tunnelpb.HeartbeatPing{}}}, nil
+ return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_HeartbeatPing{HeartbeatPing: &tunnelpb.HeartbeatPing{Id: msg.ID}}}, nil
case MessageTypeWebSocketData:
messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")
if err != nil {
@@ -268,6 +301,7 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erro
return nil, errors.Errorf("event payload is required for message type: %s", msg.Type)
}
return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_Event{Event: &tunnelpb.EventLog{
+ Id: msg.ID,
Type: msg.Event.Type,
Severity: msg.Event.Severity,
Title: msg.Event.Title,
@@ -308,19 +342,25 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erro
Data: msg.Body,
Sequence: msg.Sequence,
Eof: msg.EOF,
+ Metadata: cloneHeaderMap(msg.Metadata),
}}}, nil
case MessageTypeStreamClose:
return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_StreamClose{StreamClose: &tunnelpb.StreamClose{
StreamId: msg.ID,
Error: msg.Error,
}}}, nil
+ case MessageTypeCancelRequest:
+ // Senders must check the manager advertised tunnelCapabilityProtoParity;
+ // older managers decode this oneof case as an unknown payload.
+ return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_CancelRequest{CancelRequest: &tunnelpb.CancelRequest{
+ CommandId: msg.ID,
+ }}}, nil
case MessageTypeRequest,
MessageTypeHeartbeatAck,
MessageTypeWebSocketStart,
MessageTypeRegisterResponse,
MessageTypeCommandRequest,
- MessageTypeStreamOpen,
- MessageTypeCancelRequest:
+ MessageTypeStreamOpen:
return nil, errors.Errorf("unsupported agent message type: %s", msg.Type)
default:
return nil, errors.Errorf("unsupported agent message type: %s", msg.Type)
@@ -342,7 +382,7 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erro
Body: payload.HttpResponse.GetBody(),
}, nil
case *tunnelpb.AgentMessage_HeartbeatPing:
- return &TunnelMessage{Type: MessageTypeHeartbeat}, nil
+ return &TunnelMessage{ID: payload.HeartbeatPing.GetId(), Type: MessageTypeHeartbeat}, nil
case *tunnelpb.AgentMessage_WsData:
return &TunnelMessage{
ID: payload.WsData.GetStreamId(),
@@ -371,6 +411,7 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erro
}, nil
case *tunnelpb.AgentMessage_Event:
return &TunnelMessage{
+ ID: payload.Event.GetId(),
Type: MessageTypeEvent,
Event: &TunnelEvent{
Type: payload.Event.GetType(),
@@ -411,6 +452,7 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erro
Body: payload.FileChunk.GetData(),
Sequence: payload.FileChunk.GetSequence(),
EOF: payload.FileChunk.GetEof(),
+ Metadata: cloneHeaderMap(payload.FileChunk.GetMetadata()),
}, nil
case *tunnelpb.AgentMessage_StreamClose:
return &TunnelMessage{
@@ -418,8 +460,13 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erro
Type: MessageTypeStreamClose,
Error: payload.StreamClose.GetError(),
}, nil
+ case *tunnelpb.AgentMessage_CancelRequest:
+ return &TunnelMessage{
+ ID: payload.CancelRequest.GetCommandId(),
+ Type: MessageTypeCancelRequest,
+ }, nil
default:
- return nil, errors.Errorf("unsupported agent payload type %T", payload)
+ return nil, errors.WrapIff(errUnknownTunnelPayload, "agent payload %T", payload)
}
}
diff --git a/backend/pkg/libarcane/edge/tunnel_proto_adapter_test.go b/backend/pkg/libarcane/edge/tunnel_proto_adapter_test.go
index 656368ac7..8750adc88 100644
--- a/backend/pkg/libarcane/edge/tunnel_proto_adapter_test.go
+++ b/backend/pkg/libarcane/edge/tunnel_proto_adapter_test.go
@@ -1,8 +1,11 @@
package edge
import (
+ "context"
+ "io"
"testing"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/gorilla/websocket"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -19,7 +22,7 @@ func TestTunnelMessageToManagerProto_RoundTripRequest(t *testing.T) {
Body: []byte(`{"ok":true}`),
}
- protoMsg, err := tunnelMessageToManagerProto(original)
+ protoMsg, err := tunnelMessageToManagerProto(original, false)
require.NoError(t, err)
decoded, err := managerProtoToTunnelMessage(protoMsg)
@@ -42,7 +45,7 @@ func TestTunnelMessageToManagerProto_StreamDataUsesWebSocketPayload(t *testing.T
WSMessageType: websocket.TextMessage,
}
- protoMsg, err := tunnelMessageToManagerProto(original)
+ protoMsg, err := tunnelMessageToManagerProto(original, false)
require.NoError(t, err)
wsData := protoMsg.GetWsData()
@@ -126,7 +129,7 @@ func TestTunnelMessageToAgentProto_RoundTripEvent(t *testing.T) {
}
func TestTunnelMessageToManagerProto_UnsupportedType(t *testing.T) {
- _, err := tunnelMessageToManagerProto(&TunnelMessage{Type: MessageTypeResponse})
+ _, err := tunnelMessageToManagerProto(&TunnelMessage{Type: MessageTypeResponse}, false)
require.Error(t, err)
}
@@ -149,3 +152,111 @@ func TestTunnelMessageToAgentProto_RoundTripStreamDataPreservesWebSocketType(t *
assert.Equal(t, original.Body, decoded.Body)
assert.Equal(t, original.WSMessageType, decoded.WSMessageType)
}
+
+func TestTunnelProtoAdapter_HeartbeatIDRoundTrip(t *testing.T) {
+ ping, err := tunnelMessageToAgentProto(&TunnelMessage{ID: "hb-1", Type: MessageTypeHeartbeat})
+ require.NoError(t, err)
+ decodedPing, err := agentProtoToTunnelMessage(ping)
+ require.NoError(t, err)
+ assert.Equal(t, "hb-1", decodedPing.ID)
+
+ pong, err := tunnelMessageToManagerProto(&TunnelMessage{ID: "hb-1", Type: MessageTypeHeartbeatAck}, false)
+ require.NoError(t, err)
+ decodedPong, err := managerProtoToTunnelMessage(pong)
+ require.NoError(t, err)
+ assert.Equal(t, "hb-1", decodedPong.ID)
+}
+
+func TestTunnelProtoAdapter_FileChunkMetadataRoundTrip(t *testing.T) {
+ original := &TunnelMessage{
+ ID: "transfer-1",
+ Type: MessageTypeFileChunk,
+ Body: []byte("chunk"),
+ Sequence: 2,
+ Metadata: map[string]string{bodyTransferMetadataKey: "transfer-1"},
+ }
+
+ agentMsg, err := tunnelMessageToAgentProto(original)
+ require.NoError(t, err)
+ decoded, err := agentProtoToTunnelMessage(agentMsg)
+ require.NoError(t, err)
+ assert.Equal(t, original.Metadata, decoded.Metadata)
+
+ managerMsg, err := tunnelMessageToManagerProto(original, false)
+ require.NoError(t, err)
+ decodedManager, err := managerProtoToTunnelMessage(managerMsg)
+ require.NoError(t, err)
+ assert.Equal(t, original.Metadata, decodedManager.Metadata)
+}
+
+func TestTunnelMessageToManagerProto_StreamDataParityEncoding(t *testing.T) {
+ original := &TunnelMessage{
+ ID: "stream-1",
+ Type: MessageTypeStreamData,
+ Body: []byte("hello"),
+ WSMessageType: websocket.TextMessage,
+ }
+
+ // Legacy agents receive the ws_data re-encode and see ws_data.
+ legacy, err := tunnelMessageToManagerProto(original, false)
+ require.NoError(t, err)
+ decodedLegacy, err := managerProtoToTunnelMessage(legacy)
+ require.NoError(t, err)
+ assert.Equal(t, MessageTypeWebSocketData, decodedLegacy.Type)
+
+ // Parity-capable agents receive native stream_data with the type preserved.
+ parity, err := tunnelMessageToManagerProto(original, true)
+ require.NoError(t, err)
+ require.NotNil(t, parity.GetStreamData())
+ decodedParity, err := managerProtoToTunnelMessage(parity)
+ require.NoError(t, err)
+ assert.Equal(t, MessageTypeStreamData, decodedParity.Type)
+ assert.Equal(t, original.ID, decodedParity.ID)
+ assert.Equal(t, original.Body, decodedParity.Body)
+}
+
+func TestTunnelMessageToManagerProto_StreamEndRequiresParity(t *testing.T) {
+ original := &TunnelMessage{ID: "stream-1", Type: MessageTypeStreamEnd}
+
+ _, err := tunnelMessageToManagerProto(original, false)
+ require.Error(t, err)
+
+ parity, err := tunnelMessageToManagerProto(original, true)
+ require.NoError(t, err)
+ decoded, err := managerProtoToTunnelMessage(parity)
+ require.NoError(t, err)
+ assert.Equal(t, MessageTypeStreamEnd, decoded.Type)
+ assert.Equal(t, original.ID, decoded.ID)
+}
+
+func TestGRPCAgentTunnelConn_ReceiveSkipsUnknownPayload(t *testing.T) {
+ stream := &scriptedAgentStream{msgs: []*tunnelpb.ManagerMessage{
+ {}, // nil payload, decodes as unknown
+ {Payload: &tunnelpb.ManagerMessage_HeartbeatPong{HeartbeatPong: &tunnelpb.HeartbeatPong{Id: "hb-2"}}},
+ }}
+ conn := NewGRPCAgentTunnelConn(stream)
+
+ msg, err := conn.Receive()
+ require.NoError(t, err)
+ assert.Equal(t, MessageTypeHeartbeatAck, msg.Type)
+ assert.Equal(t, "hb-2", msg.ID)
+}
+
+type scriptedAgentStream struct {
+ msgs []*tunnelpb.ManagerMessage
+}
+
+func (s *scriptedAgentStream) Send(*tunnelpb.AgentMessage) error { return nil }
+
+func (s *scriptedAgentStream) Recv() (*tunnelpb.ManagerMessage, error) {
+ if len(s.msgs) == 0 {
+ return nil, io.EOF
+ }
+ msg := s.msgs[0]
+ s.msgs = s.msgs[1:]
+ return msg, nil
+}
+
+func (s *scriptedAgentStream) Context() context.Context { return context.Background() }
+
+func (s *scriptedAgentStream) CloseSend() error { return nil }
diff --git a/backend/pkg/libarcane/edge/tunnel_test.go b/backend/pkg/libarcane/edge/tunnel_test.go
index 3a5019217..46dd8c55e 100644
--- a/backend/pkg/libarcane/edge/tunnel_test.go
+++ b/backend/pkg/libarcane/edge/tunnel_test.go
@@ -3,13 +3,14 @@ package edge
import (
"context"
"encoding/json"
+ "io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
- tunnelpb "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/gorilla/websocket"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -96,7 +97,7 @@ func TestTunnelConn(t *testing.T) {
// Test Send on closed connection
err = tunnelConn.Send(msg)
require.Error(t, err)
- assert.Equal(t, websocket.ErrCloseSent, err)
+ assert.ErrorIs(t, err, ErrTunnelConnectionClosed)
}
func TestGRPCManagerTunnelConn_CloseCancelsReceive(t *testing.T) {
@@ -148,3 +149,93 @@ func (b *blockingGRPCManagerStream) Recv() (*tunnelpb.AgentMessage, error) {
}
func (b *blockingGRPCManagerStream) Context() context.Context { return b.ctx }
+
+func TestTunnelConn_CloseSendsCloseFrame(t *testing.T) {
+ peerRead := make(chan error, 1)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ upgrader := websocket.Upgrader{}
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ defer func() { _ = conn.Close() }()
+ _, _, err = conn.ReadMessage()
+ peerRead <- err
+ }))
+ defer server.Close()
+
+ wsURL := "ws" + strings.TrimPrefix(server.URL, "http")
+ conn, resp, err := websocket.DefaultDialer.Dial(wsURL, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ defer func() { _ = resp.Body.Close() }()
+ }
+
+ tunnelConn := NewTunnelConn(conn)
+ require.NoError(t, tunnelConn.Close())
+
+ select {
+ case err := <-peerRead:
+ assert.True(t, websocket.IsCloseError(err, websocket.CloseNormalClosure),
+ "peer should observe a normal closure, got: %v", err)
+ case <-time.After(5 * time.Second):
+ t.Fatal("timed out waiting for peer read to return")
+ }
+}
+
+func TestTunnelConn_ReceiveDeadlineMarksClosed(t *testing.T) {
+ previousReadWait := tunnelReadWait
+ tunnelReadWait = 100 * time.Millisecond
+ t.Cleanup(func() { tunnelReadWait = previousReadWait })
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ upgrader := websocket.Upgrader{}
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ defer func() { _ = conn.Close() }()
+ // Keep the socket open but never send anything.
+ _, _, _ = conn.ReadMessage()
+ }))
+ defer server.Close()
+
+ wsURL := "ws" + strings.TrimPrefix(server.URL, "http")
+ conn, resp, err := websocket.DefaultDialer.Dial(wsURL, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ defer func() { _ = resp.Body.Close() }()
+ }
+
+ tunnelConn := NewTunnelConn(conn)
+ _, err = tunnelConn.Receive()
+ require.Error(t, err)
+ assert.True(t, tunnelConn.IsClosed())
+}
+
+type closeOrderAgentStream struct {
+ events []string
+}
+
+func (s *closeOrderAgentStream) Send(*tunnelpb.AgentMessage) error { return nil }
+
+func (s *closeOrderAgentStream) Recv() (*tunnelpb.ManagerMessage, error) { return nil, io.EOF }
+
+func (s *closeOrderAgentStream) Context() context.Context { return context.Background() }
+
+func (s *closeOrderAgentStream) CloseSend() error {
+ s.events = append(s.events, "close_send")
+ return nil
+}
+
+func TestGRPCAgentTunnelConn_CloseHalfClosesBeforeCancel(t *testing.T) {
+ stream := &closeOrderAgentStream{}
+ conn := NewGRPCAgentTunnelConn(stream, func() { stream.events = append(stream.events, "cancel") })
+
+ require.NoError(t, conn.Close())
+ assert.Equal(t, []string{"close_send", "cancel"}, stream.events)
+
+ // A second close is a no-op.
+ require.NoError(t, conn.Close())
+ assert.Equal(t, []string{"close_send", "cancel"}, stream.events)
+}
diff --git a/backend/pkg/libarcane/edge/types.go b/backend/pkg/libarcane/edge/types.go
index 5ce1a8fff..85eab3715 100644
--- a/backend/pkg/libarcane/edge/types.go
+++ b/backend/pkg/libarcane/edge/types.go
@@ -38,25 +38,27 @@ type commandRequestTransfer struct {
// TunnelClient represents the agent-side tunnel client
type TunnelClient struct {
- cfg *Config
- handler http.Handler
- reconnectInterval time.Duration
- heartbeatInterval time.Duration
- grpcRegistrationTimeout time.Duration
- websocketPreferenceTTL time.Duration
- managerURL string
- managerGRPCAddr string
- localPort string // Port the agent is running on locally
- httpClient *http.Client
- conn atomic.Pointer[connBox]
- stopCh chan struct{}
- requestTimeout time.Duration
- activeStreams sync.Map // map[string]*activeWSStream
- requestTransfers sync.Map // map[string]*commandRequestTransfer
- transportPreferenceMu sync.RWMutex
- preferWebSocketUntil time.Time
- agentInstanceID string
- sessionID string
+ cfg *Config
+ handler http.Handler
+ reconnectInterval time.Duration
+ heartbeatInterval time.Duration
+ registrationTimeout time.Duration
+ websocketPreferenceTTL time.Duration
+ managerURL string
+ managerGRPCAddr string
+ localPort string // Port the agent is running on locally
+ httpClient *http.Client
+ conn atomic.Pointer[connBox]
+ stopCh chan struct{}
+ requestTimeout time.Duration
+ activeStreams sync.Map // map[string]*activeWSStream
+ requestTransfers sync.Map // map[string]*commandRequestTransfer
+ transportPreferenceMu sync.RWMutex
+ preferWebSocketUntil time.Time
+ grpcFailureStreak int
+ agentInstanceID string
+ sessionID string
+ managerCapabilities []string
}
// connBox wraps the active TunnelConnection so it can be swapped atomically on
@@ -357,6 +359,10 @@ type GRPCManagerTunnelConn struct {
cancel context.CancelFunc
mu sync.Mutex
closed atomic.Bool
+ // parityEncoding is set once the agent advertises the proto-parity-v1
+ // capability, enabling native stream_data/stream_end encodings instead of
+ // the legacy ws_data re-encode.
+ parityEncoding atomic.Bool
}
// GRPCAgentTunnelConn wraps the agent-side gRPC tunnel stream.
diff --git a/backend/pkg/libarcane/edge/proto/buf.gen.yaml b/backend/proto/buf.gen.yaml
similarity index 100%
rename from backend/pkg/libarcane/edge/proto/buf.gen.yaml
rename to backend/proto/buf.gen.yaml
diff --git a/backend/pkg/libarcane/edge/proto/buf.yaml b/backend/proto/buf.yaml
similarity index 100%
rename from backend/pkg/libarcane/edge/proto/buf.yaml
rename to backend/proto/buf.yaml
diff --git a/backend/pkg/libarcane/edge/proto/tunnel/v1/tunnel.pb.go b/backend/proto/tunnel/v1/tunnel.pb.go
similarity index 90%
rename from backend/pkg/libarcane/edge/proto/tunnel/v1/tunnel.pb.go
rename to backend/proto/tunnel/v1/tunnel.pb.go
index f5886989c..a16bfc45e 100644
--- a/backend/pkg/libarcane/edge/proto/tunnel/v1/tunnel.pb.go
+++ b/backend/proto/tunnel/v1/tunnel.pb.go
@@ -7,12 +7,11 @@
package tunnelpb
import (
+ protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+ protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
unsafe "unsafe"
-
- protoreflect "google.golang.org/protobuf/reflect/protoreflect"
- protoimpl "google.golang.org/protobuf/runtime/protoimpl"
)
const (
@@ -40,6 +39,7 @@ type AgentMessage struct {
// *AgentMessage_CommandComplete
// *AgentMessage_FileChunk
// *AgentMessage_StreamClose
+ // *AgentMessage_CancelRequest
Payload isAgentMessage_Payload `protobuf_oneof:"payload"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
@@ -199,6 +199,15 @@ func (x *AgentMessage) GetStreamClose() *StreamClose {
return nil
}
+func (x *AgentMessage) GetCancelRequest() *CancelRequest {
+ if x != nil {
+ if x, ok := x.Payload.(*AgentMessage_CancelRequest); ok {
+ return x.CancelRequest
+ }
+ }
+ return nil
+}
+
type isAgentMessage_Payload interface {
isAgentMessage_Payload()
}
@@ -255,6 +264,12 @@ type AgentMessage_StreamClose struct {
StreamClose *StreamClose `protobuf:"bytes,13,opt,name=stream_close,json=streamClose,proto3,oneof"`
}
+type AgentMessage_CancelRequest struct {
+ // Only sent when the manager advertised the proto-parity-v1 capability;
+ // older managers decode an unknown oneof case as a nil payload.
+ CancelRequest *CancelRequest `protobuf:"bytes,14,opt,name=cancel_request,json=cancelRequest,proto3,oneof"`
+}
+
func (*AgentMessage_HttpResponse) isAgentMessage_Payload() {}
func (*AgentMessage_HeartbeatPing) isAgentMessage_Payload() {}
@@ -281,6 +296,8 @@ func (*AgentMessage_FileChunk) isAgentMessage_Payload() {}
func (*AgentMessage_StreamClose) isAgentMessage_Payload() {}
+func (*AgentMessage_CancelRequest) isAgentMessage_Payload() {}
+
// Messages sent from the manager to the agent.
type ManagerMessage struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -297,6 +314,8 @@ type ManagerMessage struct {
// *ManagerMessage_StreamClose
// *ManagerMessage_CancelRequest
// *ManagerMessage_FileChunk
+ // *ManagerMessage_StreamData
+ // *ManagerMessage_StreamEnd
Payload isManagerMessage_Payload `protobuf_oneof:"payload"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
@@ -438,6 +457,24 @@ func (x *ManagerMessage) GetFileChunk() *FileChunk {
return nil
}
+func (x *ManagerMessage) GetStreamData() *StreamData {
+ if x != nil {
+ if x, ok := x.Payload.(*ManagerMessage_StreamData); ok {
+ return x.StreamData
+ }
+ }
+ return nil
+}
+
+func (x *ManagerMessage) GetStreamEnd() *StreamEnd {
+ if x != nil {
+ if x, ok := x.Payload.(*ManagerMessage_StreamEnd); ok {
+ return x.StreamEnd
+ }
+ }
+ return nil
+}
+
type isManagerMessage_Payload interface {
isManagerMessage_Payload()
}
@@ -486,6 +523,16 @@ type ManagerMessage_FileChunk struct {
FileChunk *FileChunk `protobuf:"bytes,11,opt,name=file_chunk,json=fileChunk,proto3,oneof"`
}
+type ManagerMessage_StreamData struct {
+ // Only sent when the agent advertised the proto-parity-v1 capability;
+ // older agents decode an unknown oneof case as a nil payload.
+ StreamData *StreamData `protobuf:"bytes,12,opt,name=stream_data,json=streamData,proto3,oneof"`
+}
+
+type ManagerMessage_StreamEnd struct {
+ StreamEnd *StreamEnd `protobuf:"bytes,13,opt,name=stream_end,json=streamEnd,proto3,oneof"`
+}
+
func (*ManagerMessage_HttpRequest) isManagerMessage_Payload() {}
func (*ManagerMessage_HeartbeatPong) isManagerMessage_Payload() {}
@@ -508,6 +555,10 @@ func (*ManagerMessage_CancelRequest) isManagerMessage_Payload() {}
func (*ManagerMessage_FileChunk) isManagerMessage_Payload() {}
+func (*ManagerMessage_StreamData) isManagerMessage_Payload() {}
+
+func (*ManagerMessage_StreamEnd) isManagerMessage_Payload() {}
+
type RegisterRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
AgentToken string `protobuf:"bytes,1,opt,name=agent_token,json=agentToken,proto3" json:"agent_token,omitempty"`
@@ -585,6 +636,7 @@ type RegisterResponse struct {
SecurityMode string `protobuf:"bytes,5,opt,name=security_mode,json=securityMode,proto3" json:"security_mode,omitempty"`
Capabilities []string `protobuf:"bytes,6,rep,name=capabilities,proto3" json:"capabilities,omitempty"`
DrainPrevious bool `protobuf:"varint,7,opt,name=drain_previous,json=drainPrevious,proto3" json:"drain_previous,omitempty"`
+ Id string `protobuf:"bytes,8,opt,name=id,proto3" json:"id,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -668,6 +720,13 @@ func (x *RegisterResponse) GetDrainPrevious() bool {
return false
}
+func (x *RegisterResponse) GetId() string {
+ if x != nil {
+ return x.Id
+ }
+ return ""
+}
+
type CommandRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`
@@ -986,6 +1045,7 @@ type FileChunk struct {
Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
Sequence int64 `protobuf:"varint,3,opt,name=sequence,proto3" json:"sequence,omitempty"`
Eof bool `protobuf:"varint,4,opt,name=eof,proto3" json:"eof,omitempty"`
+ Metadata map[string]string `protobuf:"bytes,5,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -1048,6 +1108,13 @@ func (x *FileChunk) GetEof() bool {
return false
}
+func (x *FileChunk) GetMetadata() map[string]string {
+ if x != nil {
+ return x.Metadata
+ }
+ return nil
+}
+
type StreamOpen struct {
state protoimpl.MessageState `protogen:"open.v1"`
StreamId string `protobuf:"bytes,1,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"`
@@ -1487,6 +1554,7 @@ func (x *StreamEnd) GetRequestId() string {
type HeartbeatPing struct {
state protoimpl.MessageState `protogen:"open.v1"`
+ Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -1521,8 +1589,16 @@ func (*HeartbeatPing) Descriptor() ([]byte, []int) {
return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{16}
}
+func (x *HeartbeatPing) GetId() string {
+ if x != nil {
+ return x.Id
+ }
+ return ""
+}
+
type HeartbeatPong struct {
state protoimpl.MessageState `protogen:"open.v1"`
+ Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -1557,6 +1633,13 @@ func (*HeartbeatPong) Descriptor() ([]byte, []int) {
return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{17}
}
+func (x *HeartbeatPong) GetId() string {
+ if x != nil {
+ return x.Id
+ }
+ return ""
+}
+
type WebSocketStart struct {
state protoimpl.MessageState `protogen:"open.v1"`
StreamId string `protobuf:"bytes,1,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"`
@@ -1742,6 +1825,7 @@ type EventLog struct {
UserId string `protobuf:"bytes,8,opt,name=user_id,json=userId,proto3" json:"user_id,omitempty"`
Username string `protobuf:"bytes,9,opt,name=username,proto3" json:"username,omitempty"`
MetadataJson []byte `protobuf:"bytes,10,opt,name=metadata_json,json=metadataJson,proto3" json:"metadata_json,omitempty"`
+ Id string `protobuf:"bytes,11,opt,name=id,proto3" json:"id,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -1846,11 +1930,18 @@ func (x *EventLog) GetMetadataJson() []byte {
return nil
}
+func (x *EventLog) GetId() string {
+ if x != nil {
+ return x.Id
+ }
+ return ""
+}
+
var File_tunnel_v1_tunnel_proto protoreflect.FileDescriptor
const file_tunnel_v1_tunnel_proto_rawDesc = "" +
"\n" +
- "\x16tunnel/v1/tunnel.proto\x12\ttunnel.v1\"\x9b\x06\n" +
+ "\x16tunnel/v1/tunnel.proto\x12\ttunnel.v1\"\xde\x06\n" +
"\fAgentMessage\x12>\n" +
"\rhttp_response\x18\x01 \x01(\v2\x17.tunnel.v1.HttpResponseH\x00R\fhttpResponse\x12A\n" +
"\x0eheartbeat_ping\x18\x02 \x01(\v2\x18.tunnel.v1.HeartbeatPingH\x00R\rheartbeatPing\x123\n" +
@@ -1869,8 +1960,9 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +
"\x10command_complete\x18\v \x01(\v2\x1a.tunnel.v1.CommandCompleteH\x00R\x0fcommandComplete\x125\n" +
"\n" +
"file_chunk\x18\f \x01(\v2\x14.tunnel.v1.FileChunkH\x00R\tfileChunk\x12;\n" +
- "\fstream_close\x18\r \x01(\v2\x16.tunnel.v1.StreamCloseH\x00R\vstreamCloseB\t\n" +
- "\apayload\"\xc3\x05\n" +
+ "\fstream_close\x18\r \x01(\v2\x16.tunnel.v1.StreamCloseH\x00R\vstreamClose\x12A\n" +
+ "\x0ecancel_request\x18\x0e \x01(\v2\x18.tunnel.v1.CancelRequestH\x00R\rcancelRequestB\t\n" +
+ "\apayload\"\xb4\x06\n" +
"\x0eManagerMessage\x12;\n" +
"\fhttp_request\x18\x01 \x01(\v2\x16.tunnel.v1.HttpRequestH\x00R\vhttpRequest\x12A\n" +
"\x0eheartbeat_pong\x18\x02 \x01(\v2\x18.tunnel.v1.HeartbeatPongH\x00R\rheartbeatPong\x126\n" +
@@ -1885,14 +1977,18 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +
"\x0ecancel_request\x18\n" +
" \x01(\v2\x18.tunnel.v1.CancelRequestH\x00R\rcancelRequest\x125\n" +
"\n" +
- "file_chunk\x18\v \x01(\v2\x14.tunnel.v1.FileChunkH\x00R\tfileChunkB\t\n" +
+ "file_chunk\x18\v \x01(\v2\x14.tunnel.v1.FileChunkH\x00R\tfileChunk\x128\n" +
+ "\vstream_data\x18\f \x01(\v2\x15.tunnel.v1.StreamDataH\x00R\n" +
+ "streamData\x125\n" +
+ "\n" +
+ "stream_end\x18\r \x01(\v2\x14.tunnel.v1.StreamEndH\x00R\tstreamEndB\t\n" +
"\apayload\"\xae\x01\n" +
"\x0fRegisterRequest\x12\x1f\n" +
"\vagent_token\x18\x01 \x01(\tR\n" +
"agentToken\x12*\n" +
"\x11agent_instance_id\x18\x02 \x01(\tR\x0fagentInstanceId\x12\"\n" +
"\fcapabilities\x18\x03 \x03(\tR\fcapabilities\x12*\n" +
- "\x11resume_session_id\x18\x04 \x01(\tR\x0fresumeSessionId\"\xfa\x01\n" +
+ "\x11resume_session_id\x18\x04 \x01(\tR\x0fresumeSessionId\"\x8a\x02\n" +
"\x10RegisterResponse\x12\x1a\n" +
"\baccepted\x18\x01 \x01(\bR\baccepted\x12%\n" +
"\x0eenvironment_id\x18\x02 \x01(\tR\renvironmentId\x12\x14\n" +
@@ -1901,7 +1997,8 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +
"session_id\x18\x04 \x01(\tR\tsessionId\x12#\n" +
"\rsecurity_mode\x18\x05 \x01(\tR\fsecurityMode\x12\"\n" +
"\fcapabilities\x18\x06 \x03(\tR\fcapabilities\x12%\n" +
- "\x0edrain_previous\x18\a \x01(\bR\rdrainPrevious\"\x9a\x04\n" +
+ "\x0edrain_previous\x18\a \x01(\bR\rdrainPrevious\x12\x0e\n" +
+ "\x02id\x18\b \x01(\tR\x02id\"\x9a\x04\n" +
"\x0eCommandRequest\x12\x1d\n" +
"\n" +
"command_id\x18\x01 \x01(\tR\tcommandId\x12!\n" +
@@ -1942,13 +2039,17 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +
"\tstreaming\x18\x06 \x01(\bR\tstreaming\x1a:\n" +
"\fHeadersEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
- "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"n\n" +
+ "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xeb\x01\n" +
"\tFileChunk\x12\x1f\n" +
"\vtransfer_id\x18\x01 \x01(\tR\n" +
"transferId\x12\x12\n" +
"\x04data\x18\x02 \x01(\fR\x04data\x12\x1a\n" +
"\bsequence\x18\x03 \x01(\x03R\bsequence\x12\x10\n" +
- "\x03eof\x18\x04 \x01(\bR\x03eof\"\x8f\x02\n" +
+ "\x03eof\x18\x04 \x01(\bR\x03eof\x12>\n" +
+ "\bmetadata\x18\x05 \x03(\v2\".tunnel.v1.FileChunk.MetadataEntryR\bmetadata\x1a;\n" +
+ "\rMetadataEntry\x12\x10\n" +
+ "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+ "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x8f\x02\n" +
"\n" +
"StreamOpen\x12\x1b\n" +
"\tstream_id\x18\x01 \x01(\tR\bstreamId\x12!\n" +
@@ -1995,9 +2096,11 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +
"\fmessage_type\x18\x03 \x01(\x05R\vmessageType\"*\n" +
"\tStreamEnd\x12\x1d\n" +
"\n" +
- "request_id\x18\x01 \x01(\tR\trequestId\"\x0f\n" +
- "\rHeartbeatPing\"\x0f\n" +
- "\rHeartbeatPong\"\xd5\x01\n" +
+ "request_id\x18\x01 \x01(\tR\trequestId\"\x1f\n" +
+ "\rHeartbeatPing\x12\x0e\n" +
+ "\x02id\x18\x01 \x01(\tR\x02id\"\x1f\n" +
+ "\rHeartbeatPong\x12\x0e\n" +
+ "\x02id\x18\x01 \x01(\tR\x02id\"\xd5\x01\n" +
"\x0eWebSocketStart\x12\x1b\n" +
"\tstream_id\x18\x01 \x01(\tR\bstreamId\x12\x12\n" +
"\x04path\x18\x02 \x01(\tR\x04path\x12\x14\n" +
@@ -2011,7 +2114,7 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +
"\x04data\x18\x02 \x01(\fR\x04data\x12!\n" +
"\fmessage_type\x18\x03 \x01(\x05R\vmessageType\"-\n" +
"\x0eWebSocketClose\x12\x1b\n" +
- "\tstream_id\x18\x01 \x01(\tR\bstreamId\"\xb7\x02\n" +
+ "\tstream_id\x18\x01 \x01(\tR\bstreamId\"\xc7\x02\n" +
"\bEventLog\x12\x12\n" +
"\x04type\x18\x01 \x01(\tR\x04type\x12\x1a\n" +
"\bseverity\x18\x02 \x01(\tR\bseverity\x12\x14\n" +
@@ -2024,9 +2127,10 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +
"\auser_id\x18\b \x01(\tR\x06userId\x12\x1a\n" +
"\busername\x18\t \x01(\tR\busername\x12#\n" +
"\rmetadata_json\x18\n" +
- " \x01(\fR\fmetadataJson2R\n" +
+ " \x01(\fR\fmetadataJson\x12\x0e\n" +
+ "\x02id\x18\v \x01(\tR\x02id2R\n" +
"\rTunnelService\x12A\n" +
- "\aConnect\x12\x17.tunnel.v1.AgentMessage\x1a\x19.tunnel.v1.ManagerMessage(\x010\x01BWZUgithub.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1;tunnelpbb\x06proto3"
+ "\aConnect\x12\x17.tunnel.v1.AgentMessage\x1a\x19.tunnel.v1.ManagerMessage(\x010\x01BDZBgithub.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1;tunnelpbb\x06proto3"
var (
file_tunnel_v1_tunnel_proto_rawDescOnce sync.Once
@@ -2040,7 +2144,7 @@ func file_tunnel_v1_tunnel_proto_rawDescGZIP() []byte {
return file_tunnel_v1_tunnel_proto_rawDescData
}
-var file_tunnel_v1_tunnel_proto_msgTypes = make([]protoimpl.MessageInfo, 29)
+var file_tunnel_v1_tunnel_proto_msgTypes = make([]protoimpl.MessageInfo, 30)
var file_tunnel_v1_tunnel_proto_goTypes = []any{
(*AgentMessage)(nil), // 0: tunnel.v1.AgentMessage
(*ManagerMessage)(nil), // 1: tunnel.v1.ManagerMessage
@@ -2067,10 +2171,11 @@ var file_tunnel_v1_tunnel_proto_goTypes = []any{
nil, // 22: tunnel.v1.CommandRequest.HeadersEntry
nil, // 23: tunnel.v1.CommandRequest.MetadataEntry
nil, // 24: tunnel.v1.CommandComplete.HeadersEntry
- nil, // 25: tunnel.v1.StreamOpen.HeadersEntry
- nil, // 26: tunnel.v1.HttpRequest.HeadersEntry
- nil, // 27: tunnel.v1.HttpResponse.HeadersEntry
- nil, // 28: tunnel.v1.WebSocketStart.HeadersEntry
+ nil, // 25: tunnel.v1.FileChunk.MetadataEntry
+ nil, // 26: tunnel.v1.StreamOpen.HeadersEntry
+ nil, // 27: tunnel.v1.HttpRequest.HeadersEntry
+ nil, // 28: tunnel.v1.HttpResponse.HeadersEntry
+ nil, // 29: tunnel.v1.WebSocketStart.HeadersEntry
}
var file_tunnel_v1_tunnel_proto_depIdxs = []int32{
13, // 0: tunnel.v1.AgentMessage.http_response:type_name -> tunnel.v1.HttpResponse
@@ -2086,31 +2191,35 @@ var file_tunnel_v1_tunnel_proto_depIdxs = []int32{
7, // 10: tunnel.v1.AgentMessage.command_complete:type_name -> tunnel.v1.CommandComplete
8, // 11: tunnel.v1.AgentMessage.file_chunk:type_name -> tunnel.v1.FileChunk
10, // 12: tunnel.v1.AgentMessage.stream_close:type_name -> tunnel.v1.StreamClose
- 12, // 13: tunnel.v1.ManagerMessage.http_request:type_name -> tunnel.v1.HttpRequest
- 17, // 14: tunnel.v1.ManagerMessage.heartbeat_pong:type_name -> tunnel.v1.HeartbeatPong
- 18, // 15: tunnel.v1.ManagerMessage.ws_start:type_name -> tunnel.v1.WebSocketStart
- 19, // 16: tunnel.v1.ManagerMessage.ws_data:type_name -> tunnel.v1.WebSocketData
- 20, // 17: tunnel.v1.ManagerMessage.ws_close:type_name -> tunnel.v1.WebSocketClose
- 3, // 18: tunnel.v1.ManagerMessage.register_response:type_name -> tunnel.v1.RegisterResponse
- 4, // 19: tunnel.v1.ManagerMessage.command_request:type_name -> tunnel.v1.CommandRequest
- 9, // 20: tunnel.v1.ManagerMessage.stream_open:type_name -> tunnel.v1.StreamOpen
- 10, // 21: tunnel.v1.ManagerMessage.stream_close:type_name -> tunnel.v1.StreamClose
- 11, // 22: tunnel.v1.ManagerMessage.cancel_request:type_name -> tunnel.v1.CancelRequest
- 8, // 23: tunnel.v1.ManagerMessage.file_chunk:type_name -> tunnel.v1.FileChunk
- 22, // 24: tunnel.v1.CommandRequest.headers:type_name -> tunnel.v1.CommandRequest.HeadersEntry
- 23, // 25: tunnel.v1.CommandRequest.metadata:type_name -> tunnel.v1.CommandRequest.MetadataEntry
- 24, // 26: tunnel.v1.CommandComplete.headers:type_name -> tunnel.v1.CommandComplete.HeadersEntry
- 25, // 27: tunnel.v1.StreamOpen.headers:type_name -> tunnel.v1.StreamOpen.HeadersEntry
- 26, // 28: tunnel.v1.HttpRequest.headers:type_name -> tunnel.v1.HttpRequest.HeadersEntry
- 27, // 29: tunnel.v1.HttpResponse.headers:type_name -> tunnel.v1.HttpResponse.HeadersEntry
- 28, // 30: tunnel.v1.WebSocketStart.headers:type_name -> tunnel.v1.WebSocketStart.HeadersEntry
- 0, // 31: tunnel.v1.TunnelService.Connect:input_type -> tunnel.v1.AgentMessage
- 1, // 32: tunnel.v1.TunnelService.Connect:output_type -> tunnel.v1.ManagerMessage
- 32, // [32:33] is the sub-list for method output_type
- 31, // [31:32] is the sub-list for method input_type
- 31, // [31:31] is the sub-list for extension type_name
- 31, // [31:31] is the sub-list for extension extendee
- 0, // [0:31] is the sub-list for field type_name
+ 11, // 13: tunnel.v1.AgentMessage.cancel_request:type_name -> tunnel.v1.CancelRequest
+ 12, // 14: tunnel.v1.ManagerMessage.http_request:type_name -> tunnel.v1.HttpRequest
+ 17, // 15: tunnel.v1.ManagerMessage.heartbeat_pong:type_name -> tunnel.v1.HeartbeatPong
+ 18, // 16: tunnel.v1.ManagerMessage.ws_start:type_name -> tunnel.v1.WebSocketStart
+ 19, // 17: tunnel.v1.ManagerMessage.ws_data:type_name -> tunnel.v1.WebSocketData
+ 20, // 18: tunnel.v1.ManagerMessage.ws_close:type_name -> tunnel.v1.WebSocketClose
+ 3, // 19: tunnel.v1.ManagerMessage.register_response:type_name -> tunnel.v1.RegisterResponse
+ 4, // 20: tunnel.v1.ManagerMessage.command_request:type_name -> tunnel.v1.CommandRequest
+ 9, // 21: tunnel.v1.ManagerMessage.stream_open:type_name -> tunnel.v1.StreamOpen
+ 10, // 22: tunnel.v1.ManagerMessage.stream_close:type_name -> tunnel.v1.StreamClose
+ 11, // 23: tunnel.v1.ManagerMessage.cancel_request:type_name -> tunnel.v1.CancelRequest
+ 8, // 24: tunnel.v1.ManagerMessage.file_chunk:type_name -> tunnel.v1.FileChunk
+ 14, // 25: tunnel.v1.ManagerMessage.stream_data:type_name -> tunnel.v1.StreamData
+ 15, // 26: tunnel.v1.ManagerMessage.stream_end:type_name -> tunnel.v1.StreamEnd
+ 22, // 27: tunnel.v1.CommandRequest.headers:type_name -> tunnel.v1.CommandRequest.HeadersEntry
+ 23, // 28: tunnel.v1.CommandRequest.metadata:type_name -> tunnel.v1.CommandRequest.MetadataEntry
+ 24, // 29: tunnel.v1.CommandComplete.headers:type_name -> tunnel.v1.CommandComplete.HeadersEntry
+ 25, // 30: tunnel.v1.FileChunk.metadata:type_name -> tunnel.v1.FileChunk.MetadataEntry
+ 26, // 31: tunnel.v1.StreamOpen.headers:type_name -> tunnel.v1.StreamOpen.HeadersEntry
+ 27, // 32: tunnel.v1.HttpRequest.headers:type_name -> tunnel.v1.HttpRequest.HeadersEntry
+ 28, // 33: tunnel.v1.HttpResponse.headers:type_name -> tunnel.v1.HttpResponse.HeadersEntry
+ 29, // 34: tunnel.v1.WebSocketStart.headers:type_name -> tunnel.v1.WebSocketStart.HeadersEntry
+ 0, // 35: tunnel.v1.TunnelService.Connect:input_type -> tunnel.v1.AgentMessage
+ 1, // 36: tunnel.v1.TunnelService.Connect:output_type -> tunnel.v1.ManagerMessage
+ 36, // [36:37] is the sub-list for method output_type
+ 35, // [35:36] is the sub-list for method input_type
+ 35, // [35:35] is the sub-list for extension type_name
+ 35, // [35:35] is the sub-list for extension extendee
+ 0, // [0:35] is the sub-list for field type_name
}
func init() { file_tunnel_v1_tunnel_proto_init() }
@@ -2132,6 +2241,7 @@ func file_tunnel_v1_tunnel_proto_init() {
(*AgentMessage_CommandComplete)(nil),
(*AgentMessage_FileChunk)(nil),
(*AgentMessage_StreamClose)(nil),
+ (*AgentMessage_CancelRequest)(nil),
}
file_tunnel_v1_tunnel_proto_msgTypes[1].OneofWrappers = []any{
(*ManagerMessage_HttpRequest)(nil),
@@ -2145,6 +2255,8 @@ func file_tunnel_v1_tunnel_proto_init() {
(*ManagerMessage_StreamClose)(nil),
(*ManagerMessage_CancelRequest)(nil),
(*ManagerMessage_FileChunk)(nil),
+ (*ManagerMessage_StreamData)(nil),
+ (*ManagerMessage_StreamEnd)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
@@ -2152,7 +2264,7 @@ func file_tunnel_v1_tunnel_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_tunnel_v1_tunnel_proto_rawDesc), len(file_tunnel_v1_tunnel_proto_rawDesc)),
NumEnums: 0,
- NumMessages: 29,
+ NumMessages: 30,
NumExtensions: 0,
NumServices: 1,
},
diff --git a/backend/pkg/libarcane/edge/proto/tunnel/v1/tunnel.proto b/backend/proto/tunnel/v1/tunnel.proto
similarity index 87%
rename from backend/pkg/libarcane/edge/proto/tunnel/v1/tunnel.proto
rename to backend/proto/tunnel/v1/tunnel.proto
index fc57740c2..49e028020 100644
--- a/backend/pkg/libarcane/edge/proto/tunnel/v1/tunnel.proto
+++ b/backend/proto/tunnel/v1/tunnel.proto
@@ -2,7 +2,7 @@ syntax = "proto3";
package tunnel.v1;
-option go_package = "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge/proto/tunnel/v1;tunnelpb";
+option go_package = "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1;tunnelpb";
// TunnelService is the reverse tunnel between an edge agent and manager.
service TunnelService {
@@ -26,6 +26,9 @@ message AgentMessage {
CommandComplete command_complete = 11;
FileChunk file_chunk = 12;
StreamClose stream_close = 13;
+ // Only sent when the manager advertised the proto-parity-v1 capability;
+ // older managers decode an unknown oneof case as a nil payload.
+ CancelRequest cancel_request = 14;
}
}
@@ -43,6 +46,10 @@ message ManagerMessage {
StreamClose stream_close = 9;
CancelRequest cancel_request = 10;
FileChunk file_chunk = 11;
+ // Only sent when the agent advertised the proto-parity-v1 capability;
+ // older agents decode an unknown oneof case as a nil payload.
+ StreamData stream_data = 12;
+ StreamEnd stream_end = 13;
}
}
@@ -61,6 +68,7 @@ message RegisterResponse {
string security_mode = 5;
repeated string capabilities = 6;
bool drain_previous = 7;
+ string id = 8;
}
message CommandRequest {
@@ -101,6 +109,7 @@ message FileChunk {
bytes data = 2;
int64 sequence = 3;
bool eof = 4;
+ map<string, string> metadata = 5;
}
message StreamOpen {
@@ -148,9 +157,13 @@ message StreamEnd {
string request_id = 1;
}
-message HeartbeatPing {}
+message HeartbeatPing {
+ string id = 1;
+}
-message HeartbeatPong {}
+message HeartbeatPong {
+ string id = 1;
+}
message WebSocketStart {
string stream_id = 1;
@@ -181,4 +194,5 @@ message EventLog {
string user_id = 8;
string username = 9;
bytes metadata_json = 10;
+ string id = 11;
}
diff --git a/backend/pkg/libarcane/edge/proto/tunnel/v1/tunnel_grpc.pb.go b/backend/proto/tunnel/v1/tunnel_grpc.pb.go
similarity index 99%
rename from backend/pkg/libarcane/edge/proto/tunnel/v1/tunnel_grpc.pb.go
rename to backend/proto/tunnel/v1/tunnel_grpc.pb.go
index 373719642..eba05fd77 100644
--- a/backend/pkg/libarcane/edge/proto/tunnel/v1/tunnel_grpc.pb.go
+++ b/backend/proto/tunnel/v1/tunnel_grpc.pb.go
@@ -8,7 +8,6 @@ package tunnelpb
import (
context "context"
-
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"