Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
package edge
import (
"context"
"encoding/json/v2"
"errors"
"io"
"log/slog"
"net"
"sync/atomic"
"time"
"github.com/coder/websocket"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"
tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
)
// TunnelMessageType represents the type of message sent over the tunnel.
type TunnelMessageType string
const (
maxGRPCTunnelMessageSize = 16 * 1024 * 1024
// maxWebSocketTunnelMessageSize caps inbound websocket tunnel frames at 2x the
// gRPC limit, since JSON base64-encodes binary bodies a gRPC peer would accept.
maxWebSocketTunnelMessageSize = 2 * maxGRPCTunnelMessageSize
// MessageTypeRegister is the first message sent by the agent on gRPC transport.
MessageTypeRegister TunnelMessageType = "register"
// MessageTypeRegisterResponse is sent by manager after register validation.
MessageTypeRegisterResponse TunnelMessageType = "register_response"
// The remaining types are the legacy edge message protocol; remove in v3.
// MessageTypeRequest is sent from manager to agent to initiate a request.
MessageTypeRequest TunnelMessageType = "request"
// MessageTypeResponse is sent from agent to manager with the response.
MessageTypeResponse TunnelMessageType = "response"
// MessageTypeHeartbeat is sent by agents to keep the connection alive.
MessageTypeHeartbeat TunnelMessageType = "heartbeat"
// MessageTypeHeartbeatAck is sent by manager to acknowledge a heartbeat.
MessageTypeHeartbeatAck TunnelMessageType = "heartbeat_ack"
// MessageTypeStreamData is sent for streaming responses (logs, stats).
MessageTypeStreamData TunnelMessageType = "stream_data"
// MessageTypeStreamEnd indicates end of a stream.
MessageTypeStreamEnd TunnelMessageType = "stream_end"
// MessageTypeWebSocketStart starts a WebSocket stream for logs/stats.
MessageTypeWebSocketStart TunnelMessageType = "ws_start"
// MessageTypeWebSocketData is a WebSocket message in either direction.
MessageTypeWebSocketData TunnelMessageType = "ws_data"
// MessageTypeWebSocketClose closes a WebSocket stream.
MessageTypeWebSocketClose TunnelMessageType = "ws_close"
// MessageTypeEvent carries an event emitted by an agent to the manager.
MessageTypeEvent TunnelMessageType = "event"
// MessageTypeCommandRequest sends a typed edge command from manager to agent.
MessageTypeCommandRequest TunnelMessageType = "command_request"
// MessageTypeCommandAck acknowledges a command was accepted by the agent.
MessageTypeCommandAck TunnelMessageType = "command_ack"
// MessageTypeCommandOutput carries chunked command output from agent to manager.
MessageTypeCommandOutput TunnelMessageType = "command_output"
// MessageTypeCommandComplete indicates final command completion.
MessageTypeCommandComplete TunnelMessageType = "command_complete"
// MessageTypeFileChunk carries chunked request or response payload data.
MessageTypeFileChunk TunnelMessageType = "file_chunk"
// MessageTypeStreamOpen opens a command-backed stream.
MessageTypeStreamOpen TunnelMessageType = "stream_open"
// MessageTypeStreamClose closes a command-backed stream.
MessageTypeStreamClose TunnelMessageType = "stream_close"
// MessageTypeCancelRequest requests cancellation of an in-flight command.
MessageTypeCancelRequest TunnelMessageType = "cancel_request"
// MessageTypeCommandCredit returns flow-control credit for consumed command output.
MessageTypeCommandCredit TunnelMessageType = "command_credit"
)
var (
// ErrTunnelConnectionClosed is returned by Send and Receive on every transport once the connection is closed.
ErrTunnelConnectionClosed = errors.New("edge tunnel connection is closed")
// tunnelReadWait bounds an idle websocket read; heartbeats arrive every DefaultHeartbeatInterval.
// It is a variable so tests can shorten it.
tunnelReadWait = 3 * DefaultHeartbeatInterval
)
// TunnelConnection is the transport contract shared by WebSocket and gRPC wrappers.
type TunnelConnection interface {
Send(msg *TunnelMessage) error
Receive() (*TunnelMessage, error)
IsExpectedReceiveError(err error) bool
Close() error
IsClosed() bool
Transport() string
}
// NewTunnelConn creates a new WebSocket tunnel connection wrapper.
func NewTunnelConn(conn *websocket.Conn) *TunnelConn {
conn.SetReadLimit(maxWebSocketTunnelMessageSize)
return &TunnelConn{conn: conn}
}
// Send sends a legacy tunnel message over the WebSocket connection.
func (t *TunnelConn) Send(msg *TunnelMessage) error {
if t.muxed.Load() {
return errTunnelMuxActive
}
data, err := json.Marshal(msg)
if err != nil {
return err
}
//nolint:forbidigo // Tunnel connection owns timed I/O independent of individual HTTP requests.
return t.write(context.Background(), websocket.MessageText, data, DefaultWriteTimeout)
}
// Receive receives a legacy tunnel message from the WebSocket connection.
func (t *TunnelConn) Receive() (*TunnelMessage, error) {
_, data, err := t.read(context.Background()) //nolint:forbidigo // Tunnel connection owns timed I/O independent of individual HTTP requests.
if err != nil {
return nil, err
}
var msg TunnelMessage
if unmarshalErr := json.Unmarshal(data, &msg); unmarshalErr != nil {
return nil, unmarshalErr
}
return &msg, nil
}
// muxConn switches the connection to binary frames carrying the multiplexed session.
func (t *TunnelConn) muxConn(ctx context.Context) io.ReadWriteCloser {
t.muxed.Store(true)
recv := func() ([]byte, error) {
for {
messageType, data, err := t.read(ctx)
if err != nil || messageType == websocket.MessageBinary {
return data, err
}
}
}
send := func(p []byte) error { return t.write(ctx, websocket.MessageBinary, p, muxWriteTimeout) }
return newMuxConn(send, recv, t.Close)
}
// write sends one frame. coder/websocket closes the connection after a failed or expired write.
func (t *TunnelConn) write(ctx context.Context, messageType websocket.MessageType, data []byte, timeout time.Duration) error {
t.mu.Lock()
defer t.mu.Unlock()
if t.closed.Load() {
return ErrTunnelConnectionClosed
}
wctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
if err := t.conn.Write(wctx, messageType, data); err != nil {
t.closed.Store(true)
return err
}
return nil
}
// read receives one frame. Read-timeout expiry and network errors are terminal: coder/websocket
// closes the connection when a read context expires.
func (t *TunnelConn) read(ctx context.Context) (websocket.MessageType, []byte, error) {
if t.closed.Load() {
return 0, nil, ErrTunnelConnectionClosed
}
rctx, cancel := context.WithTimeout(ctx, tunnelReadWait)
defer cancel()
messageType, data, err := t.conn.Read(rctx)
if err != nil {
t.closed.Store(true)
}
return messageType, data, err
}
// IsExpectedReceiveError returns true for normal WebSocket close/teardown errors.
func (t *TunnelConn) IsExpectedReceiveError(err error) bool {
return isExpectedReceiveError(err)
}
// Close closes the WebSocket tunnel connection, sending a close frame on the
// first call so the peer observes a normal closure instead of 1006.
func (t *TunnelConn) Close() error {
if t.closed.Swap(true) {
return t.conn.CloseNow()
}
// Close performs the close handshake with its own internal timeout and is
// safe concurrently with a data writer, so a stuck Send cannot block teardown.
if err := t.conn.Close(websocket.StatusNormalClosure, ""); err != nil {
return t.conn.CloseNow()
}
return nil
}
// IsClosed returns whether the connection is closed.
func (t *TunnelConn) IsClosed() bool {
return t.closed.Load()
}
// Transport identifies the underlying tunnel transport.
func (t *TunnelConn) Transport() string {
return EdgeTransportWebSocket
}
type grpcManagerStream interface {
Send(msg *tunnelpb.ManagerMessage) error
Recv() (*tunnelpb.AgentMessage, error)
Context() context.Context
}
type grpcAgentStream interface {
Send(msg *tunnelpb.AgentMessage) error
Recv() (*tunnelpb.ManagerMessage, error)
Context() context.Context
CloseSend() error
}
// NewGRPCManagerTunnelConn creates a manager-side gRPC tunnel wrapper. A nil stream yields a connection
// whose Send and Receive report ErrTunnelConnectionClosed.
func NewGRPCManagerTunnelConn(stream grpcManagerStream) *GRPCManagerTunnelConn {
if stream == nil {
return &GRPCManagerTunnelConn{}
}
recvCtx, cancel := context.WithCancel(stream.Context())
return &GRPCManagerTunnelConn{
stream: &cancelableGRPCManagerStream{
stream: stream,
ctx: recvCtx,
recvCh: make(chan grpcRecvResult),
},
cancel: cancel,
}
}
// Send sends a legacy manager->agent tunnel message over gRPC.
func (t *GRPCManagerTunnelConn) Send(msg *TunnelMessage) error {
if t.muxed.Load() {
return errTunnelMuxActive
}
protoMsg, err := tunnelMessageToManagerProto(msg, t.parity)
if err != nil {
return err
}
return t.send(protoMsg)
}
// muxConn switches the stream to MuxData messages carrying the multiplexed session.
func (t *GRPCManagerTunnelConn) muxConn(context.Context) io.ReadWriteCloser {
t.muxed.Store(true)
send := func(p []byte) error {
return t.send(&tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_MuxData{MuxData: &tunnelpb.MuxData{Data: p}}})
}
recv := func() ([]byte, error) {
if t.stream == nil {
return nil, ErrTunnelConnectionClosed
}
return receiveMuxData(t.stream.Recv, &t.closed)
}
return newMuxConn(send, recv, t.Close)
}
func (t *GRPCManagerTunnelConn) send(msg *tunnelpb.ManagerMessage) error {
t.mu.Lock()
defer t.mu.Unlock()
if t.IsClosed() || t.stream == nil {
return ErrTunnelConnectionClosed
}
if err := t.stream.Send(msg); err != nil {
t.closed.Store(true)
return err
}
return nil
}
// Receive receives an agent->manager tunnel message from gRPC.
func (t *GRPCManagerTunnelConn) Receive() (*TunnelMessage, error) {
if t.stream == nil {
return nil, ErrTunnelConnectionClosed
}
return receiveGRPC(t.stream.Context(), t.stream.Recv, agentProtoToTunnelMessage, &t.closed)
}
// IsExpectedReceiveError returns true for expected gRPC stream shutdown errors.
func (t *GRPCManagerTunnelConn) IsExpectedReceiveError(err error) bool {
return isExpectedReceiveError(err)
}
// Close marks the stream closed on manager side.
func (t *GRPCManagerTunnelConn) Close() error {
t.closed.Store(true)
if t.cancel != nil {
t.cancel()
}
return nil
}
// IsClosed returns whether the stream is closed.
func (t *GRPCManagerTunnelConn) IsClosed() bool {
return t.closed.Load()
}
// Transport identifies the underlying tunnel transport.
func (t *GRPCManagerTunnelConn) Transport() string {
return EdgeTransportGRPC
}
// NewGRPCAgentTunnelConn creates an agent-side gRPC tunnel wrapper.
func NewGRPCAgentTunnelConn(stream grpcAgentStream, cancelFns ...context.CancelFunc) *GRPCAgentTunnelConn {
var cancel context.CancelFunc
if len(cancelFns) > 0 {
cancel = cancelFns[0]
}
return &GRPCAgentTunnelConn{stream: stream, cancel: cancel}
}
// Send sends a legacy agent->manager tunnel message over gRPC.
func (t *GRPCAgentTunnelConn) Send(msg *TunnelMessage) error {
if t.muxed.Load() {
return errTunnelMuxActive
}
protoMsg, err := tunnelMessageToAgentProto(msg)
if err != nil {
return err
}
return t.send(protoMsg)
}
// muxConn switches the stream to MuxData messages carrying the multiplexed session.
func (t *GRPCAgentTunnelConn) muxConn(context.Context) io.ReadWriteCloser {
t.muxed.Store(true)
send := func(p []byte) error {
return t.send(&tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_MuxData{MuxData: &tunnelpb.MuxData{Data: p}}})
}
recv := func() ([]byte, error) {
if t.stream == nil {
return nil, ErrTunnelConnectionClosed
}
return receiveMuxData(t.stream.Recv, &t.closed)
}
return newMuxConn(send, recv, t.Close)
}
func (t *GRPCAgentTunnelConn) send(msg *tunnelpb.AgentMessage) error {
t.mu.Lock()
defer t.mu.Unlock()
if t.IsClosed() || t.stream == nil {
return ErrTunnelConnectionClosed
}
if err := t.stream.Send(msg); err != nil {
t.closed.Store(true)
return err
}
return nil
}
// Receive receives a manager->agent tunnel message from gRPC.
func (t *GRPCAgentTunnelConn) Receive() (*TunnelMessage, error) {
if t.stream == nil {
return nil, ErrTunnelConnectionClosed
}
return receiveGRPC(t.stream.Context(), t.stream.Recv, managerProtoToTunnelMessage, &t.closed)
}
// IsExpectedReceiveError returns true for expected gRPC stream shutdown errors.
func (t *GRPCAgentTunnelConn) IsExpectedReceiveError(err error) bool {
return isExpectedReceiveError(err)
}
// Close half-closes the send stream before cancelling, so the manager sees a
// clean io.EOF instead of codes.Canceled.
func (t *GRPCAgentTunnelConn) Close() error {
if t.closed.Swap(true) {
return nil
}
// Preserve a clean half-close unless an in-flight send must be cancelled first.
locked := t.mu.TryLock()
if !locked {
if t.cancel != nil {
t.cancel()
}
t.mu.Lock()
}
defer t.mu.Unlock()
var err error
if t.stream != nil {
err = t.stream.CloseSend()
}
if locked && t.cancel != nil {
t.cancel()
}
return err
}
// IsClosed returns whether the stream is closed.
func (t *GRPCAgentTunnelConn) IsClosed() bool {
return t.closed.Load()
}
// Transport identifies the underlying tunnel transport.
func (t *GRPCAgentTunnelConn) Transport() string {
return EdgeTransportGRPC
}
func (s *cancelableGRPCManagerStream) Send(msg *tunnelpb.ManagerMessage) error {
return s.stream.Send(msg)
}
// Recv returns the next message, or ctx's error once ctx is done. gRPC Recv ignores ctx,
// so a single reader goroutine waits on the stream instead.
func (s *cancelableGRPCManagerStream) Recv() (*tunnelpb.AgentMessage, error) {
s.recvOnce.Do(func() {
go func() {
for {
msg, err := s.stream.Recv()
select {
case s.recvCh <- grpcRecvResult{msg: msg, err: err}:
case <-s.ctx.Done():
return
}
if err != nil {
return
}
}
}()
})
select {
case <-s.ctx.Done():
return nil, s.ctx.Err()
case result := <-s.recvCh:
return result.msg, result.err
}
}
func (s *cancelableGRPCManagerStream) Context() context.Context {
return s.ctx
}
// receiveGRPC decodes the next stream message, skipping payloads from newer peers
// that this build cannot decode, as unknown websocket message types are skipped.
func receiveGRPC[P any](ctx context.Context, recv func() (P, error), decode func(P) (*TunnelMessage, error), closed *atomic.Bool) (*TunnelMessage, error) {
for {
protoMsg, err := recv()
if err != nil {
// A gRPC stream is dead after any Recv error, not just io.EOF.
closed.Store(true)
return nil, err
}
msg, err := decode(protoMsg)
if errors.Is(err, errUnknownTunnelPayload) {
slog.DebugContext(ctx, "Ignoring unknown edge tunnel payload", "error", err)
continue
}
return msg, err
}
}
// receiveMuxData returns the next MuxData payload, skipping any legacy message still in flight.
// A gRPC stream is dead after any Recv error.
func receiveMuxData[P interface{ GetMuxData() *tunnelpb.MuxData }](recv func() (P, error), closed *atomic.Bool) ([]byte, error) {
for {
msg, err := recv()
if err != nil {
closed.Store(true)
return nil, err
}
if data := msg.GetMuxData(); data != nil {
return data.GetData(), nil
}
}
}
// isExpectedReceiveError reports normal teardown on either transport: EOF, cancellation,
// a closed connection, a clean websocket close, or a cancelled gRPC stream.
func isExpectedReceiveError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) ||
errors.Is(err, net.ErrClosed) || errors.Is(err, ErrTunnelConnectionClosed) || wshub.IsExpectedClose(err) {
return true
}
code := status.Code(err)
return code == codes.Canceled || code == codes.DeadlineExceeded
}