Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
package edge
import (
"bytes"
"cmp"
"context"
"errors"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"net/url"
"slices"
"strings"
"sync"
"time"
"uuid"
"github.com/cenkalti/backoff/v5"
"github.com/coder/websocket"
kit "go.getarcane.app/kit/pkg"
wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
)
const (
// maxReconnectInterval caps the exponential reconnect backoff so a long-dead
// manager is still retried at a sane cadence.
maxReconnectInterval = 60 * time.Second
// healthyTunnelSessionDuration is how long a tunnel session must survive
// before the reconnect backoff is treated as recovered.
healthyTunnelSessionDuration = 60 * time.Second
// maxPollRetryInterval caps the exponential backoff applied to failed poll
// control requests. Successful polls return to the manager-advertised interval.
maxPollRetryInterval = 60 * time.Second
tunnelSessionWorkerDrainTimeout = 10 * time.Second
// DefaultHeartbeatInterval is how often the client sends heartbeats
DefaultHeartbeatInterval = 30 * time.Second
// DefaultWriteTimeout is the timeout for write operations
DefaultWriteTimeout = 10 * time.Second
// DefaultRequestTimeout is the timeout for executing local requests
DefaultRequestTimeout = 5 * time.Minute
// DefaultGRPCRegistrationTimeout bounds how long the agent waits for the
// manager to acknowledge tunnel registration on any transport.
DefaultGRPCRegistrationTimeout = 10 * time.Second
// DefaultWebSocketPreferenceTTL keeps websocket as the preferred transport
// for a short period after a successful auto-mode fallback.
DefaultWebSocketPreferenceTTL = 2 * time.Minute
// maxWebSocketPreferenceTTL caps the backoff of the websocket preference window
// while gRPC keeps failing, so a broken gRPC path is retried but never disabled.
maxWebSocketPreferenceTTL = 30 * time.Minute
defaultCommandChunkSize = 256 * 1024
)
var (
// errTunnelRegistrationTimeout marks a manager that accepted the connection but never answered registration.
errTunnelRegistrationTimeout = errors.New("timed out waiting for tunnel registration response")
// errEstablishedTunnelSessionEnded marks an accepted session the manager later dropped.
errEstablishedTunnelSessionEnded = errors.New("established edge tunnel session ended")
)
// localDialSkipHeaders lists headers not forwarded when the agent dials its own server for a
// proxied WebSocket: handshake headers and browser headers that fail the local origin check.
//
// Deprecated: legacy edge message protocol; remove in v3.
var localDialSkipHeaders = map[string]bool{
"Sec-Websocket-Key": true,
"Sec-Websocket-Version": true,
"Sec-Websocket-Extensions": true,
"Upgrade": true,
"Connection": true,
"Host": true,
"Origin": true,
"Cookie": true,
"Authorization": true,
"Referer": true,
"Sec-Fetch-Dest": true,
"Sec-Fetch-Mode": true,
"Sec-Fetch-Site": true,
"Sec-Fetch-User": true,
"Sec-Ch-Ua": true,
"Sec-Ch-Ua-Mobile": true,
"Sec-Ch-Ua-Platform": true,
}
// Deprecated: legacy edge message protocol; remove in v3.
func (t *commandRequestTransfer) stop() {
t.timerMu.Lock()
defer t.timerMu.Unlock()
if t.timer != nil {
t.timer.Stop()
}
}
// NewTunnelClient creates a new tunnel client
func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {
reconnectInterval := time.Duration(cfg.EdgeReconnectInterval) * time.Second
if reconnectInterval < time.Second {
reconnectInterval = 5 * time.Second
}
return &TunnelClient{
cfg: cfg,
handler: handler,
reconnectInterval: reconnectInterval,
heartbeatInterval: DefaultHeartbeatInterval,
registrationTimeout: DefaultGRPCRegistrationTimeout,
websocketPreferenceTTL: DefaultWebSocketPreferenceTTL,
healthySessionDuration: healthyTunnelSessionDuration,
managerGRPCAddr: httpx.ManagerGRPCAddr(cfg.ManagerApiUrl),
localPort: cmp.Or(cfg.Port, "3552"),
requestTimeout: DefaultRequestTimeout,
agentInstanceID: uuid.New().String(),
}
}
// StartWithErrorChan runs the tunnel client and optionally emits connection errors.
func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error) {
if errCh != nil {
defer close(errCh)
}
if configured := strings.ToLower(strings.TrimSpace(c.cfg.EdgeTransport)); configured != "" && NormalizeEdgeTransport(configured) != configured {
slog.WarnContext(ctx, "Unknown EDGE_TRANSPORT value; using the default transport", "configured", c.cfg.EdgeTransport, "transport", NormalizeEdgeTransport(configured))
}
if parsed, err := url.Parse(httpx.ManagerBaseURL(c.cfg.ManagerApiUrl)); err == nil && strings.Trim(parsed.Path, "/") != "" && c.managerGRPCAddr != "" {
slog.WarnContext(ctx, "MANAGER_API_URL has a path prefix that gRPC cannot route; gRPC edge tunnel transport is unavailable", "pathPrefix", parsed.Path)
c.managerGRPCAddr = ""
}
// Log only the endpoints this agent will dial.
poll := UsePollEdgeTransport(c.cfg)
attrs := []any{
"controlPlane", kit.Ternary(poll, EdgeTransportPoll, "managed"),
"securityMode", NormalizeEdgeMTLSMode(c.cfg.EdgeMTLSMode),
"muxEnabled", !c.cfg.EdgeMuxDisabled,
}
if poll {
attrs = append(attrs, "pollUrl", strings.TrimRight(httpx.ManagerBaseURL(c.cfg.ManagerApiUrl), "/")+tunnelPollPath)
}
transports := c.managedTunnelTransports()
if transports.grpc {
attrs = append(attrs, "grpcAddr", c.managerGRPCAddr)
}
if transports.websocket {
attrs = append(attrs, "websocketUrl", c.managerWebSocketURL())
}
slog.InfoContext(ctx, "Starting edge agent session client", attrs...)
// Back off exponentially (with jitter) so a broken or unauthorized agent does not hammer the
// manager, resetting only once a session has stayed up long enough to count as healthy.
reconnectBackoff := backoff.NewExponentialBackOff()
reconnectBackoff.InitialInterval = c.reconnectInterval
reconnectBackoff.MaxInterval = maxReconnectInterval
serve := func(ctx context.Context) error {
if err := EnsureAgentMTLSAssets(ctx, c.cfg); err != nil {
return err
}
if err := ValidateAgentMTLSConfig(c.cfg); err != nil {
return err
}
return kit.Ternary(poll, c.connectAndServePoll, c.connectAndServeManagedTunnel)(ctx)
}
for {
if ctx.Err() != nil {
slog.InfoContext(ctx, "Edge tunnel client shutting down")
return
}
sessionStart := time.Now()
if err := serve(ctx); err != nil {
slog.WarnContext(ctx, "Edge tunnel disconnected", "error", err)
if errCh != nil {
select {
case errCh <- err:
default:
}
}
}
if time.Since(sessionStart) >= c.healthySessionDuration {
reconnectBackoff.Reset()
}
delay := reconnectBackoff.NextBackOff()
select {
case <-ctx.Done():
return
case <-time.After(delay):
slog.InfoContext(ctx, "Attempting to reconnect edge tunnel", "delay", delay)
}
}
}
func (c *TunnelClient) connectAndServeManagedTunnel(ctx context.Context) error {
transports := c.managedTunnelTransports()
if transports.grpc {
c.transportPreferenceMu.RLock()
preferredUntil, grpcFailureStreak := c.preferWebSocketUntil, c.grpcFailureStreak
c.transportPreferenceMu.RUnlock()
if transports.websocket && time.Now().Before(preferredUntil) {
slog.InfoContext(ctx, "Temporarily preferring websocket edge tunnel transport after recent websocket success",
"preferredUntil", preferredUntil,
"grpcFailureStreak", grpcFailureStreak,
"managerWsUrl", c.managerWebSocketURL(),
)
return c.connectAndServeWebSocket(ctx)
}
sessionStart := time.Now()
if err := c.connectAndServeGRPC(ctx); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
// A healthy session the manager dropped (restart, redeploy) is not a broken gRPC path:
// let the outer loop back off and retry gRPC instead of racing a websocket dial.
if errors.Is(err, errEstablishedTunnelSessionEnded) && time.Since(sessionStart) >= c.healthySessionDuration {
return err
}
c.transportPreferenceMu.Lock()
c.grpcFailureStreak++
c.transportPreferenceMu.Unlock()
if transports.websocket {
slog.WarnContext(ctx, "gRPC edge tunnel connection failed, falling back to websocket transport",
"error", err,
"managerGrpcAddr", c.managerGRPCAddr,
"managerWsUrl", c.managerWebSocketURL(),
)
if wsErr := c.connectAndServeWebSocket(ctx); wsErr != nil {
// Keep both transport failures in the chain for errors.Is/errors.As traversal.
return fmt.Errorf("gRPC edge tunnel failed: %w; websocket fallback failed: %w", err, wsErr)
}
return nil
}
return err
}
return nil
}
if transports.websocket {
return c.connectAndServeWebSocket(ctx)
}
return errors.New("no edge tunnel transport is available")
}
func (c *TunnelClient) managedTunnelTransports() tunnelTransports {
if c == nil || c.cfg == nil {
return tunnelTransports{}
}
managerGRPCAvailable := strings.TrimSpace(c.managerGRPCAddr) != ""
managerWebSocketAvailable := c.managerWebSocketURL() != ""
switch NormalizeEdgeTransport(c.cfg.EdgeTransport) {
case EdgeTransportAuto, EdgeTransportPoll:
return tunnelTransports{grpc: managerGRPCAvailable, websocket: managerWebSocketAvailable}
case EdgeTransportWebSocket:
return tunnelTransports{websocket: managerWebSocketAvailable}
case EdgeTransportGRPC:
return tunnelTransports{grpc: managerGRPCAvailable}
default:
return tunnelTransports{}
}
}
func (c *TunnelClient) managerWebSocketURL() string {
if managerURL := strings.TrimSpace(c.managerURL); managerURL != "" {
return managerURL
}
if c.cfg == nil {
return ""
}
managerBaseURL := strings.TrimRight(strings.TrimSpace(httpx.ManagerBaseURL(c.cfg.ManagerApiUrl)), "/")
if managerBaseURL == "" {
return ""
}
return HTTPToWebSocketURL(managerBaseURL) + tunnelConnectPath
}
func (c *TunnelClient) awaitRegistration(ctx context.Context, conn TunnelConnection) (*TunnelMessage, error) {
type registrationResult struct {
msg *TunnelMessage
err error
}
recvCh := make(chan registrationResult, 1)
go func() {
msg, err := conn.Receive()
recvCh <- registrationResult{msg: msg, err: err}
}()
timer := time.NewTimer(c.registrationTimeout)
defer timer.Stop()
select {
case <-ctx.Done():
_ = conn.Close()
return nil, ctx.Err()
case <-timer.C:
_ = conn.Close()
return nil, fmt.Errorf("after %s: %w", c.registrationTimeout, errTunnelRegistrationTimeout)
case result := <-recvCh:
if result.err != nil {
return nil, fmt.Errorf("failed to receive tunnel registration response: %w", result.err)
}
if result.msg == nil {
return nil, errors.New("received empty tunnel registration response")
}
if result.msg.Type != MessageTypeRegisterResponse {
return nil, fmt.Errorf("unexpected first tunnel message: %s", result.msg.Type)
}
if !result.msg.Accepted {
return nil, fmt.Errorf("manager rejected tunnel registration: %s", result.msg.Error)
}
c.registration.Store(clientRegistration{
sessionID: result.msg.SessionID,
commandCredit: slices.Contains(result.msg.Capabilities, tunnelCapabilityCommandCreditGrant),
})
return result.msg, nil
}
}
// serveTunnelSession runs the shared register/heartbeat/message lifecycle on an established
// tunnel connection, for every transport.
func (c *TunnelClient) serveTunnelSession(ctx context.Context, conn TunnelConnection, managerAddr string) error {
box := &connBox{conn: conn}
c.conn.Store(box)
connCtx, connCancel := context.WithCancel(ctx)
var workers sync.WaitGroup
stopCloseWatch := context.AfterFunc(connCtx, func() { _ = conn.Close() })
defer func() {
connCancel()
// Tear down streams and pending transfers so a reconnect cannot leak goroutines or sockets.
c.activeStreams.Range(func(key, value any) bool {
streamID, idOK := key.(string)
stream, streamOK := value.(*activeWSStream)
if idOK && streamOK {
c.closeWebSocketStream(streamID, stream)
}
return true
})
c.requestTransfers.Range(func(transferID, value any) bool {
switch transfer, ok := value.(*commandRequestTransfer); {
case !ok:
c.requestTransfers.Delete(transferID)
case transfer.conn == conn && c.requestTransfers.CompareAndDelete(transferID, transfer):
transfer.stop()
}
return true
})
_ = conn.Close()
stopCloseWatch()
drainCtx, cancelDrain := context.WithTimeout(context.WithoutCancel(ctx), tunnelSessionWorkerDrainTimeout)
if err := utils.WaitGroup(drainCtx, &workers); err != nil {
slog.WarnContext(context.WithoutCancel(ctx), "Timed out waiting for edge tunnel session workers", "error", err)
}
cancelDrain()
c.conn.CompareAndSwap(box, nil)
}()
registration, _ := c.registration.Load()
capabilities := append(AdvertisedEdgeCommands(), tunnelCapabilityChunkedRequest, tunnelCapabilityProtoParity, tunnelCapabilityCommandCredit)
if !c.cfg.EdgeMuxDisabled {
capabilities = append(capabilities, tunnelCapabilityMux)
}
registerMsg := &TunnelMessage{
Type: MessageTypeRegister,
AgentToken: c.cfg.AgentToken,
AgentInstance: c.agentInstanceID,
Capabilities: capabilities,
ResumeSession: registration.sessionID,
}
if err := conn.Send(registerMsg); err != nil {
// A rejected gRPC stream can report EOF from Send; Recv carries the RPC status.
if conn.Transport() != EdgeTransportGRPC || !errors.Is(err, io.EOF) {
return fmt.Errorf("failed to send %s tunnel register message: %w", conn.Transport(), err)
}
}
registerResp, err := c.awaitRegistration(ctx, conn)
if err != nil {
return err
}
transport, useMux := conn.(muxTransport)
useMux = useMux && slices.Contains(registerResp.Capabilities, tunnelCapabilityMuxGrant)
slog.InfoContext(ctx, "Edge tunnel connected to manager",
"transport", conn.Transport(),
"dataPlane", kit.Ternary(useMux, EdgeDataPlaneMux, EdgeDataPlaneLegacy),
"managerAddr", managerAddr,
"environmentId", registerResp.EnvironmentID,
"sessionId", registerResp.SessionID,
)
c.transportPreferenceMu.Lock()
switch conn.Transport() {
case EdgeTransportGRPC:
c.preferWebSocketUntil, c.grpcFailureStreak = time.Time{}, 0
case EdgeTransportWebSocket:
if transports := c.managedTunnelTransports(); transports.grpc && transports.websocket {
// Back off the preference window while gRPC keeps failing so reconnects do not re-pay its timeout.
ttl := min(c.websocketPreferenceTTL<<min(c.grpcFailureStreak, 4), maxWebSocketPreferenceTTL)
c.preferWebSocketUntil = time.Now().Add(ttl)
}
}
c.transportPreferenceMu.Unlock()
if useMux {
return fmt.Errorf("%w: %w", errEstablishedTunnelSessionEnded, c.serveMuxSession(connCtx, transport.muxConn(connCtx)))
}
// Deprecated: legacy edge message protocol; remove in v3.
defer setActiveAgentEvents(&agentEventSink{conn: conn})()
workers.Go(func() { c.heartbeatLoop(connCtx, conn) })
return fmt.Errorf("%w: %w", errEstablishedTunnelSessionEnded, c.messageLoop(connCtx, conn, &workers))
}
// heartbeatLoop sends periodic heartbeats.
//
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) heartbeatLoop(ctx context.Context, conn TunnelConnection) {
ticker := time.NewTicker(c.heartbeatInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if conn == nil || conn.IsClosed() {
return
}
if err := conn.Send(&TunnelMessage{ID: uuid.New().String(), Type: MessageTypeHeartbeat}); err != nil {
slog.WarnContext(ctx, "Failed to send heartbeat", "error", err)
// Force reconnect so the manager does not keep stale state without heartbeats.
if closeErr := conn.Close(); closeErr != nil {
slog.DebugContext(ctx, "Failed to close tunnel connection after heartbeat failure", "error", closeErr)
}
return
}
slog.DebugContext(ctx, "Sent heartbeat to manager")
}
}
}
// messageLoop processes incoming messages from the manager.
//
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) messageLoop(ctx context.Context, conn TunnelConnection, workers *sync.WaitGroup) error {
for {
if err := ctx.Err(); err != nil {
return err
}
msg, err := conn.Receive()
if err != nil {
return fmt.Errorf("failed to receive message: %w", err)
}
switch msg.Type {
case MessageTypeRequest:
workers.Go(func() { c.handleRequest(ctx, conn, msg) })
case MessageTypeCommandRequest:
if transferID := strings.TrimSpace(msg.Metadata[bodyTransferMetadataKey]); transferID != "" {
c.beginCommandRequestTransfer(ctx, conn, transferID, msg)
} else {
c.startCommandRequest(ctx, conn, msg, workers)
}
case MessageTypeFileChunk:
c.handleCommandRequestChunk(ctx, msg, workers)
case MessageTypeWebSocketStart, MessageTypeStreamOpen:
c.handleWebSocketStart(ctx, conn, msg, workers)
case MessageTypeWebSocketData, MessageTypeStreamData:
c.handleStreamData(ctx, msg)
case MessageTypeWebSocketClose, MessageTypeStreamClose:
if value, ok := c.activeStreams.Load(msg.ID); ok {
if stream, isStream := value.(*activeWSStream); isStream {
c.closeWebSocketStream(msg.ID, stream)
slog.DebugContext(ctx, "Closed WebSocket stream", "streamId", msg.ID)
}
}
case MessageTypeCancelRequest:
if value, ok := c.commandCancels.LoadAndDelete(msg.ID); ok {
if cancel, isCancel := value.(context.CancelFunc); isCancel {
cancel()
slog.DebugContext(ctx, "Cancelled edge command on manager request", "id", msg.ID)
}
}
case MessageTypeCommandCredit:
if value, ok := c.commandRecorders.Load(msg.ID); ok {
if recorder, isRecorder := value.(*commandResponseRecorder); isRecorder {
recorder.inFlight.Add(-msg.Credit)
select {
case recorder.credited <- struct{}{}:
default:
}
}
}
case MessageTypeResponse, MessageTypeHeartbeat, MessageTypeStreamEnd, MessageTypeEvent, MessageTypeCommandAck, MessageTypeCommandOutput, MessageTypeCommandComplete:
slog.DebugContext(ctx, "Ignoring message type on agent", "type", msg.Type)
case MessageTypeHeartbeatAck:
slog.DebugContext(ctx, "Received heartbeat ack")
case MessageTypeRegisterResponse:
if !msg.Accepted {
return fmt.Errorf("manager rejected tunnel registration: %s", msg.Error)
}
slog.InfoContext(ctx, "Edge tunnel re-registered",
"transport", conn.Transport(),
"environmentId", msg.EnvironmentID,
)
case MessageTypeRegister:
slog.DebugContext(ctx, "Ignoring register message on agent")
default:
slog.WarnContext(ctx, "Unknown message type", "type", msg.Type)
}
}
}
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) beginCommandRequestTransfer(ctx context.Context, conn TunnelConnection, transferID string, msg *TunnelMessage) {
if c == nil || conn == nil || msg == nil {
return
}
transfer := &commandRequestTransfer{request: msg, conn: conn}
transfer.timerMu.Lock()
previous, loaded := c.requestTransfers.Swap(transferID, transfer)
transfer.timer = time.AfterFunc(c.requestTimeout, func() {
if c.requestTransfers.CompareAndDelete(transferID, transfer) {
slog.WarnContext(ctx, "Command body transfer expired", "transferId", transferID, "commandId", msg.ID)
c.sendCommandComplete(conn, msg.ID, http.StatusRequestTimeout, "command body transfer timed out")
}
})
transfer.timerMu.Unlock()
if loaded {
if previous, ok := previous.(*commandRequestTransfer); ok {
previous.stop()
c.sendCommandComplete(previous.conn, previous.request.ID, http.StatusBadRequest, "duplicate command body transfer ID")
}
}
}
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) handleCommandRequestChunk(ctx context.Context, msg *TunnelMessage, workers *sync.WaitGroup) {
if c == nil || msg == nil {
return
}
value, ok := c.requestTransfers.Load(msg.ID)
if !ok {
slog.WarnContext(ctx, "Received command body chunk for unknown transfer", "transferId", msg.ID)
return
}
transfer, ok := value.(*commandRequestTransfer)
if !ok {
c.requestTransfers.Delete(msg.ID)
slog.WarnContext(ctx, "Discarded invalid command body transfer state", "transferId", msg.ID)
return
}
if msg.Sequence != transfer.nextSequence {
if c.requestTransfers.CompareAndDelete(msg.ID, transfer) {
transfer.stop()
c.sendCommandComplete(transfer.conn, transfer.request.ID, http.StatusBadRequest, "command body chunks arrived out of order")
}
return
}
if transfer.body.Len()+len(msg.Body) > maxLegacyCommandBodyBytes {
if c.requestTransfers.CompareAndDelete(msg.ID, transfer) {
transfer.stop()
c.sendCommandComplete(transfer.conn, transfer.request.ID, http.StatusRequestEntityTooLarge, "command body is too large for a legacy edge tunnel; upgrade the manager and agent")
}
return
}
_, _ = transfer.body.Write(msg.Body)
transfer.nextSequence++
if !msg.EOF {
return
}
if c.requestTransfers.CompareAndDelete(msg.ID, transfer) {
transfer.stop()
transfer.request.Body = transfer.body.Bytes()
c.startCommandRequest(ctx, transfer.conn, transfer.request, workers)
}
}
func (c *TunnelClient) startCommandRequest(ctx context.Context, conn TunnelConnection, msg *TunnelMessage, workers *sync.WaitGroup) {
commandCtx, cancel := context.WithCancel(ctx)
c.commandCancels.Store(msg.ID, cancel)
workers.Go(func() {
defer c.commandCancels.Delete(msg.ID)
defer cancel()
c.handleCommandRequest(commandCtx, conn, msg)
})
}
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {
if !ValidateEdgeCommand(msg.Command, msg.Method, msg.Path, false) {
c.sendCommandComplete(conn, msg.ID, http.StatusBadRequest, "unsupported edge command")
return
}
if conn == nil {
return
}
if err := conn.Send(&TunnelMessage{ID: msg.ID, Type: MessageTypeCommandAck, Command: msg.Command}); err != nil {
slog.WarnContext(ctx, "Failed to acknowledge edge command", "id", msg.ID, "command", msg.Command, "error", err)
return
}
reqCtx, cancel := context.WithTimeout(ctx, kit.Ternary(msg.TimeoutMillis > 0, min(time.Duration(msg.TimeoutMillis)*time.Millisecond, c.requestTimeout), c.requestTimeout))
defer cancel()
req, err := c.buildLocalHTTPRequest(reqCtx, msg)
if err != nil {
c.sendCommandComplete(conn, msg.ID, http.StatusInternalServerError, fmt.Sprintf("failed to create request: %v", err))
return
}
recorder := &commandResponseRecorder{
commandID: msg.ID,
commandName: msg.Command,
conn: conn,
headers: make(http.Header),
statusCode: http.StatusOK,
ctx: reqCtx,
credited: make(chan struct{}, 1),
}
if registration, _ := c.registration.Load(); registration.commandCredit {
recorder.window = commandCreditWindow
c.commandRecorders.Store(msg.ID, recorder)
defer c.commandRecorders.Delete(msg.ID)
}
c.handler.ServeHTTP(recorder, req)
if closeErr := recorder.Close(); closeErr != nil {
slog.WarnContext(reqCtx, "Failed to finalize command response", "id", msg.ID, "command", msg.Command, "error", closeErr)
}
}
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) buildLocalHTTPRequest(ctx context.Context, msg *TunnelMessage) (*http.Request, error) {
var body io.Reader
if len(msg.Body) > 0 {
body = bytes.NewReader(append([]byte(nil), msg.Body...))
}
path := msg.Path + kit.Ternary(msg.Query != "", "?"+msg.Query, "")
req, err := http.NewRequestWithContext(ctx, msg.Method, path, body)
if err != nil {
return nil, err
}
// net/http ignores Header.Set("Host", ...); the field must be set explicitly.
for k, v := range msg.Headers {
if http.CanonicalHeaderKey(k) == "Host" {
req.Host = v
continue
}
req.Header.Set(k, v)
}
return req, nil
}
// agentAuthCredentials returns the canonical header/metadata credential
// set every agent transport presents, so the three transports cannot drift.
func agentAuthCredentials(token string) map[string]string {
return map[string]string{
HeaderAgentToken: token,
HeaderAPIKey: token,
HeaderAuthorization: "Bearer " + token,
}
}
// handleRequest serves a tunneled request locally, streaming the response on gRPC connections.
//
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) handleRequest(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {
reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeout)
defer cancel()
reqCtx = context.WithValue(reqCtx, internalTunnelRequestContextKey{}, true)
streaming := conn.Transport() == EdgeTransportGRPC
slog.DebugContext(reqCtx, "Processing tunneled request", "id", msg.ID, "method", msg.Method, "path", msg.Path, "bodyLength", len(msg.Body), "streaming", streaming)
req, err := c.buildLocalHTTPRequest(reqCtx, msg)
if err != nil {
_ = conn.Send(&TunnelMessage{
ID: msg.ID,
Type: MessageTypeResponse,
Status: http.StatusInternalServerError,
Body: fmt.Appendf(nil, "failed to create request: %v", err),
})
return
}
if streaming {
recorder := &streamingResponseRecorder{requestID: msg.ID, conn: conn, headers: make(http.Header), statusCode: http.StatusOK}
c.handler.ServeHTTP(recorder, req)
if closeErr := recorder.Close(); closeErr != nil {
slog.WarnContext(reqCtx, "Failed to finalize streamed response", "id", msg.ID, "error", closeErr)
}
return
}
rw := &responseRecorder{headers: make(http.Header), statusCode: http.StatusOK}
c.handler.ServeHTTP(rw, req)
resp := &TunnelMessage{
ID: msg.ID,
Type: MessageTypeResponse,
Status: rw.statusCode,
Headers: flattenResponseHeaders(rw.headers),
Body: rw.body.Bytes(),
}
if sendErr := conn.Send(resp); sendErr != nil {
slog.ErrorContext(reqCtx, "Failed to send response", "id", msg.ID, "error", sendErr)
} else {
slog.DebugContext(reqCtx, "Sent tunneled response", "id", msg.ID, "status", rw.statusCode)
}
}
// handleWebSocketStart proxies a manager WebSocket or stream-open request to the local server.
//
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) handleWebSocketStart(ctx context.Context, conn TunnelConnection, msg *TunnelMessage, workers *sync.WaitGroup) {
streamID := msg.ID
if msg.Type == MessageTypeStreamOpen && !ValidateEdgeCommand(msg.Command, http.MethodGet, msg.Path, true) {
c.sendStreamCloseMessage(conn, streamID, "unsupported edge stream")
return
}
slog.DebugContext(ctx, "Starting WebSocket stream", "streamId", streamID, "path", msg.Path)
localURL := c.buildLocalWebSocketURL(msg)
dialCtx, cancelDial := context.WithTimeout(ctx, 30*time.Second)
defer cancelDial()
ws, resp, err := websocket.Dial(dialCtx, localURL, &websocket.DialOptions{HTTPHeader: c.buildLocalWebSocketHeaders(msg)})
// coder/websocket leaves resp.Body nil on a successful handshake.
if resp != nil && resp.Body != nil {
defer func() { _ = resp.Body.Close() }()
}
if err != nil {
attrs := []any{"error", err, "url", localURL}
if resp != nil {
attrs = append(attrs, "status", resp.StatusCode)
if resp.Body != nil {
buf := make([]byte, 512)
if n, _ := resp.Body.Read(buf); n > 0 {
attrs = append(attrs, "response_body", string(buf[:n]))
}
}
}
slog.ErrorContext(ctx, "Failed to dial local WebSocket", attrs...)
c.sendStreamCloseMessage(conn, streamID, "")
return
}
// Local frames are forwarded into tunnel messages, so allow that size instead of the 32KB default.
ws.SetReadLimit(maxGRPCTunnelMessageSize)
streamCtx, cancel := context.WithCancel(ctx)
stream := &activeWSStream{ws: ws, conn: conn, cancel: cancel, dataCh: make(chan wsPayload, 100)}
c.activeStreams.Store(streamID, stream)
workers.Go(func() {
defer c.closeWebSocketStream(streamID, stream)
for {
msgType, data, readErr := ws.Read(streamCtx)
if readErr != nil {
if !wshub.IsExpectedClose(readErr) {
slog.DebugContext(ctx, "Local WebSocket read error", "error", readErr)
}
c.sendStreamCloseMessage(conn, streamID, "")
return
}
if sendErr := conn.Send(&TunnelMessage{ID: streamID, Type: MessageTypeWebSocketData, Body: data, WSMessageType: int(msgType)}); sendErr != nil {
slog.DebugContext(ctx, "Failed to send WebSocket data to manager", "error", sendErr)
return
}
}
})
workers.Go(func() { c.startLocalWebSocketWriteLoop(ctx, streamCtx, ws, stream, cancel) })
}
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) buildLocalWebSocketURL(msg *TunnelMessage) string {
// LISTEN may be a host, host:port, IPv6, or bracketed IPv6 with port.
host := strings.TrimSpace(c.cfg.Listen)
if splitHost, _, err := net.SplitHostPort(host); err == nil {
host = splitHost
}
if host = strings.Trim(host, "[]"); slices.Contains([]string{"", "0.0.0.0", "::"}, host) {
host = "localhost"
}
return "ws://" + net.JoinHostPort(host, c.localPort) + msg.Path + kit.Ternary(msg.Query != "", "?"+msg.Query, "")
}
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) buildLocalWebSocketHeaders(msg *TunnelMessage) http.Header {
headers := http.Header{}
for k, v := range msg.Headers {
if canonicalKey := http.CanonicalHeaderKey(k); !localDialSkipHeaders[canonicalKey] {
headers.Set(canonicalKey, v)
}
}
remenv.ApplyAgentTokenHeaders(headers, &c.cfg.AgentToken)
return headers
}
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) closeWebSocketStream(streamID string, stream *activeWSStream) {
stream.mu.Lock()
if stream.closed {
stream.mu.Unlock()
return
}
stream.closed = true
close(stream.dataCh)
stream.mu.Unlock()
stream.cancel()
_ = stream.ws.CloseNow()
c.activeStreams.Delete(streamID)
}
// startLocalWebSocketWriteLoop writes manager frames to the local WebSocket; kept apart to bound handleWebSocketStart's complexity.
//
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) startLocalWebSocketWriteLoop(ctx, streamCtx context.Context, ws *websocket.Conn, stream *activeWSStream, cancel context.CancelFunc) {
for {
select {
case <-streamCtx.Done():
return
case payload, ok := <-stream.dataCh:
if !ok {
return
}
if !isForwardableWSMessage(payload.messageType) {
slog.WarnContext(ctx, "Dropping WebSocket message with unsupported type", "messageType", payload.messageType)
continue
}
if err := ws.Write(streamCtx, websocket.MessageType(payload.messageType), payload.data); err != nil {
slog.DebugContext(ctx, "Failed to write to local WebSocket", "error", err)
cancel()
return
}
}
}
}
// handleStreamData queues manager frames for a local WebSocket, dropping them when its buffer is full; kept apart to
// bound messageLoop's complexity.
//
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) handleStreamData(ctx context.Context, msg *TunnelMessage) {
value, ok := c.activeStreams.Load(msg.ID)
stream, isStream := value.(*activeWSStream)
if !ok || !isStream {
slog.DebugContext(ctx, "Received WebSocket data for unknown stream", "streamId", msg.ID)
return
}
stream.mu.Lock()
defer stream.mu.Unlock()
if stream.closed {
return
}
select {
case stream.dataCh <- wsPayload{messageType: msg.WSMessageType, data: msg.Body}:
default:
slog.DebugContext(ctx, "Dropping WebSocket data due to backpressure", "streamId", msg.ID)
}
}
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) sendStreamCloseMessage(conn TunnelConnection, streamID, message string) {
if conn == nil {
return
}
_ = conn.Send(&TunnelMessage{ID: streamID, Type: MessageTypeStreamClose, Error: message})
}
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) sendCommandComplete(conn TunnelConnection, commandID string, status int, message string) {
if conn == nil {
return
}
_ = conn.Send(&TunnelMessage{
ID: commandID,
Type: MessageTypeCommandComplete,
Status: status,
Error: message,
})
}
// Deprecated: legacy edge message protocol; remove in v3.
func (r *commandResponseRecorder) Header() http.Header {
return r.headers
}
func (r *commandResponseRecorder) Write(b []byte) (int, error) {
r.mu.Lock()
defer r.mu.Unlock()
if r.flushErr != nil {
return 0, r.flushErr
}
if len(b) == 0 {
return 0, nil
}
if !r.streaming && r.buffer.Len()+len(b) <= defaultCommandChunkSize {
return r.buffer.Write(b)
}
r.streaming = true
if err := r.flushBufferLocked(); err != nil {
return 0, err
}
for chunk := range slices.Chunk(b, defaultCommandChunkSize) {
if err := r.sendOutputLocked(chunk); err != nil {
return 0, err
}
}
return len(b), nil
}
func (r *commandResponseRecorder) WriteHeader(statusCode int) {
r.mu.Lock()
defer r.mu.Unlock()
r.statusCode = statusCode
r.wroteHeader = true
}
func (r *commandResponseRecorder) Flush() {
r.mu.Lock()
defer r.mu.Unlock()
r.streaming = true
if r.flushErr != nil {
return
}
// An empty first flush still sends the status and headers so the manager can start the response.
if r.sequence == 0 && r.buffer.Len() == 0 {
r.flushErr = r.sendOutputLocked(nil)
return
}
r.flushErr = r.flushBufferLocked()
}
func (r *commandResponseRecorder) Close() error {
r.mu.Lock()
defer r.mu.Unlock()
if r.closed {
return nil
}
if r.flushErr != nil {
return r.flushErr
}
complete := &TunnelMessage{
ID: r.commandID,
Type: MessageTypeCommandComplete,
Status: r.statusCode,
Headers: flattenResponseHeaders(r.headers),
Streaming: r.streaming,
Command: r.commandName,
}
if r.streaming {
if err := r.flushBufferLocked(); err != nil {
return err
}
} else {
complete.Body = append([]byte(nil), r.buffer.Bytes()...)
}
if err := r.conn.Send(complete); err != nil {
return err
}
r.closed = true
return nil
}
func (r *commandResponseRecorder) flushBufferLocked() error {
if r.buffer.Len() == 0 {
return nil
}
if err := r.sendOutputLocked(r.buffer.Bytes()); err != nil {
return err
}
r.buffer.Reset()
return nil
}
// sendOutputLocked sends one output chunk, first waiting for manager credit when the window is
// full. The first chunk also carries the status and headers so the manager can stream the response.
func (r *commandResponseRecorder) sendOutputLocked(chunk []byte) error {
for r.window > 0 && r.inFlight.Load() > 0 && r.inFlight.Load()+int64(len(chunk)) > r.window {
select {
case <-r.credited:
case <-r.ctx.Done():
return r.ctx.Err()
}
}
r.inFlight.Add(int64(len(chunk)))
msg := &TunnelMessage{
ID: r.commandID,
Type: MessageTypeCommandOutput,
Body: append([]byte(nil), chunk...),
Sequence: r.sequence,
Command: r.commandName,
}
if r.sequence == 0 {
msg.Status = r.statusCode
msg.Headers = flattenResponseHeaders(r.headers)
}
if err := r.conn.Send(msg); err != nil {
return err
}
r.sequence++
return nil
}
func flattenResponseHeaders(headers http.Header) map[string]string {
out := make(map[string]string, len(headers))
for k, vs := range headers {
if len(vs) > 0 {
out[k] = vs[0]
}
}
return out
}
// Deprecated: legacy edge message protocol; remove in v3.
func (r *responseRecorder) Header() http.Header {
return r.headers
}
func (r *responseRecorder) Write(b []byte) (int, error) {
return r.body.Write(b)
}
func (r *responseRecorder) WriteHeader(statusCode int) {
r.statusCode = statusCode
}
// Deprecated: legacy edge message protocol; remove in v3.
func (r *streamingResponseRecorder) Header() http.Header {
return r.headers
}
func (r *streamingResponseRecorder) Write(b []byte) (int, error) {
r.mu.Lock()
defer r.mu.Unlock()
if err := r.writeHeaderLocked(); err != nil {
return 0, err
}
for chunk := range slices.Chunk(b, defaultCommandChunkSize) {
if err := r.conn.Send(&TunnelMessage{
ID: r.requestID,
Type: MessageTypeStreamData,
Body: append([]byte(nil), chunk...),
}); err != nil {
return 0, err
}
}
return len(b), nil
}
func (r *streamingResponseRecorder) WriteHeader(statusCode int) {
r.mu.Lock()
defer r.mu.Unlock()
r.statusCode = statusCode
_ = r.writeHeaderLocked()
}
func (r *streamingResponseRecorder) Flush() {
r.mu.Lock()
defer r.mu.Unlock()
_ = r.writeHeaderLocked()
}
func (r *streamingResponseRecorder) Close() error {
r.mu.Lock()
defer r.mu.Unlock()
if r.closed {
return nil
}
if err := r.writeHeaderLocked(); err != nil {
return err
}
if err := r.conn.Send(&TunnelMessage{ID: r.requestID, Type: MessageTypeStreamEnd}); err != nil {
return err
}
r.closed = true
return nil
}
// writeHeaderLocked sends the response status and headers once, marked as a streamed response.
func (r *streamingResponseRecorder) writeHeaderLocked() error {
if r.wroteHeader {
return nil
}
respHeaders := flattenResponseHeaders(r.headers)
respHeaders[tunnelStreamHeader] = "1"
if err := r.conn.Send(&TunnelMessage{
ID: r.requestID,
Type: MessageTypeResponse,
Status: r.statusCode,
Headers: respHeaders,
}); err != nil {
return err
}
r.wroteHeader = true
return nil
}
// StartTunnelClient starts the tunnel client with an owned background worker.
func StartTunnelClient(ctx context.Context, cfg *Config, handler http.Handler) (func(context.Context) error, error) {
if !cfg.EdgeAgent {
return nil, errors.New("edge tunnel disabled")
}
if UseGRPCEdgeTransport(cfg) && httpx.ManagerGRPCAddr(cfg.ManagerApiUrl) == "" {
return nil, errors.New("MANAGER_API_URL with a valid host is required for gRPC transport")
}
if UseWebSocketEdgeTransport(cfg) && strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)) == "" {
return nil, errors.New("MANAGER_API_URL is required for websocket transport")
}
if UsePollEdgeTransport(cfg) && strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)) == "" {
return nil, errors.New("MANAGER_API_URL is required for poll transport")
}
if cfg.AgentToken == "" {
return nil, errors.New("AGENT_TOKEN is required")
}
if parsed, err := url.Parse(httpx.ManagerBaseURL(cfg.ManagerApiUrl)); NormalizeEdgeTransport(cfg.EdgeTransport) == EdgeTransportGRPC && err == nil && strings.Trim(parsed.Path, "/") != "" {
return nil, errors.New("gRPC transport cannot use a MANAGER_API_URL with a path prefix; use EDGE_TRANSPORT=websocket or auto")
}
if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeDisabled && !managerUsesTLS(cfg) {
return nil, errors.New("EDGE_MTLS_MODE requires MANAGER_API_URL to use https")
}
client := NewTunnelClient(cfg, handler)
return concurrency.StartSupervised(ctx, "Edge tunnel client", func(runCtx context.Context) error {
client.StartWithErrorChan(runCtx, nil)
return nil
})
}
func (c *TunnelClient) connectAndServeWebSocket(ctx context.Context) error {
managerWSURL := c.managerWebSocketURL()
if managerWSURL == "" {
return errors.New("manager WebSocket URL is empty")
}
httpClient, err := NewManagerHTTPClient(c.cfg, 0)
if err != nil {
return fmt.Errorf("failed to configure edge websocket TLS: %w", err)
}
headers := http.Header{}
for header, value := range agentAuthCredentials(c.cfg.AgentToken) {
headers.Set(header, value)
}
slog.DebugContext(ctx, "Dialing manager for websocket edge tunnel", "url", managerWSURL)
// The dial context bounds only the handshake; the connection outlives it.
dialCtx, dialCancel := context.WithTimeout(ctx, 30*time.Second)
defer dialCancel()
conn, resp, err := websocket.Dial(dialCtx, managerWSURL, &websocket.DialOptions{
HTTPClient: httpClient,
HTTPHeader: headers,
})
if err != nil {
if resp != nil && resp.Body != nil {
defer func() { _ = resp.Body.Close() }()
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return fmt.Errorf("failed to connect to manager websocket endpoint (status: %d, body: %s): %w", resp.StatusCode, strings.TrimSpace(string(body)), err)
}
return fmt.Errorf("failed to connect to manager websocket endpoint: %w", err)
}
return c.serveTunnelSession(ctx, NewTunnelConn(conn), managerWSURL)
}