Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
package edge
import (
"context"
"errors"
"fmt"
"io"
"log"
"log/slog"
"net"
"net/http"
"slices"
"time"
"github.com/hashicorp/yamux"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)
const (
// tunnelCapabilityMux is offered by agents that can run the multiplexed data plane.
tunnelCapabilityMux = "mux-v1"
// tunnelCapabilityMuxGrant switches both peers to mux; only managers that serve mux send it, since older ones echo offers.
tunnelCapabilityMuxGrant = "mux-grant-v1"
// EdgeDataPlaneMux and EdgeDataPlaneLegacy name the negotiated data plane in runtime state.
EdgeDataPlaneMux = "mux"
EdgeDataPlaneLegacy = "legacy"
// MuxAgentBaseURL is the synthetic origin for requests a manager sends through a multiplexed tunnel.
MuxAgentBaseURL = "http://agent.edge.arcane.internal"
muxManagerBaseURL = "http://manager.edge.arcane.internal"
muxEventsPath = "/api/tunnel/events"
muxStreamWindowSize = 4 << 20
muxMaxFrameSize = 256 << 10
muxMaxPendingWrite = 1 << 20
muxWriteTimeout = 30 * time.Second
// muxDialTimeout bounds a stream open on its own, because http.Transport detaches dials from request cancellation.
muxDialTimeout = 30 * time.Second
// ProxyResponseHeaderTimeout is generous because proxied operations such as deploys can run for minutes before responding.
ProxyResponseHeaderTimeout = 30 * time.Minute
)
// errTunnelMuxActive rejects legacy messages once a connection carries a multiplexed session.
var errTunnelMuxActive = errors.New("edge tunnel connection carries a multiplexed session")
// muxTransport is a tunnel connection that can switch to carrying a multiplexed session.
type muxTransport interface {
muxConn(ctx context.Context) io.ReadWriteCloser
}
// newMuxConfig returns the yamux settings both peers use; only the agent sends keepalive pings.
func newMuxConfig(keepAlive bool) *yamux.Config {
cfg := yamux.DefaultConfig()
cfg.EnableKeepAlive = keepAlive
cfg.KeepAliveInterval = DefaultHeartbeatInterval
cfg.ConnectionWriteTimeout = muxWriteTimeout
cfg.MaxStreamWindowSize = muxStreamWindowSize
cfg.LogOutput = nil
cfg.Logger = newMuxLogger()
return cfg
}
// newMuxLogger routes yamux and mux HTTP server diagnostics to slog at debug level.
func newMuxLogger() *log.Logger {
return slog.NewLogLogger(slog.Default().Handler(), slog.LevelDebug)
}
// newMuxHTTPClient returns a traced client whose connections are streams opened by dial.
func newMuxHTTPClient(dial func(ctx context.Context) (net.Conn, error)) *http.Client {
return &http.Client{Transport: otelhttp.NewTransport(&http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
dialCtx, cancel := context.WithTimeout(ctx, muxDialTimeout)
defer cancel()
return dial(dialCtx)
},
// Stays below the agent server's IdleTimeout so a pooled stream is never reused as the server closes it.
IdleConnTimeout: 30 * time.Second,
MaxIdleConnsPerHost: 32,
ExpectContinueTimeout: time.Second,
ResponseHeaderTimeout: ProxyResponseHeaderTimeout,
ReadBufferSize: 64 << 10,
WriteBufferSize: 64 << 10,
})}
}
// openMuxStream opens a stream but returns as soon as ctx ends. yamux's Open cannot be cancelled while it
// waits for an in-flight SYN slot, so a stream that opens after the caller left is closed; no other stream is touched.
func openMuxStream(ctx context.Context, session *yamux.Session) (net.Conn, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
type openResult struct {
conn net.Conn
err error
}
opened := make(chan openResult, 1)
go func() {
conn, err := session.Open()
opened <- openResult{conn: conn, err: err}
}()
select {
case result := <-opened:
if result.err != nil {
return nil, fmt.Errorf("open edge tunnel stream: %w: %w", ErrTunnelConnectionClosed, result.err)
}
return result.conn, nil
case <-ctx.Done():
go func() {
if result := <-opened; result.conn != nil {
_ = result.conn.Close()
}
}()
// A session closing races the dial context; report it as a closed tunnel either way.
if session.IsClosed() {
return nil, fmt.Errorf("open edge tunnel stream: %w", ErrTunnelConnectionClosed)
}
return nil, ctx.Err()
}
}
// newMuxConn starts the sender that coalesces writes, so a yamux header and body share one transport message.
// Batches are never reused, since gRPC forbids modifying a message after Send.
func newMuxConn(send func([]byte) error, recv func() ([]byte, error), closeTransport func() error) *muxConn {
c := &muxConn{recv: recv, closeTransport: closeTransport}
c.cond.L = &c.mu
go func() {
for {
c.mu.Lock()
for c.err == nil && len(c.pending) == 0 {
c.cond.Wait()
}
batch, err := c.pending, c.err
c.pending = nil
c.cond.Broadcast()
c.mu.Unlock()
if err != nil {
return
}
for frame := range slices.Chunk(batch, muxMaxFrameSize) {
if sendErr := send(frame); sendErr != nil {
c.fail(sendErr)
return
}
}
}
}()
return c
}
func (c *muxConn) Write(p []byte) (int, error) {
c.mu.Lock()
defer c.mu.Unlock()
for c.err == nil && len(c.pending) >= muxMaxPendingWrite {
c.cond.Wait()
}
if c.err != nil {
return 0, c.err
}
c.pending = append(c.pending, p...)
c.cond.Broadcast()
return len(p), nil
}
// Read is called only by the yamux receive loop.
func (c *muxConn) Read(p []byte) (int, error) {
for len(c.unread) == 0 {
frame, err := c.recv()
if err != nil {
c.fail(err)
return 0, err
}
c.unread = frame
}
n := copy(p, c.unread)
c.unread = c.unread[n:]
return n, nil
}
func (c *muxConn) Close() error {
c.fail(net.ErrClosed)
var err error
c.closeOnce.Do(func() { err = c.closeTransport() })
return err
}
// fail records the first error and wakes blocked writers and the sender.
func (c *muxConn) fail(err error) {
c.mu.Lock()
if c.err == nil {
c.err = err
}
c.cond.Broadcast()
c.mu.Unlock()
}