Skip to content

Modern Docker Management, Designed for Everyone

26.1 KB
Raw Blame
package edge

import (
	"cmp"
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"log/slog"
	"net/http"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"sync"
	"time"
	"uuid"

	certgen "github.com/getarcaneapp/arcane/cli/v2/pkg/generate"
	"github.com/labstack/echo/v5"
	"go.getarcane.app/acfs/atomic"
	kit "go.getarcane.app/kit/pkg"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/trace"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/peer"
	"google.golang.org/grpc/status"

	wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
	tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
)

const (
	// TunnelStaleTimeout is how long before a tunnel is considered stale.
	TunnelStaleTimeout = 2 * time.Minute
	// tunnelStaleSweepInterval backs up read-deadline liveness, so it runs well under TunnelStaleTimeout.
	tunnelStaleSweepInterval = time.Minute
	// pendingDeliveryTimeout and the backlog limits bound legacy pending consumers; remove in v3.
	// pendingDeliveryTimeout is how long one queued message may wait for a slow consumer.
	pendingDeliveryTimeout = 5 * time.Second
	// maxPendingBacklogBytes and maxPendingBacklogMessages bound how far a consumer may fall behind.
	maxPendingBacklogBytes    = 32 << 20
	maxPendingBacklogMessages = 4096
)

// observeAgentsOnce registers the connected-agents gauge for the first (manager) tunnel server.
var observeAgentsOnce sync.Once

// EnvironmentResolver resolves an agent token to an environment ID.
type EnvironmentResolver func(ctx context.Context, token string) (environmentID string, err error)

// EnvironmentNameResolver resolves a display name for an environment ID.
type EnvironmentNameResolver func(ctx context.Context, environmentID string) (environmentName string, err error)

// StatusUpdateCallback is called when an edge agent connects or disconnects.
// The connected parameter is true on connect, false on disconnect.
type StatusUpdateCallback func(ctx context.Context, environmentID string, connected bool)

// EventCallback is called when an edge agent publishes an event.
type EventCallback func(ctx context.Context, environmentID string, event *TunnelEvent) error

// EnrollmentCallback is called after a successful manager-side edge mTLS enrollment.
// reenrolled is true when the environment had already enrolled before the cooldown.
type EnrollmentCallback func(ctx context.Context, environmentID, remoteAddr string, certIssued, caGenerated, reenrolled bool)

// NewTunnelServerWithRegistry creates a new tunnel server using an injected tunnel registry.
func NewTunnelServerWithRegistry(registry *TunnelRegistry, resolver EnvironmentResolver, statusCallback StatusUpdateCallback) *TunnelServer {
	if registry == nil {
		registry = NewTunnelRegistry()
	}
	observeAgentsOnce.Do(func() {
		if _, err := registry.ObserveAgents(otel.Meter(tracing.InstrumentationName)); err != nil {
			otel.Handle(err)
		}
	})

	return &TunnelServer{
		registry:       registry,
		resolver:       resolver,
		statusCallback: statusCallback,
		cleanupDone:    make(chan struct{}),
	}
}

// GRPCServerOptions returns the receive limit and stream interceptors for the tunnel service.
// It is served through ServeHTTP, so transport keepalive options belong on net/http.
func (s *TunnelServer) GRPCServerOptions() []grpc.ServerOption {
	recovery := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) (err error) {
		defer func() {
			if panicErr := utils.PanicToError(recover()); panicErr != nil {
				slog.ErrorContext(ss.Context(), "panic in gRPC tunnel stream", "method", info.FullMethod, "error", panicErr)
				err = status.Error(codes.Internal, "internal tunnel error")
			}
		}()
		return handler(srv, ss)
	}

	logging := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
		start := time.Now()
		err := handler(srv, ss)
		duration := time.Since(start)
		switch {
		case err == nil:
			slog.DebugContext(ss.Context(), "gRPC stream completed", "method", info.FullMethod, "duration", duration)
		case isExpectedReceiveError(err):
			slog.DebugContext(ss.Context(), "gRPC stream closed", "method", info.FullMethod, "duration", duration, "error", err)
		default:
			slog.WarnContext(ss.Context(), "gRPC stream failed", "method", info.FullMethod, "duration", duration, "error", err)
		}
		return err
	}

	auth := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
		// TunnelService only exposes Connect; keep this gate aligned with tunnel.proto.
		if info.FullMethod != tunnelpb.TunnelService_Connect_FullMethodName {
			return handler(srv, ss)
		}

		ctx := ss.Context()
		md, _ := metadata.FromIncomingContext(ctx)
		token, _ := agentToken(md.Get)
		envID, err := s.resolveEnvironment(ctx, token)
		if err != nil {
			return status.Error(codes.Unauthenticated, "invalid agent token")
		}

		var state *tls.ConnectionState
		if p, ok := peer.FromContext(ctx); ok {
			if tlsInfo, isTLS := p.AuthInfo.(credentials.TLSInfo); isTLS {
				state = &tlsInfo.State
			}
		}
		if identityErr := s.requireCertificateIdentity(state, envID); identityErr != nil {
			return status.Error(codes.Unauthenticated, identityErr.Error())
		}

		identity := streamIdentity{environmentID: envID, securityMode: kit.Ternary(hasVerifiedPeerCertificate(state), "mtls", "token")}
		return handler(srv, &contextualServerStream{ServerStream: ss, ctx: context.WithValue(ctx, streamIdentityKey{}, identity)})
	}

	return []grpc.ServerOption{
		grpc.MaxRecvMsgSize(maxGRPCTunnelMessageSize),
		grpc.ChainStreamInterceptor(recovery, logging, auth),
	}
}

// HandleConnect is the WebSocket handler for edge agent connections.
// This is registered at /api/tunnel/connect.
func (s *TunnelServer) HandleConnect(c *echo.Context) error {
	req := c.Request()
	ctx := req.Context()

	// Agents authenticate with tokens or mTLS rather than browser cookies, so no Origin check is needed.
	conn, err := wshub.Accept(c.Response(), req, nil)
	if err != nil {
		slog.ErrorContext(ctx, "Failed to upgrade edge tunnel connection", "error", err)
		return nil
	}

	tunnelConn := NewTunnelConn(conn)
	reject := func(reason string) error {
		if reason != "" {
			_ = tunnelConn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, Error: reason})
		}
		_ = tunnelConn.Close()
		return nil
	}

	registerMsg, err := tunnelConn.Receive()
	if err != nil {
		slog.WarnContext(ctx, "Failed to receive websocket edge tunnel registration", "error", err)
		return reject("")
	}
	if registerMsg == nil || registerMsg.Type != MessageTypeRegister {
		slog.WarnContext(ctx, "Websocket edge tunnel missing register message")
		return reject("")
	}

	// Headers can be lost on the upgrade path, and proxy-terminated mTLS consumes the client
	// certificate, so the register message's token is the fallback environment claim.
	token, _ := agentToken(req.Header.Values)
	token = cmp.Or(token, strings.TrimSpace(registerMsg.AgentToken))
	if token == "" {
		slog.WarnContext(ctx, "Edge tunnel connection attempt without token")
		return reject("agent token required")
	}

	envID, err := s.resolveEnvironment(ctx, token)
	if err != nil {
		slog.WarnContext(ctx, "Failed to resolve agent token", "error", err)
		return reject("invalid agent token")
	}
	if identityErr := s.requireCertificateIdentity(req.TLS, envID); identityErr != nil {
		slog.WarnContext(ctx, "Rejected websocket edge tunnel with mismatched client certificate", "environmentId", envID, "error", identityErr)
		return reject("client certificate does not match environment")
	}

	s.manageConnectedTunnel(ctx, envID, tunnelConn, registerMsg, kit.Ternary(hasVerifiedPeerCertificate(req.TLS), "mtls", "token"))
	return nil
}

// HandleMTLSEnroll returns manager-generated edge client certificates for the calling environment.
// The response contains private keys, so response-body logging must stay disabled here.
func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {
	req := c.Request()
	ctx := req.Context()
	c.Response().Header().Set("Cache-Control", "no-store")
	c.Response().Header().Set("Pragma", "no-cache")

	if !usesGeneratedManagerCA(s.Config) {
		return c.JSON(http.StatusNotFound, map[string]any{"error": "edge mTLS enrollment is not available"})
	}

	token, _ := agentToken(req.Header.Values)
	if token == "" {
		return c.JSON(http.StatusUnauthorized, map[string]any{"error": "agent token required"})
	}

	envID, err := s.resolveEnvironment(ctx, token)
	if err != nil {
		slog.WarnContext(ctx, "Failed to resolve edge token for mTLS enrollment", "error", err)
		return c.JSON(http.StatusUnauthorized, map[string]any{"error": "invalid agent token"})
	}

	envName := ""
	if s.NameResolver != nil {
		resolvedName, resolveErr := s.NameResolver(ctx, envID)
		if resolveErr != nil {
			slog.WarnContext(ctx, "Failed to resolve environment name for edge mTLS enrollment", "environmentId", envID, "error", resolveErr)
		} else {
			envName = resolvedName
		}
	}

	// The marker holds the last enrollment time; within the cooldown the existing assets are re-served.
	failState := func(stateErr error) error {
		slog.ErrorContext(ctx, "Failed to read edge mTLS enrollment state", "environmentId", envID, "error", stateErr)
		return c.JSON(http.StatusInternalServerError, map[string]any{"error": "failed to read edge mTLS enrollment state"})
	}
	assetsDir, err := edgeMTLSAssetsDir(s.Config, managerMTLSDirName)
	if err != nil {
		return failState(err)
	}
	safeEnvID := sanitizedEnvID(envID)
	if safeEnvID == "" {
		return failState(errors.New("environment ID is required"))
	}
	markerPath := filepath.Join(assetsDir, generatedClientMTLSSubdir, safeEnvID, generatedMTLSEnrolledName)
	marker, readErr := os.ReadFile(markerPath)
	if readErr != nil && !errors.Is(readErr, fs.ErrNotExist) {
		return failState(readErr)
	}
	now := time.Now()
	previouslyEnrolled := readErr == nil
	enrollmentLimited := false
	if trimmed := strings.TrimSpace(string(marker)); trimmed != "" {
		enrolledAt, parseErr := time.Parse(time.RFC3339Nano, trimmed)
		if parseErr != nil {
			return failState(fmt.Errorf("failed to parse edge mTLS enrollment marker %s: %w", markerPath, parseErr))
		}
		enrollmentLimited = now.Sub(enrolledAt) < managerMTLSReenrollCooldown
	}

	if enrollmentLimited {
		cachedAssets, cacheErr := GenerateManagerClientMTLSAssetsWithContext(ctx, s.Config, envID, envName)
		if cacheErr == nil && cachedAssets != nil {
			slog.InfoContext(ctx, "Served cached edge mTLS enrollment during cooldown", "environmentId", envID, "remoteAddr", c.RealIP())
			return c.JSON(http.StatusOK, enrollMTLSResponse{Files: cachedAssets.Files})
		}
		if cacheErr != nil {
			slog.WarnContext(ctx, "Failed to re-serve cached edge mTLS enrollment during cooldown", "environmentId", envID, "error", cacheErr)
		}
		slog.WarnContext(ctx, "Rejected repeated edge mTLS enrollment during cooldown", "environmentId", envID, "remoteAddr", c.RealIP())
		return c.JSON(http.StatusTooManyRequests, map[string]any{"error": "edge mTLS enrollment was recently completed; retry later"})
	}

	assets, err := GenerateManagerClientMTLSAssetsWithContext(ctx, s.Config, envID, envName)
	if err != nil {
		slog.ErrorContext(ctx, "Failed to generate edge mTLS enrollment assets", "environmentId", envID, "error", err)
		return c.JSON(http.StatusInternalServerError, map[string]any{"error": "failed to generate edge mTLS assets"})
	}
	if assets == nil {
		return c.JSON(http.StatusNotFound, map[string]any{"error": "edge mTLS enrollment assets unavailable"})
	}
	assets.Reenrolled = previouslyEnrolled
	if recordErr := atomic.WriteFile(markerPath, []byte(now.UTC().Format(time.RFC3339Nano)+"\n"), 0o600); recordErr != nil {
		slog.ErrorContext(ctx, "Failed to record edge mTLS enrollment state", "environmentId", envID, "error", recordErr)
		return c.JSON(http.StatusInternalServerError, map[string]any{"error": "failed to record edge mTLS enrollment state"})
	}
	if assets.Reenrolled {
		slog.WarnContext(ctx, "Edge mTLS certificate assets re-enrolled", "environmentId", envID, "remoteAddr", c.RealIP(), "certIssued", assets.CertIssued)
	} else {
		slog.InfoContext(ctx, "Edge mTLS certificate assets enrolled", "environmentId", envID, "remoteAddr", c.RealIP(), "certIssued", assets.CertIssued)
	}

	if s.EnrollmentCallback != nil {
		s.EnrollmentCallback(context.WithoutCancel(ctx), envID, c.RealIP(), assets.CertIssued, assets.CAGenerated, assets.Reenrolled)
	}

	return c.JSON(http.StatusOK, enrollMTLSResponse{Files: assets.Files})
}

// Connect is the gRPC bidi stream handler for edge agent connections.
func (s *TunnelServer) Connect(stream grpc.BidiStreamingServer[tunnelpb.AgentMessage, tunnelpb.ManagerMessage]) error {
	ctx := stream.Context()

	firstMsg, err := stream.Recv()
	if err != nil {
		if errors.Is(err, io.EOF) {
			return status.Error(codes.Unauthenticated, "register message required")
		}
		return err
	}

	registerMsg, err := agentProtoToTunnelMessage(firstMsg)
	if err != nil || registerMsg.Type != MessageTypeRegister {
		return status.Error(codes.Unauthenticated, "first message must be register")
	}

	// The auth interceptor already verified the token and client certificate for this stream.
	identity, _ := ctx.Value(streamIdentityKey{}).(streamIdentity)
	if strings.TrimSpace(identity.environmentID) == "" {
		return status.Error(codes.Unauthenticated, "authenticated environment is missing from stream context")
	}

	managerConn := NewGRPCManagerTunnelConn(stream)
	managerConn.parity = slices.Contains(registerMsg.Capabilities, tunnelCapabilityProtoParity)
	s.manageConnectedTunnel(ctx, identity.environmentID, managerConn, registerMsg, identity.securityMode)
	return nil
}

func (s *TunnelServer) resolveEnvironment(ctx context.Context, token string) (string, error) {
	if s.resolver == nil {
		return "", errors.New("edge resolver is not configured")
	}
	token = strings.TrimSpace(token)
	if token == "" {
		return "", errors.New("agent token required")
	}
	return s.resolver(ctx, token)
}

// agentToken returns the first agent token from X-Arcane-Agent-Token, X-API-Key, then
// "Authorization: Bearer", for proxies that strip custom headers, plus the header that supplied it.
func agentToken(values func(key string) []string) (string, string) {
	for _, header := range []string{HeaderAgentToken, HeaderAPIKey} {
		if tokens := kit.TrimNonEmpty(values(header)); len(tokens) > 0 {
			return tokens[0], header
		}
	}
	for _, value := range values(HeaderAuthorization) {
		if token := remenv.ExtractBearerToken(value); token != "" {
			return token, HeaderAuthorization
		}
	}
	return "", ""
}

// manageConnectedTunnel registers an authenticated agent session and serves its messages until it disconnects.
func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, envID string, conn TunnelConnection, registerMsg *TunnelMessage, securityMode string) {
	callbackCtx := context.WithoutCancel(ctx)
	tunnel := NewAgentTunnelWithConn(envID, conn)
	tunnel.SessionID = uuid.New().String()
	tunnel.SecurityMode = securityMode
	tunnel.AgentInstance = strings.TrimSpace(registerMsg.AgentInstance)
	tunnel.Capabilities = registerMsg.Capabilities
	tunnel.Transport = conn.Transport()
	// The session exists before registration so callers that find the tunnel mid-handshake wait for it.
	transport, useMux := conn.(muxTransport)
	useMux = useMux && slices.Contains(registerMsg.Capabilities, tunnelCapabilityMux) && (s.Config == nil || !s.Config.EdgeMuxDisabled)
	if useMux {
		tunnel.mux = &muxSession{ready: make(chan struct{}), closed: make(chan struct{})}
		tunnel.mux.client = newMuxHTTPClient(tunnel.mux.dial)
	}

	// The span covers the connection lifetime; ctx stays unparented so per-message work doesn't nest under it.
	_, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "edge.tunnel "+tunnel.Transport,
		trace.WithSpanKind(trace.SpanKindServer),
		trace.WithAttributes(
			attribute.String("arcane.edge.transport", tunnel.Transport),
			attribute.String("arcane.environment.id", envID),
			attribute.String("arcane.edge.session_id", tunnel.SessionID),
			attribute.String("arcane.edge.security_mode", securityMode),
			attribute.String("arcane.edge.agent_instance", tunnel.AgentInstance),
		))
	var spanErr error
	defer func() { tracing.End(span, spanErr) }()

	accepted, drainPrevious, rejectReason, err := s.registry.RegisterSession(callbackCtx, tunnel, TunnelStaleTimeout)
	if err != nil {
		slog.ErrorContext(ctx, "Failed to register edge agent session", "environmentId", envID, "agentInstanceId", tunnel.AgentInstance, "error", err)
		_ = tunnel.CloseWithReason("edge agent session registration unavailable")
		spanErr = err
		return
	}
	if !accepted {
		slog.WarnContext(ctx, "Rejected duplicate edge agent session", "environmentId", envID, "agentInstanceId", tunnel.AgentInstance, "reason", rejectReason)
		_ = conn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, EnvironmentID: envID, Error: rejectReason})
		_ = tunnel.CloseWithReason(rejectReason)
		spanErr = errors.New(rejectReason)
		return
	}

	defer func() {
		removed, active := s.registry.UnregisterCurrent(callbackCtx, envID, tunnel)
		if !removed {
			return
		}
		slog.InfoContext(ctx, "Edge agent disconnected", "environmentId", envID, "sessionId", tunnel.SessionID)
		if !active {
			s.updateConnectionStatus(callbackCtx, tunnel, false)
		}
	}()

	slog.InfoContext(ctx, "Edge agent connected", "environmentId", envID, "sessionId", tunnel.SessionID, "securityMode", tunnel.SecurityMode,
		"transport", tunnel.Transport, "dataPlane", kit.Ternary(useMux, EdgeDataPlaneMux, EdgeDataPlaneLegacy))

	// Echo the agent's capabilities plus the manager's own; older agents ignore extra strings.
	capabilities := kit.Unique(append(slices.Clone(tunnel.Capabilities), tunnelCapabilityProtoParity, tunnelCapabilityChunkedRequest, tunnelCapabilityCommandCreditGrant))
	if useMux {
		capabilities = append(capabilities, tunnelCapabilityMuxGrant)
	}
	if sendErr := conn.Send(&TunnelMessage{
		Type:          MessageTypeRegisterResponse,
		Accepted:      true,
		EnvironmentID: envID,
		SessionID:     tunnel.SessionID,
		SecurityMode:  tunnel.SecurityMode,
		Capabilities:  capabilities,
		DrainPrevious: drainPrevious,
	}); sendErr != nil {
		slog.WarnContext(ctx, "Failed to send register response", "environmentId", envID, "error", sendErr)
		_ = tunnel.CloseWithReason("")
		spanErr = sendErr
		return
	}
	if useMux {
		spanErr = s.serveMuxTunnel(ctx, tunnel, transport.muxConn(ctx))
		return
	}

	// Deprecated: legacy edge message protocol; remove in v3.
	s.updateConnectionStatus(callbackCtx, tunnel, true)
	deliver := func(msg *TunnelMessage, closeStream bool) {
		value, _ := tunnel.Pending.Load(msg.ID)
		if pending, ok := value.(*PendingRequest); ok {
			pending.enqueue(ctx, tunnel, msg, closeStream)
			return
		}
		slog.DebugContext(ctx, "Received message for unknown request", "id", msg.ID, "type", msg.Type)
	}

	for ctx.Err() == nil {
		msg, receiveErr := conn.Receive()
		if receiveErr != nil {
			if !conn.IsExpectedReceiveError(receiveErr) {
				slog.WarnContext(ctx, "Error receiving from edge tunnel", "environmentId", envID, "error", receiveErr)
				spanErr = receiveErr
			}
			return
		}

		switch msg.Type {
		case MessageTypeHeartbeat:
			tunnel.UpdateHeartbeat()
			if ackErr := conn.Send(&TunnelMessage{ID: msg.ID, Type: MessageTypeHeartbeatAck}); ackErr != nil {
				slog.WarnContext(ctx, "Failed to send heartbeat ack", "error", ackErr)
			}
		case MessageTypeResponse, MessageTypeCommandAck, MessageTypeCommandOutput, MessageTypeCommandComplete, MessageTypeFileChunk:
			deliver(msg, false)
		case MessageTypeStreamData, MessageTypeStreamEnd, MessageTypeWebSocketData, MessageTypeWebSocketClose, MessageTypeStreamClose:
			deliver(msg, true)
		case MessageTypeEvent:
			if msg.Event == nil {
				slog.WarnContext(ctx, "Received event message without payload", "environmentId", envID)
				continue
			}
			if s.EventCallback == nil {
				continue
			}
			event := msg.Event
			go func() {
				eventCtx, cancel := context.WithTimeout(callbackCtx, edgeEventTimeout)
				defer cancel()
				if eventErr := s.EventCallback(eventCtx, envID, event); eventErr != nil {
					slog.WarnContext(eventCtx, "Failed to process edge event", "environmentId", envID, "type", event.Type, "error", eventErr)
				}
			}()
		case MessageTypeRequest, MessageTypeHeartbeatAck, MessageTypeWebSocketStart, MessageTypeRegisterResponse,
			MessageTypeCommandRequest, MessageTypeStreamOpen, MessageTypeCancelRequest, MessageTypeRegister, MessageTypeCommandCredit:
			slog.DebugContext(ctx, "Ignoring message type from agent", "type", msg.Type, "environmentId", envID)
		default:
			slog.WarnContext(ctx, "Unknown message type from agent", "type", msg.Type, "environmentId", envID)
		}
	}
}

// enqueue hands msg to the consumer without blocking the tunnel; queued messages wait in order and outlive
// a tunnel close. A consumer that stalls past pendingDeliveryTimeout or the backlog limits is failed once.
//
// Deprecated: legacy edge message protocol; remove in v3.
func (p *PendingRequest) enqueue(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage, closeStream bool) {
	retained := func(m *TunnelMessage) int {
		size := len(m.Body) + len(m.Error)
		for k, v := range m.Headers {
			size += len(k) + len(v)
		}
		for k, v := range m.Metadata {
			size += len(k) + len(v)
		}
		return size
	}

	fail := func(stalled error) {
		// A consumer that already finished has nothing left to fail.
		if !tunnel.Pending.CompareAndDelete(msg.ID, p) {
			return
		}
		select {
		case p.failureCh <- stalled:
		default:
		}
		slog.WarnContext(ctx, "Failed slow pending request consumer", "id", msg.ID, "type", msg.Type, "error", stalled)
		if !closeStream {
			return
		}
		if closeErr := tunnel.Conn.Send(&TunnelMessage{ID: msg.ID, Type: MessageTypeStreamClose, Error: stalled.Error()}); closeErr != nil {
			slog.DebugContext(ctx, "Failed to close slow edge stream", "id", msg.ID, "error", closeErr)
		}
	}

	p.mu.Lock()
	if p.failed {
		p.mu.Unlock()
		return
	}
	if !p.draining {
		select {
		case p.ResponseCh <- msg:
			p.mu.Unlock()
			return
		default:
		}
		p.draining = true
		go func() {
			timer := time.NewTimer(pendingDeliveryTimeout)
			defer timer.Stop()
			for {
				p.mu.Lock()
				if p.failed || len(p.backlog) == 0 {
					p.draining = false
					p.mu.Unlock()
					return
				}
				next := p.backlog[0]
				p.backlog[0] = nil
				p.backlog = p.backlog[1:]
				p.backlogBytes -= retained(next)
				p.mu.Unlock()

				timer.Reset(pendingDeliveryTimeout)
				select {
				case p.ResponseCh <- next:
				case <-timer.C:
					p.mu.Lock()
					stalled := !p.failed
					p.failed, p.backlog = true, nil
					p.mu.Unlock()
					if stalled {
						fail(fmt.Errorf("consumer for pending request %s stalled for %s", next.ID, pendingDeliveryTimeout))
					}
					return
				}
			}
		}()
	}

	p.backlog = append(p.backlog, msg)
	p.backlogBytes += retained(msg)
	overflow := p.backlogBytes > maxPendingBacklogBytes || len(p.backlog) > maxPendingBacklogMessages
	if overflow {
		p.failed, p.backlog = true, nil
	}
	p.mu.Unlock()
	if overflow {
		fail(fmt.Errorf("consumer for pending request %s fell more than %d messages or %d bytes behind", msg.ID, maxPendingBacklogMessages, maxPendingBacklogBytes))
	}
}

// StartCleanupLoop periodically cleans up stale tunnels.
func (s *TunnelServer) StartCleanupLoop(ctx context.Context) {
	defer s.cleanupDoneOnce.Do(func() { close(s.cleanupDone) })
	ticker := time.NewTicker(tunnelStaleSweepInterval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			removed := s.registry.CleanupStale(ctx, TunnelStaleTimeout)
			for _, tunnel := range removed {
				s.updateConnectionStatus(context.WithoutCancel(ctx), tunnel, false)
			}
			if len(removed) > 0 {
				slog.InfoContext(ctx, "Cleaned up stale tunnels", "count", len(removed))
			}
		}
	}
}

// updateConnectionStatus reports a connect or disconnect, skipping a disconnect
// when a newer live tunnel already serves the environment.
func (s *TunnelServer) updateConnectionStatus(ctx context.Context, tunnel *AgentTunnel, connected bool) {
	if s.statusCallback == nil {
		return
	}

	s.statusMu.Lock()
	defer s.statusMu.Unlock()
	if !connected {
		if active, _ := s.registry.Get(tunnel.EnvironmentID).Get(); active != tunnel && active.connected() {
			return
		}
	}
	s.statusCallback(ctx, tunnel.EnvironmentID, connected)
}

// WaitForCleanupDone blocks until the cleanup loop has stopped.
func (s *TunnelServer) WaitForCleanupDone() {
	<-s.cleanupDone
}

func (s *contextualServerStream) Context() context.Context {
	return s.ctx
}

// requireCertificateIdentity checks a direct-TLS client certificate against envID. A nil state
// means TLS ended before Arcane, such as at a proxy, so only the token applies.
func (s *TunnelServer) requireCertificateIdentity(state *tls.ConnectionState, envID string) error {
	mode := EdgeMTLSModeDisabled
	if s.Config != nil {
		mode = NormalizeEdgeMTLSMode(s.Config.EdgeMTLSMode)
	}
	if mode == EdgeMTLSModeDisabled || state == nil {
		return nil
	}
	if !hasVerifiedPeerCertificate(state) {
		if mode == EdgeMTLSModeRequired {
			return errors.New("verified edge mTLS client certificate is required")
		}
		return nil
	}

	safeEnvID := sanitizedEnvID(envID)
	if safeEnvID == "" {
		return errors.New("environment ID is required for edge mTLS certificate identity check")
	}
	appURL := cmp.Or(strings.TrimSpace(s.Config.AppURL), strings.TrimSpace(httpx.ManagerBaseURL(s.Config.ManagerApiUrl)))
	expected := certgen.BuildEdgeMTLSURISAN(appURL, safeEnvID)
	if expected == nil {
		return errors.New("edge mTLS trust domain is required for certificate identity check")
	}
	if !certificateHasURISAN(state.VerifiedChains[0][0], expected) {
		return fmt.Errorf("verified edge mTLS client certificate does not match environment %s", strings.TrimSpace(envID))
	}
	return nil
}

// IsInternalTunnelRequest reports whether a request is dispatched by the legacy in-process tunnel client, which
// skips request logging. Only the legacy edge message protocol sets it; remove in v3.
func IsInternalTunnelRequest(ctx context.Context) bool {
	if ctx == nil {
		return false
	}
	isInternal, _ := ctx.Value(internalTunnelRequestContextKey{}).(bool)
	return isInternal
}