Skip to content

Modern Docker Management, Designed for Everyone

12.4 KB
Raw Blame
package edge

import (
	"context"
	"errors"
	"log/slog"
	"sync"
	"time"

	"github.com/samber/mo"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/metric"

	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
)

// NewAgentTunnelWithConn creates a new agent tunnel from a transport-agnostic connection.
func NewAgentTunnelWithConn(envID string, conn TunnelConnection) *AgentTunnel {
	now := time.Now()
	return &AgentTunnel{
		EnvironmentID: envID,
		Conn:          conn,
		ConnectedAt:   now,
		LastHeartbeat: now,
		State:         "connected",
		done:          make(chan struct{}),
	}
}

// UpdateHeartbeat updates the last heartbeat timestamp
func (t *AgentTunnel) UpdateHeartbeat() {
	t.mu.Lock()
	t.LastHeartbeat = time.Now()
	t.mu.Unlock()
}

// connected reports whether the tunnel has a live, open connection.
func (t *AgentTunnel) connected() bool {
	return t != nil && t.Conn != nil && !t.Conn.IsClosed()
}

// openStreams counts open mux streams, or pending requests on a legacy tunnel.
func (t *AgentTunnel) openStreams() int {
	if t.mux != nil {
		t.mux.mu.Lock()
		defer t.mux.mu.Unlock()
		if t.mux.session == nil {
			return 0
		}
		return t.mux.session.NumStreams()
	}
	count := 0
	t.Pending.Range(func(_, _ any) bool {
		count++
		return true
	})
	return count
}

// GetLastHeartbeat returns the last heartbeat time
func (t *AgentTunnel) GetLastHeartbeat() time.Time {
	t.mu.RLock()
	defer t.mu.RUnlock()
	return t.LastHeartbeat
}

// TunnelMetadataSnapshot is a consistent copy of a tunnel's mutable metadata.
type TunnelMetadataSnapshot struct {
	ConnectedAt   time.Time
	LastHeartbeat time.Time
	SessionID     string
	AgentInstance string
	SecurityMode  string
	Capabilities  []string
	State         string
}

// MetadataSnapshot returns the tunnel's mutable metadata under one read lock, so readers never race
// heartbeat, registration, or close writers.
func (t *AgentTunnel) MetadataSnapshot() TunnelMetadataSnapshot {
	if t == nil {
		return TunnelMetadataSnapshot{}
	}

	t.mu.RLock()
	defer t.mu.RUnlock()

	return TunnelMetadataSnapshot{
		ConnectedAt:   t.ConnectedAt,
		LastHeartbeat: t.LastHeartbeat,
		SessionID:     t.SessionID,
		AgentInstance: t.AgentInstance,
		SecurityMode:  t.SecurityMode,
		Capabilities:  append([]string(nil), t.Capabilities...),
		State:         t.State,
	}
}

// CloseWithReason closes the tunnel connection and records the disconnect
// reason. Pass "" when there is no reason to record.
func (t *AgentTunnel) CloseWithReason(reason string) error {
	if t == nil {
		return nil
	}
	t.mu.Lock()
	t.State = "closed"
	if reason != "" {
		t.DisconnectErr = reason
	}
	t.mu.Unlock()
	t.closeOnce.Do(func() {
		if t.done != nil {
			close(t.done)
		}
	})
	if t.mux != nil {
		t.mux.close()
	}
	if t.Conn == nil {
		return nil
	}
	return t.Conn.Close()
}

// Active returns the environment's tunnel when its connection is still open.
func (r *TunnelRegistry) Active(envID string) mo.Option[*AgentTunnel] {
	if tunnel, _ := r.Get(envID).Get(); tunnel.connected() {
		return mo.Some(tunnel)
	}
	return mo.None[*AgentTunnel]()
}

// WaitForActive polls until the environment has an open tunnel or timeout elapses.
func (r *TunnelRegistry) WaitForActive(ctx context.Context, envID string, timeout time.Duration) mo.Option[*AgentTunnel] {
	waitCtx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()
	ticker := time.NewTicker(DefaultTunnelAcquirePollEvery)
	defer ticker.Stop()
	for {
		if tunnel := r.Active(envID); tunnel.IsPresent() {
			return tunnel
		}
		select {
		case <-waitCtx.Done():
			return mo.None[*AgentTunnel]()
		case <-ticker.C:
		}
	}
}

// NewTunnelRegistry creates a new tunnel registry
func NewTunnelRegistry() *TunnelRegistry {
	return &TunnelRegistry{
		tunnels: concurrency.NewStateMap[string, *AgentTunnel](),
	}
}

// Get retrieves a tunnel by environment ID
func (r *TunnelRegistry) Get(envID string) mo.Option[*AgentTunnel] {
	tunnel, ok := r.tunnels.Get(envID)
	return mo.TupleToOption(tunnel, ok)
}

// Register adds a tunnel to the registry, closing any existing tunnel for the same env
func (r *TunnelRegistry) Register(envID string, tunnel *AgentTunnel) {
	ctx := context.Background() //nolint:forbidigo // Legacy registration path runs outside any request context.
	if r == nil || tunnel == nil {
		slog.ErrorContext(ctx, "Failed to register edge agent tunnel", "environmentId", envID, "error", "tunnel is required")
		return
	}
	previous, err := r.tunnels.ApplyTyped(ctx, "register edge tunnel", func(tunnels map[string]*AgentTunnel) (*AgentTunnel, bool, error) {
		if r.stopped.Load() {
			return nil, false, errors.New("edge tunnel registry stopped")
		}
		previous := tunnels[envID]
		tunnels[envID] = tunnel
		return previous, true, nil
	})
	if err != nil {
		slog.ErrorContext(ctx, "Failed to register edge agent tunnel", "environmentId", envID, "error", err)
		return
	}
	if previous != nil && previous != tunnel {
		slog.InfoContext(ctx, "Replacing existing edge tunnel")
		_ = previous.CloseWithReason("")
	}
	slog.InfoContext(ctx, "Edge agent tunnel registered")
}

type registerSessionResult struct {
	accepted      bool
	drainPrevious bool
	reason        string
	previous      *AgentTunnel
}

// RegisterSession adds a tunnel with duplicate-session handling.
func (r *TunnelRegistry) RegisterSession(ctx context.Context, tunnel *AgentTunnel, staleAfter time.Duration) (bool, bool, string, error) {
	if r == nil || tunnel == nil {
		return false, false, "tunnel is required", nil
	}

	envID := tunnel.EnvironmentID
	if envID == "" {
		return false, false, "environment ID is required", nil
	}
	if existing, ok := r.Get(envID).Get(); ok && existing != tunnel && existing.mux != nil && existing.connected() && !existing.mux.alive(ctx, muxRegistrationPingTimeout) {
		slog.InfoContext(ctx, "Existing edge tunnel session failed a liveness ping; replacing it", "environmentId", envID)
		_ = existing.CloseWithReason("edge tunnel liveness ping failed")
	}

	result, err := r.tunnels.ApplyTyped(ctx, "register edge tunnel session", func(tunnels map[string]*AgentTunnel) (registerSessionResult, bool, error) {
		var result registerSessionResult
		if r.stopped.Load() {
			return result, false, errors.New("edge tunnel registry stopped")
		}
		if existing := tunnels[envID]; existing != nil {
			if existing == tunnel {
				result.accepted = true
				return result, false, nil
			}

			existingMetadata := existing.MetadataSnapshot()
			tunnelMetadata := tunnel.MetadataSnapshot()
			existingStale := staleAfter > 0 && time.Since(existingMetadata.LastHeartbeat) > staleAfter
			sameAgentInstance := existingMetadata.AgentInstance != "" && tunnelMetadata.AgentInstance != "" && existingMetadata.AgentInstance == tunnelMetadata.AgentInstance
			if existing.connected() && !existingStale && !sameAgentInstance {
				result.reason = "another edge agent session is already active"
				return result, false, nil
			}
			result.previous = existing
			result.drainPrevious = sameAgentInstance
		}

		tunnels[envID] = tunnel
		result.accepted = true
		return result, true, nil
	})
	if err != nil {
		return false, false, "", err
	}
	if result.previous != nil {
		_ = result.previous.CloseWithReason("replaced by newer edge tunnel session")
	}
	metadata := tunnel.MetadataSnapshot()
	slog.InfoContext(ctx, "Edge agent tunnel registered", "environmentId", envID, "sessionId", metadata.SessionID, "securityMode", metadata.SecurityMode)
	return result.accepted, result.drainPrevious, result.reason, nil
}

// Unregister removes a tunnel from the registry
func (r *TunnelRegistry) Unregister(envID string) {
	ctx := context.Background() //nolint:forbidigo // Legacy unregistration path runs outside any request context.
	tunnel, removed, err := r.tunnels.Remove(ctx, "unregister edge tunnel", envID)
	if err != nil {
		slog.ErrorContext(ctx, "Failed to unregister edge agent tunnel", "environmentId", envID, "error", err)
		return
	}
	if removed {
		_ = tunnel.CloseWithReason("")
		slog.InfoContext(ctx, "Edge agent tunnel unregistered")
	}
}

// UnregisterCurrent removes the environment's tunnel only if it is still current. It reports whether it was
// removed and whether another live session already replaced it.
func (r *TunnelRegistry) UnregisterCurrent(ctx context.Context, envID string, current *AgentTunnel) (bool, bool) {
	if r == nil || current == nil {
		return false, false
	}

	type unregisterResult struct {
		removed           bool
		activeReplacement bool
	}
	result, err := r.tunnels.ApplyTyped(ctx, "unregister current edge tunnel", func(tunnels map[string]*AgentTunnel) (unregisterResult, bool, error) {
		var result unregisterResult
		existing := tunnels[envID]
		if existing != current {
			result.activeReplacement = existing.connected()
			return result, false, nil
		}
		delete(tunnels, envID)
		result.removed = true
		return result, true, nil
	})
	if err != nil {
		slog.ErrorContext(ctx, "Failed to unregister current edge agent tunnel", "environmentId", envID, "error", err)
		return false, false
	}
	if !result.removed {
		return false, result.activeReplacement
	}

	_ = current.CloseWithReason("")
	metadata := current.MetadataSnapshot()
	slog.InfoContext(ctx, "Edge agent tunnel unregistered", "environmentId", envID, "sessionId", metadata.SessionID)
	return true, false
}

// CleanupStale removes tunnels that haven't had a heartbeat within the given duration.
func (r *TunnelRegistry) CleanupStale(ctx context.Context, maxAge time.Duration) []*AgentTunnel {
	now := time.Now()
	removed, err := r.tunnels.RemoveWhere(ctx, "clean up stale edge tunnels", func(_ string, tunnel *AgentTunnel) bool {
		return tunnel == nil || now.Sub(tunnel.GetLastHeartbeat()) > maxAge
	})
	if err != nil {
		slog.ErrorContext(ctx, "Failed to clean up stale edge tunnels", "error", err)
		return nil
	}
	for _, tunnel := range removed {
		if tunnel == nil {
			continue
		}
		slog.WarnContext(ctx, "Removing stale edge tunnel", "lastHeartbeat", tunnel.GetLastHeartbeat())
		_ = tunnel.CloseWithReason("edge tunnel heartbeat expired")
	}

	return removed
}

// Stop drains registry mutations and closes active tunnels.
func (r *TunnelRegistry) Stop(ctx context.Context) error {
	r.stopped.Store(true)
	fallback := r.tunnels.Values()
	removed, err := r.tunnels.Drain(ctx, "stop edge tunnel registry")
	if err != nil {
		removed = fallback
	}
	for _, tunnel := range removed {
		if tunnel != nil {
			err = errors.Join(err, tunnel.CloseWithReason("edge tunnel registry stopped"))
		}
	}
	return err
}

// ObserveAgents reports connected edge agents by transport through the arcane.edge.agents gauge.
// Poll-mode agents count as "poll" while their check-ins are fresh and no tunnel is open.
func (r *TunnelRegistry) ObserveAgents(meter metric.Meter) (metric.Registration, error) {
	gauge, err := meter.Int64ObservableGauge("arcane.edge.agents",
		metric.WithDescription("Connected edge agents by transport"),
		metric.WithUnit("{agent}"),
	)
	if err != nil {
		return nil, err
	}
	return meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
		counts := map[string]int64{EdgeTransportWebSocket: 0, EdgeTransportGRPC: 0, EdgeTransportPoll: 0}
		tunneled := make(map[string]bool)
		for _, tunnel := range r.tunnels.Values() {
			if tunnel.connected() {
				counts[tunnel.Conn.Transport()]++
				tunneled[tunnel.EnvironmentID] = true
			}
		}
		polls, now := GetPollRuntimeRegistry(), time.Now()
		polls.mu.RLock()
		for envID, state := range polls.states {
			if !tunneled[envID] && state.LastPollAt != nil && now.Sub(*state.LastPollAt) <= pollRuntimeTTL(state) {
				counts[EdgeTransportPoll]++
			}
		}
		polls.mu.RUnlock()
		for transport, count := range counts {
			o.ObserveInt64(gauge, count, metric.WithAttributes(attribute.String("arcane.edge.transport", transport)))
		}
		return nil
	}, gauge)
}

var (
	defaultRegistryMu sync.RWMutex
	defaultRegistry   = NewTunnelRegistry()
)

// GetRegistry returns the global tunnel registry
func GetRegistry() *TunnelRegistry {
	defaultRegistryMu.RLock()
	registry := defaultRegistry
	defaultRegistryMu.RUnlock()
	return registry
}

// SetDefaultRegistry replaces the process-wide default tunnel registry.
func SetDefaultRegistry(registry *TunnelRegistry) {
	if registry == nil {
		registry = NewTunnelRegistry()
	}

	defaultRegistryMu.Lock()
	defaultRegistry = registry
	defaultRegistryMu.Unlock()
}

// ClearDefaultRegistry replaces registry only when it is still the active default.
func ClearDefaultRegistry(registry *TunnelRegistry) {
	defaultRegistryMu.Lock()
	if defaultRegistry == registry {
		defaultRegistry = NewTunnelRegistry()
	}
	defaultRegistryMu.Unlock()
}