Skip to content

Modern Docker Management, Designed for Everyone

9.6 KB
Raw Blame
package ws

import (
	"cmp"
	"context"
	"encoding/json/v2"
	"fmt"
	"log/slog"
	"net/http"
	"net/http/pprof"
	"time"

	"github.com/coder/websocket"
	systemtypes "github.com/getarcaneapp/arcane/types/v2/system"
	"github.com/labstack/echo/v5"
	"go.getarcane.app/streams/logs"

	"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
	"github.com/getarcaneapp/arcane/backend/v2/internal/diagnostics"
	"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
	wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"
)

const (
	// diagnosticsStreamInterval is how often the live diagnostics stream pushes a snapshot.
	diagnosticsStreamInterval = 2 * time.Second
	// actorDiagnosticsStreamInterval is slower because each push queries every environment.
	actorDiagnosticsStreamInterval = 10 * time.Second

	// Keepalive/liveness bounds for the diagnostics sockets, mirroring the system
	// stats stream. Without them a silently dead peer wedges the writer forever,
	// which also strands the writer's deferred cleanup (log subscription, conn).
	diagnosticsReadLimit  = 512
	diagnosticsWriteWait  = 10 * time.Second
	diagnosticsPingWait   = 10 * time.Second
	diagnosticsPingPeriod = 54 * time.Second
)

// buildDiagnostics assembles a snapshot: runtime/memory/GC from the
// DiagnosticsService plus this package's WebSocket metrics and worker-goroutine count.
func buildDiagnostics(diag *diagnostics.DiagnosticsService) systemtypes.Diagnostics {
	d := systemtypes.Diagnostics{Timestamp: time.Now().UTC()}
	if diag != nil {
		d.Runtime, d.Memory, d.GC = diag.Collect()
	}
	d.Runtime.WSWorkerGoroutines = wshub.CountWorkerGoroutines()
	d.WebSocket = systemtypes.WebSocketDiagnostics{
		Snapshot:    defaultWebSocketMetrics.Snapshot(),
		Connections: defaultWebSocketMetrics.Connections(),
	}
	return d
}

// registerDiagnosticsRoutes wires the global diagnostics WebSocket streams and the
// net/http/pprof debug endpoints. Streams require the diagnostics permission;
// pprof keeps the stricter admin-required gate. Called from NewWebSocketHandler.
func (h *WebSocketHandler) registerDiagnosticsRoutes(group *echo.Group, authMiddleware *auth.AuthMiddleware) {
	diag := group.Group("/diagnostics",
		authMiddleware.WithAdminNotRequired().Add(),
		middleware.RequireEchoPermission(authz.PermDiagnosticsRead),
	)
	diag.GET("/stream", h.DiagnosticsStream)
	diag.GET("/actors/stream", h.ActorDiagnosticsStream)
	diag.GET("/logs/stream", h.ServerLogsStream)

	pprofGroup := group.Group("/debug/pprof", authMiddleware.WithAdminRequired().Add())
	pprofGroup.GET("", echo.WrapHandler(http.HandlerFunc(pprof.Index)))
	pprofGroup.GET("/", echo.WrapHandler(http.HandlerFunc(pprof.Index)))
	pprofGroup.GET("/cmdline", echo.WrapHandler(http.HandlerFunc(pprof.Cmdline)))
	pprofGroup.GET("/profile", echo.WrapHandler(http.HandlerFunc(pprof.Profile)))
	pprofGroup.POST("/symbol", echo.WrapHandler(http.HandlerFunc(pprof.Symbol)))
	pprofGroup.GET("/symbol", echo.WrapHandler(http.HandlerFunc(pprof.Symbol)))
	pprofGroup.GET("/trace", echo.WrapHandler(http.HandlerFunc(pprof.Trace)))
	// pprof.Index only serves a named profile when the request path starts with the
	// literal "/debug/pprof/". This group is mounted under /api, so that never
	// matches and every named profile would render the index page instead.
	pprofGroup.GET("/:name", func(c *echo.Context) error {
		pprof.Handler(c.Param("name")).ServeHTTP(c.Response(), c.Request())
		return nil
	})
}

// DiagnosticsStream pushes a snapshot on connect and every diagnosticsStreamInterval,
// and answers refresh, dump, leak-scan, and profile commands from the client.
func (h *WebSocketHandler) DiagnosticsStream(c *echo.Context) error {
	ps, _ := c.Get(string(middleware.ContextKeyUserPermissions)).(*authz.PermissionSet)
	snapshot := func(context.Context) any {
		d := buildDiagnostics(h.diagnosticsService)
		return systemtypes.DiagnosticsMessage{Type: "snapshot", Snapshot: &d}
	}
	return h.streamSnapshots(c, diagnosticsStreamInterval, snapshot, func(ctx context.Context, cmd systemtypes.DiagnosticsCommand) any {
		if cmd.Type == "refresh" {
			return snapshot(ctx)
		}
		result := systemtypes.DiagnosticsMessage{Type: "result", ID: cmd.ID}
		var err error
		switch cmd.Type {
		case "leakScan":
			report, scanErr := h.diagnosticsService.ScanGoroutineLeaks()
			result.LeakReport, err = &report, scanErr
		case "dump", "profile":
			if !ps.IsGlobalAdmin() {
				result.Error = "profiles require a global administrator"
				return result
			}
			if cmd.Type == "dump" {
				var text []byte
				text, err = h.diagnosticsService.Profile(ctx, cmd.Name, 0, 2)
				result.Text = string(text)
			} else {
				result.Data, err = h.diagnosticsService.Profile(ctx, cmd.Name, min(cmp.Or(cmd.Seconds, 30), 120), 0)
			}
		default:
			err = fmt.Errorf("unknown diagnostics command %q", cmd.Type)
		}
		if err != nil {
			result.Error = err.Error()
		}
		return result
	})
}

// ActorDiagnosticsStream pushes actor diagnostics for every environment on
// connect and then every actorDiagnosticsStreamInterval.
func (h *WebSocketHandler) ActorDiagnosticsStream(c *echo.Context) error {
	return h.streamSnapshots(c, actorDiagnosticsStreamInterval, func(ctx context.Context) any {
		return h.diagnosticsService.CollectAllActors(ctx)
	}, nil)
}

// streamSnapshots writes build's result on connect and every interval until
// the client disconnects. A non-nil handle answers client commands concurrently.
func (h *WebSocketHandler) streamSnapshots(c *echo.Context, interval time.Duration, build func(context.Context) any, handle func(context.Context, systemtypes.DiagnosticsCommand) any) error {
	conn, unregister, accepted := h.acceptWS(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)
	if !accepted {
		return nil
	}
	defer unregister()
	defer func() {
		if closeNowErr := conn.CloseNow(); closeNowErr != nil {
			slog.DebugContext(c.Request().Context(), "Failed to close diagnostics websocket connection", "error", closeNowErr)
		}
	}()

	ctx := c.Request().Context()
	write := func(v any) bool {
		b, marshalErr := json.Marshal(v)
		if marshalErr != nil {
			return true
		}
		wctx, cancel := context.WithTimeout(ctx, diagnosticsWriteWait)
		defer cancel()
		return conn.Write(wctx, websocket.MessageText, b) == nil
	}
	var onMessage func([]byte)
	if handle != nil {
		commands := make(chan systemtypes.DiagnosticsCommand, 1)
		go func() {
			for {
				select {
				case <-ctx.Done():
					return
				case cmd := <-commands:
					write(handle(ctx, cmd))
				}
			}
		}()
		onMessage = func(b []byte) {
			var cmd systemtypes.DiagnosticsCommand
			if json.Unmarshal(b, &cmd) != nil {
				return
			}
			select {
			case commands <- cmd:
			default:
				write(systemtypes.DiagnosticsMessage{
					Type:  "result",
					ID:    cmd.ID,
					Error: "busy",
				})
			}
		}
	}
	done := diagnosticsReadLoop(ctx, conn, onMessage)

	if !write(build(ctx)) {
		return nil
	}
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	pingTicker := time.NewTicker(diagnosticsPingPeriod)
	defer pingTicker.Stop()
	for {
		select {
		case <-done:
			return nil
		case <-ticker.C:
			if !write(build(ctx)) {
				return nil
			}
		case <-pingTicker.C:
			if !pingDiagnosticsConn(ctx, conn) {
				return nil
			}
		}
	}
}

// ServerLogsStream replays the recent backend log backlog then streams new entries live.
func (h *WebSocketHandler) ServerLogsStream(c *echo.Context) error {
	conn, unregister, accepted := h.acceptWS(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)
	if !accepted {
		return nil
	}
	defer unregister()
	defer func() {
		if closeNowErr := conn.CloseNow(); closeNowErr != nil {
			slog.DebugContext(c.Request().Context(), "Failed to close server logs websocket connection", "error", closeNowErr)
		}
	}()

	// Subscribe before replaying the backlog so no entry is missed in the gap; at
	// worst the newest backlog entry is delivered twice, which is harmless.
	ch, cancel := defaultLogBroadcaster.Subscribe()
	defer cancel()

	ctx := c.Request().Context()
	done := diagnosticsReadLoop(ctx, conn, nil)
	write := func(e logs.Entry) bool {
		b, marshalErr := json.Marshal(e)
		if marshalErr != nil {
			return true
		}
		wctx, wcancel := context.WithTimeout(ctx, diagnosticsWriteWait)
		defer wcancel()
		return conn.Write(wctx, websocket.MessageText, b) == nil
	}

	for _, e := range defaultLogBroadcaster.Recent() {
		if !write(e) {
			return nil
		}
	}
	pingTicker := time.NewTicker(diagnosticsPingPeriod)
	defer pingTicker.Stop()
	for {
		select {
		case <-done:
			return nil
		case e, ok := <-ch:
			if !ok {
				return nil
			}
			if !write(e) {
				return nil
			}
		case <-pingTicker.C:
			if !pingDiagnosticsConn(ctx, conn) {
				return nil
			}
		}
	}
}

// pingDiagnosticsConn reports whether the peer answered a ping in time.
// The pong is serviced by the diagnosticsReadLoop reader, so a peer
// that stops answering fails here within diagnosticsPingWait.
func pingDiagnosticsConn(ctx context.Context, conn *websocket.Conn) bool {
	pctx, cancel := context.WithTimeout(ctx, diagnosticsPingWait)
	defer cancel()
	return conn.Ping(pctx) == nil
}

// diagnosticsReadLoop passes incoming frames to onMessage, or drains them when
// it is nil; the returned channel closes when the peer disconnects.
func diagnosticsReadLoop(ctx context.Context, conn *websocket.Conn, onMessage func([]byte)) <-chan struct{} {
	conn.SetReadLimit(diagnosticsReadLimit)

	done := make(chan struct{})
	go func() {
		defer close(done)
		for {
			_, b, err := conn.Read(ctx)
			if err != nil {
				return
			}
			if onMessage != nil {
				onMessage(b)
			}
		}
	}()
	return done
}

var defaultLogBroadcaster = logs.New(1000)

// LogBroadcaster returns the backend-wide log broadcaster used by diagnostics.
func LogBroadcaster() *logs.Broadcaster {
	return defaultLogBroadcaster
}