Skip to content

Modern Docker Management, Designed for Everyone

13.8 KB
Raw Blame
package diagnostics

import (
	"bytes"
	"cmp"
	"context"
	"errors"
	"fmt"
	"log/slog"
	"net/http"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"runtime/trace"
	"slices"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/getarcaneapp/arcane/types/v2/system"
	kit "go.getarcane.app/kit/pkg"
	"gorm.io/gorm"

	"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
)

const (
	recentGCPauseSamples     = 16
	goroutineLeakProfileName = "goroutineleak"
	leakScanCacheTTL         = 30 * time.Second

	// maxActorDetailRowsInternal caps the instances and failures listed per actor type.
	maxActorDetailRowsInternal = 10
)

// DiagnosticsService gathers Go runtime, memory, and garbage-collector
// statistics for the diagnostics endpoints. It holds no external dependencies;
// WebSocket metrics and worker-goroutine counts are merged in at the handler
// layer to avoid an import cycle with the api/ws package.
type DiagnosticsService struct {
	startedAt    time.Time
	actors       *francis.Runtime
	coordinator  *runs.Coordinator
	environments *environment.EnvironmentService

	leakMu     sync.Mutex
	leakReport system.GoroutineLeakReport
}

func NewDiagnosticsService(actors *francis.Runtime, coordinator *runs.Coordinator, environments *environment.EnvironmentService) *DiagnosticsService {
	return &DiagnosticsService{startedAt: time.Now(), actors: actors, coordinator: coordinator, environments: environments}
}

func (s *DiagnosticsService) Collect() (system.RuntimeInfo, system.MemoryInfo, system.GCInfo) {
	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)

	var gc debug.GCStats
	debug.ReadGCStats(&gc)

	s.leakMu.Lock()
	leakScannedAt := s.leakReport.ScannedAt
	s.leakMu.Unlock()

	rt := system.RuntimeInfo{
		Goroutines:       runtime.NumGoroutine(),
		LeakedGoroutines: leakedGoroutineCountInternal(),
		LeakScannedAt:    leakScannedAt,
		GOMAXPROCS:       runtime.GOMAXPROCS(0),
		NumCPU:           runtime.NumCPU(),
		GoVersion:        runtime.Version(),
		OS:               runtime.GOOS,
		Arch:             runtime.GOARCH,
		NumCgoCall:       runtime.NumCgoCall(),
		UptimeSeconds:    int64(time.Since(s.startedAt).Seconds()),
	}

	mi := system.MemoryInfo{
		Alloc:         mem.Alloc,
		TotalAlloc:    mem.TotalAlloc,
		Sys:           mem.Sys,
		HeapAlloc:     mem.HeapAlloc,
		HeapSys:       mem.HeapSys,
		HeapInuse:     mem.HeapInuse,
		HeapIdle:      mem.HeapIdle,
		HeapReleased:  mem.HeapReleased,
		HeapObjects:   mem.HeapObjects,
		StackInuse:    mem.StackInuse,
		StackSys:      mem.StackSys,
		MSpanInuse:    mem.MSpanInuse,
		MCacheInuse:   mem.MCacheInuse,
		NextGC:        mem.NextGC,
		NumGC:         mem.NumGC,
		NumForcedGC:   mem.NumForcedGC,
		GCCPUFraction: mem.GCCPUFraction,
	}

	// gc.Pause is ordered most-recent-first; cap the slice we expose.
	pauses := gc.Pause
	if len(pauses) > recentGCPauseSamples {
		pauses = pauses[:recentGCPauseSamples]
	}
	recent := make([]int64, len(pauses))
	for i, p := range pauses {
		recent[i] = p.Nanoseconds()
	}

	gi := system.GCInfo{
		LastGC:         gc.LastGC,
		NumGC:          gc.NumGC,
		PauseTotalNs:   gc.PauseTotal.Nanoseconds(),
		RecentPausesNs: recent,
	}

	return rt, mi, gi
}

// ScanGoroutineLeaks runs a leak-detection GC and returns the leaked-goroutine
// pprof text (debug=1). debug=2 would dump every goroutine, not only leaks.
func (s *DiagnosticsService) ScanGoroutineLeaks() (system.GoroutineLeakReport, error) {
	s.leakMu.Lock()
	defer s.leakMu.Unlock()
	if time.Since(s.leakReport.ScannedAt) < leakScanCacheTTL {
		return s.leakReport, nil
	}

	p := pprof.Lookup(goroutineLeakProfileName)
	if p == nil {
		return system.GoroutineLeakReport{}, errors.New("goroutineleak profile is not available")
	}

	var buf bytes.Buffer
	if err := p.WriteTo(&buf, 1); err != nil {
		return system.GoroutineLeakReport{}, fmt.Errorf("write goroutineleak profile: %w", err)
	}

	s.leakReport = system.GoroutineLeakReport{
		Count:     p.Count(),
		Profile:   buf.String(),
		ScannedAt: time.Now().UTC(),
	}
	return s.leakReport, nil
}

// Goroutines groups the live goroutines by identical stack, largest group first.
// Records that differ only by pprof labels share a stack and merge into one group.
func (s *DiagnosticsService) Goroutines() (system.GoroutineProfile, error) {
	var buf bytes.Buffer
	if err := pprof.Lookup("goroutine").WriteTo(&buf, 1); err != nil {
		return system.GoroutineProfile{}, fmt.Errorf("write goroutine profile: %w", err)
	}

	profile := system.GoroutineProfile{Groups: []system.GoroutineGroup{}, CapturedAt: time.Now().UTC()}
	index := map[string]int{}
	// debug=1 text holds "N @ pcs" records of "#\tpc\tfunc+off\tfile:line" frames, separated by blank lines.
	// A tabwriter aligns the columns, so a run of tabs is one separator.
	for record := range strings.SplitSeq(buf.String(), "\n\n") {
		group := system.GoroutineGroup{Frames: []system.GoroutineFrame{}}
		for line := range strings.Lines(record) {
			fields := strings.FieldsFunc(line, func(r rune) bool { return r == '\t' || r == '\n' })
			if len(fields) == 0 {
				continue
			}
			switch head, _, isHead := strings.Cut(fields[0], " @"); {
			case isHead && !strings.HasPrefix(head, "#"):
				group.Count, _ = strconv.Atoi(head)
			case len(fields) == 4 && fields[0] == "#":
				function, _, _ := strings.Cut(fields[2], "+0x")
				sep := strings.LastIndexByte(fields[3], ':')
				lineNo, err := strconv.Atoi(fields[3][sep+1:])
				hasLine := sep >= 0 && err == nil
				group.Frames = append(group.Frames, system.GoroutineFrame{
					Function: function,
					File:     kit.Ternary(hasLine, fields[3][:max(sep, 0)], fields[3]),
					Line:     kit.Ternary(hasLine, lineNo, 0),
				})
				// Frames run innermost first, so the last non-runtime frame is the entry point.
				group.Label = kit.Ternary(group.Label == "" || !strings.HasPrefix(function, "runtime."), function, group.Label)
			}
		}
		if group.Count == 0 {
			continue
		}
		profile.Total += group.Count
		key := fmt.Sprint(group.Frames)
		if i, ok := index[key]; ok {
			profile.Groups[i].Count += group.Count
			continue
		}
		index[key] = len(profile.Groups)
		profile.Groups = append(profile.Groups, group)
	}
	slices.SortStableFunc(profile.Groups, func(a, b system.GoroutineGroup) int { return cmp.Compare(b.Count, a.Count) })
	return profile, nil
}

// Profile captures a pprof profile in the given debug format. The CPU profile
// and execution trace sample for seconds or until ctx ends.
func (s *DiagnosticsService) Profile(ctx context.Context, name string, seconds, debugLevel int) ([]byte, error) {
	var buf bytes.Buffer
	sample := func() {
		timer := time.NewTimer(time.Duration(seconds) * time.Second)
		defer timer.Stop()
		select {
		case <-timer.C:
		case <-ctx.Done():
		}
	}
	switch name {
	case "profile":
		if err := pprof.StartCPUProfile(&buf); err != nil {
			return nil, fmt.Errorf("start CPU profile: %w", err)
		}
		sample()
		pprof.StopCPUProfile()
	case "trace":
		if err := trace.Start(&buf); err != nil {
			return nil, fmt.Errorf("start trace: %w", err)
		}
		sample()
		trace.Stop()
	default:
		p := pprof.Lookup(name)
		if p == nil {
			return nil, fmt.Errorf("unknown profile %q", name)
		}
		if err := p.WriteTo(&buf, debugLevel); err != nil {
			return nil, fmt.Errorf("write %s profile: %w", name, err)
		}
	}
	return buf.Bytes(), ctx.Err()
}

// leakedGoroutineCountInternal returns the last-known leak count without
// triggering a leak-detection GC.
func leakedGoroutineCountInternal() int {
	p := pprof.Lookup(goroutineLeakProfileName)
	if p == nil {
		return 0
	}
	return p.Count()
}

// CollectActors reports this process's Francis host, actor types, and run queue.
func (s *DiagnosticsService) CollectActors(ctx context.Context) (system.ActorDiagnostics, error) {
	d := system.ActorDiagnostics{EnvironmentID: environment.LocalEnvironmentID, Runs: map[string]int{}, CheckedAt: time.Now().UTC()}
	select {
	case <-s.actors.Ready():
		d.Ready = true
	default:
		return d, nil
	}

	store, err := s.actors.Store()
	if err != nil {
		return d, err
	}
	db := store.WithContext(ctx)
	var hosts []actorHostModel
	if err = db.Order("host_id").Find(&hosts).Error; err != nil {
		return d, fmt.Errorf("list actor hosts: %w", err)
	}
	for _, host := range hosts {
		d.Hosts = append(d.Hosts, system.ActorHostInfo{HostID: host.HostID, Address: host.Address, LastHealthCheck: time.Time(host.LastHealthCheck)})
	}

	// Records names coordinator actors, so load it before resolving instance names.
	records, err := s.coordinator.Records(ctx)
	if err != nil {
		return d, err
	}
	for _, record := range records {
		for _, run := range record.Runs {
			d.Runs[string(run.Status)]++
		}
	}

	d.ActorTypes, err = s.collectActorTypesInternal(db)
	return d, err
}

// collectActorTypesInternal reports counts, recent instances, and dead jobs per registered actor type.
func (s *DiagnosticsService) collectActorTypesInternal(db *gorm.DB) ([]system.ActorTypeInfo, error) {
	var actorTypes []actorTypeModel
	err := db.Table(francis.TablePrefix + "_host_actor_types").
		Select("actor_type, MAX(actor_concurrency_limit) AS concurrency_limit").
		Group("actor_type").Order("actor_type").Scan(&actorTypes).Error
	if err != nil {
		return nil, fmt.Errorf("list actor types: %w", err)
	}
	var active, states, alarms, leased, terminal []actorCountModel
	var instances []activeActorModel
	var failures []terminalJobModel
	err = errors.Join(
		db.Table(francis.TablePrefix+"_active_actors").Select("actor_type, COUNT(*) AS count").Group("actor_type").Scan(&active).Error,
		db.Table(francis.TablePrefix+"_actor_state").Select("actor_type, COUNT(*) AS count").Group("actor_type").Scan(&states).Error,
		db.Table(francis.TablePrefix+"_alarms").Select("actor_type, alarm_kind AS kind, COUNT(*) AS count, MIN(alarm_due_time) AS next_due").Group("actor_type, alarm_kind").Scan(&alarms).Error,
		db.Table(francis.TablePrefix+"_alarms").Select("actor_type, COUNT(*) AS count").Where("alarm_lease_id IS NOT NULL").Group("actor_type").Scan(&leased).Error,
		db.Table(francis.TablePrefix+"_terminal_jobs").Select("actor_type, job_status AS kind, COUNT(*) AS count").Group("actor_type, job_status").Scan(&terminal).Error,
		db.Order("actor_activation DESC").Limit(500).Find(&instances).Error,
		db.Where("job_status = ?", "dead").Order("ended_at DESC").Limit(100).Find(&failures).Error,
	)
	if err != nil {
		return nil, fmt.Errorf("collect actor activity: %w", err)
	}

	out := make([]system.ActorTypeInfo, len(actorTypes))
	byType := make(map[string]*system.ActorTypeInfo, len(actorTypes))
	for i, actorType := range actorTypes {
		out[i] = system.ActorTypeInfo{ActorType: actorType.ActorType, ConcurrencyLimit: actorType.ConcurrencyLimit, Instances: []system.ActorInstance{}, RecentFailures: []system.ActorJobFailure{}}
		byType[actorType.ActorType] = &out[i]
	}
	addCountsInternal(byType, active, func(info *system.ActorTypeInfo, row actorCountModel) { info.ActiveActors = row.Count })
	addCountsInternal(byType, states, func(info *system.ActorTypeInfo, row actorCountModel) { info.StoredStates = row.Count })
	addCountsInternal(byType, leased, func(info *system.ActorTypeInfo, row actorCountModel) { info.LeasedAlarms = row.Count })
	addCountsInternal(byType, alarms, func(info *system.ActorTypeInfo, row actorCountModel) {
		if row.Kind == "job" {
			info.PendingJobs += row.Count
		} else {
			info.PendingAlarms += row.Count
		}
		if due := time.Time(row.NextDue); !due.IsZero() && (info.NextAlarmAt.IsZero() || due.Before(info.NextAlarmAt)) {
			info.NextAlarmAt = due
		}
	})
	addCountsInternal(byType, terminal, func(info *system.ActorTypeInfo, row actorCountModel) {
		switch row.Kind {
		case "dead":
			info.DeadJobs += row.Count
		case "completed":
			info.CompletedJobs += row.Count
		}
	})
	for _, row := range instances {
		if info := byType[row.ActorType]; info != nil && len(info.Instances) < maxActorDetailRowsInternal {
			info.Instances = append(info.Instances, system.ActorInstance{ActorID: row.ActorID, Name: s.actors.ActorName(row.ActorID), HostID: row.HostID, ActivatedAt: time.Time(row.Activation)})
		}
	}
	for _, row := range failures {
		if info := byType[row.ActorType]; info != nil && len(info.RecentFailures) < maxActorDetailRowsInternal {
			info.RecentFailures = append(info.RecentFailures, system.ActorJobFailure{
				JobID: row.JobID, ActorID: row.ActorID, Name: s.actors.ActorName(row.ActorID), Method: row.JobMethod,
				Attempts: row.Attempts, LastError: kit.FromPtr(row.LastError), FailedAt: time.Time(row.EndedAt),
			})
		}
	}
	return out, nil
}

// addCountsInternal applies grouped rows to their registered actor type.
func addCountsInternal(byType map[string]*system.ActorTypeInfo, rows []actorCountModel, apply func(*system.ActorTypeInfo, actorCountModel)) {
	for _, row := range rows {
		if info := byType[row.ActorType]; info != nil {
			apply(info, row)
		}
	}
}

// CollectAllActors gathers actor diagnostics from the local and every active remote environment.
func (s *DiagnosticsService) CollectAllActors(ctx context.Context) []system.ActorDiagnostics {
	local, err := s.CollectActors(ctx)
	if err != nil {
		local.Error = err.Error()
	}
	if env, envErr := s.environments.GetEnvironmentByIDCached(ctx, environment.LocalEnvironmentID); envErr == nil {
		local.EnvironmentName = env.Name
	}

	remotes, err := s.environments.ListActiveRemoteEnvironments(ctx)
	if err != nil {
		slog.WarnContext(ctx, "Failed to list environments for actor diagnostics", "error", err)
	}
	out := make([]system.ActorDiagnostics, len(remotes)+1)
	out[0] = local
	var wg sync.WaitGroup
	for i, env := range remotes {
		wg.Go(func() {
			var d system.ActorDiagnostics
			if proxyErr := s.environments.ProxyJSONRequestForEnvironment(ctx, env, http.MethodGet, "/api/environments/0/diagnostics/actors", nil, &d); proxyErr != nil {
				d = system.ActorDiagnostics{Error: proxyErr.Error(), CheckedAt: time.Now().UTC()}
			}
			d.EnvironmentID, d.EnvironmentName = env.ID, env.Name
			out[i+1] = d
		})
	}
	wg.Wait()
	return out
}