Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
feat: add actor statuses to diagnostics page
Kyle Mendell01a8964d69ba
diff --git a/backend/api/handlers/diagnostics.go b/backend/api/handlers/diagnostics.goindex d2d93667f..43b4feb78 100644--- a/backend/api/handlers/diagnostics.go+++ b/backend/api/handlers/diagnostics.go@@ -6,88 +6,46 @@ import ("github.com/danielgtaylor/huma/v2""github.com/getarcaneapp/arcane/types/v2/system"- "go.getarcane.app/streams/logs"- "github.com/getarcaneapp/arcane/backend/v2/api/ws""github.com/getarcaneapp/arcane/backend/v2/internal/diagnostics""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil")-// DiagnosticsHandler serves the REST diagnostics endpoints. The live WebSocket-// streams and pprof routes live in the api/ws package alongside the other-// streaming endpoints; the snapshot is assembled there too (ws.BuildDiagnostics).+// DiagnosticsHandler serves the environment-scoped actor diagnostics the manager+// collects from each agent. The diagnostics page itself uses the api/ws streams.type DiagnosticsHandler struct {diag *diagnostics.DiagnosticsService}-type DiagnosticsInput struct{}--type GetDiagnosticsOutput struct {- Body system.Diagnostics-}--type GetDiagnosticsLogsOutput struct {- Body []logs.Entry+type GetEnvironmentActorDiagnosticsInput struct {+ EnvironmentID string `path:"id" doc:"Environment ID"`}-type ScanGoroutineLeaksInput struct{}--type ScanGoroutineLeaksOutput struct {- Body system.GoroutineLeakReport+type GetEnvironmentActorDiagnosticsOutput struct {+ Body system.ActorDiagnostics}// RegisterDiagnostics registers the Huma diagnostics REST endpoints.func RegisterDiagnostics(api huma.API, diag *diagnostics.DiagnosticsService) {h := &DiagnosticsHandler{diag: diag}- huma.Register(api, huma.Operation{- OperationID: "get-diagnostics",- Method: http.MethodGet,- Path: "/diagnostics",- Summary: "Get runtime diagnostics",- Description: "Returns Go runtime, memory, garbage-collector, and WebSocket connection statistics.",- Tags: []string{"Diagnostics"},- Security: handlerutil.DefaultOperationSecurity(),- Middlewares: middleware.RequirePermission(api, authz.PermDiagnosticsRead),- }, h.GetDiagnostics)-- huma.Register(api, huma.Operation{- OperationID: "get-diagnostics-logs",+ middleware.RegisterWithPermission(api, huma.Operation{+ OperationID: "get-environment-actor-diagnostics",Method: http.MethodGet,- Path: "/diagnostics/logs",- Summary: "Get recent backend logs",- Description: "Returns the most recent buffered backend log entries (oldest first).",+ Path: "/environments/{id}/diagnostics/actors",+ Summary: "Get actor diagnostics for an environment",+ Description: "Returns Francis actor host, actor type, and run queue diagnostics for one environment.",Tags: []string{"Diagnostics"},Security: handlerutil.DefaultOperationSecurity(),- Middlewares: middleware.RequirePermission(api, authz.PermDiagnosticsRead),- }, h.GetRecentLogs)-- huma.Register(api, huma.Operation{- OperationID: "scan-goroutine-leaks",- Method: http.MethodPost,- Path: "/diagnostics/goroutineleak",- Summary: "Scan for leaked goroutines",- Description: "Runs a goroutine leak-detection GC cycle and returns the goroutineleak pprof profile as text (leaked stacks only).",- Tags: []string{"Diagnostics"},- Security: handlerutil.DefaultOperationSecurity(),- Middlewares: middleware.RequirePermission(api, authz.PermDiagnosticsRead),- }, h.ScanGoroutineLeaks)-}--func (h *DiagnosticsHandler) GetDiagnostics(_ context.Context, _ *DiagnosticsInput) (*GetDiagnosticsOutput, error) {- return &GetDiagnosticsOutput{Body: ws.BuildDiagnostics(h.diag)}, nil-}--func (h *DiagnosticsHandler) GetRecentLogs(_ context.Context, _ *DiagnosticsInput) (*GetDiagnosticsLogsOutput, error) {- return &GetDiagnosticsLogsOutput{Body: ws.LogBroadcaster().Recent()}, nil+ }, authz.PermDiagnosticsRead, h.GetEnvironmentActorDiagnostics)}-func (h *DiagnosticsHandler) ScanGoroutineLeaks(_ context.Context, _ *ScanGoroutineLeaksInput) (*ScanGoroutineLeaksOutput, error) {- report, err := h.diag.ScanGoroutineLeaks()+func (h *DiagnosticsHandler) GetEnvironmentActorDiagnostics(ctx context.Context, _ *GetEnvironmentActorDiagnosticsInput) (*GetEnvironmentActorDiagnosticsOutput, error) {+ d, err := h.diag.CollectActors(ctx)if err != nil {- return nil, huma.Error500InternalServerError("Failed to collect goroutine leak profile: " + err.Error())+ return nil, huma.Error500InternalServerError("Failed to collect actor diagnostics: " + err.Error())}- return &ScanGoroutineLeaksOutput{Body: report}, nil+ return &GetEnvironmentActorDiagnosticsOutput{Body: d}, nil}diff --git a/backend/api/ws/diagnostics.go b/backend/api/ws/diagnostics.goindex 51bd1e5d9..522ed96c6 100644--- a/backend/api/ws/diagnostics.go+++ b/backend/api/ws/diagnostics.go@@ -1,8 +1,10 @@package wsimport (+ "cmp""context""encoding/json/v2"+ "fmt""log/slog""net/http""net/http/pprof"@@ -23,6 +25,8 @@ import (const (// diagnosticsStreamInterval is how often the live diagnostics stream pushes a snapshot.diagnosticsStreamInterval = 2 * time.Second+ // actorDiagnosticsStreamInterval is slower because each push queries every environment.+ actorDiagnosticsStreamInterval = 10 * time.Second// Keepalive/liveness bounds for the diagnostics sockets, mirroring the system// stats stream. Without them a silently dead peer wedges the writer forever,@@ -33,10 +37,9 @@ const (diagnosticsPingPeriod = 54 * time.Second)-// BuildDiagnostics assembles a full diagnostics snapshot: runtime/memory/GC from-// the DiagnosticsService plus this package's WebSocket metrics and worker-goroutine-// count. Shared by the REST endpoint (via handlers) and the live WebSocket stream.-func BuildDiagnostics(diag *diagnostics.DiagnosticsService) systemtypes.Diagnostics {+// buildDiagnosticsInternal assembles a snapshot: runtime/memory/GC from the+// DiagnosticsService plus this package's WebSocket metrics and worker-goroutine count.+func buildDiagnosticsInternal(diag *diagnostics.DiagnosticsService) systemtypes.Diagnostics {d := systemtypes.Diagnostics{Timestamp: time.Now().UTC()}if diag != nil {d.Runtime, d.Memory, d.GC = diag.Collect()@@ -58,6 +61,7 @@ func (h *WebSocketHandler) registerDiagnosticsRoutesInternal(group *echo.Group,middleware.RequireEchoPermission(authz.PermDiagnosticsRead),)diag.GET("/stream", h.DiagnosticsStream)+ diag.GET("/actors/stream", h.ActorDiagnosticsStream)diag.GET("/logs/stream", h.ServerLogsStream)pprofGroup := group.Group("/debug/pprof", authMiddleware.WithAdminRequired().Add())@@ -77,13 +81,62 @@ func (h *WebSocketHandler) registerDiagnosticsRoutesInternal(group *echo.Group,})}-// DiagnosticsStream pushes a fresh diagnostics snapshot on connect and then every-// diagnosticsStreamInterval until the client disconnects.+// DiagnosticsStream pushes a snapshot on connect and every diagnosticsStreamInterval,+// and answers refresh, dump, leak-scan, and profile commands from the client.func (h *WebSocketHandler) DiagnosticsStream(c *echo.Context) error {- conn, err := wshub.Accept(c.Response(), c.Request(), h.checkWSOrigin)- if err != nil {+ ps, _ := c.Get(string(middleware.ContextKeyUserPermissions)).(*authz.PermissionSet)+ snapshot := func(context.Context) any {+ d := buildDiagnosticsInternal(h.diagnosticsService)+ return systemtypes.DiagnosticsMessage{Type: "snapshot", Snapshot: &d}+ }+ return h.streamSnapshotsInternal(c, diagnosticsStreamInterval, snapshot, func(ctx context.Context, cmd systemtypes.DiagnosticsCommand) any {+ if cmd.Type == "refresh" {+ return snapshot(ctx)+ }+ result := systemtypes.DiagnosticsMessage{Type: "result", ID: cmd.ID}+ var err error+ switch cmd.Type {+ case "leakScan":+ report, scanErr := h.diagnosticsService.ScanGoroutineLeaks()+ result.LeakReport, err = &report, scanErr+ case "dump", "profile":+ if !ps.IsGlobalAdmin() {+ result.Error = "profiles require a global administrator"+ return result+ }+ if cmd.Type == "dump" {+ var text []byte+ text, err = h.diagnosticsService.Profile(ctx, cmd.Name, 0, 2)+ result.Text = string(text)+ } else {+ result.Data, err = h.diagnosticsService.Profile(ctx, cmd.Name, min(cmp.Or(cmd.Seconds, 30), 120), 0)+ }+ default:+ err = fmt.Errorf("unknown diagnostics command %q", cmd.Type)+ }+ if err != nil {+ result.Error = err.Error()+ }+ return result+ })+}++// ActorDiagnosticsStream pushes actor diagnostics for every environment on+// connect and then every actorDiagnosticsStreamInterval.+func (h *WebSocketHandler) ActorDiagnosticsStream(c *echo.Context) error {+ return h.streamSnapshotsInternal(c, actorDiagnosticsStreamInterval, func(ctx context.Context) any {+ return h.diagnosticsService.CollectAllActors(ctx)+ }, nil)+}++// streamSnapshotsInternal writes build's result on connect and every interval until+// the client disconnects. A non-nil handle answers client commands concurrently.+func (h *WebSocketHandler) streamSnapshotsInternal(c *echo.Context, interval time.Duration, build func(context.Context) any, handle func(context.Context, systemtypes.DiagnosticsCommand) any) error {+ conn, unregister, accepted := h.acceptWSInternal(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)+ if !accepted {return nil}+ defer unregister()defer func() {if closeNowErr := conn.CloseNow(); closeNowErr != nil {slog.Debug("Failed to close diagnostics websocket connection", "error", closeNowErr)@@ -91,9 +144,8 @@ func (h *WebSocketHandler) DiagnosticsStream(c *echo.Context) error {}()ctx := c.Request().Context()- done := diagnosticsReadLoopInternal(ctx, conn)- write := func() bool {- b, marshalErr := json.Marshal(BuildDiagnostics(h.diagnosticsService))+ write := func(v any) bool {+ b, marshalErr := json.Marshal(v)if marshalErr != nil {return true}@@ -101,11 +153,22 @@ func (h *WebSocketHandler) DiagnosticsStream(c *echo.Context) error {defer cancel()return conn.Write(wctx, websocket.MessageText, b) == nil}+ var onMessage func([]byte)+ if handle != nil {+ onMessage = func(b []byte) {+ var cmd systemtypes.DiagnosticsCommand+ if json.Unmarshal(b, &cmd) != nil {+ return+ }+ go write(handle(ctx, cmd))+ }+ }+ done := diagnosticsReadLoopInternal(ctx, conn, onMessage)- if !write() {+ if !write(build(ctx)) {return nil}- ticker := time.NewTicker(diagnosticsStreamInterval)+ ticker := time.NewTicker(interval)defer ticker.Stop()pingTicker := time.NewTicker(diagnosticsPingPeriod)defer pingTicker.Stop()@@ -114,7 +177,7 @@ func (h *WebSocketHandler) DiagnosticsStream(c *echo.Context) error {case <-done:return nilcase <-ticker.C:- if !write() {+ if !write(build(ctx)) {return nil}case <-pingTicker.C:@@ -127,10 +190,11 @@ func (h *WebSocketHandler) DiagnosticsStream(c *echo.Context) error {// ServerLogsStream replays the recent backend log backlog then streams new entries live.func (h *WebSocketHandler) ServerLogsStream(c *echo.Context) error {- conn, err := wshub.Accept(c.Response(), c.Request(), h.checkWSOrigin)- if err != nil {+ conn, unregister, accepted := h.acceptWSInternal(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)+ if !accepted {return nil}+ defer unregister()defer func() {if closeNowErr := conn.CloseNow(); closeNowErr != nil {slog.Debug("Failed to close server logs websocket connection", "error", closeNowErr)@@ -143,7 +207,7 @@ func (h *WebSocketHandler) ServerLogsStream(c *echo.Context) error {defer cancel()ctx := c.Request().Context()- done := diagnosticsReadLoopInternal(ctx, conn)+ done := diagnosticsReadLoopInternal(ctx, conn, nil)write := func(e logs.Entry) bool {b, marshalErr := json.Marshal(e)if marshalErr != nil {@@ -189,18 +253,22 @@ func pingDiagnosticsConnInternal(ctx context.Context, conn *websocket.Conn) boolreturn conn.Ping(pctx) == nil}-// diagnosticsReadLoopInternal drains incoming frames; the returned channel closes-// when the peer disconnects, signaling the writer loop to stop.-func diagnosticsReadLoopInternal(ctx context.Context, conn *websocket.Conn) <-chan struct{} {+// diagnosticsReadLoopInternal passes incoming frames to onMessage, or drains them when+// it is nil; the returned channel closes when the peer disconnects.+func diagnosticsReadLoopInternal(ctx context.Context, conn *websocket.Conn, onMessage func([]byte)) <-chan struct{} {conn.SetReadLimit(diagnosticsReadLimit)done := make(chan struct{})go func() {defer close(done)for {- if _, _, err := conn.Read(ctx); err != nil {+ _, b, err := conn.Read(ctx)+ if err != nil {return}+ if onMessage != nil {+ onMessage(b)+ }}}()return donediff --git a/backend/internal/diagnostics/model.go b/backend/internal/diagnostics/model.gonew file mode 100644index 000000000..7e9921d57--- /dev/null+++ b/backend/internal/diagnostics/model.go@@ -0,0 +1,71 @@+package diagnostics++import (+ "fmt"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"+)++// actorHostModel is a read-only view of a registered Francis host.+type actorHostModel struct {+ HostID string `gorm:"column:host_id"`+ Address string `gorm:"column:host_address"`+ LastHealthCheck actorTimeInternal `gorm:"column:host_last_health_check"`+}++func (actorHostModel) TableName() string { return francis.TablePrefix + "_hosts" }++// actorTypeModel is a registered actor type with its concurrency limit.+type actorTypeModel struct {+ ActorType string+ ConcurrencyLimit int+}++// actorCountModel is one grouped count row from a Francis table.+type actorCountModel struct {+ ActorType string+ Kind string+ Count int+ NextDue actorTimeInternal+}++// activeActorModel is a read-only view of an activated actor.+type activeActorModel struct {+ ActorType string `gorm:"column:actor_type"`+ ActorID string `gorm:"column:actor_id"`+ HostID string `gorm:"column:host_id"`+ Activation actorTimeInternal `gorm:"column:actor_activation"`+}++func (activeActorModel) TableName() string { return francis.TablePrefix + "_active_actors" }++// terminalJobModel is a read-only view of a finished actor job.+type terminalJobModel struct {+ JobID string `gorm:"column:job_id"`+ ActorType string `gorm:"column:actor_type"`+ ActorID string `gorm:"column:actor_id"`+ JobMethod string `gorm:"column:job_method"`+ Attempts int `gorm:"column:attempts"`+ LastError *string `gorm:"column:last_error"`+ EndedAt actorTimeInternal `gorm:"column:ended_at"`+}++func (terminalJobModel) TableName() string { return francis.TablePrefix + "_terminal_jobs" }++// actorTimeInternal scans Francis timestamps: SQLite stores epoch milliseconds, Postgres a timestamp.+type actorTimeInternal time.Time++func (t *actorTimeInternal) Scan(value any) error {+ switch v := value.(type) {+ case int64:+ *t = actorTimeInternal(time.UnixMilli(v).UTC())+ case time.Time:+ *t = actorTimeInternal(v.UTC())+ case nil:+ *t = actorTimeInternal{}+ default:+ return fmt.Errorf("unsupported actor timestamp %T", value)+ }+ return nil+}diff --git a/backend/internal/diagnostics/service.go b/backend/internal/diagnostics/service.goindex ec1ec6797..13408a00e 100644--- a/backend/internal/diagnostics/service.go+++ b/backend/internal/diagnostics/service.go@@ -2,24 +2,34 @@ package diagnosticsimport ("bytes"+ "context""errors""fmt"+ "log/slog"+ "net/http""runtime""runtime/debug""runtime/pprof"+ "runtime/trace""sync""time""github.com/getarcaneapp/arcane/types/v2/system"+ kit "go.getarcane.app/kit/pkg"+ "gorm.io/gorm"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis")const (- // recentGCPauseSamples is the number of recent GC pause durations reported.- recentGCPauseSamples = 16-- // goroutineLeakProfileName is the runtime/pprof profile that reports goroutines- // blocked on concurrency primitives that cannot be unblocked.+ recentGCPauseSamples = 16goroutineLeakProfileName = "goroutineleak"++ // maxActorDetailRowsInternal caps the instances and failures listed per actor type.+ maxActorDetailRowsInternal = 10)// DiagnosticsService gathers Go runtime, memory, and garbage-collector@@ -27,22 +37,20 @@ const (// WebSocket metrics and worker-goroutine counts are merged in at the handler// layer to avoid an import cycle with the api/ws package.type DiagnosticsService struct {- startedAt time.Time+ startedAt time.Time+ db *database.DB+ actors *francis.Runtime+ coordinator *runs.Coordinator+ environments *environment.EnvironmentServiceleakMu sync.MutexleakScannedAt time.Time}-// NewDiagnosticsService returns a DiagnosticsService. startedAt is captured at-// construction (≈ process start) and used to report uptime.-func NewDiagnosticsService() *DiagnosticsService {- return &DiagnosticsService{startedAt: time.Now()}+func NewDiagnosticsService(db *database.DB, actors *francis.Runtime, coordinator *runs.Coordinator, environments *environment.EnvironmentService) *DiagnosticsService {+ return &DiagnosticsService{startedAt: time.Now(), db: db, actors: actors, coordinator: coordinator, environments: environments}}-// Collect samples the current runtime, memory, and GC state.-//-// LeakedGoroutines is the last-known leak count from the runtime; it does not-// trigger a leak-detection GC. Use ScanGoroutineLeaks for a fresh scan.func (s *DiagnosticsService) Collect() (system.RuntimeInfo, system.MemoryInfo, system.GCInfo) {var mem runtime.MemStatsruntime.ReadMemStats(&mem)@@ -132,6 +140,43 @@ func (s *DiagnosticsService) ScanGoroutineLeaks() (system.GoroutineLeakReport, e}, nil}+// Profile captures a pprof profile in the given debug format. The CPU profile+// and execution trace sample for seconds or until ctx ends.+func (s *DiagnosticsService) Profile(ctx context.Context, name string, seconds, debugLevel int) ([]byte, error) {+ var buf bytes.Buffer+ sample := func() {+ timer := time.NewTimer(time.Duration(seconds) * time.Second)+ defer timer.Stop()+ select {+ case <-timer.C:+ case <-ctx.Done():+ }+ }+ switch name {+ case "profile":+ if err := pprof.StartCPUProfile(&buf); err != nil {+ return nil, fmt.Errorf("start CPU profile: %w", err)+ }+ sample()+ pprof.StopCPUProfile()+ case "trace":+ if err := trace.Start(&buf); err != nil {+ return nil, fmt.Errorf("start trace: %w", err)+ }+ sample()+ trace.Stop()+ default:+ p := pprof.Lookup(name)+ if p == nil {+ return nil, fmt.Errorf("unknown profile %q", name)+ }+ if err := p.WriteTo(&buf, debugLevel); err != nil {+ return nil, fmt.Errorf("write %s profile: %w", name, err)+ }+ }+ return buf.Bytes(), ctx.Err()+}+// leakedGoroutineCountInternal returns the last-known leak count without// triggering a leak-detection GC.func leakedGoroutineCountInternal() int {@@ -141,3 +186,145 @@ func leakedGoroutineCountInternal() int {}return p.Count()}++// CollectActors reports this process's Francis host, actor types, and run queue.+func (s *DiagnosticsService) CollectActors(ctx context.Context) (system.ActorDiagnostics, error) {+ d := system.ActorDiagnostics{EnvironmentID: environment.LocalEnvironmentID, Runs: map[string]int{}, CheckedAt: time.Now().UTC()}+ select {+ case <-s.actors.Ready():+ d.Ready = true+ default:+ return d, nil+ }++ db := s.db.WithContext(ctx)+ var hosts []actorHostModel+ if err := db.Order("host_id").Find(&hosts).Error; err != nil {+ return d, fmt.Errorf("list actor hosts: %w", err)+ }+ for _, host := range hosts {+ d.Hosts = append(d.Hosts, system.ActorHostInfo{HostID: host.HostID, Address: host.Address, LastHealthCheck: time.Time(host.LastHealthCheck)})+ }++ // Records names coordinator actors, so load it before resolving instance names.+ records, err := s.coordinator.Records(ctx)+ if err != nil {+ return d, err+ }+ for _, record := range records {+ for _, run := range record.Runs {+ d.Runs[string(run.Status)]+++ }+ }++ d.ActorTypes, err = s.collectActorTypesInternal(db)+ return d, err+}++// collectActorTypesInternal reports counts, recent instances, and dead jobs per registered actor type.+func (s *DiagnosticsService) collectActorTypesInternal(db *gorm.DB) ([]system.ActorTypeInfo, error) {+ var actorTypes []actorTypeModel+ err := db.Table(francis.TablePrefix + "_host_actor_types").+ Select("actor_type, MAX(actor_concurrency_limit) AS concurrency_limit").+ Group("actor_type").Order("actor_type").Scan(&actorTypes).Error+ if err != nil {+ return nil, fmt.Errorf("list actor types: %w", err)+ }+ var active, states, alarms, leased, terminal []actorCountModel+ var instances []activeActorModel+ var failures []terminalJobModel+ err = errors.Join(+ db.Table(francis.TablePrefix+"_active_actors").Select("actor_type, COUNT(*) AS count").Group("actor_type").Scan(&active).Error,+ db.Table(francis.TablePrefix+"_actor_state").Select("actor_type, COUNT(*) AS count").Group("actor_type").Scan(&states).Error,+ db.Table(francis.TablePrefix+"_alarms").Select("actor_type, alarm_kind AS kind, COUNT(*) AS count, MIN(alarm_due_time) AS next_due").Group("actor_type, alarm_kind").Scan(&alarms).Error,+ db.Table(francis.TablePrefix+"_alarms").Select("actor_type, COUNT(*) AS count").Where("alarm_lease_id IS NOT NULL").Group("actor_type").Scan(&leased).Error,+ db.Table(francis.TablePrefix+"_terminal_jobs").Select("actor_type, job_status AS kind, COUNT(*) AS count").Group("actor_type, job_status").Scan(&terminal).Error,+ db.Order("actor_activation DESC").Limit(500).Find(&instances).Error,+ db.Where("job_status = ?", "dead").Order("ended_at DESC").Limit(100).Find(&failures).Error,+ )+ if err != nil {+ return nil, fmt.Errorf("collect actor activity: %w", err)+ }++ out := make([]system.ActorTypeInfo, len(actorTypes))+ byType := make(map[string]*system.ActorTypeInfo, len(actorTypes))+ for i, actorType := range actorTypes {+ out[i] = system.ActorTypeInfo{ActorType: actorType.ActorType, ConcurrencyLimit: actorType.ConcurrencyLimit, Instances: []system.ActorInstance{}, RecentFailures: []system.ActorJobFailure{}}+ byType[actorType.ActorType] = &out[i]+ }+ addCountsInternal(byType, active, func(info *system.ActorTypeInfo, row actorCountModel) { info.ActiveActors = row.Count })+ addCountsInternal(byType, states, func(info *system.ActorTypeInfo, row actorCountModel) { info.StoredStates = row.Count })+ addCountsInternal(byType, leased, func(info *system.ActorTypeInfo, row actorCountModel) { info.LeasedAlarms = row.Count })+ addCountsInternal(byType, alarms, func(info *system.ActorTypeInfo, row actorCountModel) {+ if row.Kind == "job" {+ info.PendingJobs += row.Count+ } else {+ info.PendingAlarms += row.Count+ }+ if due := time.Time(row.NextDue); !due.IsZero() && (info.NextAlarmAt.IsZero() || due.Before(info.NextAlarmAt)) {+ info.NextAlarmAt = due+ }+ })+ addCountsInternal(byType, terminal, func(info *system.ActorTypeInfo, row actorCountModel) {+ switch row.Kind {+ case "dead":+ info.DeadJobs += row.Count+ case "completed":+ info.CompletedJobs += row.Count+ }+ })+ for _, row := range instances {+ if info := byType[row.ActorType]; info != nil && len(info.Instances) < maxActorDetailRowsInternal {+ info.Instances = append(info.Instances, system.ActorInstance{ActorID: row.ActorID, Name: s.actors.ActorName(row.ActorID), HostID: row.HostID, ActivatedAt: time.Time(row.Activation)})+ }+ }+ for _, row := range failures {+ if info := byType[row.ActorType]; info != nil && len(info.RecentFailures) < maxActorDetailRowsInternal {+ info.RecentFailures = append(info.RecentFailures, system.ActorJobFailure{+ JobID: row.JobID, ActorID: row.ActorID, Name: s.actors.ActorName(row.ActorID), Method: row.JobMethod,+ Attempts: row.Attempts, LastError: kit.FromPtr(row.LastError), FailedAt: time.Time(row.EndedAt),+ })+ }+ }+ return out, nil+}++// addCountsInternal applies grouped rows to their registered actor type.+func addCountsInternal(byType map[string]*system.ActorTypeInfo, rows []actorCountModel, apply func(*system.ActorTypeInfo, actorCountModel)) {+ for _, row := range rows {+ if info := byType[row.ActorType]; info != nil {+ apply(info, row)+ }+ }+}++// CollectAllActors gathers actor diagnostics from the local and every active remote environment.+func (s *DiagnosticsService) CollectAllActors(ctx context.Context) []system.ActorDiagnostics {+ local, err := s.CollectActors(ctx)+ if err != nil {+ local.Error = err.Error()+ }+ if env, envErr := s.environments.GetEnvironmentByIDCached(ctx, environment.LocalEnvironmentID); envErr == nil {+ local.EnvironmentName = env.Name+ }++ remotes, err := s.environments.ListActiveRemoteEnvironments(ctx)+ if err != nil {+ slog.WarnContext(ctx, "Failed to list environments for actor diagnostics", "error", err)+ }+ out := make([]system.ActorDiagnostics, len(remotes)+1)+ out[0] = local+ var wg sync.WaitGroup+ for i, env := range remotes {+ wg.Go(func() {+ var d system.ActorDiagnostics+ if proxyErr := s.environments.ProxyJSONRequestForEnvironment(ctx, env, http.MethodGet, "/api/environments/0/diagnostics/actors", nil, &d); proxyErr != nil {+ d = system.ActorDiagnostics{Error: proxyErr.Error(), CheckedAt: time.Now().UTC()}+ }+ d.EnvironmentID, d.EnvironmentName = env.ID, env.Name+ out[i+1] = d+ })+ }+ wg.Wait()+ return out+}diff --git a/backend/internal/diagnostics/service_test.go b/backend/internal/diagnostics/service_test.goindex 4fceabdf6..bffb9334e 100644--- a/backend/internal/diagnostics/service_test.go+++ b/backend/internal/diagnostics/service_test.go@@ -12,7 +12,7 @@ import ()func TestDiagnosticsServiceCollect(t *testing.T) {- s := NewDiagnosticsService()+ s := NewDiagnosticsService(nil, nil, nil, nil)rt, mem, _ := s.Collect()@@ -43,7 +43,7 @@ func TestGoroutineLeakProfileAvailable(t *testing.T) {}func TestDiagnosticsServiceScanGoroutineLeaks(t *testing.T) {- s := NewDiagnosticsService()+ s := NewDiagnosticsService(nil, nil, nil, nil)report, err := s.ScanGoroutineLeaks()require.NoError(t, err)@@ -58,7 +58,7 @@ func TestDiagnosticsServiceScanGoroutineLeaks(t *testing.T) {}func TestDiagnosticsServiceScanGoroutineLeaksDetectsLeak(t *testing.T) {- s := NewDiagnosticsService()+ s := NewDiagnosticsService(nil, nil, nil, nil)leakUnbufferedChannelInternal()deadline := time.Now().Add(5 * time.Second)diff --git a/backend/pkg/libarcane/ws/metrics.go b/backend/pkg/libarcane/ws/metrics.goindex 4f6db634e..fbc5826ef 100644--- a/backend/pkg/libarcane/ws/metrics.go+++ b/backend/pkg/libarcane/ws/metrics.go@@ -17,6 +17,7 @@ type WebSocketMetrics struct {containerExec atomic.Int64systemStats atomic.Int64serviceLogsActive atomic.Int64+ diagnosticsActive atomic.Int64seq atomic.Uint64mu sync.RWMutexconnections map[string]systemtypes.WebSocketConnectionInfo@@ -38,6 +39,7 @@ func (m *WebSocketMetrics) Snapshot() systemtypes.WebSocketMetricsSnapshot {ContainerExec: m.containerExec.Load(),SystemStats: m.systemStats.Load(),ServiceLogsActive: m.serviceLogsActive.Load(),+ DiagnosticsActive: m.diagnosticsActive.Load(),}}@@ -100,5 +102,7 @@ func (m *WebSocketMetrics) applyDelta(kind string, delta int64) {m.systemStats.Add(delta)case systemtypes.WSKindServiceLogs:m.serviceLogsActive.Add(delta)+ case systemtypes.WSKindDiagnostics:+ m.diagnosticsActive.Add(delta)}}diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.goindex 4f175f23a..d07b3acd7 100644--- a/backend/pkg/scheduler/runs/actors.go+++ b/backend/pkg/scheduler/runs/actors.go@@ -6,6 +6,7 @@ import ("errors""fmt""log/slog"+ "slices""strings""time""uuid"@@ -342,12 +343,17 @@ func (a *coordinatorActorInternal) Alarm(ctx context.Context, name string, datareturn a.flushInternal(ctx, &state)}-func (q *Coordinator) repairInternal(ctx context.Context) error {+// repairInternal flushes coordinators so schedule alarms and dispatches are restored.+// Unless all is set, it skips records without unfinished runs so idle coordinators can deactivate.+func (q *Coordinator) repairInternal(ctx context.Context, all bool) error {records, err := q.Records(ctx)if err != nil {return err}for _, record := range records {+ if !all && !slices.ContainsFunc(record.Runs, func(run st.Run) bool { return !run.Status.Terminal() && run.Status != st.NeedsAttention }) {+ continue+ }if _, invokeErr := q.service.Invoke(ctx, coordinatorTypeInternal, kit.SHA256Hex(record.EnvironmentID+"\x00"+record.JobID), "repair", nil); invokeErr != nil {return invokeErr}diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.goindex ef28ef7fd..6265fda00 100644--- a/backend/pkg/scheduler/runs/actors_test.go+++ b/backend/pkg/scheduler/runs/actors_test.go@@ -268,10 +268,10 @@ func TestReconciliationRepairsMissingAlarm(t *testing.T) {record, err := q.ScheduleState(t.Context(), "repair")require.NoError(t, err)id := kit.SHA256Hex("0\x00repair")- err = q.repairInternal(t.Context())+ err = q.repairInternal(t.Context(), true)require.NoError(t, err)require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorTypeInternal, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))- err = q.repairInternal(t.Context())+ err = q.repairInternal(t.Context(), true)require.NoError(t, err)require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorTypeInternal, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))after, err := q.ScheduleState(t.Context(), "repair")@@ -358,7 +358,7 @@ func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {state.Dispatches[key] = intentstate.Revision++require.NoError(t, q.service.SetState(t.Context(), coordinatorTypeInternal, id, state, nil))- err = q.repairInternal(t.Context())+ err = q.repairInternal(t.Context(), false)require.NoError(t, err)require.NoError(t, q.service.GetState(t.Context(), coordinatorTypeInternal, id, &state))require.Equal(t, original, state.Dispatches[key].JobID)diff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.goindex cbce1e781..f4b63ac40 100644--- a/backend/pkg/scheduler/runs/coordinator.go+++ b/backend/pkg/scheduler/runs/coordinator.go@@ -287,6 +287,8 @@ func (q *Coordinator) dispatchInternal(ctx context.Context) {defer close(q.done)initialRetention := true+ // Flush every coordinator once while enabled to restore schedule alarms; later passes only cover unfinished work.+ fullRepair := truenextRetention := time.Now()for ctx.Err() == nil {q.retryActivitySyncInternal(ctx)@@ -301,9 +303,14 @@ func (q *Coordinator) dispatchInternal(ctx context.Context) {nextRetention = time.Now().Add(time.Hour)}nextWake := time.Now().Add(15 * time.Second)- if err := q.repairInternal(ctx); err != nil && ctx.Err() == nil {- slog.ErrorContext(ctx, "Job queue dispatch failed", "error", err)+ enabled := q.enabled.Load()+ if err := q.repairInternal(ctx, fullRepair); err != nil {+ if ctx.Err() == nil {+ slog.ErrorContext(ctx, "Job queue dispatch failed", "error", err)+ }nextWake = time.Now().Add(5 * time.Second)+ } else if enabled {+ fullRepair = false}if nextRetention.Before(nextWake) {nextWake = nextRetentiondiff --git a/backend/pkg/utils/francis/logging.go b/backend/pkg/utils/francis/logging.goindex 0fb39c0ef..44543bc18 100644--- a/backend/pkg/utils/francis/logging.go+++ b/backend/pkg/utils/francis/logging.go@@ -13,6 +13,13 @@ func (r *Runtime) NameActor(actorID, name string) {r.actorNames.Store(actorID, name)}+// ActorName returns the display name set by NameActor, or an empty string.+func (r *Runtime) ActorName(actorID string) string {+ name, _ := r.actorNames.Load(actorID)+ displayName, _ := name.(string)+ return displayName+}+type actorLogHandlerInternal struct {slog.Handlerdiff --git a/backend/pkg/utils/francis/provider.go b/backend/pkg/utils/francis/provider.goindex e6e85d491..526797ef5 100644--- a/backend/pkg/utils/francis/provider.go+++ b/backend/pkg/utils/francis/provider.go@@ -28,9 +28,9 @@ func providerOptionInternal(databaseURL string) (local.HostOption, error) {if err != nil {return nil, err}- return local.WithSQLiteProvider(sqlite.SQLiteProviderOptions{ConnectionString: dsn, TablePrefix: tablePrefixInternal}), nil+ return local.WithSQLiteProvider(sqlite.SQLiteProviderOptions{ConnectionString: dsn, TablePrefix: TablePrefix}), nilcase strings.HasPrefix(databaseURL, "postgres"):- return local.WithPostgresProvider(postgres.PostgresProviderOptions{ConnectionString: databaseURL, TablePrefix: tablePrefixInternal}), nil+ return local.WithPostgresProvider(postgres.PostgresProviderOptions{ConnectionString: databaseURL, TablePrefix: TablePrefix}), nildefault:return nil, errors.New("unsupported actor database URL")}@@ -53,9 +53,9 @@ func ClearRestoredHosts(ctx context.Context, databaseURL string) (err error) {if dsnErr != nil {return dsnErr}- provider, err = sqlite.NewSQLiteProvider(slog.Default(), sqlite.SQLiteProviderOptions{ConnectionString: dsn, TablePrefix: tablePrefixInternal}, cfg)+ provider, err = sqlite.NewSQLiteProvider(slog.Default(), sqlite.SQLiteProviderOptions{ConnectionString: dsn, TablePrefix: TablePrefix}, cfg)case strings.HasPrefix(databaseURL, "postgres"):- provider, err = postgres.NewPostgresProvider(slog.Default(), postgres.PostgresProviderOptions{ConnectionString: databaseURL, TablePrefix: tablePrefixInternal}, cfg)+ provider, err = postgres.NewPostgresProvider(slog.Default(), postgres.PostgresProviderOptions{ConnectionString: databaseURL, TablePrefix: TablePrefix}, cfg)default:return errors.New("unsupported restored actor database URL")}diff --git a/backend/pkg/utils/francis/runtime.go b/backend/pkg/utils/francis/runtime.goindex e490e8159..4efbee8e7 100644--- a/backend/pkg/utils/francis/runtime.go+++ b/backend/pkg/utils/francis/runtime.go@@ -22,7 +22,8 @@ import ("github.com/quic-go/quic-go/http3")-const tablePrefixInternal = "arcane_francis"+// TablePrefix namespaces the Francis tables inside the Arcane database.+const TablePrefix = "arcane_francis"// Runtime starts one durable actor host and publishes its service after readiness.// Services must wait for Ready before using Service, including during startup retries.@@ -69,7 +70,7 @@ func New(databaseURL, encryptionKey, instanceID, port string, options ...local.H},}runtime.options = append(runtime.options, local.WithLogger(slog.New(&actorLogHandlerInternal{- Handler: slog.Default().With("scope", "actor-host").Handler(),+ Handler: slog.Default().With("scope", "francis.arcane.internal").Handler(),names: &runtime.actorNames,})))if encryptionKey != "" && instanceID != "" {diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex 8c3aa79f9..7c82457a3 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -3268,9 +3268,6 @@"diagnostics": "Diagnostics","diagnostics_description": "Live Go runtime, profiling, and backend logs.","diagnostics_status_connecting": "Connecting…",- "diagnostics_updated_ago": "updated {ago}",- "diagnostics_just_now": "just now",- "diagnostics_seconds_ago": "{seconds}s ago","diagnostics_resume": "Resume","diagnostics_loading": "Loading diagnostics…","diagnostics_error_load": "Failed to load diagnostics",@@ -3278,6 +3275,8 @@"diagnostics_error_dump": "Failed to load dump","diagnostics_tab_connections": "Connections","diagnostics_tab_profiling": "Profiling",+ "diagnostics_tab_actors": "Actors",+ "diagnostics_actors_alarms": "Alarms","diagnostics_connections_empty": "No active WebSocket connections.","diagnostics_section_system": "System status","diagnostics_download": "Download",@@ -3335,7 +3334,6 @@"diagnostics_conn_client_ip": "Client IP","diagnostics_conn_since": "Since","diagnostics_section_logs": "Live Backend Logs",- "diagnostics_logs_streaming": "Streaming","disconnected": "Disconnected","diagnostics_logs_count": "{count} lines","diagnostics_logs_all_levels": "All levels",diff --git a/frontend/src/lib/layouts/settings-page-layout.svelte b/frontend/src/lib/layouts/settings-page-layout.svelteindex 2c7050fde..8edc1e2d3 100644--- a/frontend/src/lib/layouts/settings-page-layout.svelte+++ b/frontend/src/lib/layouts/settings-page-layout.svelte@@ -63,7 +63,7 @@{/if}{#if pageType === 'management' && statCards.length > 0}<div class="mt-2 flex flex-wrap items-center gap-x-4 gap-y-1">- {#each statCards as card, i (card)}+ {#each statCards as card, i (i)}{#if i > 0}<div class="h-4 w-px bg-border/50"></div>{/if}diff --git a/frontend/src/lib/services/diagnostics-service.ts b/frontend/src/lib/services/diagnostics-service.tsdeleted file mode 100644index 07948224f..000000000--- a/frontend/src/lib/services/diagnostics-service.ts+++ /dev/null@@ -1,71 +0,0 @@-import type { Diagnostics, GoroutineLeakReport, LogEntry, PprofProfile } from '#lib/types/diagnostics.js';--import BaseAPIService from './api-service';--class DiagnosticsAPIService extends BaseAPIService {- /** One-shot runtime/memory/GC + WebSocket snapshot (used for initial paint). */- async getDiagnostics(): Promise<Diagnostics> {- const res = await this.api.get('/diagnostics');- return res.data as Diagnostics;- }-- /** Recent buffered backend log entries (oldest first). */- async getRecentLogs(): Promise<LogEntry[]> {- const res = await this.api.get('/diagnostics/logs');- return (res.data ?? []) as LogEntry[];- }-- /** Fetch a human-readable pprof dump (debug=2 text) for inline display. */- async getDump(name: 'goroutine' | 'heap'): Promise<string> {- const res = await this.api.get(`/debug/pprof/${name}`, {- params: { debug: 2 },- responseType: 'text'- });- return (res.data ?? '') as string;- }-- /** Run a leak-detection GC and return leaked-goroutine stacks (debug=1 text). */- async scanGoroutineLeaks(): Promise<GoroutineLeakReport> {- const res = await this.api.post('/diagnostics/goroutineleak', undefined, {- timeout: 60000- });- return res.data as GoroutineLeakReport;- }-- /**- * Download a raw pprof profile via the authed client (so bearer/API-key auth- * is attached) and trigger a browser save. `profile` and `trace` are- * time-sampled; pass an optional duration in seconds.- */- async downloadProfile(profile: PprofProfile, seconds?: number): Promise<void> {- const params: Record<string, number> = {};- if (profile === 'profile') params['seconds'] = seconds ?? 30;- if (profile === 'trace') params['seconds'] = seconds ?? 5;-- const defaultSeconds = profile === 'trace' ? 5 : 30;- const res = await this.api.get(`/debug/pprof/${profile}`, {- params,- responseType: 'blob',- // Sampled profiles block for their full duration; goroutineleak runs a GC.- timeout:- profile === 'profile' || profile === 'trace'- ? (seconds ?? defaultSeconds) * 1000 + 15000- : profile === 'goroutineleak'- ? 60000- : undefined- });-- const blob = res.data as Blob;- const url = URL.createObjectURL(blob);- const ext = profile === 'trace' ? 'out' : 'pprof';- const anchor = document.createElement('a');- anchor.href = url;- anchor.download = `${profile}.${ext}`;- document.body.appendChild(anchor);- anchor.click();- anchor.remove();- URL.revokeObjectURL(url);- }-}--export const diagnosticsService = new DiagnosticsAPIService();diff --git a/frontend/src/lib/types/diagnostics.ts b/frontend/src/lib/types/diagnostics.tsindex bb008565f..bc2948a8f 100644--- a/frontend/src/lib/types/diagnostics.ts+++ b/frontend/src/lib/types/diagnostics.ts@@ -59,6 +59,7 @@ export interface WebSocketMetricsSnapshot {containerExec: number;systemStats: number;serviceLogsActive: number;+ diagnosticsActive: number;}export interface WebSocketDiagnostics {@@ -74,6 +75,73 @@ export interface Diagnostics {websocket: WebSocketDiagnostics;}+export interface ActorHostInfo {+ hostId: string;+ address: string;+ lastHealthCheck: string;+}++export interface ActorInstance {+ actorId: string;+ name?: string;+ hostId: string;+ activatedAt: string;+}++export interface ActorJobFailure {+ jobId: string;+ actorId: string;+ name?: string;+ method: string;+ attempts: number;+ lastError?: string;+ failedAt: string;+}++export interface ActorTypeInfo {+ actorType: string;+ concurrencyLimit: number;+ activeActors: number;+ storedStates: number;+ pendingAlarms: number;+ pendingJobs: number;+ leasedAlarms: number;+ completedJobs: number;+ deadJobs: number;+ nextAlarmAt?: string;+ instances: ActorInstance[] | null;+ recentFailures: ActorJobFailure[] | null;+}++export interface ActorDiagnostics {+ environmentId: string;+ environmentName: string;+ error?: string;+ ready: boolean;+ hosts: ActorHostInfo[] | null;+ actorTypes: ActorTypeInfo[] | null;+ runs: Record<string, number> | null;+ checkedAt: string;+}++export interface DiagnosticsCommand {+ id: string;+ type: 'refresh' | 'dump' | 'leakScan' | 'profile';+ name?: string;+ seconds?: number;+}++export interface DiagnosticsMessage {+ type: 'snapshot' | 'result';+ id?: string;+ snapshot?: Diagnostics;+ leakReport?: GoroutineLeakReport;+ text?: string;+ /** Base64-encoded pprof bytes for profile results. */+ data?: string;+ error?: string;+}+export interface LogEntry {time: string;level: string;diff --git a/frontend/src/lib/utils/ws.ts b/frontend/src/lib/utils/ws.tsindex cedd5107e..ac627e19d 100644--- a/frontend/src/lib/utils/ws.ts+++ b/frontend/src/lib/utils/ws.ts@@ -1,5 +1,14 @@-import type { Diagnostics, LogEntry } from '#lib/types/diagnostics.js';+import { m } from '#lib/paraglide/messages.js';+import type {+ ActorDiagnostics,+ Diagnostics,+ DiagnosticsCommand,+ DiagnosticsMessage,+ LogEntry,+ PprofProfile+} from '#lib/types/diagnostics.js';import type { SystemStats } from '#lib/types/shared.js';+import { downloadBlob } from '#lib/utils/browser-download.js';import { tryCatch } from '#lib/utils/try-catch.js';export interface ReconnectWSOptions<T> {@@ -129,6 +138,12 @@ export class ReconnectingWebSocket<T = unknown> {}, backoff);}+ send(data: string): boolean {+ if (this.ws?.readyState !== WebSocket.OPEN) return false;+ this.ws.send(data);+ return true;+ }+close() {this.closed = true;this.attempt = 0;@@ -243,14 +258,75 @@ export function createContainerStatsWebSocket(opts: {});}+/** Diagnostics stream carrying live snapshots plus request/response commands. */export function createDiagnosticsWebSocket(opts: {- onMessage: (data: Diagnostics) => void;+ onSnapshot: (data: Diagnostics) => void;+ onOpen?: () => void;+ onClose?: () => void;+}) {+ let nextId = 0;+ const pending = new Map<string, { resolve: (msg: DiagnosticsMessage) => void; reject: (err: Error) => void }>();+ const rejectPending = () => {+ for (const entry of pending.values()) entry.reject(new Error(m.disconnected()));+ pending.clear();+ };++ const socket = createDiagnosticsStreamWebSocket<DiagnosticsMessage>('/api/diagnostics/stream', {+ onMessage: (msg) => {+ if (msg.type === 'snapshot') {+ if (msg.snapshot) opts.onSnapshot(msg.snapshot);+ return;+ }+ const entry = pending.get(msg.id ?? '');+ if (!entry) return;+ pending.delete(msg.id ?? '');+ if (msg.error) entry.reject(new Error(msg.error));+ else entry.resolve(msg);+ },+ onOpen: opts.onOpen,+ onClose: () => {+ rejectPending();+ opts.onClose?.();+ }+ });++ const request = (command: Omit<DiagnosticsCommand, 'id'>) =>+ new Promise<DiagnosticsMessage>((resolve, reject) => {+ const id = String(++nextId);+ if (!socket.send(JSON.stringify({ ...command, id }))) {+ reject(new Error(m.disconnected()));+ return;+ }+ pending.set(id, { resolve, reject });+ });++ return {+ connect: () => socket.connect(),+ close: () => {+ socket.close();+ rejectPending();+ },+ refresh: () => socket.send(JSON.stringify({ id: '', type: 'refresh' })),+ request,+ /** Captures a pprof profile over the stream and saves it as a file. */+ async downloadProfile(profile: PprofProfile) {+ const msg = await request({ type: 'profile', name: profile });+ const bytes = Uint8Array.from(atob(msg.data ?? ''), (char) => char.charCodeAt(0));+ downloadBlob(bytes, `${profile}.${profile === 'trace' ? 'out' : 'pprof'}`);+ }+ };+}++export type DiagnosticsWebSocket = ReturnType<typeof createDiagnosticsWebSocket>;++export function createActorDiagnosticsWebSocket(opts: {+ onMessage: (data: ActorDiagnostics[]) => void;onOpen?: () => void;onClose?: () => void;onError?: (err: Event | Error) => void;maxBackoff?: number;}) {- return createDiagnosticsStreamWebSocket('/api/diagnostics/stream', opts);+ return createDiagnosticsStreamWebSocket('/api/diagnostics/actors/stream', opts);}export function createBackendLogsWebSocket(opts: {@@ -272,7 +348,7 @@ function createDiagnosticsStreamWebSocket<T>(onError?: (err: Event | Error) => void;maxBackoff?: number;}-) {+): ReconnectingWebSocket<T> {const buildUrl = () => {const protocol = location.protocol === 'https:' ? 'wss' : 'ws';return `${protocol}://${location.host}${path}`;diff --git a/frontend/src/routes/(app)/images/+page.svelte b/frontend/src/routes/(app)/images/+page.svelteindex 319320ce1..3133c5e24 100644--- a/frontend/src/routes/(app)/images/+page.svelte+++ b/frontend/src/routes/(app)/images/+page.svelte@@ -428,7 +428,12 @@<Dialog.Description>{m.images_prune_confirm_description({ mode: imagePruneMode })}</Dialog.Description></Dialog.Header><SettingsSection class="py-4">- <SettingsRow label={m.images()} description={m.prune_images_dialog_description()} layout="wide">+ <SettingsRow+ label={m.images()}+ description={m.prune_images_dialog_description()}+ layout="wide"+ contentClass="flex justify-center"+ ><PruneModePickermodeOptions={imagePruneModes}bind:value={imagePruneMode}diff --git a/frontend/src/routes/(app)/settings/diagnostics/+page.svelte b/frontend/src/routes/(app)/settings/diagnostics/+page.svelteindex 34e40f469..33c2498f6 100644--- a/frontend/src/routes/(app)/settings/diagnostics/+page.svelte+++ b/frontend/src/routes/(app)/settings/diagnostics/+page.svelte@@ -17,32 +17,29 @@ClockIcon,ConnectionIcon,DownloadIcon,+ EnvironmentsIcon,ArrowDownIcon} from '#lib/icons/index.js';import { SettingsPageLayout } from '#lib/layouts/index.js';import type { SettingsStatCard } from '#lib/layouts/types.js';import { m } from '#lib/paraglide/messages.js';- import { diagnosticsService } from '#lib/services/diagnostics-service.js';import type { Diagnostics, GoroutineLeakReport, PprofProfile } from '#lib/types/diagnostics.js';import { cn } from '#lib/utils.js';import { formatTime } from '#lib/utils/formatting.js';import { tryCatch } from '#lib/utils/try-catch.js';- import { createDiagnosticsWebSocket, ReconnectingWebSocket } from '#lib/utils/ws.js';+ import { createDiagnosticsWebSocket, type DiagnosticsWebSocket } from '#lib/utils/ws.js';+ import DiagnosticActorPanel from './components/diagnostic-actor-panel.svelte';import DiagnosticLeakPanel from './components/diagnostic-leak-panel.svelte';import DiagnosticLogPanel from './components/diagnostic-log-panel.svelte';- type DiagnosticsTab = 'overview' | 'connections' | 'logs' | 'profiling';+ type DiagnosticsTab = 'overview' | 'connections' | 'actors' | 'logs' | 'profiling';let diag = $state<Diagnostics | null>(null);- let connected = $state(false);let paused = $state(false);- let lastUpdated = $state(0);- let now = $state(performance.now());let error = $state<string | null>(null);- let ws: ReconnectingWebSocket<Diagnostics> | null = null;- let tick: ReturnType<typeof setInterval> | null = null;+ let ws = $state.raw<DiagnosticsWebSocket | null>(null);const wsKindLabels: Record<string, string> = {project_logs: m.diagnostics_ws_project_logs(),@@ -50,7 +47,8 @@container_stats: m.diagnostics_ws_container_stats(),container_exec: m.diagnostics_ws_kind_terminal(),system_stats: m.diagnostics_ws_system_stats(),- service_logs: m.diagnostics_ws_service_logs()+ service_logs: m.diagnostics_ws_service_logs(),+ diagnostics: m.diagnostics()};function fmtBytes(n: number): string {@@ -92,12 +90,6 @@return m.diagnostics_leaks_not_scanned();}- const agoText = $derived.by(() => {- if (!lastUpdated) return '—';- const s = Math.max(0, Math.round((now - lastUpdated) / 1000));- return s <= 1 ? m.diagnostics_just_now() : m.diagnostics_seconds_ago({ seconds: s });- });-const totalConnections = $derived.by(() => {const s = diag?.websocket?.snapshot;if (!s) return 0;@@ -113,7 +105,8 @@{ label: m.diagnostics_ws_container_stats(), value: s.containerStats },{ label: m.diagnostics_ws_terminals(), value: s.containerExec },{ label: m.diagnostics_ws_system_stats(), value: s.systemStats },- { label: m.diagnostics_ws_service_logs(), value: s.serviceLogsActive }+ { label: m.diagnostics_ws_service_logs(), value: s.serviceLogsActive },+ { label: m.diagnostics(), value: s.diagnosticsActive }];});@@ -127,6 +120,7 @@icon: ConnectionIcon,badge: totalConnections || undefined},+ { value: 'actors', label: m.diagnostics_tab_actors(), icon: EnvironmentsIcon },{ value: 'logs', label: m.common_logs(), icon: ActivityIcon },{value: 'profiling',@@ -165,12 +159,6 @@const headerStats = $derived.by((): SettingsStatCard[] => {if (!diag) return [];return [- {- title: m.diagnostics_updated_ago({ ago: agoText }),- value: paused ? m.paused() : connected ? m.common_live() : m.diagnostics_status_connecting(),- icon: ActivityIcon,- iconColor: paused ? 'text-warning' : connected ? 'text-success' : 'text-muted-foreground'- },{title: m.goroutines(),value: fmtNum(diag.runtime.goroutines),@@ -211,43 +199,19 @@]);function applySnapshot(d: Diagnostics) {+ if (paused) return;diag = d;- lastUpdated = performance.now();error = null;}- function openStream() {- ws = createDiagnosticsWebSocket({- onMessage: applySnapshot,- onOpen: () => (connected = true),- onClose: () => (connected = false)- });- ws.connect();- }-- function closeStream() {- ws?.close();- ws = null;- connected = false;- }-+ // Pausing only freezes the view so on-demand commands keep working.function togglePause() {paused = !paused;- if (paused) closeStream();- else openStream();+ if (!paused) refresh();}- async function refresh() {- const operationResult = await tryCatch(- (async () => {- applySnapshot(await diagnosticsService.getDiagnostics());- })()- );- if (operationResult.error !== null) {- const e = operationResult.error;-- error = e instanceof Error ? e.message : m.diagnostics_error_load();- }+ function refresh() {+ if (!ws?.refresh()) error = m.disconnected();}let dumpOpen = $state<{ goroutine: boolean; heap: boolean }>({ goroutine: false, heap: false });@@ -257,13 +221,15 @@async function loadDump(name: 'goroutine' | 'heap') {dumpLoading[name] = true;try {- const operationResult = await tryCatch((async () => diagnosticsService.getDump(name))());+ const operationResult = await tryCatch(+ ws ? ws.request({ type: 'dump', name }) : Promise.reject(new Error(m.disconnected()))+ );if (operationResult.error !== null) {const e = operationResult.error;dumpText[name] = e instanceof Error ? e.message : m.diagnostics_error_dump();} else {- dumpText[name] = operationResult.data;+ dumpText[name] = operationResult.data.text ?? '';}} finally {dumpLoading[name] = false;@@ -305,7 +271,8 @@try {const operationResult = await tryCatch((async () => {- await diagnosticsService.downloadProfile(p);+ if (!ws) throw new Error(m.disconnected());+ await ws.downloadProfile(p);})());if (operationResult.error !== null) {@@ -319,12 +286,13 @@}onMount(() => {- refresh();- openStream();- tick = setInterval(() => (now = performance.now()), 1000);+ ws = createDiagnosticsWebSocket({+ onSnapshot: applySnapshot+ });+ ws.connect();return () => {- closeStream();- if (tick) clearInterval(tick);+ ws?.close();+ ws = null;};});</script>@@ -487,7 +455,7 @@<Tabs.Content value="connections" class="mt-6"><div class="space-y-4">{@render sectionHeading(m.diagnostics_section_connections({ count: diag.websocket.connections?.length ?? 0 }))}- <div class="grid grid-cols-2 gap-x-6 gap-y-3 sm:grid-cols-3 lg:grid-cols-6">+ <div class="grid grid-cols-2 gap-x-6 gap-y-3 sm:grid-cols-4 lg:grid-cols-7">{#each wsCounts as c (c.label)}<div><div class="text-xs text-muted-foreground">{c.label}</div>@@ -531,6 +499,10 @@</div></Tabs.Content>+ <Tabs.Content value="actors" class="mt-6">+ <DiagnosticActorPanel />+ </Tabs.Content>+<Tabs.Content value="logs" class="mt-6"><DiagnosticLogPanel height="min(70vh, 640px)" /></Tabs.Content>@@ -542,6 +514,7 @@<DiagnosticLeakPanelleakedGoroutines={diag.runtime.leakedGoroutines}leakScannedAt={diag.runtime.leakScannedAt}+ socket={ws}onscanned={onLeakScanned}/></section>diff --git a/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-actor-panel.svelte b/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-actor-panel.sveltenew file mode 100644index 000000000..d2e3cd00e--- /dev/null+++ b/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-actor-panel.svelte@@ -0,0 +1,197 @@+<script lang="ts">+ import { onDestroy, onMount } from 'svelte';+ import { SvelteSet } from 'svelte/reactivity';+ import { slide } from 'svelte/transition';++ import { jobStatusLabel } from '#lib/components/job-card/job-status.js';+ import * as Alert from '#lib/components/ui/alert/index.js';+ import * as Table from '#lib/components/ui/table/index.js';+ import { ArrowRightIcon } from '#lib/icons/index.js';+ import { m } from '#lib/paraglide/messages.js';+ import type { ActorDiagnostics, ActorTypeInfo } from '#lib/types/diagnostics.js';+ import { cn } from '#lib/utils.js';+ import { formatTime } from '#lib/utils/formatting.js';+ import { createActorDiagnosticsWebSocket, ReconnectingWebSocket } from '#lib/utils/ws.js';++ let environments = $state<ActorDiagnostics[]>([]);+ const expanded = new SvelteSet<string>();+ let ws: ReconnectingWebSocket<ActorDiagnostics[]> | null = null;++ function toggle(key: string) {+ if (expanded.has(key)) expanded.delete(key);+ else expanded.add(key);+ }++ function hasDetails(t: ActorTypeInfo): boolean {+ return Boolean(t.instances?.length || t.recentFailures?.length);+ }++ onMount(() => {+ ws = createActorDiagnosticsWebSocket({ onMessage: (data) => (environments = data) });+ ws.connect();+ });++ onDestroy(() => ws?.close());+</script>++<div class="space-y-4">+ {#each environments as env (env.environmentId)}+ <div class="space-y-3 rounded-xl border border-border/60 p-4">+ <div class="flex flex-wrap items-center gap-2">+ <h3 class="text-sm font-semibold">{env.environmentName || env.environmentId}</h3>+ {#if !env.error && !env.ready}+ <span class="text-xs text-warning">{m.jobs_status_starting()}</span>+ {/if}+ {#each env.hosts ?? [] as host (host.hostId)}+ <span class="font-mono text-xs text-muted-foreground">+ {host.address} · {m.last_check()}+ {formatTime(host.lastHealthCheck) || '—'}+ </span>+ {/each}+ </div>++ {#if env.error}+ <Alert.Root variant="destructive">+ <Alert.Title>{m.common_unavailable()}</Alert.Title>+ <Alert.Description>{env.error}</Alert.Description>+ </Alert.Root>+ {:else if env.ready}+ <div class="flex flex-wrap items-center gap-x-4 gap-y-1 text-xs">+ <span class="text-muted-foreground">{m.jobs_run_history()}</span>+ {#each Object.entries(env.runs ?? {}) as [status, count] (status)}+ <span><span class="font-semibold tabular-nums">{count}</span> {jobStatusLabel(status, true)}</span>+ {:else}+ <span class="text-muted-foreground">{m.none()}</span>+ {/each}+ </div>++ {#if env.actorTypes?.length}+ <div class="overflow-x-auto">+ <Table.Root>+ <Table.Header>+ <Table.Row variant="static">+ <Table.Head variant="expander"></Table.Head>+ <Table.Head>{m.common_type()}</Table.Head>+ <Table.Head>{m.common_active()}</Table.Head>+ <Table.Head>{m.common_state()}</Table.Head>+ <Table.Head>{m.diagnostics_actors_alarms()}</Table.Head>+ <Table.Head>{m.common_pending()}</Table.Head>+ <Table.Head>{m.common_running()}</Table.Head>+ <Table.Head>{m.completed()}</Table.Head>+ <Table.Head>{m.common_failed()}</Table.Head>+ <Table.Head>{m.jobs_next_run()}</Table.Head>+ </Table.Row>+ </Table.Header>+ <Table.Body>+ {#each env.actorTypes as t (t.actorType)}+ {@const key = `${env.environmentId}:${t.actorType}`}+ {@const expandable = hasDetails(t)}+ {@const open = expandable && expanded.has(key)}+ <Table.Row+ variant={expandable ? 'default' : 'static'}+ data-expanded={open ? true : undefined}+ class={cn(!expandable && 'cursor-default')}+ onclick={() => expandable && toggle(key)}+ >+ <Table.Cell variant="expander">+ {#if expandable}+ <ArrowRightIcon+ class={cn('size-4 text-muted-foreground transition-transform duration-200', open && 'rotate-90')}+ aria-hidden="true"+ />+ {/if}+ </Table.Cell>+ <Table.Cell class="font-mono text-xs">{t.actorType}</Table.Cell>+ <Table.Cell class="tabular-nums">+ {t.activeActors}{#if t.concurrencyLimit > 0}<span class="text-muted-foreground">+ / {m.common_limit()} {t.concurrencyLimit}</span+ >{/if}+ </Table.Cell>+ <Table.Cell class="tabular-nums">{t.storedStates}</Table.Cell>+ <Table.Cell class="tabular-nums">{t.pendingAlarms}</Table.Cell>+ <Table.Cell class="tabular-nums">{t.pendingJobs}</Table.Cell>+ <Table.Cell class="tabular-nums">{t.leasedAlarms}</Table.Cell>+ <Table.Cell class="tabular-nums">{t.completedJobs}</Table.Cell>+ <Table.Cell class={cn('tabular-nums', t.deadJobs > 0 && 'text-destructive')}>{t.deadJobs}</Table.Cell>+ <Table.Cell class="text-xs text-muted-foreground tabular-nums">+ {t.nextAlarmAt ? formatTime(t.nextAlarmAt) : '—'}+ </Table.Cell>+ </Table.Row>+ {#if open}+ <Table.Row variant="detail">+ <Table.Cell colspan={10} variant="flush">+ <div transition:slide={{ duration: 200 }} class="space-y-4 px-6 py-4">+ {#if t.instances?.length}+ {@render instanceList(t)}+ {/if}+ {#if t.recentFailures?.length}+ {@render failureList(t)}+ {/if}+ </div>+ </Table.Cell>+ </Table.Row>+ {/if}+ {/each}+ </Table.Body>+ </Table.Root>+ </div>+ {:else}+ <p class="text-sm text-muted-foreground">{m.none()}</p>+ {/if}+ {/if}+ </div>+ {:else}+ <p class="py-8 text-sm text-muted-foreground">{m.diagnostics_status_connecting()}</p>+ {/each}+</div>++{#snippet instanceList(t: ActorTypeInfo)}+ <section class="space-y-1.5">+ <h4 class="text-2xs font-semibold tracking-wide text-muted-foreground uppercase">+ {m.common_active()} · {t.activeActors}+ </h4>+ <ul class="ml-1 border-l border-border/40">+ {#each t.instances as instance (instance.actorId)}+ <li class="flex items-baseline gap-3 py-1 pl-3 text-xs">+ <span class="font-medium">{instance.name || instance.actorId.slice(0, 12)}</span>+ <span class="min-w-0 flex-1 truncate font-mono text-2xs text-muted-foreground" title={instance.actorId}+ >{instance.actorId}</span+ >+ <span class="shrink-0 text-2xs text-muted-foreground tabular-nums">+ {m.common_started()}+ <span class="text-foreground">{formatTime(instance.activatedAt)}</span>+ </span>+ </li>+ {/each}+ </ul>+ </section>+{/snippet}++{#snippet failureList(t: ActorTypeInfo)}+ <section class="space-y-1.5">+ <h4 class="text-2xs font-semibold tracking-wide text-destructive uppercase">+ {m.common_failed()} · {t.deadJobs}+ </h4>+ <ul class="ml-1 border-l border-destructive/40">+ {#each t.recentFailures as failure (failure.jobId)}+ <li class="space-y-1 py-1.5 pl-3 text-xs">+ <div class="flex items-baseline gap-3">+ <span class="font-medium">{failure.name || failure.actorId.slice(0, 12)}</span>+ <span class="font-mono text-2xs text-muted-foreground">{failure.method}</span>+ <span class="ml-auto shrink-0 text-2xs text-muted-foreground tabular-nums">+ {m.jobs_run_attempts()} <span class="text-foreground">{failure.attempts}</span> ·+ <span class="text-foreground">{formatTime(failure.failedAt)}</span>+ </span>+ </div>+ {#if failure.lastError}+ <p+ class="rounded-md border border-destructive/30 bg-destructive/10 px-2 py-1 font-mono text-2xs break-words text-destructive"+ >+ {failure.lastError}+ </p>+ {/if}+ </li>+ {/each}+ </ul>+ </section>+{/snippet}diff --git a/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-leak-panel.svelte b/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-leak-panel.svelteindex b149fce5c..b8f87410c 100644--- a/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-leak-panel.svelte+++ b/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-leak-panel.svelte@@ -2,19 +2,20 @@import { ArcaneButton } from '#lib/components/arcane-button/index.js';import { AlertTriangleIcon, DownloadIcon } from '#lib/icons/index.js';import { m } from '#lib/paraglide/messages.js';- import { diagnosticsService } from '#lib/services/diagnostics-service.js';import type { GoroutineLeakReport } from '#lib/types/diagnostics.js';import { cn } from '#lib/utils.js';import { formatTime } from '#lib/utils/formatting.js';import { tryCatch } from '#lib/utils/try-catch.js';+ import type { DiagnosticsWebSocket } from '#lib/utils/ws.js';interface Props {leakedGoroutines: number;leakScannedAt?: string;+ socket: DiagnosticsWebSocket | null;onscanned?: (report: GoroutineLeakReport) => void;}- let { leakedGoroutines, leakScannedAt, onscanned }: Props = $props();+ let { leakedGoroutines, leakScannedAt, socket, onscanned }: Props = $props();let scanning = $state(false);let downloading = $state(false);@@ -32,7 +33,9 @@try {const operationResult = await tryCatch((async () => {- const next = await diagnosticsService.scanGoroutineLeaks();+ if (!socket) throw new Error(m.disconnected());+ const next = (await socket.request({ type: 'leakScan' })).leakReport;+ if (!next) return;report = next;onscanned?.(next);})()@@ -53,7 +56,8 @@try {const operationResult = await tryCatch((async () => {- await diagnosticsService.downloadProfile('goroutineleak');+ if (!socket) throw new Error(m.disconnected());+ await socket.downloadProfile('goroutineleak');})());if (operationResult.error !== null) {diff --git a/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-log-panel.svelte b/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-log-panel.svelteindex a8acb62ed..77d3526bf 100644--- a/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-log-panel.svelte+++ b/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-log-panel.svelte@@ -87,10 +87,6 @@<div class="overflow-hidden rounded-xl border border-border/60"><div class="flex flex-wrap items-center gap-2 border-b border-border/60 bg-muted/30 px-3 py-2">- <span class="flex items-center gap-1.5 text-xs font-medium">- <span class={cn('size-2 rounded-full', connected ? 'bg-success' : 'bg-muted-foreground')}></span>- {connected ? m.diagnostics_logs_streaming() : m.disconnected()}- </span><span class="text-xs text-muted-foreground tabular-nums">{m.diagnostics_logs_count({ count: filtered.length })}</span><div class="ml-auto flex items-center gap-2">diff --git a/types/system/diagnostics.go b/types/system/diagnostics.goindex e3b49e4df..299a56d6b 100644--- a/types/system/diagnostics.go+++ b/types/system/diagnostics.go@@ -92,3 +92,81 @@ type WebSocketDiagnostics struct {Snapshot WebSocketMetricsSnapshot `json:"snapshot"`Connections []WebSocketConnectionInfo `json:"connections"`}++// ActorDiagnostics describes the Francis actor host of one environment.+type ActorDiagnostics struct {+ EnvironmentID string `json:"environmentId"`+ EnvironmentName string `json:"environmentName"`+ // Error is set when the environment's actor diagnostics could not be collected.+ Error string `json:"error,omitempty"`+ Ready bool `json:"ready"`+ Hosts []ActorHostInfo `json:"hosts"`+ ActorTypes []ActorTypeInfo `json:"actorTypes"`+ Runs map[string]int `json:"runs"`+ CheckedAt time.Time `json:"checkedAt"`+}++// ActorHostInfo is a registered Francis host.+type ActorHostInfo struct {+ HostID string `json:"hostId"`+ Address string `json:"address"`+ LastHealthCheck time.Time `json:"lastHealthCheck"`+}++// ActorTypeInfo holds per actor type activation, alarm, and job details.+type ActorTypeInfo struct {+ ActorType string `json:"actorType"`+ // ConcurrencyLimit caps concurrent invocations per actor; 0 is unlimited.+ ConcurrencyLimit int `json:"concurrencyLimit"`+ ActiveActors int `json:"activeActors"`+ StoredStates int `json:"storedStates"`+ PendingAlarms int `json:"pendingAlarms"`+ PendingJobs int `json:"pendingJobs"`+ LeasedAlarms int `json:"leasedAlarms"`+ CompletedJobs int `json:"completedJobs"`+ DeadJobs int `json:"deadJobs"`+ NextAlarmAt time.Time `json:"nextAlarmAt,omitzero"`+ Instances []ActorInstance `json:"instances"`+ RecentFailures []ActorJobFailure `json:"recentFailures"`+}++// ActorInstance is an activated actor.+type ActorInstance struct {+ ActorID string `json:"actorId"`+ // Name is Arcane's display name for the actor, when known.+ Name string `json:"name,omitempty"`+ HostID string `json:"hostId"`+ ActivatedAt time.Time `json:"activatedAt"`+}++// ActorJobFailure is a dead-lettered actor job.+type ActorJobFailure struct {+ JobID string `json:"jobId"`+ ActorID string `json:"actorId"`+ Name string `json:"name,omitempty"`+ Method string `json:"method"`+ Attempts int `json:"attempts"`+ LastError string `json:"lastError,omitempty"`+ FailedAt time.Time `json:"failedAt"`+}++// DiagnosticsCommand is a client request sent over the diagnostics stream.+type DiagnosticsCommand struct {+ ID string `json:"id"`+ // Type is refresh, dump, leakScan, or profile.+ Type string `json:"type"`+ Name string `json:"name,omitempty"`+ Seconds int `json:"seconds,omitempty"`+}++// DiagnosticsMessage is a snapshot push or command result on the diagnostics stream.+type DiagnosticsMessage struct {+ // Type is snapshot or result.+ Type string `json:"type"`+ ID string `json:"id,omitempty"`+ Snapshot *Diagnostics `json:"snapshot,omitempty"`+ LeakReport *GoroutineLeakReport `json:"leakReport,omitempty"`+ Text string `json:"text,omitempty"`+ Data []byte `json:"data,omitempty"`+ Error string `json:"error,omitempty"`+}diff --git a/types/system/websocket.go b/types/system/websocket.goindex f42648b0d..2746d014c 100644--- a/types/system/websocket.go+++ b/types/system/websocket.go@@ -10,6 +10,7 @@ const (WSKindContainerExec = "container_exec"WSKindSystemStats = "system_stats"WSKindServiceLogs = "service_logs"+ WSKindDiagnostics = "diagnostics")// WebSocketConnectionInfo describes a single active WebSocket connection.@@ -51,6 +52,8 @@ type WebSocketMetricsSnapshot struct {ContainerExec int64 `json:"containerExec"`// SystemStats is the number of active system-stats streams.SystemStats int64 `json:"systemStats"`+ // DiagnosticsActive is the number of active diagnostics streams.+ DiagnosticsActive int64 `json:"diagnosticsActive"`// ServiceLogsActive is the number of active swarm service-log streams.ServiceLogsActive int64 `json:"serviceLogsActive"`}