Skip to content

Modern Docker Management, Designed for Everyone

feat: add actor statuses to diagnostics page

Kyle Mendell01a8964d69ba
diff --git a/backend/api/handlers/diagnostics.go b/backend/api/handlers/diagnostics.go
index d2d93667f..43b4feb78 100644
--- a/backend/api/handlers/diagnostics.go
+++ b/backend/api/handlers/diagnostics.go
@@ -6,88 +6,46 @@ import (
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/types/v2/system"
- "go.getarcane.app/streams/logs"
- "github.com/getarcaneapp/arcane/backend/v2/api/ws"
"github.com/getarcaneapp/arcane/backend/v2/internal/diagnostics"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
)
-// DiagnosticsHandler serves the REST diagnostics endpoints. The live WebSocket
-// streams and pprof routes live in the api/ws package alongside the other
-// streaming endpoints; the snapshot is assembled there too (ws.BuildDiagnostics).
+// DiagnosticsHandler serves the environment-scoped actor diagnostics the manager
+// collects from each agent. The diagnostics page itself uses the api/ws streams.
type DiagnosticsHandler struct {
diag *diagnostics.DiagnosticsService
}
-type DiagnosticsInput struct{}
-
-type GetDiagnosticsOutput struct {
- Body system.Diagnostics
-}
-
-type GetDiagnosticsLogsOutput struct {
- Body []logs.Entry
+type GetEnvironmentActorDiagnosticsInput struct {
+ EnvironmentID string `path:"id" doc:"Environment ID"`
}
-type ScanGoroutineLeaksInput struct{}
-
-type ScanGoroutineLeaksOutput struct {
- Body system.GoroutineLeakReport
+type GetEnvironmentActorDiagnosticsOutput struct {
+ Body system.ActorDiagnostics
}
// RegisterDiagnostics registers the Huma diagnostics REST endpoints.
func RegisterDiagnostics(api huma.API, diag *diagnostics.DiagnosticsService) {
h := &DiagnosticsHandler{diag: diag}
- huma.Register(api, huma.Operation{
- OperationID: "get-diagnostics",
- Method: http.MethodGet,
- Path: "/diagnostics",
- Summary: "Get runtime diagnostics",
- Description: "Returns Go runtime, memory, garbage-collector, and WebSocket connection statistics.",
- Tags: []string{"Diagnostics"},
- Security: handlerutil.DefaultOperationSecurity(),
- Middlewares: middleware.RequirePermission(api, authz.PermDiagnosticsRead),
- }, h.GetDiagnostics)
-
- huma.Register(api, huma.Operation{
- OperationID: "get-diagnostics-logs",
+ middleware.RegisterWithPermission(api, huma.Operation{
+ OperationID: "get-environment-actor-diagnostics",
Method: http.MethodGet,
- Path: "/diagnostics/logs",
- Summary: "Get recent backend logs",
- Description: "Returns the most recent buffered backend log entries (oldest first).",
+ Path: "/environments/{id}/diagnostics/actors",
+ Summary: "Get actor diagnostics for an environment",
+ Description: "Returns Francis actor host, actor type, and run queue diagnostics for one environment.",
Tags: []string{"Diagnostics"},
Security: handlerutil.DefaultOperationSecurity(),
- Middlewares: middleware.RequirePermission(api, authz.PermDiagnosticsRead),
- }, h.GetRecentLogs)
-
- huma.Register(api, huma.Operation{
- OperationID: "scan-goroutine-leaks",
- Method: http.MethodPost,
- Path: "/diagnostics/goroutineleak",
- Summary: "Scan for leaked goroutines",
- Description: "Runs a goroutine leak-detection GC cycle and returns the goroutineleak pprof profile as text (leaked stacks only).",
- Tags: []string{"Diagnostics"},
- Security: handlerutil.DefaultOperationSecurity(),
- Middlewares: middleware.RequirePermission(api, authz.PermDiagnosticsRead),
- }, h.ScanGoroutineLeaks)
-}
-
-func (h *DiagnosticsHandler) GetDiagnostics(_ context.Context, _ *DiagnosticsInput) (*GetDiagnosticsOutput, error) {
- return &GetDiagnosticsOutput{Body: ws.BuildDiagnostics(h.diag)}, nil
-}
-
-func (h *DiagnosticsHandler) GetRecentLogs(_ context.Context, _ *DiagnosticsInput) (*GetDiagnosticsLogsOutput, error) {
- return &GetDiagnosticsLogsOutput{Body: ws.LogBroadcaster().Recent()}, nil
+ }, authz.PermDiagnosticsRead, h.GetEnvironmentActorDiagnostics)
}
-func (h *DiagnosticsHandler) ScanGoroutineLeaks(_ context.Context, _ *ScanGoroutineLeaksInput) (*ScanGoroutineLeaksOutput, error) {
- report, err := h.diag.ScanGoroutineLeaks()
+func (h *DiagnosticsHandler) GetEnvironmentActorDiagnostics(ctx context.Context, _ *GetEnvironmentActorDiagnosticsInput) (*GetEnvironmentActorDiagnosticsOutput, error) {
+ d, err := h.diag.CollectActors(ctx)
if err != nil {
- return nil, huma.Error500InternalServerError("Failed to collect goroutine leak profile: " + err.Error())
+ return nil, huma.Error500InternalServerError("Failed to collect actor diagnostics: " + err.Error())
}
- return &ScanGoroutineLeaksOutput{Body: report}, nil
+ return &GetEnvironmentActorDiagnosticsOutput{Body: d}, nil
}
diff --git a/backend/api/ws/diagnostics.go b/backend/api/ws/diagnostics.go
index 51bd1e5d9..522ed96c6 100644
--- a/backend/api/ws/diagnostics.go
+++ b/backend/api/ws/diagnostics.go
@@ -1,8 +1,10 @@
package ws
import (
+ "cmp"
"context"
"encoding/json/v2"
+ "fmt"
"log/slog"
"net/http"
"net/http/pprof"
@@ -23,6 +25,8 @@ import (
const (
// diagnosticsStreamInterval is how often the live diagnostics stream pushes a snapshot.
diagnosticsStreamInterval = 2 * time.Second
+ // actorDiagnosticsStreamInterval is slower because each push queries every environment.
+ actorDiagnosticsStreamInterval = 10 * time.Second
// Keepalive/liveness bounds for the diagnostics sockets, mirroring the system
// stats stream. Without them a silently dead peer wedges the writer forever,
@@ -33,10 +37,9 @@ const (
diagnosticsPingPeriod = 54 * time.Second
)
-// BuildDiagnostics assembles a full diagnostics snapshot: runtime/memory/GC from
-// the DiagnosticsService plus this package's WebSocket metrics and worker-goroutine
-// count. Shared by the REST endpoint (via handlers) and the live WebSocket stream.
-func BuildDiagnostics(diag *diagnostics.DiagnosticsService) systemtypes.Diagnostics {
+// buildDiagnosticsInternal assembles a snapshot: runtime/memory/GC from the
+// DiagnosticsService plus this package's WebSocket metrics and worker-goroutine count.
+func buildDiagnosticsInternal(diag *diagnostics.DiagnosticsService) systemtypes.Diagnostics {
d := systemtypes.Diagnostics{Timestamp: time.Now().UTC()}
if diag != nil {
d.Runtime, d.Memory, d.GC = diag.Collect()
@@ -58,6 +61,7 @@ func (h *WebSocketHandler) registerDiagnosticsRoutesInternal(group *echo.Group,
middleware.RequireEchoPermission(authz.PermDiagnosticsRead),
)
diag.GET("/stream", h.DiagnosticsStream)
+ diag.GET("/actors/stream", h.ActorDiagnosticsStream)
diag.GET("/logs/stream", h.ServerLogsStream)
pprofGroup := group.Group("/debug/pprof", authMiddleware.WithAdminRequired().Add())
@@ -77,13 +81,62 @@ func (h *WebSocketHandler) registerDiagnosticsRoutesInternal(group *echo.Group,
})
}
-// DiagnosticsStream pushes a fresh diagnostics snapshot on connect and then every
-// diagnosticsStreamInterval until the client disconnects.
+// DiagnosticsStream pushes a snapshot on connect and every diagnosticsStreamInterval,
+// and answers refresh, dump, leak-scan, and profile commands from the client.
func (h *WebSocketHandler) DiagnosticsStream(c *echo.Context) error {
- conn, err := wshub.Accept(c.Response(), c.Request(), h.checkWSOrigin)
- if err != nil {
+ ps, _ := c.Get(string(middleware.ContextKeyUserPermissions)).(*authz.PermissionSet)
+ snapshot := func(context.Context) any {
+ d := buildDiagnosticsInternal(h.diagnosticsService)
+ return systemtypes.DiagnosticsMessage{Type: "snapshot", Snapshot: &d}
+ }
+ return h.streamSnapshotsInternal(c, diagnosticsStreamInterval, snapshot, func(ctx context.Context, cmd systemtypes.DiagnosticsCommand) any {
+ if cmd.Type == "refresh" {
+ return snapshot(ctx)
+ }
+ result := systemtypes.DiagnosticsMessage{Type: "result", ID: cmd.ID}
+ var err error
+ switch cmd.Type {
+ case "leakScan":
+ report, scanErr := h.diagnosticsService.ScanGoroutineLeaks()
+ result.LeakReport, err = &report, scanErr
+ case "dump", "profile":
+ if !ps.IsGlobalAdmin() {
+ result.Error = "profiles require a global administrator"
+ return result
+ }
+ if cmd.Type == "dump" {
+ var text []byte
+ text, err = h.diagnosticsService.Profile(ctx, cmd.Name, 0, 2)
+ result.Text = string(text)
+ } else {
+ result.Data, err = h.diagnosticsService.Profile(ctx, cmd.Name, min(cmp.Or(cmd.Seconds, 30), 120), 0)
+ }
+ default:
+ err = fmt.Errorf("unknown diagnostics command %q", cmd.Type)
+ }
+ if err != nil {
+ result.Error = err.Error()
+ }
+ return result
+ })
+}
+
+// ActorDiagnosticsStream pushes actor diagnostics for every environment on
+// connect and then every actorDiagnosticsStreamInterval.
+func (h *WebSocketHandler) ActorDiagnosticsStream(c *echo.Context) error {
+ return h.streamSnapshotsInternal(c, actorDiagnosticsStreamInterval, func(ctx context.Context) any {
+ return h.diagnosticsService.CollectAllActors(ctx)
+ }, nil)
+}
+
+// streamSnapshotsInternal writes build's result on connect and every interval until
+// the client disconnects. A non-nil handle answers client commands concurrently.
+func (h *WebSocketHandler) streamSnapshotsInternal(c *echo.Context, interval time.Duration, build func(context.Context) any, handle func(context.Context, systemtypes.DiagnosticsCommand) any) error {
+ conn, unregister, accepted := h.acceptWSInternal(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)
+ if !accepted {
return nil
}
+ defer unregister()
defer func() {
if closeNowErr := conn.CloseNow(); closeNowErr != nil {
slog.Debug("Failed to close diagnostics websocket connection", "error", closeNowErr)
@@ -91,9 +144,8 @@ func (h *WebSocketHandler) DiagnosticsStream(c *echo.Context) error {
}()
ctx := c.Request().Context()
- done := diagnosticsReadLoopInternal(ctx, conn)
- write := func() bool {
- b, marshalErr := json.Marshal(BuildDiagnostics(h.diagnosticsService))
+ write := func(v any) bool {
+ b, marshalErr := json.Marshal(v)
if marshalErr != nil {
return true
}
@@ -101,11 +153,22 @@ func (h *WebSocketHandler) DiagnosticsStream(c *echo.Context) error {
defer cancel()
return conn.Write(wctx, websocket.MessageText, b) == nil
}
+ var onMessage func([]byte)
+ if handle != nil {
+ onMessage = func(b []byte) {
+ var cmd systemtypes.DiagnosticsCommand
+ if json.Unmarshal(b, &cmd) != nil {
+ return
+ }
+ go write(handle(ctx, cmd))
+ }
+ }
+ done := diagnosticsReadLoopInternal(ctx, conn, onMessage)
- if !write() {
+ if !write(build(ctx)) {
return nil
}
- ticker := time.NewTicker(diagnosticsStreamInterval)
+ ticker := time.NewTicker(interval)
defer ticker.Stop()
pingTicker := time.NewTicker(diagnosticsPingPeriod)
defer pingTicker.Stop()
@@ -114,7 +177,7 @@ func (h *WebSocketHandler) DiagnosticsStream(c *echo.Context) error {
case <-done:
return nil
case <-ticker.C:
- if !write() {
+ if !write(build(ctx)) {
return nil
}
case <-pingTicker.C:
@@ -127,10 +190,11 @@ func (h *WebSocketHandler) DiagnosticsStream(c *echo.Context) error {
// ServerLogsStream replays the recent backend log backlog then streams new entries live.
func (h *WebSocketHandler) ServerLogsStream(c *echo.Context) error {
- conn, err := wshub.Accept(c.Response(), c.Request(), h.checkWSOrigin)
- if err != nil {
+ conn, unregister, accepted := h.acceptWSInternal(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)
+ if !accepted {
return nil
}
+ defer unregister()
defer func() {
if closeNowErr := conn.CloseNow(); closeNowErr != nil {
slog.Debug("Failed to close server logs websocket connection", "error", closeNowErr)
@@ -143,7 +207,7 @@ func (h *WebSocketHandler) ServerLogsStream(c *echo.Context) error {
defer cancel()
ctx := c.Request().Context()
- done := diagnosticsReadLoopInternal(ctx, conn)
+ done := diagnosticsReadLoopInternal(ctx, conn, nil)
write := func(e logs.Entry) bool {
b, marshalErr := json.Marshal(e)
if marshalErr != nil {
@@ -189,18 +253,22 @@ func pingDiagnosticsConnInternal(ctx context.Context, conn *websocket.Conn) bool
return conn.Ping(pctx) == nil
}
-// diagnosticsReadLoopInternal drains incoming frames; the returned channel closes
-// when the peer disconnects, signaling the writer loop to stop.
-func diagnosticsReadLoopInternal(ctx context.Context, conn *websocket.Conn) <-chan struct{} {
+// diagnosticsReadLoopInternal passes incoming frames to onMessage, or drains them when
+// it is nil; the returned channel closes when the peer disconnects.
+func diagnosticsReadLoopInternal(ctx context.Context, conn *websocket.Conn, onMessage func([]byte)) <-chan struct{} {
conn.SetReadLimit(diagnosticsReadLimit)
done := make(chan struct{})
go func() {
defer close(done)
for {
- if _, _, err := conn.Read(ctx); err != nil {
+ _, b, err := conn.Read(ctx)
+ if err != nil {
return
}
+ if onMessage != nil {
+ onMessage(b)
+ }
}
}()
return done
diff --git a/backend/internal/diagnostics/model.go b/backend/internal/diagnostics/model.go
new file mode 100644
index 000000000..7e9921d57
--- /dev/null
+++ b/backend/internal/diagnostics/model.go
@@ -0,0 +1,71 @@
+package diagnostics
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
+)
+
+// actorHostModel is a read-only view of a registered Francis host.
+type actorHostModel struct {
+ HostID string `gorm:"column:host_id"`
+ Address string `gorm:"column:host_address"`
+ LastHealthCheck actorTimeInternal `gorm:"column:host_last_health_check"`
+}
+
+func (actorHostModel) TableName() string { return francis.TablePrefix + "_hosts" }
+
+// actorTypeModel is a registered actor type with its concurrency limit.
+type actorTypeModel struct {
+ ActorType string
+ ConcurrencyLimit int
+}
+
+// actorCountModel is one grouped count row from a Francis table.
+type actorCountModel struct {
+ ActorType string
+ Kind string
+ Count int
+ NextDue actorTimeInternal
+}
+
+// activeActorModel is a read-only view of an activated actor.
+type activeActorModel struct {
+ ActorType string `gorm:"column:actor_type"`
+ ActorID string `gorm:"column:actor_id"`
+ HostID string `gorm:"column:host_id"`
+ Activation actorTimeInternal `gorm:"column:actor_activation"`
+}
+
+func (activeActorModel) TableName() string { return francis.TablePrefix + "_active_actors" }
+
+// terminalJobModel is a read-only view of a finished actor job.
+type terminalJobModel struct {
+ JobID string `gorm:"column:job_id"`
+ ActorType string `gorm:"column:actor_type"`
+ ActorID string `gorm:"column:actor_id"`
+ JobMethod string `gorm:"column:job_method"`
+ Attempts int `gorm:"column:attempts"`
+ LastError *string `gorm:"column:last_error"`
+ EndedAt actorTimeInternal `gorm:"column:ended_at"`
+}
+
+func (terminalJobModel) TableName() string { return francis.TablePrefix + "_terminal_jobs" }
+
+// actorTimeInternal scans Francis timestamps: SQLite stores epoch milliseconds, Postgres a timestamp.
+type actorTimeInternal time.Time
+
+func (t *actorTimeInternal) Scan(value any) error {
+ switch v := value.(type) {
+ case int64:
+ *t = actorTimeInternal(time.UnixMilli(v).UTC())
+ case time.Time:
+ *t = actorTimeInternal(v.UTC())
+ case nil:
+ *t = actorTimeInternal{}
+ default:
+ return fmt.Errorf("unsupported actor timestamp %T", value)
+ }
+ return nil
+}
diff --git a/backend/internal/diagnostics/service.go b/backend/internal/diagnostics/service.go
index ec1ec6797..13408a00e 100644
--- a/backend/internal/diagnostics/service.go
+++ b/backend/internal/diagnostics/service.go
@@ -2,24 +2,34 @@ package diagnostics
import (
"bytes"
+ "context"
"errors"
"fmt"
+ "log/slog"
+ "net/http"
"runtime"
"runtime/debug"
"runtime/pprof"
+ "runtime/trace"
"sync"
"time"
"github.com/getarcaneapp/arcane/types/v2/system"
+ kit "go.getarcane.app/kit/pkg"
+ "gorm.io/gorm"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
)
const (
- // recentGCPauseSamples is the number of recent GC pause durations reported.
- recentGCPauseSamples = 16
-
- // goroutineLeakProfileName is the runtime/pprof profile that reports goroutines
- // blocked on concurrency primitives that cannot be unblocked.
+ recentGCPauseSamples = 16
goroutineLeakProfileName = "goroutineleak"
+
+ // maxActorDetailRowsInternal caps the instances and failures listed per actor type.
+ maxActorDetailRowsInternal = 10
)
// DiagnosticsService gathers Go runtime, memory, and garbage-collector
@@ -27,22 +37,20 @@ const (
// WebSocket metrics and worker-goroutine counts are merged in at the handler
// layer to avoid an import cycle with the api/ws package.
type DiagnosticsService struct {
- startedAt time.Time
+ startedAt time.Time
+ db *database.DB
+ actors *francis.Runtime
+ coordinator *runs.Coordinator
+ environments *environment.EnvironmentService
leakMu sync.Mutex
leakScannedAt time.Time
}
-// NewDiagnosticsService returns a DiagnosticsService. startedAt is captured at
-// construction (≈ process start) and used to report uptime.
-func NewDiagnosticsService() *DiagnosticsService {
- return &DiagnosticsService{startedAt: time.Now()}
+func NewDiagnosticsService(db *database.DB, actors *francis.Runtime, coordinator *runs.Coordinator, environments *environment.EnvironmentService) *DiagnosticsService {
+ return &DiagnosticsService{startedAt: time.Now(), db: db, actors: actors, coordinator: coordinator, environments: environments}
}
-// Collect samples the current runtime, memory, and GC state.
-//
-// LeakedGoroutines is the last-known leak count from the runtime; it does not
-// trigger a leak-detection GC. Use ScanGoroutineLeaks for a fresh scan.
func (s *DiagnosticsService) Collect() (system.RuntimeInfo, system.MemoryInfo, system.GCInfo) {
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
@@ -132,6 +140,43 @@ func (s *DiagnosticsService) ScanGoroutineLeaks() (system.GoroutineLeakReport, e
}, nil
}
+// Profile captures a pprof profile in the given debug format. The CPU profile
+// and execution trace sample for seconds or until ctx ends.
+func (s *DiagnosticsService) Profile(ctx context.Context, name string, seconds, debugLevel int) ([]byte, error) {
+ var buf bytes.Buffer
+ sample := func() {
+ timer := time.NewTimer(time.Duration(seconds) * time.Second)
+ defer timer.Stop()
+ select {
+ case <-timer.C:
+ case <-ctx.Done():
+ }
+ }
+ switch name {
+ case "profile":
+ if err := pprof.StartCPUProfile(&buf); err != nil {
+ return nil, fmt.Errorf("start CPU profile: %w", err)
+ }
+ sample()
+ pprof.StopCPUProfile()
+ case "trace":
+ if err := trace.Start(&buf); err != nil {
+ return nil, fmt.Errorf("start trace: %w", err)
+ }
+ sample()
+ trace.Stop()
+ default:
+ p := pprof.Lookup(name)
+ if p == nil {
+ return nil, fmt.Errorf("unknown profile %q", name)
+ }
+ if err := p.WriteTo(&buf, debugLevel); err != nil {
+ return nil, fmt.Errorf("write %s profile: %w", name, err)
+ }
+ }
+ return buf.Bytes(), ctx.Err()
+}
+
// leakedGoroutineCountInternal returns the last-known leak count without
// triggering a leak-detection GC.
func leakedGoroutineCountInternal() int {
@@ -141,3 +186,145 @@ func leakedGoroutineCountInternal() int {
}
return p.Count()
}
+
+// CollectActors reports this process's Francis host, actor types, and run queue.
+func (s *DiagnosticsService) CollectActors(ctx context.Context) (system.ActorDiagnostics, error) {
+ d := system.ActorDiagnostics{EnvironmentID: environment.LocalEnvironmentID, Runs: map[string]int{}, CheckedAt: time.Now().UTC()}
+ select {
+ case <-s.actors.Ready():
+ d.Ready = true
+ default:
+ return d, nil
+ }
+
+ db := s.db.WithContext(ctx)
+ var hosts []actorHostModel
+ if err := db.Order("host_id").Find(&hosts).Error; err != nil {
+ return d, fmt.Errorf("list actor hosts: %w", err)
+ }
+ for _, host := range hosts {
+ d.Hosts = append(d.Hosts, system.ActorHostInfo{HostID: host.HostID, Address: host.Address, LastHealthCheck: time.Time(host.LastHealthCheck)})
+ }
+
+ // Records names coordinator actors, so load it before resolving instance names.
+ records, err := s.coordinator.Records(ctx)
+ if err != nil {
+ return d, err
+ }
+ for _, record := range records {
+ for _, run := range record.Runs {
+ d.Runs[string(run.Status)]++
+ }
+ }
+
+ d.ActorTypes, err = s.collectActorTypesInternal(db)
+ return d, err
+}
+
+// collectActorTypesInternal reports counts, recent instances, and dead jobs per registered actor type.
+func (s *DiagnosticsService) collectActorTypesInternal(db *gorm.DB) ([]system.ActorTypeInfo, error) {
+ var actorTypes []actorTypeModel
+ err := db.Table(francis.TablePrefix + "_host_actor_types").
+ Select("actor_type, MAX(actor_concurrency_limit) AS concurrency_limit").
+ Group("actor_type").Order("actor_type").Scan(&actorTypes).Error
+ if err != nil {
+ return nil, fmt.Errorf("list actor types: %w", err)
+ }
+ var active, states, alarms, leased, terminal []actorCountModel
+ var instances []activeActorModel
+ var failures []terminalJobModel
+ err = errors.Join(
+ db.Table(francis.TablePrefix+"_active_actors").Select("actor_type, COUNT(*) AS count").Group("actor_type").Scan(&active).Error,
+ db.Table(francis.TablePrefix+"_actor_state").Select("actor_type, COUNT(*) AS count").Group("actor_type").Scan(&states).Error,
+ db.Table(francis.TablePrefix+"_alarms").Select("actor_type, alarm_kind AS kind, COUNT(*) AS count, MIN(alarm_due_time) AS next_due").Group("actor_type, alarm_kind").Scan(&alarms).Error,
+ db.Table(francis.TablePrefix+"_alarms").Select("actor_type, COUNT(*) AS count").Where("alarm_lease_id IS NOT NULL").Group("actor_type").Scan(&leased).Error,
+ db.Table(francis.TablePrefix+"_terminal_jobs").Select("actor_type, job_status AS kind, COUNT(*) AS count").Group("actor_type, job_status").Scan(&terminal).Error,
+ db.Order("actor_activation DESC").Limit(500).Find(&instances).Error,
+ db.Where("job_status = ?", "dead").Order("ended_at DESC").Limit(100).Find(&failures).Error,
+ )
+ if err != nil {
+ return nil, fmt.Errorf("collect actor activity: %w", err)
+ }
+
+ out := make([]system.ActorTypeInfo, len(actorTypes))
+ byType := make(map[string]*system.ActorTypeInfo, len(actorTypes))
+ for i, actorType := range actorTypes {
+ out[i] = system.ActorTypeInfo{ActorType: actorType.ActorType, ConcurrencyLimit: actorType.ConcurrencyLimit, Instances: []system.ActorInstance{}, RecentFailures: []system.ActorJobFailure{}}
+ byType[actorType.ActorType] = &out[i]
+ }
+ addCountsInternal(byType, active, func(info *system.ActorTypeInfo, row actorCountModel) { info.ActiveActors = row.Count })
+ addCountsInternal(byType, states, func(info *system.ActorTypeInfo, row actorCountModel) { info.StoredStates = row.Count })
+ addCountsInternal(byType, leased, func(info *system.ActorTypeInfo, row actorCountModel) { info.LeasedAlarms = row.Count })
+ addCountsInternal(byType, alarms, func(info *system.ActorTypeInfo, row actorCountModel) {
+ if row.Kind == "job" {
+ info.PendingJobs += row.Count
+ } else {
+ info.PendingAlarms += row.Count
+ }
+ if due := time.Time(row.NextDue); !due.IsZero() && (info.NextAlarmAt.IsZero() || due.Before(info.NextAlarmAt)) {
+ info.NextAlarmAt = due
+ }
+ })
+ addCountsInternal(byType, terminal, func(info *system.ActorTypeInfo, row actorCountModel) {
+ switch row.Kind {
+ case "dead":
+ info.DeadJobs += row.Count
+ case "completed":
+ info.CompletedJobs += row.Count
+ }
+ })
+ for _, row := range instances {
+ if info := byType[row.ActorType]; info != nil && len(info.Instances) < maxActorDetailRowsInternal {
+ info.Instances = append(info.Instances, system.ActorInstance{ActorID: row.ActorID, Name: s.actors.ActorName(row.ActorID), HostID: row.HostID, ActivatedAt: time.Time(row.Activation)})
+ }
+ }
+ for _, row := range failures {
+ if info := byType[row.ActorType]; info != nil && len(info.RecentFailures) < maxActorDetailRowsInternal {
+ info.RecentFailures = append(info.RecentFailures, system.ActorJobFailure{
+ JobID: row.JobID, ActorID: row.ActorID, Name: s.actors.ActorName(row.ActorID), Method: row.JobMethod,
+ Attempts: row.Attempts, LastError: kit.FromPtr(row.LastError), FailedAt: time.Time(row.EndedAt),
+ })
+ }
+ }
+ return out, nil
+}
+
+// addCountsInternal applies grouped rows to their registered actor type.
+func addCountsInternal(byType map[string]*system.ActorTypeInfo, rows []actorCountModel, apply func(*system.ActorTypeInfo, actorCountModel)) {
+ for _, row := range rows {
+ if info := byType[row.ActorType]; info != nil {
+ apply(info, row)
+ }
+ }
+}
+
+// CollectAllActors gathers actor diagnostics from the local and every active remote environment.
+func (s *DiagnosticsService) CollectAllActors(ctx context.Context) []system.ActorDiagnostics {
+ local, err := s.CollectActors(ctx)
+ if err != nil {
+ local.Error = err.Error()
+ }
+ if env, envErr := s.environments.GetEnvironmentByIDCached(ctx, environment.LocalEnvironmentID); envErr == nil {
+ local.EnvironmentName = env.Name
+ }
+
+ remotes, err := s.environments.ListActiveRemoteEnvironments(ctx)
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to list environments for actor diagnostics", "error", err)
+ }
+ out := make([]system.ActorDiagnostics, len(remotes)+1)
+ out[0] = local
+ var wg sync.WaitGroup
+ for i, env := range remotes {
+ wg.Go(func() {
+ var d system.ActorDiagnostics
+ if proxyErr := s.environments.ProxyJSONRequestForEnvironment(ctx, env, http.MethodGet, "/api/environments/0/diagnostics/actors", nil, &d); proxyErr != nil {
+ d = system.ActorDiagnostics{Error: proxyErr.Error(), CheckedAt: time.Now().UTC()}
+ }
+ d.EnvironmentID, d.EnvironmentName = env.ID, env.Name
+ out[i+1] = d
+ })
+ }
+ wg.Wait()
+ return out
+}
diff --git a/backend/internal/diagnostics/service_test.go b/backend/internal/diagnostics/service_test.go
index 4fceabdf6..bffb9334e 100644
--- a/backend/internal/diagnostics/service_test.go
+++ b/backend/internal/diagnostics/service_test.go
@@ -12,7 +12,7 @@ import (
)
func TestDiagnosticsServiceCollect(t *testing.T) {
- s := NewDiagnosticsService()
+ s := NewDiagnosticsService(nil, nil, nil, nil)
rt, mem, _ := s.Collect()
@@ -43,7 +43,7 @@ func TestGoroutineLeakProfileAvailable(t *testing.T) {
}
func TestDiagnosticsServiceScanGoroutineLeaks(t *testing.T) {
- s := NewDiagnosticsService()
+ s := NewDiagnosticsService(nil, nil, nil, nil)
report, err := s.ScanGoroutineLeaks()
require.NoError(t, err)
@@ -58,7 +58,7 @@ func TestDiagnosticsServiceScanGoroutineLeaks(t *testing.T) {
}
func TestDiagnosticsServiceScanGoroutineLeaksDetectsLeak(t *testing.T) {
- s := NewDiagnosticsService()
+ s := NewDiagnosticsService(nil, nil, nil, nil)
leakUnbufferedChannelInternal()
deadline := time.Now().Add(5 * time.Second)
diff --git a/backend/pkg/libarcane/ws/metrics.go b/backend/pkg/libarcane/ws/metrics.go
index 4f6db634e..fbc5826ef 100644
--- a/backend/pkg/libarcane/ws/metrics.go
+++ b/backend/pkg/libarcane/ws/metrics.go
@@ -17,6 +17,7 @@ type WebSocketMetrics struct {
containerExec atomic.Int64
systemStats atomic.Int64
serviceLogsActive atomic.Int64
+ diagnosticsActive atomic.Int64
seq atomic.Uint64
mu sync.RWMutex
connections map[string]systemtypes.WebSocketConnectionInfo
@@ -38,6 +39,7 @@ func (m *WebSocketMetrics) Snapshot() systemtypes.WebSocketMetricsSnapshot {
ContainerExec: m.containerExec.Load(),
SystemStats: m.systemStats.Load(),
ServiceLogsActive: m.serviceLogsActive.Load(),
+ DiagnosticsActive: m.diagnosticsActive.Load(),
}
}
@@ -100,5 +102,7 @@ func (m *WebSocketMetrics) applyDelta(kind string, delta int64) {
m.systemStats.Add(delta)
case systemtypes.WSKindServiceLogs:
m.serviceLogsActive.Add(delta)
+ case systemtypes.WSKindDiagnostics:
+ m.diagnosticsActive.Add(delta)
}
}
diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.go
index 4f175f23a..d07b3acd7 100644
--- a/backend/pkg/scheduler/runs/actors.go
+++ b/backend/pkg/scheduler/runs/actors.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"log/slog"
+ "slices"
"strings"
"time"
"uuid"
@@ -342,12 +343,17 @@ func (a *coordinatorActorInternal) Alarm(ctx context.Context, name string, data
return a.flushInternal(ctx, &state)
}
-func (q *Coordinator) repairInternal(ctx context.Context) error {
+// repairInternal flushes coordinators so schedule alarms and dispatches are restored.
+// Unless all is set, it skips records without unfinished runs so idle coordinators can deactivate.
+func (q *Coordinator) repairInternal(ctx context.Context, all bool) error {
records, err := q.Records(ctx)
if err != nil {
return err
}
for _, record := range records {
+ if !all && !slices.ContainsFunc(record.Runs, func(run st.Run) bool { return !run.Status.Terminal() && run.Status != st.NeedsAttention }) {
+ continue
+ }
if _, invokeErr := q.service.Invoke(ctx, coordinatorTypeInternal, kit.SHA256Hex(record.EnvironmentID+"\x00"+record.JobID), "repair", nil); invokeErr != nil {
return invokeErr
}
diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.go
index ef28ef7fd..6265fda00 100644
--- a/backend/pkg/scheduler/runs/actors_test.go
+++ b/backend/pkg/scheduler/runs/actors_test.go
@@ -268,10 +268,10 @@ func TestReconciliationRepairsMissingAlarm(t *testing.T) {
record, err := q.ScheduleState(t.Context(), "repair")
require.NoError(t, err)
id := kit.SHA256Hex("0\x00repair")
- err = q.repairInternal(t.Context())
+ err = q.repairInternal(t.Context(), true)
require.NoError(t, err)
require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorTypeInternal, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))
- err = q.repairInternal(t.Context())
+ err = q.repairInternal(t.Context(), true)
require.NoError(t, err)
require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorTypeInternal, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))
after, err := q.ScheduleState(t.Context(), "repair")
@@ -358,7 +358,7 @@ func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {
state.Dispatches[key] = intent
state.Revision++
require.NoError(t, q.service.SetState(t.Context(), coordinatorTypeInternal, id, state, nil))
- err = q.repairInternal(t.Context())
+ err = q.repairInternal(t.Context(), false)
require.NoError(t, err)
require.NoError(t, q.service.GetState(t.Context(), coordinatorTypeInternal, id, &state))
require.Equal(t, original, state.Dispatches[key].JobID)
diff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.go
index cbce1e781..f4b63ac40 100644
--- a/backend/pkg/scheduler/runs/coordinator.go
+++ b/backend/pkg/scheduler/runs/coordinator.go
@@ -287,6 +287,8 @@ func (q *Coordinator) dispatchInternal(ctx context.Context) {
defer close(q.done)
initialRetention := true
+ // Flush every coordinator once while enabled to restore schedule alarms; later passes only cover unfinished work.
+ fullRepair := true
nextRetention := time.Now()
for ctx.Err() == nil {
q.retryActivitySyncInternal(ctx)
@@ -301,9 +303,14 @@ func (q *Coordinator) dispatchInternal(ctx context.Context) {
nextRetention = time.Now().Add(time.Hour)
}
nextWake := time.Now().Add(15 * time.Second)
- if err := q.repairInternal(ctx); err != nil && ctx.Err() == nil {
- slog.ErrorContext(ctx, "Job queue dispatch failed", "error", err)
+ enabled := q.enabled.Load()
+ if err := q.repairInternal(ctx, fullRepair); err != nil {
+ if ctx.Err() == nil {
+ slog.ErrorContext(ctx, "Job queue dispatch failed", "error", err)
+ }
nextWake = time.Now().Add(5 * time.Second)
+ } else if enabled {
+ fullRepair = false
}
if nextRetention.Before(nextWake) {
nextWake = nextRetention
diff --git a/backend/pkg/utils/francis/logging.go b/backend/pkg/utils/francis/logging.go
index 0fb39c0ef..44543bc18 100644
--- a/backend/pkg/utils/francis/logging.go
+++ b/backend/pkg/utils/francis/logging.go
@@ -13,6 +13,13 @@ func (r *Runtime) NameActor(actorID, name string) {
r.actorNames.Store(actorID, name)
}
+// ActorName returns the display name set by NameActor, or an empty string.
+func (r *Runtime) ActorName(actorID string) string {
+ name, _ := r.actorNames.Load(actorID)
+ displayName, _ := name.(string)
+ return displayName
+}
+
type actorLogHandlerInternal struct {
slog.Handler
diff --git a/backend/pkg/utils/francis/provider.go b/backend/pkg/utils/francis/provider.go
index e6e85d491..526797ef5 100644
--- a/backend/pkg/utils/francis/provider.go
+++ b/backend/pkg/utils/francis/provider.go
@@ -28,9 +28,9 @@ func providerOptionInternal(databaseURL string) (local.HostOption, error) {
if err != nil {
return nil, err
}
- return local.WithSQLiteProvider(sqlite.SQLiteProviderOptions{ConnectionString: dsn, TablePrefix: tablePrefixInternal}), nil
+ return local.WithSQLiteProvider(sqlite.SQLiteProviderOptions{ConnectionString: dsn, TablePrefix: TablePrefix}), nil
case strings.HasPrefix(databaseURL, "postgres"):
- return local.WithPostgresProvider(postgres.PostgresProviderOptions{ConnectionString: databaseURL, TablePrefix: tablePrefixInternal}), nil
+ return local.WithPostgresProvider(postgres.PostgresProviderOptions{ConnectionString: databaseURL, TablePrefix: TablePrefix}), nil
default:
return nil, errors.New("unsupported actor database URL")
}
@@ -53,9 +53,9 @@ func ClearRestoredHosts(ctx context.Context, databaseURL string) (err error) {
if dsnErr != nil {
return dsnErr
}
- provider, err = sqlite.NewSQLiteProvider(slog.Default(), sqlite.SQLiteProviderOptions{ConnectionString: dsn, TablePrefix: tablePrefixInternal}, cfg)
+ provider, err = sqlite.NewSQLiteProvider(slog.Default(), sqlite.SQLiteProviderOptions{ConnectionString: dsn, TablePrefix: TablePrefix}, cfg)
case strings.HasPrefix(databaseURL, "postgres"):
- provider, err = postgres.NewPostgresProvider(slog.Default(), postgres.PostgresProviderOptions{ConnectionString: databaseURL, TablePrefix: tablePrefixInternal}, cfg)
+ provider, err = postgres.NewPostgresProvider(slog.Default(), postgres.PostgresProviderOptions{ConnectionString: databaseURL, TablePrefix: TablePrefix}, cfg)
default:
return errors.New("unsupported restored actor database URL")
}
diff --git a/backend/pkg/utils/francis/runtime.go b/backend/pkg/utils/francis/runtime.go
index e490e8159..4efbee8e7 100644
--- a/backend/pkg/utils/francis/runtime.go
+++ b/backend/pkg/utils/francis/runtime.go
@@ -22,7 +22,8 @@ import (
"github.com/quic-go/quic-go/http3"
)
-const tablePrefixInternal = "arcane_francis"
+// TablePrefix namespaces the Francis tables inside the Arcane database.
+const TablePrefix = "arcane_francis"
// Runtime starts one durable actor host and publishes its service after readiness.
// Services must wait for Ready before using Service, including during startup retries.
@@ -69,7 +70,7 @@ func New(databaseURL, encryptionKey, instanceID, port string, options ...local.H
},
}
runtime.options = append(runtime.options, local.WithLogger(slog.New(&actorLogHandlerInternal{
- Handler: slog.Default().With("scope", "actor-host").Handler(),
+ Handler: slog.Default().With("scope", "francis.arcane.internal").Handler(),
names: &runtime.actorNames,
})))
if encryptionKey != "" && instanceID != "" {
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 8c3aa79f9..7c82457a3 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -3268,9 +3268,6 @@
"diagnostics": "Diagnostics",
"diagnostics_description": "Live Go runtime, profiling, and backend logs.",
"diagnostics_status_connecting": "Connecting…",
- "diagnostics_updated_ago": "updated {ago}",
- "diagnostics_just_now": "just now",
- "diagnostics_seconds_ago": "{seconds}s ago",
"diagnostics_resume": "Resume",
"diagnostics_loading": "Loading diagnostics…",
"diagnostics_error_load": "Failed to load diagnostics",
@@ -3278,6 +3275,8 @@
"diagnostics_error_dump": "Failed to load dump",
"diagnostics_tab_connections": "Connections",
"diagnostics_tab_profiling": "Profiling",
+ "diagnostics_tab_actors": "Actors",
+ "diagnostics_actors_alarms": "Alarms",
"diagnostics_connections_empty": "No active WebSocket connections.",
"diagnostics_section_system": "System status",
"diagnostics_download": "Download",
@@ -3335,7 +3334,6 @@
"diagnostics_conn_client_ip": "Client IP",
"diagnostics_conn_since": "Since",
"diagnostics_section_logs": "Live Backend Logs",
- "diagnostics_logs_streaming": "Streaming",
"disconnected": "Disconnected",
"diagnostics_logs_count": "{count} lines",
"diagnostics_logs_all_levels": "All levels",
diff --git a/frontend/src/lib/layouts/settings-page-layout.svelte b/frontend/src/lib/layouts/settings-page-layout.svelte
index 2c7050fde..8edc1e2d3 100644
--- a/frontend/src/lib/layouts/settings-page-layout.svelte
+++ b/frontend/src/lib/layouts/settings-page-layout.svelte
@@ -63,7 +63,7 @@
{/if}
{#if pageType === 'management' && statCards.length > 0}
<div class="mt-2 flex flex-wrap items-center gap-x-4 gap-y-1">
- {#each statCards as card, i (card)}
+ {#each statCards as card, i (i)}
{#if i > 0}
<div class="h-4 w-px bg-border/50"></div>
{/if}
diff --git a/frontend/src/lib/services/diagnostics-service.ts b/frontend/src/lib/services/diagnostics-service.ts
deleted file mode 100644
index 07948224f..000000000
--- a/frontend/src/lib/services/diagnostics-service.ts
+++ /dev/null
@@ -1,71 +0,0 @@
-import type { Diagnostics, GoroutineLeakReport, LogEntry, PprofProfile } from '#lib/types/diagnostics.js';
-
-import BaseAPIService from './api-service';
-
-class DiagnosticsAPIService extends BaseAPIService {
- /** One-shot runtime/memory/GC + WebSocket snapshot (used for initial paint). */
- async getDiagnostics(): Promise<Diagnostics> {
- const res = await this.api.get('/diagnostics');
- return res.data as Diagnostics;
- }
-
- /** Recent buffered backend log entries (oldest first). */
- async getRecentLogs(): Promise<LogEntry[]> {
- const res = await this.api.get('/diagnostics/logs');
- return (res.data ?? []) as LogEntry[];
- }
-
- /** Fetch a human-readable pprof dump (debug=2 text) for inline display. */
- async getDump(name: 'goroutine' | 'heap'): Promise<string> {
- const res = await this.api.get(`/debug/pprof/${name}`, {
- params: { debug: 2 },
- responseType: 'text'
- });
- return (res.data ?? '') as string;
- }
-
- /** Run a leak-detection GC and return leaked-goroutine stacks (debug=1 text). */
- async scanGoroutineLeaks(): Promise<GoroutineLeakReport> {
- const res = await this.api.post('/diagnostics/goroutineleak', undefined, {
- timeout: 60000
- });
- return res.data as GoroutineLeakReport;
- }
-
- /**
- * Download a raw pprof profile via the authed client (so bearer/API-key auth
- * is attached) and trigger a browser save. `profile` and `trace` are
- * time-sampled; pass an optional duration in seconds.
- */
- async downloadProfile(profile: PprofProfile, seconds?: number): Promise<void> {
- const params: Record<string, number> = {};
- if (profile === 'profile') params['seconds'] = seconds ?? 30;
- if (profile === 'trace') params['seconds'] = seconds ?? 5;
-
- const defaultSeconds = profile === 'trace' ? 5 : 30;
- const res = await this.api.get(`/debug/pprof/${profile}`, {
- params,
- responseType: 'blob',
- // Sampled profiles block for their full duration; goroutineleak runs a GC.
- timeout:
- profile === 'profile' || profile === 'trace'
- ? (seconds ?? defaultSeconds) * 1000 + 15000
- : profile === 'goroutineleak'
- ? 60000
- : undefined
- });
-
- const blob = res.data as Blob;
- const url = URL.createObjectURL(blob);
- const ext = profile === 'trace' ? 'out' : 'pprof';
- const anchor = document.createElement('a');
- anchor.href = url;
- anchor.download = `${profile}.${ext}`;
- document.body.appendChild(anchor);
- anchor.click();
- anchor.remove();
- URL.revokeObjectURL(url);
- }
-}
-
-export const diagnosticsService = new DiagnosticsAPIService();
diff --git a/frontend/src/lib/types/diagnostics.ts b/frontend/src/lib/types/diagnostics.ts
index bb008565f..bc2948a8f 100644
--- a/frontend/src/lib/types/diagnostics.ts
+++ b/frontend/src/lib/types/diagnostics.ts
@@ -59,6 +59,7 @@ export interface WebSocketMetricsSnapshot {
containerExec: number;
systemStats: number;
serviceLogsActive: number;
+ diagnosticsActive: number;
}
export interface WebSocketDiagnostics {
@@ -74,6 +75,73 @@ export interface Diagnostics {
websocket: WebSocketDiagnostics;
}
+export interface ActorHostInfo {
+ hostId: string;
+ address: string;
+ lastHealthCheck: string;
+}
+
+export interface ActorInstance {
+ actorId: string;
+ name?: string;
+ hostId: string;
+ activatedAt: string;
+}
+
+export interface ActorJobFailure {
+ jobId: string;
+ actorId: string;
+ name?: string;
+ method: string;
+ attempts: number;
+ lastError?: string;
+ failedAt: string;
+}
+
+export interface ActorTypeInfo {
+ actorType: string;
+ concurrencyLimit: number;
+ activeActors: number;
+ storedStates: number;
+ pendingAlarms: number;
+ pendingJobs: number;
+ leasedAlarms: number;
+ completedJobs: number;
+ deadJobs: number;
+ nextAlarmAt?: string;
+ instances: ActorInstance[] | null;
+ recentFailures: ActorJobFailure[] | null;
+}
+
+export interface ActorDiagnostics {
+ environmentId: string;
+ environmentName: string;
+ error?: string;
+ ready: boolean;
+ hosts: ActorHostInfo[] | null;
+ actorTypes: ActorTypeInfo[] | null;
+ runs: Record<string, number> | null;
+ checkedAt: string;
+}
+
+export interface DiagnosticsCommand {
+ id: string;
+ type: 'refresh' | 'dump' | 'leakScan' | 'profile';
+ name?: string;
+ seconds?: number;
+}
+
+export interface DiagnosticsMessage {
+ type: 'snapshot' | 'result';
+ id?: string;
+ snapshot?: Diagnostics;
+ leakReport?: GoroutineLeakReport;
+ text?: string;
+ /** Base64-encoded pprof bytes for profile results. */
+ data?: string;
+ error?: string;
+}
+
export interface LogEntry {
time: string;
level: string;
diff --git a/frontend/src/lib/utils/ws.ts b/frontend/src/lib/utils/ws.ts
index cedd5107e..ac627e19d 100644
--- a/frontend/src/lib/utils/ws.ts
+++ b/frontend/src/lib/utils/ws.ts
@@ -1,5 +1,14 @@
-import type { Diagnostics, LogEntry } from '#lib/types/diagnostics.js';
+import { m } from '#lib/paraglide/messages.js';
+import type {
+ ActorDiagnostics,
+ Diagnostics,
+ DiagnosticsCommand,
+ DiagnosticsMessage,
+ LogEntry,
+ PprofProfile
+} from '#lib/types/diagnostics.js';
import type { SystemStats } from '#lib/types/shared.js';
+import { downloadBlob } from '#lib/utils/browser-download.js';
import { tryCatch } from '#lib/utils/try-catch.js';
export interface ReconnectWSOptions<T> {
@@ -129,6 +138,12 @@ export class ReconnectingWebSocket<T = unknown> {
}, backoff);
}
+ send(data: string): boolean {
+ if (this.ws?.readyState !== WebSocket.OPEN) return false;
+ this.ws.send(data);
+ return true;
+ }
+
close() {
this.closed = true;
this.attempt = 0;
@@ -243,14 +258,75 @@ export function createContainerStatsWebSocket(opts: {
});
}
+/** Diagnostics stream carrying live snapshots plus request/response commands. */
export function createDiagnosticsWebSocket(opts: {
- onMessage: (data: Diagnostics) => void;
+ onSnapshot: (data: Diagnostics) => void;
+ onOpen?: () => void;
+ onClose?: () => void;
+}) {
+ let nextId = 0;
+ const pending = new Map<string, { resolve: (msg: DiagnosticsMessage) => void; reject: (err: Error) => void }>();
+ const rejectPending = () => {
+ for (const entry of pending.values()) entry.reject(new Error(m.disconnected()));
+ pending.clear();
+ };
+
+ const socket = createDiagnosticsStreamWebSocket<DiagnosticsMessage>('/api/diagnostics/stream', {
+ onMessage: (msg) => {
+ if (msg.type === 'snapshot') {
+ if (msg.snapshot) opts.onSnapshot(msg.snapshot);
+ return;
+ }
+ const entry = pending.get(msg.id ?? '');
+ if (!entry) return;
+ pending.delete(msg.id ?? '');
+ if (msg.error) entry.reject(new Error(msg.error));
+ else entry.resolve(msg);
+ },
+ onOpen: opts.onOpen,
+ onClose: () => {
+ rejectPending();
+ opts.onClose?.();
+ }
+ });
+
+ const request = (command: Omit<DiagnosticsCommand, 'id'>) =>
+ new Promise<DiagnosticsMessage>((resolve, reject) => {
+ const id = String(++nextId);
+ if (!socket.send(JSON.stringify({ ...command, id }))) {
+ reject(new Error(m.disconnected()));
+ return;
+ }
+ pending.set(id, { resolve, reject });
+ });
+
+ return {
+ connect: () => socket.connect(),
+ close: () => {
+ socket.close();
+ rejectPending();
+ },
+ refresh: () => socket.send(JSON.stringify({ id: '', type: 'refresh' })),
+ request,
+ /** Captures a pprof profile over the stream and saves it as a file. */
+ async downloadProfile(profile: PprofProfile) {
+ const msg = await request({ type: 'profile', name: profile });
+ const bytes = Uint8Array.from(atob(msg.data ?? ''), (char) => char.charCodeAt(0));
+ downloadBlob(bytes, `${profile}.${profile === 'trace' ? 'out' : 'pprof'}`);
+ }
+ };
+}
+
+export type DiagnosticsWebSocket = ReturnType<typeof createDiagnosticsWebSocket>;
+
+export function createActorDiagnosticsWebSocket(opts: {
+ onMessage: (data: ActorDiagnostics[]) => void;
onOpen?: () => void;
onClose?: () => void;
onError?: (err: Event | Error) => void;
maxBackoff?: number;
}) {
- return createDiagnosticsStreamWebSocket('/api/diagnostics/stream', opts);
+ return createDiagnosticsStreamWebSocket('/api/diagnostics/actors/stream', opts);
}
export function createBackendLogsWebSocket(opts: {
@@ -272,7 +348,7 @@ function createDiagnosticsStreamWebSocket<T>(
onError?: (err: Event | Error) => void;
maxBackoff?: number;
}
-) {
+): ReconnectingWebSocket<T> {
const buildUrl = () => {
const protocol = location.protocol === 'https:' ? 'wss' : 'ws';
return `${protocol}://${location.host}${path}`;
diff --git a/frontend/src/routes/(app)/images/+page.svelte b/frontend/src/routes/(app)/images/+page.svelte
index 319320ce1..3133c5e24 100644
--- a/frontend/src/routes/(app)/images/+page.svelte
+++ b/frontend/src/routes/(app)/images/+page.svelte
@@ -428,7 +428,12 @@
<Dialog.Description>{m.images_prune_confirm_description({ mode: imagePruneMode })}</Dialog.Description>
</Dialog.Header>
<SettingsSection class="py-4">
- <SettingsRow label={m.images()} description={m.prune_images_dialog_description()} layout="wide">
+ <SettingsRow
+ label={m.images()}
+ description={m.prune_images_dialog_description()}
+ layout="wide"
+ contentClass="flex justify-center"
+ >
<PruneModePicker
modeOptions={imagePruneModes}
bind:value={imagePruneMode}
diff --git a/frontend/src/routes/(app)/settings/diagnostics/+page.svelte b/frontend/src/routes/(app)/settings/diagnostics/+page.svelte
index 34e40f469..33c2498f6 100644
--- a/frontend/src/routes/(app)/settings/diagnostics/+page.svelte
+++ b/frontend/src/routes/(app)/settings/diagnostics/+page.svelte
@@ -17,32 +17,29 @@
ClockIcon,
ConnectionIcon,
DownloadIcon,
+ EnvironmentsIcon,
ArrowDownIcon
} from '#lib/icons/index.js';
import { SettingsPageLayout } from '#lib/layouts/index.js';
import type { SettingsStatCard } from '#lib/layouts/types.js';
import { m } from '#lib/paraglide/messages.js';
- import { diagnosticsService } from '#lib/services/diagnostics-service.js';
import type { Diagnostics, GoroutineLeakReport, PprofProfile } from '#lib/types/diagnostics.js';
import { cn } from '#lib/utils.js';
import { formatTime } from '#lib/utils/formatting.js';
import { tryCatch } from '#lib/utils/try-catch.js';
- import { createDiagnosticsWebSocket, ReconnectingWebSocket } from '#lib/utils/ws.js';
+ import { createDiagnosticsWebSocket, type DiagnosticsWebSocket } from '#lib/utils/ws.js';
+ import DiagnosticActorPanel from './components/diagnostic-actor-panel.svelte';
import DiagnosticLeakPanel from './components/diagnostic-leak-panel.svelte';
import DiagnosticLogPanel from './components/diagnostic-log-panel.svelte';
- type DiagnosticsTab = 'overview' | 'connections' | 'logs' | 'profiling';
+ type DiagnosticsTab = 'overview' | 'connections' | 'actors' | 'logs' | 'profiling';
let diag = $state<Diagnostics | null>(null);
- let connected = $state(false);
let paused = $state(false);
- let lastUpdated = $state(0);
- let now = $state(performance.now());
let error = $state<string | null>(null);
- let ws: ReconnectingWebSocket<Diagnostics> | null = null;
- let tick: ReturnType<typeof setInterval> | null = null;
+ let ws = $state.raw<DiagnosticsWebSocket | null>(null);
const wsKindLabels: Record<string, string> = {
project_logs: m.diagnostics_ws_project_logs(),
@@ -50,7 +47,8 @@
container_stats: m.diagnostics_ws_container_stats(),
container_exec: m.diagnostics_ws_kind_terminal(),
system_stats: m.diagnostics_ws_system_stats(),
- service_logs: m.diagnostics_ws_service_logs()
+ service_logs: m.diagnostics_ws_service_logs(),
+ diagnostics: m.diagnostics()
};
function fmtBytes(n: number): string {
@@ -92,12 +90,6 @@
return m.diagnostics_leaks_not_scanned();
}
- const agoText = $derived.by(() => {
- if (!lastUpdated) return '—';
- const s = Math.max(0, Math.round((now - lastUpdated) / 1000));
- return s <= 1 ? m.diagnostics_just_now() : m.diagnostics_seconds_ago({ seconds: s });
- });
-
const totalConnections = $derived.by(() => {
const s = diag?.websocket?.snapshot;
if (!s) return 0;
@@ -113,7 +105,8 @@
{ label: m.diagnostics_ws_container_stats(), value: s.containerStats },
{ label: m.diagnostics_ws_terminals(), value: s.containerExec },
{ label: m.diagnostics_ws_system_stats(), value: s.systemStats },
- { label: m.diagnostics_ws_service_logs(), value: s.serviceLogsActive }
+ { label: m.diagnostics_ws_service_logs(), value: s.serviceLogsActive },
+ { label: m.diagnostics(), value: s.diagnosticsActive }
];
});
@@ -127,6 +120,7 @@
icon: ConnectionIcon,
badge: totalConnections || undefined
},
+ { value: 'actors', label: m.diagnostics_tab_actors(), icon: EnvironmentsIcon },
{ value: 'logs', label: m.common_logs(), icon: ActivityIcon },
{
value: 'profiling',
@@ -165,12 +159,6 @@
const headerStats = $derived.by((): SettingsStatCard[] => {
if (!diag) return [];
return [
- {
- title: m.diagnostics_updated_ago({ ago: agoText }),
- value: paused ? m.paused() : connected ? m.common_live() : m.diagnostics_status_connecting(),
- icon: ActivityIcon,
- iconColor: paused ? 'text-warning' : connected ? 'text-success' : 'text-muted-foreground'
- },
{
title: m.goroutines(),
value: fmtNum(diag.runtime.goroutines),
@@ -211,43 +199,19 @@
]);
function applySnapshot(d: Diagnostics) {
+ if (paused) return;
diag = d;
- lastUpdated = performance.now();
error = null;
}
- function openStream() {
- ws = createDiagnosticsWebSocket({
- onMessage: applySnapshot,
- onOpen: () => (connected = true),
- onClose: () => (connected = false)
- });
- ws.connect();
- }
-
- function closeStream() {
- ws?.close();
- ws = null;
- connected = false;
- }
-
+ // Pausing only freezes the view so on-demand commands keep working.
function togglePause() {
paused = !paused;
- if (paused) closeStream();
- else openStream();
+ if (!paused) refresh();
}
- async function refresh() {
- const operationResult = await tryCatch(
- (async () => {
- applySnapshot(await diagnosticsService.getDiagnostics());
- })()
- );
- if (operationResult.error !== null) {
- const e = operationResult.error;
-
- error = e instanceof Error ? e.message : m.diagnostics_error_load();
- }
+ function refresh() {
+ if (!ws?.refresh()) error = m.disconnected();
}
let dumpOpen = $state<{ goroutine: boolean; heap: boolean }>({ goroutine: false, heap: false });
@@ -257,13 +221,15 @@
async function loadDump(name: 'goroutine' | 'heap') {
dumpLoading[name] = true;
try {
- const operationResult = await tryCatch((async () => diagnosticsService.getDump(name))());
+ const operationResult = await tryCatch(
+ ws ? ws.request({ type: 'dump', name }) : Promise.reject(new Error(m.disconnected()))
+ );
if (operationResult.error !== null) {
const e = operationResult.error;
dumpText[name] = e instanceof Error ? e.message : m.diagnostics_error_dump();
} else {
- dumpText[name] = operationResult.data;
+ dumpText[name] = operationResult.data.text ?? '';
}
} finally {
dumpLoading[name] = false;
@@ -305,7 +271,8 @@
try {
const operationResult = await tryCatch(
(async () => {
- await diagnosticsService.downloadProfile(p);
+ if (!ws) throw new Error(m.disconnected());
+ await ws.downloadProfile(p);
})()
);
if (operationResult.error !== null) {
@@ -319,12 +286,13 @@
}
onMount(() => {
- refresh();
- openStream();
- tick = setInterval(() => (now = performance.now()), 1000);
+ ws = createDiagnosticsWebSocket({
+ onSnapshot: applySnapshot
+ });
+ ws.connect();
return () => {
- closeStream();
- if (tick) clearInterval(tick);
+ ws?.close();
+ ws = null;
};
});
</script>
@@ -487,7 +455,7 @@
<Tabs.Content value="connections" class="mt-6">
<div class="space-y-4">
{@render sectionHeading(m.diagnostics_section_connections({ count: diag.websocket.connections?.length ?? 0 }))}
- <div class="grid grid-cols-2 gap-x-6 gap-y-3 sm:grid-cols-3 lg:grid-cols-6">
+ <div class="grid grid-cols-2 gap-x-6 gap-y-3 sm:grid-cols-4 lg:grid-cols-7">
{#each wsCounts as c (c.label)}
<div>
<div class="text-xs text-muted-foreground">{c.label}</div>
@@ -531,6 +499,10 @@
</div>
</Tabs.Content>
+ <Tabs.Content value="actors" class="mt-6">
+ <DiagnosticActorPanel />
+ </Tabs.Content>
+
<Tabs.Content value="logs" class="mt-6">
<DiagnosticLogPanel height="min(70vh, 640px)" />
</Tabs.Content>
@@ -542,6 +514,7 @@
<DiagnosticLeakPanel
leakedGoroutines={diag.runtime.leakedGoroutines}
leakScannedAt={diag.runtime.leakScannedAt}
+ socket={ws}
onscanned={onLeakScanned}
/>
</section>
diff --git a/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-actor-panel.svelte b/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-actor-panel.svelte
new file mode 100644
index 000000000..d2e3cd00e
--- /dev/null
+++ b/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-actor-panel.svelte
@@ -0,0 +1,197 @@
+<script lang="ts">
+ import { onDestroy, onMount } from 'svelte';
+ import { SvelteSet } from 'svelte/reactivity';
+ import { slide } from 'svelte/transition';
+
+ import { jobStatusLabel } from '#lib/components/job-card/job-status.js';
+ import * as Alert from '#lib/components/ui/alert/index.js';
+ import * as Table from '#lib/components/ui/table/index.js';
+ import { ArrowRightIcon } from '#lib/icons/index.js';
+ import { m } from '#lib/paraglide/messages.js';
+ import type { ActorDiagnostics, ActorTypeInfo } from '#lib/types/diagnostics.js';
+ import { cn } from '#lib/utils.js';
+ import { formatTime } from '#lib/utils/formatting.js';
+ import { createActorDiagnosticsWebSocket, ReconnectingWebSocket } from '#lib/utils/ws.js';
+
+ let environments = $state<ActorDiagnostics[]>([]);
+ const expanded = new SvelteSet<string>();
+ let ws: ReconnectingWebSocket<ActorDiagnostics[]> | null = null;
+
+ function toggle(key: string) {
+ if (expanded.has(key)) expanded.delete(key);
+ else expanded.add(key);
+ }
+
+ function hasDetails(t: ActorTypeInfo): boolean {
+ return Boolean(t.instances?.length || t.recentFailures?.length);
+ }
+
+ onMount(() => {
+ ws = createActorDiagnosticsWebSocket({ onMessage: (data) => (environments = data) });
+ ws.connect();
+ });
+
+ onDestroy(() => ws?.close());
+</script>
+
+<div class="space-y-4">
+ {#each environments as env (env.environmentId)}
+ <div class="space-y-3 rounded-xl border border-border/60 p-4">
+ <div class="flex flex-wrap items-center gap-2">
+ <h3 class="text-sm font-semibold">{env.environmentName || env.environmentId}</h3>
+ {#if !env.error && !env.ready}
+ <span class="text-xs text-warning">{m.jobs_status_starting()}</span>
+ {/if}
+ {#each env.hosts ?? [] as host (host.hostId)}
+ <span class="font-mono text-xs text-muted-foreground">
+ {host.address} · {m.last_check()}
+ {formatTime(host.lastHealthCheck) || '—'}
+ </span>
+ {/each}
+ </div>
+
+ {#if env.error}
+ <Alert.Root variant="destructive">
+ <Alert.Title>{m.common_unavailable()}</Alert.Title>
+ <Alert.Description>{env.error}</Alert.Description>
+ </Alert.Root>
+ {:else if env.ready}
+ <div class="flex flex-wrap items-center gap-x-4 gap-y-1 text-xs">
+ <span class="text-muted-foreground">{m.jobs_run_history()}</span>
+ {#each Object.entries(env.runs ?? {}) as [status, count] (status)}
+ <span><span class="font-semibold tabular-nums">{count}</span> {jobStatusLabel(status, true)}</span>
+ {:else}
+ <span class="text-muted-foreground">{m.none()}</span>
+ {/each}
+ </div>
+
+ {#if env.actorTypes?.length}
+ <div class="overflow-x-auto">
+ <Table.Root>
+ <Table.Header>
+ <Table.Row variant="static">
+ <Table.Head variant="expander"></Table.Head>
+ <Table.Head>{m.common_type()}</Table.Head>
+ <Table.Head>{m.common_active()}</Table.Head>
+ <Table.Head>{m.common_state()}</Table.Head>
+ <Table.Head>{m.diagnostics_actors_alarms()}</Table.Head>
+ <Table.Head>{m.common_pending()}</Table.Head>
+ <Table.Head>{m.common_running()}</Table.Head>
+ <Table.Head>{m.completed()}</Table.Head>
+ <Table.Head>{m.common_failed()}</Table.Head>
+ <Table.Head>{m.jobs_next_run()}</Table.Head>
+ </Table.Row>
+ </Table.Header>
+ <Table.Body>
+ {#each env.actorTypes as t (t.actorType)}
+ {@const key = `${env.environmentId}:${t.actorType}`}
+ {@const expandable = hasDetails(t)}
+ {@const open = expandable && expanded.has(key)}
+ <Table.Row
+ variant={expandable ? 'default' : 'static'}
+ data-expanded={open ? true : undefined}
+ class={cn(!expandable && 'cursor-default')}
+ onclick={() => expandable && toggle(key)}
+ >
+ <Table.Cell variant="expander">
+ {#if expandable}
+ <ArrowRightIcon
+ class={cn('size-4 text-muted-foreground transition-transform duration-200', open && 'rotate-90')}
+ aria-hidden="true"
+ />
+ {/if}
+ </Table.Cell>
+ <Table.Cell class="font-mono text-xs">{t.actorType}</Table.Cell>
+ <Table.Cell class="tabular-nums">
+ {t.activeActors}{#if t.concurrencyLimit > 0}<span class="text-muted-foreground">
+ / {m.common_limit()} {t.concurrencyLimit}</span
+ >{/if}
+ </Table.Cell>
+ <Table.Cell class="tabular-nums">{t.storedStates}</Table.Cell>
+ <Table.Cell class="tabular-nums">{t.pendingAlarms}</Table.Cell>
+ <Table.Cell class="tabular-nums">{t.pendingJobs}</Table.Cell>
+ <Table.Cell class="tabular-nums">{t.leasedAlarms}</Table.Cell>
+ <Table.Cell class="tabular-nums">{t.completedJobs}</Table.Cell>
+ <Table.Cell class={cn('tabular-nums', t.deadJobs > 0 && 'text-destructive')}>{t.deadJobs}</Table.Cell>
+ <Table.Cell class="text-xs text-muted-foreground tabular-nums">
+ {t.nextAlarmAt ? formatTime(t.nextAlarmAt) : '—'}
+ </Table.Cell>
+ </Table.Row>
+ {#if open}
+ <Table.Row variant="detail">
+ <Table.Cell colspan={10} variant="flush">
+ <div transition:slide={{ duration: 200 }} class="space-y-4 px-6 py-4">
+ {#if t.instances?.length}
+ {@render instanceList(t)}
+ {/if}
+ {#if t.recentFailures?.length}
+ {@render failureList(t)}
+ {/if}
+ </div>
+ </Table.Cell>
+ </Table.Row>
+ {/if}
+ {/each}
+ </Table.Body>
+ </Table.Root>
+ </div>
+ {:else}
+ <p class="text-sm text-muted-foreground">{m.none()}</p>
+ {/if}
+ {/if}
+ </div>
+ {:else}
+ <p class="py-8 text-sm text-muted-foreground">{m.diagnostics_status_connecting()}</p>
+ {/each}
+</div>
+
+{#snippet instanceList(t: ActorTypeInfo)}
+ <section class="space-y-1.5">
+ <h4 class="text-2xs font-semibold tracking-wide text-muted-foreground uppercase">
+ {m.common_active()} · {t.activeActors}
+ </h4>
+ <ul class="ml-1 border-l border-border/40">
+ {#each t.instances as instance (instance.actorId)}
+ <li class="flex items-baseline gap-3 py-1 pl-3 text-xs">
+ <span class="font-medium">{instance.name || instance.actorId.slice(0, 12)}</span>
+ <span class="min-w-0 flex-1 truncate font-mono text-2xs text-muted-foreground" title={instance.actorId}
+ >{instance.actorId}</span
+ >
+ <span class="shrink-0 text-2xs text-muted-foreground tabular-nums">
+ {m.common_started()}
+ <span class="text-foreground">{formatTime(instance.activatedAt)}</span>
+ </span>
+ </li>
+ {/each}
+ </ul>
+ </section>
+{/snippet}
+
+{#snippet failureList(t: ActorTypeInfo)}
+ <section class="space-y-1.5">
+ <h4 class="text-2xs font-semibold tracking-wide text-destructive uppercase">
+ {m.common_failed()} · {t.deadJobs}
+ </h4>
+ <ul class="ml-1 border-l border-destructive/40">
+ {#each t.recentFailures as failure (failure.jobId)}
+ <li class="space-y-1 py-1.5 pl-3 text-xs">
+ <div class="flex items-baseline gap-3">
+ <span class="font-medium">{failure.name || failure.actorId.slice(0, 12)}</span>
+ <span class="font-mono text-2xs text-muted-foreground">{failure.method}</span>
+ <span class="ml-auto shrink-0 text-2xs text-muted-foreground tabular-nums">
+ {m.jobs_run_attempts()} <span class="text-foreground">{failure.attempts}</span> ·
+ <span class="text-foreground">{formatTime(failure.failedAt)}</span>
+ </span>
+ </div>
+ {#if failure.lastError}
+ <p
+ class="rounded-md border border-destructive/30 bg-destructive/10 px-2 py-1 font-mono text-2xs break-words text-destructive"
+ >
+ {failure.lastError}
+ </p>
+ {/if}
+ </li>
+ {/each}
+ </ul>
+ </section>
+{/snippet}
diff --git a/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-leak-panel.svelte b/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-leak-panel.svelte
index b149fce5c..b8f87410c 100644
--- a/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-leak-panel.svelte
+++ b/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-leak-panel.svelte
@@ -2,19 +2,20 @@
import { ArcaneButton } from '#lib/components/arcane-button/index.js';
import { AlertTriangleIcon, DownloadIcon } from '#lib/icons/index.js';
import { m } from '#lib/paraglide/messages.js';
- import { diagnosticsService } from '#lib/services/diagnostics-service.js';
import type { GoroutineLeakReport } from '#lib/types/diagnostics.js';
import { cn } from '#lib/utils.js';
import { formatTime } from '#lib/utils/formatting.js';
import { tryCatch } from '#lib/utils/try-catch.js';
+ import type { DiagnosticsWebSocket } from '#lib/utils/ws.js';
interface Props {
leakedGoroutines: number;
leakScannedAt?: string;
+ socket: DiagnosticsWebSocket | null;
onscanned?: (report: GoroutineLeakReport) => void;
}
- let { leakedGoroutines, leakScannedAt, onscanned }: Props = $props();
+ let { leakedGoroutines, leakScannedAt, socket, onscanned }: Props = $props();
let scanning = $state(false);
let downloading = $state(false);
@@ -32,7 +33,9 @@
try {
const operationResult = await tryCatch(
(async () => {
- const next = await diagnosticsService.scanGoroutineLeaks();
+ if (!socket) throw new Error(m.disconnected());
+ const next = (await socket.request({ type: 'leakScan' })).leakReport;
+ if (!next) return;
report = next;
onscanned?.(next);
})()
@@ -53,7 +56,8 @@
try {
const operationResult = await tryCatch(
(async () => {
- await diagnosticsService.downloadProfile('goroutineleak');
+ if (!socket) throw new Error(m.disconnected());
+ await socket.downloadProfile('goroutineleak');
})()
);
if (operationResult.error !== null) {
diff --git a/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-log-panel.svelte b/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-log-panel.svelte
index a8acb62ed..77d3526bf 100644
--- a/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-log-panel.svelte
+++ b/frontend/src/routes/(app)/settings/diagnostics/components/diagnostic-log-panel.svelte
@@ -87,10 +87,6 @@
<div class="overflow-hidden rounded-xl border border-border/60">
<div class="flex flex-wrap items-center gap-2 border-b border-border/60 bg-muted/30 px-3 py-2">
- <span class="flex items-center gap-1.5 text-xs font-medium">
- <span class={cn('size-2 rounded-full', connected ? 'bg-success' : 'bg-muted-foreground')}></span>
- {connected ? m.diagnostics_logs_streaming() : m.disconnected()}
- </span>
<span class="text-xs text-muted-foreground tabular-nums">{m.diagnostics_logs_count({ count: filtered.length })}</span>
<div class="ml-auto flex items-center gap-2">
diff --git a/types/system/diagnostics.go b/types/system/diagnostics.go
index e3b49e4df..299a56d6b 100644
--- a/types/system/diagnostics.go
+++ b/types/system/diagnostics.go
@@ -92,3 +92,81 @@ type WebSocketDiagnostics struct {
Snapshot WebSocketMetricsSnapshot `json:"snapshot"`
Connections []WebSocketConnectionInfo `json:"connections"`
}
+
+// ActorDiagnostics describes the Francis actor host of one environment.
+type ActorDiagnostics struct {
+ EnvironmentID string `json:"environmentId"`
+ EnvironmentName string `json:"environmentName"`
+ // Error is set when the environment's actor diagnostics could not be collected.
+ Error string `json:"error,omitempty"`
+ Ready bool `json:"ready"`
+ Hosts []ActorHostInfo `json:"hosts"`
+ ActorTypes []ActorTypeInfo `json:"actorTypes"`
+ Runs map[string]int `json:"runs"`
+ CheckedAt time.Time `json:"checkedAt"`
+}
+
+// ActorHostInfo is a registered Francis host.
+type ActorHostInfo struct {
+ HostID string `json:"hostId"`
+ Address string `json:"address"`
+ LastHealthCheck time.Time `json:"lastHealthCheck"`
+}
+
+// ActorTypeInfo holds per actor type activation, alarm, and job details.
+type ActorTypeInfo struct {
+ ActorType string `json:"actorType"`
+ // ConcurrencyLimit caps concurrent invocations per actor; 0 is unlimited.
+ ConcurrencyLimit int `json:"concurrencyLimit"`
+ ActiveActors int `json:"activeActors"`
+ StoredStates int `json:"storedStates"`
+ PendingAlarms int `json:"pendingAlarms"`
+ PendingJobs int `json:"pendingJobs"`
+ LeasedAlarms int `json:"leasedAlarms"`
+ CompletedJobs int `json:"completedJobs"`
+ DeadJobs int `json:"deadJobs"`
+ NextAlarmAt time.Time `json:"nextAlarmAt,omitzero"`
+ Instances []ActorInstance `json:"instances"`
+ RecentFailures []ActorJobFailure `json:"recentFailures"`
+}
+
+// ActorInstance is an activated actor.
+type ActorInstance struct {
+ ActorID string `json:"actorId"`
+ // Name is Arcane's display name for the actor, when known.
+ Name string `json:"name,omitempty"`
+ HostID string `json:"hostId"`
+ ActivatedAt time.Time `json:"activatedAt"`
+}
+
+// ActorJobFailure is a dead-lettered actor job.
+type ActorJobFailure struct {
+ JobID string `json:"jobId"`
+ ActorID string `json:"actorId"`
+ Name string `json:"name,omitempty"`
+ Method string `json:"method"`
+ Attempts int `json:"attempts"`
+ LastError string `json:"lastError,omitempty"`
+ FailedAt time.Time `json:"failedAt"`
+}
+
+// DiagnosticsCommand is a client request sent over the diagnostics stream.
+type DiagnosticsCommand struct {
+ ID string `json:"id"`
+ // Type is refresh, dump, leakScan, or profile.
+ Type string `json:"type"`
+ Name string `json:"name,omitempty"`
+ Seconds int `json:"seconds,omitempty"`
+}
+
+// DiagnosticsMessage is a snapshot push or command result on the diagnostics stream.
+type DiagnosticsMessage struct {
+ // Type is snapshot or result.
+ Type string `json:"type"`
+ ID string `json:"id,omitempty"`
+ Snapshot *Diagnostics `json:"snapshot,omitempty"`
+ LeakReport *GoroutineLeakReport `json:"leakReport,omitempty"`
+ Text string `json:"text,omitempty"`
+ Data []byte `json:"data,omitempty"`
+ Error string `json:"error,omitempty"`
+}
diff --git a/types/system/websocket.go b/types/system/websocket.go
index f42648b0d..2746d014c 100644
--- a/types/system/websocket.go
+++ b/types/system/websocket.go
@@ -10,6 +10,7 @@ const (
WSKindContainerExec = "container_exec"
WSKindSystemStats = "system_stats"
WSKindServiceLogs = "service_logs"
+ WSKindDiagnostics = "diagnostics"
)
// WebSocketConnectionInfo describes a single active WebSocket connection.
@@ -51,6 +52,8 @@ type WebSocketMetricsSnapshot struct {
ContainerExec int64 `json:"containerExec"`
// SystemStats is the number of active system-stats streams.
SystemStats int64 `json:"systemStats"`
+ // DiagnosticsActive is the number of active diagnostics streams.
+ DiagnosticsActive int64 `json:"diagnosticsActive"`
// ServiceLogsActive is the number of active swarm service-log streams.
ServiceLogsActive int64 `json:"serviceLogsActive"`
}