Skip to content

Modern Docker Management, Designed for Everyone

fix: stream environment liveness over a multiplexed client stream (#3406)

Kyle Mendell1d054e1ac1f5
diff --git a/backend/api/api.go b/backend/api/api.go
index d86beec46..402f3857c 100644
--- a/backend/api/api.go
+++ b/backend/api/api.go
@@ -397,4 +397,5 @@ func registerHandlersInternal(api huma.API, deps HandlerDeps, handlerAppCtx hand
handlers.RegisterWebhooks(api, deps.Webhook)
handlers.RegisterVulnerability(api, deps.Vulnerability, handlerAppCtx)
handlers.RegisterDashboard(api, deps.Dashboard, deps.Environment)
+ handlers.RegisterStream(api, deps.Dashboard, deps.Activity, deps.Environment)
}
diff --git a/backend/api/handlers/activities.go b/backend/api/handlers/activities.go
index 6624048cd..8cb3cbceb 100644
--- a/backend/api/handlers/activities.go
+++ b/backend/api/handlers/activities.go
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"hash/fnv"
- "io"
"net/http"
"net/url"
"strconv"
@@ -19,7 +18,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
"github.com/getarcaneapp/arcane/types/v2/activity"
"github.com/getarcaneapp/arcane/types/v2/base"
"go.getarcane.app/streams/agg"
@@ -74,7 +72,6 @@ const (
activityStreamRemotePollInterval = 5 * time.Second
activityStreamEnvReconcileInterval = 30 * time.Second
activityStreamRemotePollTimeout = 15 * time.Second
- activityStreamEventBuffer = 256
)
type CancelActivityInput struct {
@@ -87,6 +84,7 @@ type CancelActivityOutput struct {
Body base.ApiResponse[activity.Activity]
}
+//nolint:dupl // Huma operation registrations are intentionally explicit.
func RegisterActivities(api huma.API, activityService *services.ActivityService, environmentService *services.EnvironmentService) {
h := &ActivityHandler{
activityService: activityService,
@@ -113,17 +111,6 @@ func RegisterActivities(api huma.API, activityService *services.ActivityService,
Security: defaultOperationSecurityInternal(),
}, authz.PermActivitiesRead, h.GetActivity)
- huma.Register(api, huma.Operation{
- OperationID: "stream-all-activities",
- Method: http.MethodGet,
- Path: "/activities/stream",
- Summary: "Stream background activities across all environments",
- Description: "Stream background activity updates for the local environment and all enabled remote environments as JSON lines",
- Tags: []string{"Activities"},
- Security: defaultOperationSecurityInternal(),
- Middlewares: humamw.RequireAnyEnvironmentPermission(api, authz.PermActivitiesRead),
- }, h.StreamAllActivities)
-
humamw.RegisterWithPermission(api, huma.Operation{
OperationID: "cancel-activity",
Method: http.MethodPost,
@@ -277,47 +264,6 @@ func (h *ActivityHandler) cancelRequestedByInternal(ctx context.Context, forward
return strings.TrimSpace(forwarded)
}
-func (h *ActivityHandler) StreamAllActivities(ctx context.Context, input *StreamAllActivitiesInput) (*huma.StreamResponse, error) {
- return &huma.StreamResponse{
- Body: func(humaCtx huma.Context) { //nolint:contextcheck // streaming work must use humaCtx.Context()
- httpx.SetJSONStreamHeaders(humaCtx)
-
- writer := humaCtx.BodyWriter()
- flush := func() {
- if f, ok := writer.(http.Flusher); ok {
- f.Flush()
- }
- }
-
- ps, _ := humamw.PermissionsFromContext(humaCtx.Context())
- h.streamAllActivitiesInternal(humaCtx.Context(), ps, input.Limit, writer, flush)
- },
- }, nil
-}
-
-// streamAllActivitiesInternal multiplexes activity events for the local
-// environment and every enabled remote environment over a single response so
-// the browser needs one connection regardless of environment count.
-func (h *ActivityHandler) streamAllActivitiesInternal(ctx context.Context, ps *authz.PermissionSet, limit int, writer io.Writer, flush func()) {
- // RunAuthorizedAggregateStream logs how the stream ended; there is nothing
- // left to report to a client whose response headers were sent long ago.
- _ = httpx.RunAuthorizedAggregateStream(ctx, ps, authz.PermActivitiesRead, agg.Config[activity.StreamEvent]{
- Writer: writer,
- Flush: flush,
- Buffer: activityStreamEventBuffer,
- HeartbeatInterval: activityStreamHeartbeatInterval,
- MakeHeartbeat: func() activity.StreamEvent {
- return activity.StreamEvent{Type: "heartbeat", Timestamp: time.Now()}
- },
- },
- func(ctx context.Context, events chan<- activity.StreamEvent) {
- h.runLocalActivityStreamProducerInternal(ctx, limit, events)
- },
- func(ctx context.Context, events chan<- activity.StreamEvent) {
- h.runRemoteActivityStreamPollersInternal(ctx, ps, limit, events)
- })
-}
-
func (h *ActivityHandler) runLocalActivityStreamProducerInternal(ctx context.Context, limit int, events chan<- activity.StreamEvent) {
sendSnapshot := func() bool {
activities, _, err := h.activityService.ListActivitiesPaginated(ctx, "0", pagination.QueryParams{
diff --git a/backend/api/handlers/activities_test.go b/backend/api/handlers/activities_test.go
index 68b4a6df4..93665c272 100644
--- a/backend/api/handlers/activities_test.go
+++ b/backend/api/handlers/activities_test.go
@@ -21,6 +21,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/types/v2/activity"
+ streamtypes "github.com/getarcaneapp/arcane/types/v2/stream"
)
func setupActivityHandlerTestDBInternal(t *testing.T) *database.DB {
@@ -146,10 +147,9 @@ func createStreamTestRemoteEnvironmentInternal(t *testing.T, db *database.DB, en
}).Error)
}
-// runStreamAllInternal drives streamAllActivitiesInternal through a pipe and
-// returns each decoded event to onEvent until it reports done or the stream
-// ends; remaining output is drained so a blocked writer can always finish.
-func runStreamAllInternal(t *testing.T, ctx context.Context, cancel context.CancelFunc, handler *ActivityHandler, ps *authz.PermissionSet, onEvent func(activity.StreamEvent) bool) {
+// runActivityClientStreamInternal drives the activities channel through a pipe
+// and returns each decoded activity event to onEvent until it reports done.
+func runActivityClientStreamInternal(t *testing.T, ctx context.Context, cancel context.CancelFunc, handler *ActivityHandler, ps *authz.PermissionSet, onEvent func(activity.StreamEvent) bool) {
t.Helper()
pr, pw := io.Pipe()
@@ -157,14 +157,18 @@ func runStreamAllInternal(t *testing.T, ctx context.Context, cancel context.Canc
go func() {
defer close(done)
defer func() { _ = pw.Close() }()
- handler.streamAllActivitiesInternal(ctx, ps, 50, pw, func() {})
+ streamHandler := &StreamHandler{activity: handler}
+ streamHandler.streamClientInternal(ctx, ps, &StreamClientInput{
+ Channels: streamtypes.ChannelActivities,
+ Limit: 50,
+ }, pw, func() {})
}()
scanner := bufio.NewScanner(pr)
for scanner.Scan() {
- var event activity.StreamEvent
+ var event streamtypes.Event
require.NoError(t, json.Unmarshal(scanner.Bytes(), &event))
- if onEvent(event) {
+ if event.Activity != nil && onEvent(*event.Activity) {
cancel()
break
}
@@ -207,7 +211,7 @@ func TestActivityHandlerStreamAllEmitsEnvironmentScopedEventsInternal(t *testing
}
var localSnapshot, remoteSnapshot bool
- runStreamAllInternal(t, ctx, cancel, handler, authz.SudoPermissionSet(), func(event activity.StreamEvent) bool {
+ runActivityClientStreamInternal(t, ctx, cancel, handler, authz.SudoPermissionSet(), func(event activity.StreamEvent) bool {
if event.Type == "snapshot" && event.EnvironmentID == "0" && len(event.Activities) == 1 {
require.Equal(t, local.ID, event.Activities[0].ID)
require.Equal(t, "0", event.Activities[0].SourceEnvironmentID)
@@ -262,7 +266,7 @@ func TestActivityHandlerStreamAllReusesRemoteEnvironmentAfterInitialPollInternal
}
remoteSnapshotCount := 0
- runStreamAllInternal(t, ctx, cancel, handler, authz.SudoPermissionSet(), func(event activity.StreamEvent) bool {
+ runActivityClientStreamInternal(t, ctx, cancel, handler, authz.SudoPermissionSet(), func(event activity.StreamEvent) bool {
if event.Type != "snapshot" || event.EnvironmentID != "remote-1" {
return false
}
@@ -315,7 +319,7 @@ func TestActivityHandlerStreamAllRemoteFailureEmitsErrorAndKeepsStreamingInterna
}
var localSnapshot, remoteError bool
- runStreamAllInternal(t, ctx, cancel, handler, authz.SudoPermissionSet(), func(event activity.StreamEvent) bool {
+ runActivityClientStreamInternal(t, ctx, cancel, handler, authz.SudoPermissionSet(), func(event activity.StreamEvent) bool {
if event.Type == "snapshot" && event.EnvironmentID == "0" {
localSnapshot = true
}
@@ -366,7 +370,7 @@ func TestActivityHandlerStreamAllFiltersUnauthorizedEnvironmentsInternal(t *test
ps.AddEnv("remote-allowed", authz.PermActivitiesRead)
seenEnvironments := make(map[string]struct{})
- runStreamAllInternal(t, ctx, cancel, handler, ps, func(event activity.StreamEvent) bool {
+ runActivityClientStreamInternal(t, ctx, cancel, handler, ps, func(event activity.StreamEvent) bool {
if event.EnvironmentID != "" {
seenEnvironments[event.EnvironmentID] = struct{}{}
}
diff --git a/backend/api/handlers/dashboard.go b/backend/api/handlers/dashboard.go
index 6825d3e59..70c971892 100644
--- a/backend/api/handlers/dashboard.go
+++ b/backend/api/handlers/dashboard.go
@@ -3,7 +3,6 @@ package handlers
import (
"context"
stderrors "errors"
- "io"
"net/http"
"time"
@@ -16,7 +15,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
"github.com/getarcaneapp/arcane/types/v2/base"
containertypes "github.com/getarcaneapp/arcane/types/v2/container"
dashboardtypes "github.com/getarcaneapp/arcane/types/v2/dashboard"
@@ -40,10 +38,6 @@ type GetDashboardOutput struct {
Body base.ApiResponse[dashboardtypes.Snapshot]
}
-type StreamAllDashboardsInput struct {
- DebugAllGood bool `query:"debugAllGood" default:"false" doc:"Debug mode: force an empty action item list"`
-}
-
const (
dashboardStreamHeartbeatInterval = 15 * time.Second
dashboardStreamLocalPollInterval = 15 * time.Second
@@ -68,17 +62,6 @@ func RegisterDashboard(api huma.API, dashboardService *services.DashboardService
Tags: []string{"Dashboard"},
Security: defaultOperationSecurityInternal(),
}, authz.PermDashboardRead, h.GetDashboard)
-
- huma.Register(api, huma.Operation{
- OperationID: "stream-all-dashboards",
- Method: http.MethodGet,
- Path: "/dashboard/stream",
- Summary: "Stream dashboard snapshots across all environments",
- Description: "Stream dashboard snapshot updates for the local environment and all enabled remote environments as JSON lines",
- Tags: []string{"Dashboard"},
- Security: defaultOperationSecurityInternal(),
- Middlewares: humamw.RequireAnyEnvironmentPermission(api, authz.PermDashboardRead),
- }, h.StreamAllDashboards)
}
func (h *DashboardHandler) GetDashboard(ctx context.Context, input *GetDashboardInput) (*GetDashboardOutput, error) {
@@ -104,47 +87,6 @@ func (h *DashboardHandler) GetDashboard(ctx context.Context, input *GetDashboard
}, nil
}
-func (h *DashboardHandler) StreamAllDashboards(ctx context.Context, input *StreamAllDashboardsInput) (*huma.StreamResponse, error) {
- return &huma.StreamResponse{
- Body: func(humaCtx huma.Context) { //nolint:contextcheck // streaming work must use humaCtx.Context()
- httpx.SetJSONStreamHeaders(humaCtx)
-
- writer := humaCtx.BodyWriter()
- flush := func() {
- if f, ok := writer.(http.Flusher); ok {
- f.Flush()
- }
- }
-
- ps, _ := humamw.PermissionsFromContext(humaCtx.Context())
- h.streamAllDashboardsInternal(humaCtx.Context(), ps, input.DebugAllGood, writer, flush)
- },
- }, nil
-}
-
-// streamAllDashboardsInternal multiplexes dashboard snapshots for the local
-// environment and every enabled remote environment over a single response so
-// the browser needs one connection regardless of environment count.
-func (h *DashboardHandler) streamAllDashboardsInternal(ctx context.Context, ps *authz.PermissionSet, debugAllGood bool, writer io.Writer, flush func()) {
- // RunAuthorizedAggregateStream logs how the stream ended; there is nothing
- // left to report to a client whose response headers were sent long ago.
- _ = httpx.RunAuthorizedAggregateStream(ctx, ps, authz.PermDashboardRead, agg.Config[dashboardtypes.StreamEvent]{
- Writer: writer,
- Flush: flush,
- Buffer: dashboardStreamEventBuffer,
- HeartbeatInterval: dashboardStreamHeartbeatInterval,
- MakeHeartbeat: func() dashboardtypes.StreamEvent {
- return dashboardtypes.StreamEvent{Type: "heartbeat", Timestamp: time.Now()}
- },
- },
- func(ctx context.Context, events chan<- dashboardtypes.StreamEvent) {
- h.runLocalDashboardStreamProducerInternal(ctx, debugAllGood, events)
- },
- func(ctx context.Context, events chan<- dashboardtypes.StreamEvent) {
- h.runRemoteDashboardStreamPollersInternal(ctx, ps, debugAllGood, events)
- })
-}
-
// trimDashboardStreamSnapshotInternal drops the first-page container/image
// tables: the all-environments dashboard only reads the aggregate counters,
// and re-sending table rows for every environment on every poll would bloat
diff --git a/backend/api/handlers/dashboard_test.go b/backend/api/handlers/dashboard_test.go
index 2822b7f78..9aca9e1b8 100644
--- a/backend/api/handlers/dashboard_test.go
+++ b/backend/api/handlers/dashboard_test.go
@@ -18,6 +18,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
dashboardtypes "github.com/getarcaneapp/arcane/types/v2/dashboard"
+ streamtypes "github.com/getarcaneapp/arcane/types/v2/stream"
"github.com/libtnb/sqlite"
dockercontainer "github.com/moby/moby/api/types/container"
dockerimage "github.com/moby/moby/api/types/image"
@@ -137,25 +138,33 @@ func TestDashboardHandlerGetDashboardReturnsSnapshot(t *testing.T) {
}, snapshot.ActionItems.Items)
}
-// runDashboardStreamAllInternal drives streamAllDashboardsInternal through a
-// pipe and returns each decoded event to onEvent until it reports done or the
-// stream ends; remaining output is drained so a blocked writer can finish.
+// runDashboardStreamAllInternal drives the dashboard channel of the multiplexed
+// client stream through a pipe and returns each decoded event to onEvent until
+// it reports done or the stream ends; remaining output is drained so a blocked
+// writer can finish.
func runDashboardStreamAllInternal(t *testing.T, ctx context.Context, cancel context.CancelFunc, handler *DashboardHandler, ps *authz.PermissionSet, onEvent func(dashboardtypes.StreamEvent) bool) {
t.Helper()
+ streamHandler := &StreamHandler{dashboard: handler}
+ input := &StreamClientInput{Channels: streamtypes.ChannelDashboard}
+
pr, pw := io.Pipe()
done := make(chan struct{})
go func() {
defer close(done)
defer func() { _ = pw.Close() }()
- handler.streamAllDashboardsInternal(ctx, ps, false, pw, func() {})
+ streamHandler.streamClientInternal(ctx, ps, input, pw, func() {})
}()
scanner := bufio.NewScanner(pr)
for scanner.Scan() {
- var event dashboardtypes.StreamEvent
- require.NoError(t, json.Unmarshal(scanner.Bytes(), &event))
- if onEvent(event) {
+ var envelope streamtypes.Event
+ require.NoError(t, json.Unmarshal(scanner.Bytes(), &envelope))
+ // Heartbeats are connection keepalives, not dashboard events.
+ if envelope.Dashboard == nil {
+ continue
+ }
+ if onEvent(*envelope.Dashboard) {
cancel()
break
}
diff --git a/backend/api/handlers/environments.go b/backend/api/handlers/environments.go
index 89b3e2b7b..e7b2f80c0 100644
--- a/backend/api/handlers/environments.go
+++ b/backend/api/handlers/environments.go
@@ -29,10 +29,18 @@ import (
"github.com/getarcaneapp/arcane/types/v2/base"
"github.com/getarcaneapp/arcane/types/v2/environment"
"github.com/getarcaneapp/arcane/types/v2/version"
+ "go.getarcane.app/streams/agg"
)
const localDockerEnvironmentID = "0"
+const (
+ // Only covers poll-mode TTL expiry; tunnel and health-check changes arrive
+ // on the service's runtime-change signal instead.
+ environmentStreamPollInterval = 5 * time.Second
+ environmentStreamRefreshFloor = 30 * time.Second
+)
+
// EnvironmentHandler handles environment management endpoints.
type EnvironmentHandler struct {
environmentService *services.EnvironmentService
@@ -430,6 +438,93 @@ func accessibleEnvironmentIDsInternal(ps *authz.PermissionSet) []string {
return ids
}
+// visibleEnvironmentsForInternal returns the environments the caller may see,
+// with the manager's runtime overlay already applied. It reuses the same access
+// rules as ListEnvironments so the stream and the REST list can never disagree
+// about which environments a caller has.
+func (h *EnvironmentHandler) visibleEnvironmentsForInternal(ctx context.Context, ps *authz.PermissionSet) ([]environment.Environment, error) {
+ envs, err := h.environmentService.ListVisibleEnvironments(ctx)
+ if err != nil {
+ return nil, err
+ }
+ if environmentListerSeesAllInternal(ps) {
+ return envs, nil
+ }
+
+ allowed := make(map[string]struct{}, len(ps.PerEnv))
+ for _, envID := range accessibleEnvironmentIDsInternal(ps) {
+ allowed[envID] = struct{}{}
+ }
+
+ filtered := envs[:0]
+ for _, env := range envs {
+ if _, ok := allowed[env.ID]; ok {
+ filtered = append(filtered, env)
+ }
+ }
+ return filtered, nil
+}
+
+func (h *EnvironmentHandler) runEnvironmentStreamProducerInternal(ctx context.Context, ps *authz.PermissionSet, events chan<- environment.StreamEvent) {
+ changes, unsubscribe := h.environmentService.SubscribeRuntimeChanges()
+ defer unsubscribe()
+
+ var lastFingerprint []byte
+ var lastSentAt time.Time
+
+ send := func() bool {
+ envs, err := h.visibleEnvironmentsForInternal(ctx, ps)
+ if err != nil {
+ // A failed read must not end the stream; the next tick retries.
+ if ctx.Err() == nil {
+ slog.WarnContext(ctx, "environment stream failed to list environments", "error", err)
+ }
+ return ctx.Err() == nil
+ }
+
+ fingerprint, err := json.Marshal(envs)
+ if err != nil {
+ slog.WarnContext(ctx, "environment stream failed to encode environments", "error", err)
+ return ctx.Err() == nil
+ }
+ // Re-send unchanged state on a floor so relative timestamps in the UI
+ // ("last seen 2 minutes ago") keep advancing.
+ if bytes.Equal(fingerprint, lastFingerprint) && time.Since(lastSentAt) < environmentStreamRefreshFloor {
+ return true
+ }
+ lastFingerprint = fingerprint
+ lastSentAt = time.Now()
+
+ return agg.Send(ctx, events, environment.StreamEvent{
+ Type: "snapshot",
+ Environments: envs,
+ Timestamp: time.Now(),
+ })
+ }
+
+ if !send() {
+ return
+ }
+
+ ticker := time.NewTicker(environmentStreamPollInterval)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-changes:
+ if !send() {
+ return
+ }
+ case <-ticker.C:
+ if !send() {
+ return
+ }
+ }
+ }
+}
+
// CreateEnvironment creates a new environment.
func (h *EnvironmentHandler) CreateEnvironment(ctx context.Context, input *CreateEnvironmentInput) (*CreateEnvironmentOutput, error) {
user, err := requireUserInternal(ctx)
diff --git a/backend/api/handlers/job_schedules.go b/backend/api/handlers/job_schedules.go
index 97ce19b53..f1e60d81f 100644
--- a/backend/api/handlers/job_schedules.go
+++ b/backend/api/handlers/job_schedules.go
@@ -46,6 +46,7 @@ type RunJobOutput struct {
Body jobschedule.JobRunResponse
}
+//nolint:dupl // Huma operation registrations are intentionally explicit.
func RegisterJobSchedules(api huma.API, jobSvc *services.JobService, envSvc *services.EnvironmentService) {
h := &JobSchedulesHandler{
jobService: jobSvc,
diff --git a/backend/api/handlers/stream.go b/backend/api/handlers/stream.go
new file mode 100644
index 000000000..ee1f6f8bf
--- /dev/null
+++ b/backend/api/handlers/stream.go
@@ -0,0 +1,220 @@
+package handlers
+
+import (
+ "context"
+ "io"
+ "log/slog"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/danielgtaylor/huma/v2"
+ humamw "github.com/getarcaneapp/arcane/backend/v2/api/middleware"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/services"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
+ httputils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ dashboardtypes "github.com/getarcaneapp/arcane/types/v2/dashboard"
+ environmenttypes "github.com/getarcaneapp/arcane/types/v2/environment"
+ streamtypes "github.com/getarcaneapp/arcane/types/v2/stream"
+ "go.getarcane.app/streams/agg"
+)
+
+const (
+ clientStreamHeartbeatInterval = 15 * time.Second
+ clientStreamDeadPeerTimeout = 45 * time.Second
+ clientStreamEventBuffer = 64
+ // Per-channel handoff buffer between a channel's producer and the envelope
+ // forwarder. Small on purpose: the shared writer is the real backpressure
+ // point, and a deep per-channel buffer would only delay noticing a stalled
+ // client.
+ clientStreamChannelBuffer = 16
+)
+
+// StreamHandler multiplexes every live feed onto one connection.
+type StreamHandler struct {
+ dashboard *DashboardHandler
+ activity *ActivityHandler
+ environment *EnvironmentHandler
+}
+
+type StreamClientInput struct {
+ Channels string `query:"channels" doc:"Comma-separated channels to subscribe to: environments, dashboard, activities"`
+ DebugAllGood bool `query:"debugAllGood" default:"false" doc:"Debug mode for the dashboard channel: force an empty action item list"`
+ Limit int `query:"limit" default:"0" doc:"Maximum activities to include in each activities-channel snapshot"`
+}
+
+// RegisterStream registers the multiplexed client stream. It builds its own
+// per-domain handlers from the same services the individual handlers use, so
+// each channel's producer stays the single implementation of that feed.
+func RegisterStream(
+ api huma.API,
+ dashboardService *services.DashboardService,
+ activityService *services.ActivityService,
+ environmentService *services.EnvironmentService,
+) {
+ h := &StreamHandler{
+ dashboard: &DashboardHandler{
+ dashboardService: dashboardService,
+ environmentService: environmentService,
+ },
+ activity: &ActivityHandler{
+ activityService: activityService,
+ environmentService: environmentService,
+ },
+ // Only the environment service is needed: the channel producer lists
+ // environments and derives liveness, nothing more.
+ environment: &EnvironmentHandler{environmentService: environmentService},
+ }
+
+ huma.Register(api, huma.Operation{
+ OperationID: "streamClient",
+ Method: http.MethodGet,
+ Path: "/stream",
+ Summary: "Multiplexed client stream",
+ Description: "Streams the requested channels (environments, dashboard, activities) over a single JSON-lines connection",
+ Tags: []string{"Stream"},
+ Security: defaultOperationSecurityInternal(),
+ // Ungated: each channel applies its own permission check below, and a
+ // caller with access to none simply gets a heartbeat-only stream.
+ }, h.StreamClient)
+}
+
+func (h *StreamHandler) StreamClient(ctx context.Context, input *StreamClientInput) (*huma.StreamResponse, error) {
+ return &huma.StreamResponse{
+ Body: func(humaCtx huma.Context) { //nolint:contextcheck // streaming work must use humaCtx.Context()
+ httputils.SetJSONStreamHeaders(humaCtx)
+
+ writer := humaCtx.BodyWriter()
+ flush := func() {
+ if f, ok := writer.(http.Flusher); ok {
+ f.Flush()
+ }
+ }
+
+ ps, _ := humamw.PermissionsFromContext(humaCtx.Context())
+ h.streamClientInternal(humaCtx.Context(), ps, input, writer, flush)
+ },
+ }, nil
+}
+
+func (h *StreamHandler) streamClientInternal(ctx context.Context, ps *authz.PermissionSet, input *StreamClientInput, writer io.Writer, flush func()) {
+ releaseDeadPeerTimeout, timeoutErr := httputils.AcquireDeadPeerTimeout(ctx, clientStreamDeadPeerTimeout)
+ if timeoutErr != nil {
+ slog.DebugContext(ctx, "could not bound client stream dead-peer timeout", "error", timeoutErr)
+ }
+ defer releaseDeadPeerTimeout()
+
+ // Nothing can be reported to a client whose headers were sent long ago.
+ _ = agg.Run(ctx, agg.Config[streamtypes.Event]{
+ Writer: writer,
+ Flush: flush,
+ Buffer: clientStreamEventBuffer,
+ HeartbeatInterval: clientStreamHeartbeatInterval,
+ MakeHeartbeat: func() streamtypes.Event {
+ return streamtypes.Event{Type: "heartbeat", Timestamp: time.Now()}
+ },
+ Producers: h.producersForInternal(ps, input),
+ })
+}
+
+// producersForInternal selects the producers for the requested channels,
+// dropping any the caller lacks permission for.
+func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *StreamClientInput) []agg.Producer[streamtypes.Event] {
+ requested := parseStreamChannelsInternal(input.Channels)
+ producers := make([]agg.Producer[streamtypes.Event], 0, 4)
+
+ if requested[streamtypes.ChannelEnvironments] {
+ // Ungated like listEnvironments; the producer filters to the caller's
+ // accessible environments.
+ producers = append(producers, forwardStreamChannelInternal(
+ streamtypes.ChannelEnvironments,
+ func(channel string, event environmenttypes.StreamEvent) streamtypes.Event {
+ return streamtypes.Event{Channel: channel, Environment: &event, Timestamp: event.Timestamp}
+ },
+ func(producerCtx context.Context, events chan<- environmenttypes.StreamEvent) {
+ h.environment.runEnvironmentStreamProducerInternal(producerCtx, ps, events)
+ },
+ ))
+ }
+
+ if requested[streamtypes.ChannelDashboard] {
+ wrapDashboard := func(channel string, event dashboardtypes.StreamEvent) streamtypes.Event {
+ return streamtypes.Event{Channel: channel, Dashboard: &event, Timestamp: event.Timestamp}
+ }
+ if ps.Allows(authz.PermDashboardRead, localDockerEnvironmentID) {
+ producers = append(producers, forwardStreamChannelInternal(
+ streamtypes.ChannelDashboard, wrapDashboard,
+ func(producerCtx context.Context, events chan<- dashboardtypes.StreamEvent) {
+ h.dashboard.runLocalDashboardStreamProducerInternal(producerCtx, input.DebugAllGood, events)
+ },
+ ))
+ }
+ if ps.AllowsAny(authz.PermDashboardRead) {
+ producers = append(producers, forwardStreamChannelInternal(
+ streamtypes.ChannelDashboard, wrapDashboard,
+ func(producerCtx context.Context, events chan<- dashboardtypes.StreamEvent) {
+ h.dashboard.runRemoteDashboardStreamPollersInternal(producerCtx, ps, input.DebugAllGood, events)
+ },
+ ))
+ }
+ }
+
+ if requested[streamtypes.ChannelActivities] {
+ wrapActivity := func(channel string, event activitytypes.StreamEvent) streamtypes.Event {
+ return streamtypes.Event{Channel: channel, Activity: &event, Timestamp: event.Timestamp}
+ }
+ if ps.Allows(authz.PermActivitiesRead, localDockerEnvironmentID) {
+ producers = append(producers, forwardStreamChannelInternal(
+ streamtypes.ChannelActivities, wrapActivity,
+ func(producerCtx context.Context, events chan<- activitytypes.StreamEvent) {
+ h.activity.runLocalActivityStreamProducerInternal(producerCtx, input.Limit, events)
+ },
+ ))
+ }
+ if ps.AllowsAny(authz.PermActivitiesRead) {
+ producers = append(producers, forwardStreamChannelInternal(
+ streamtypes.ChannelActivities, wrapActivity,
+ func(producerCtx context.Context, events chan<- activitytypes.StreamEvent) {
+ h.activity.runRemoteActivityStreamPollersInternal(producerCtx, ps, input.Limit, events)
+ },
+ ))
+ }
+ }
+
+ return producers
+}
+
+func parseStreamChannelsInternal(raw string) map[string]bool {
+ channels := make(map[string]bool, 3)
+ for name := range strings.SplitSeq(raw, ",") {
+ if trimmed := strings.TrimSpace(strings.ToLower(name)); trimmed != "" {
+ channels[trimmed] = true
+ }
+ }
+ return channels
+}
+
+// forwardStreamChannelInternal adapts a channel's own producer to the shared
+// envelope. Each producer keeps emitting its native event type, so adding a
+// channel never means rewriting its feed.
+func forwardStreamChannelInternal[T any](
+ channel string,
+ wrap func(string, T) streamtypes.Event,
+ producer func(context.Context, chan<- T),
+) agg.Producer[streamtypes.Event] {
+ return func(ctx context.Context, out chan<- streamtypes.Event) {
+ inner := make(chan T, clientStreamChannelBuffer)
+
+ go func() {
+ defer close(inner)
+ producer(ctx, inner)
+ }()
+
+ for event := range inner {
+ if !agg.Send(ctx, out, wrap(channel, event)) {
+ return
+ }
+ }
+ }
+}
diff --git a/backend/api/handlers/stream_test.go b/backend/api/handlers/stream_test.go
new file mode 100644
index 000000000..c9a87ba2a
--- /dev/null
+++ b/backend/api/handlers/stream_test.go
@@ -0,0 +1,83 @@
+package handlers
+
+import (
+ "bufio"
+ "context"
+ "encoding/json"
+ "io"
+ "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/services"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
+ streamtypes "github.com/getarcaneapp/arcane/types/v2/stream"
+ "github.com/stretchr/testify/require"
+)
+
+// The environments channel must report an environment the manager cannot reach.
+//
+// This is the property the whole live-status fix rests on, and the one neither
+// of the other channels can provide: the dashboard channel only emits when it
+// gets a snapshot back from an agent, so an environment that is down produces
+// nothing. Without a pushed status an offline agent stays offline in the UI
+// forever, which is exactly what happens after a fleet update.
+func TestStreamHandlerEnvironmentsChannelEmitsUnreachableEnvironmentInternal(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ db := setupActivityHandlerTestDBInternal(t)
+ limitStreamTestDBToSingleConnInternal(t, db)
+
+ // An edge environment with no tunnel registered: unreachable by construction.
+ createStreamTestRemoteEnvironmentInternal(t, db, "remote-1", "Remote", "edge://remote-1", "token")
+ require.NoError(t, db.Exec(`UPDATE environments SET is_edge = ? WHERE id = ?`, true, "remote-1").Error)
+
+ handler := &StreamHandler{
+ environment: &EnvironmentHandler{
+ environmentService: services.NewEnvironmentService(db, nil, nil, nil, nil, nil),
+ },
+ }
+ ps := authz.NewPermissionSet()
+ ps.Sudo = true
+
+ pr, pw := io.Pipe()
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ defer func() { _ = pw.Close() }()
+ handler.streamClientInternal(ctx, ps, &StreamClientInput{Channels: streamtypes.ChannelEnvironments}, pw, func() {})
+ }()
+
+ var seen *streamtypes.Event
+ scanner := bufio.NewScanner(pr)
+ for scanner.Scan() {
+ var envelope streamtypes.Event
+ require.NoError(t, json.Unmarshal(scanner.Bytes(), &envelope))
+ if envelope.Environment == nil {
+ continue
+ }
+ seen = &envelope
+ cancel()
+ break
+ }
+
+ go func() { _, _ = io.Copy(io.Discard, pr) }()
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("stream did not terminate after cancel")
+ }
+
+ require.NotNil(t, seen, "expected an environments snapshot for an unreachable environment")
+ require.Equal(t, streamtypes.ChannelEnvironments, seen.Channel)
+
+ var remote *string
+ for _, env := range seen.Environment.Environments {
+ if env.ID == "remote-1" {
+ status := env.Status
+ remote = &status
+ }
+ }
+ require.NotNil(t, remote, "unreachable environment must still appear in the snapshot")
+ require.Equal(t, "offline", *remote)
+}
diff --git a/backend/api/middleware/role.go b/backend/api/middleware/role.go
index d0d1b5c0e..8e2826e0a 100644
--- a/backend/api/middleware/role.go
+++ b/backend/api/middleware/role.go
@@ -55,23 +55,6 @@ func RequirePermission(api huma.API, perm string) huma.Middlewares {
}}
}
-// RequireAnyEnvironmentPermission protects aggregate operations that span
-// environments but do not carry an environment ID in their path. The caller
-// must hold perm globally or for at least one environment; handlers remain
-// responsible for filtering aggregate output to the exact allowed scopes.
-func RequireAnyEnvironmentPermission(api huma.API, perm string) huma.Middlewares {
- return huma.Middlewares{func(ctx huma.Context, next func(huma.Context)) {
- ps, _ := PermissionsFromContext(ctx.Context())
- if !ps.AllowsAny(perm) {
- if err := huma.WriteErr(api, ctx, http.StatusForbidden, "permission denied: "+perm); err != nil {
- slog.WarnContext(ctx.Context(), "failed to write 403 response", "error", err)
- }
- return
- }
- next(ctx)
- }}
-}
-
// RequireGlobalAdmin returns a per-operation Huma middleware that rejects any
// caller who is not a global admin (or sudo). Used for operations that are
// intentionally not exposed as delegated permissions — role creation/edits,
diff --git a/backend/api/middleware/role_test.go b/backend/api/middleware/role_test.go
index e9089729e..d9b1980cd 100644
--- a/backend/api/middleware/role_test.go
+++ b/backend/api/middleware/role_test.go
@@ -219,58 +219,3 @@ func TestRequireSudoRejectsHumanAdminAndAllowsAgent(t *testing.T) {
})
}
}
-
-func TestRequireAnyEnvironmentPermission(t *testing.T) {
- tests := []struct {
- name string
- permission *authz.PermissionSet
- wantStatus int
- wantRan bool
- }{
- {name: "missing", permission: authz.NewPermissionSet(), wantStatus: http.StatusForbidden},
- {name: "other permission", permission: func() *authz.PermissionSet {
- ps := authz.NewPermissionSet()
- ps.AddEnv("env-1", authz.PermContainersList)
- return ps
- }(), wantStatus: http.StatusForbidden},
- {name: "environment permission", permission: func() *authz.PermissionSet {
- ps := authz.NewPermissionSet()
- ps.AddEnv("env-1", authz.PermActivitiesRead)
- return ps
- }(), wantStatus: http.StatusNoContent, wantRan: true},
- {name: "global permission", permission: func() *authz.PermissionSet {
- ps := authz.NewPermissionSet()
- ps.AddGlobal(authz.PermActivitiesRead)
- return ps
- }(), wantStatus: http.StatusNoContent, wantRan: true},
- {name: "sudo", permission: authz.SudoPermissionSet(), wantStatus: http.StatusNoContent, wantRan: true},
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- router := echo.New()
- api := humaecho.NewWithGroup(router, router.Group("/api"), huma.DefaultConfig("test", "1.0.0"))
- api.UseMiddleware(func(ctx huma.Context, next func(huma.Context)) {
- next(huma.WithContext(ctx, context.WithValue(ctx.Context(), ContextKeyUserPermissions, tt.permission)))
- })
-
- handlerRan := false
- huma.Register(api, huma.Operation{
- OperationID: "aggregate-permission-" + tt.name,
- Method: http.MethodGet,
- Path: "/activities/stream",
- Middlewares: RequireAnyEnvironmentPermission(api, authz.PermActivitiesRead),
- }, func(_ context.Context, _ *struct{}) (*struct{}, error) {
- handlerRan = true
- return &struct{}{}, nil
- })
-
- req := httptest.NewRequest(http.MethodGet, "/api/activities/stream", nil)
- rec := httptest.NewRecorder()
- router.ServeHTTP(rec, req)
-
- require.Equal(t, tt.wantStatus, rec.Code)
- require.Equal(t, tt.wantRan, handlerRan)
- })
- }
-}
diff --git a/backend/internal/bootstrap/edge_bootstrap.go b/backend/internal/bootstrap/edge_bootstrap.go
index 6ec79cbba..f08131e91 100644
--- a/backend/internal/bootstrap/edge_bootstrap.go
+++ b/backend/internal/bootstrap/edge_bootstrap.go
@@ -50,6 +50,11 @@ func registerEdgeTunnelRoutes(
if err := createEdgeConnectionEvent(ctx, eventService, envID, envName, connected); err != nil {
slog.WarnContext(ctx, "Failed to create edge connection event", "environment_id", envID, "connected", connected, "error", err)
}
+
+ // This is the only funnel for "an edge tunnel came up or went down"
+ // (register, unregister and stale reaping all route through it), so it
+ // is where open status streams learn about it without polling.
+ environmentService.NotifyRuntimeStateChanged()
}
eventCallback := func(ctx context.Context, envID string, evt *edge.TunnelEvent) error {
diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.go
index 542ab95bd..0ed0508ed 100644
--- a/backend/internal/bootstrap/router_bootstrap.go
+++ b/backend/internal/bootstrap/router_bootstrap.go
@@ -44,8 +44,7 @@ var loggerSkipPatterns = []string{
// ends, so it reports the full connection lifetime as request latency —
// which reads as a multi-minute hung request. The handlers log their own
// termination instead.
- "GET /api/dashboard/stream",
- "GET /api/activities/stream",
+ "GET /api/stream",
"GET /_app/*",
"GET /img",
"GET /api/health",
diff --git a/backend/internal/services/environment_runtime_notify_test.go b/backend/internal/services/environment_runtime_notify_test.go
new file mode 100644
index 000000000..f34cdd260
--- /dev/null
+++ b/backend/internal/services/environment_runtime_notify_test.go
@@ -0,0 +1,45 @@
+package services
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// A watcher must see a change signal, and a burst must not block the notifier —
+// it runs on the edge tunnel's connect/disconnect path.
+func TestEnvironmentServiceRuntimeChangeSignalCoalescesInternal(t *testing.T) {
+ s := &EnvironmentService{}
+
+ changes, unsubscribe := s.SubscribeRuntimeChanges()
+ defer unsubscribe()
+
+ // More notifications than the channel can hold: the extras are dropped
+ // rather than blocking, because a watcher re-derives live state on wake.
+ for range 5 {
+ s.NotifyRuntimeStateChanged()
+ }
+
+ select {
+ case <-changes:
+ default:
+ t.Fatal("expected a pending runtime change signal")
+ }
+
+ select {
+ case <-changes:
+ t.Fatal("expected coalesced signals, got a second wake-up")
+ default:
+ }
+
+ // Unsubscribing must stop delivery so a closed stream cannot leak.
+ unsubscribe()
+ s.NotifyRuntimeStateChanged()
+ select {
+ case <-changes:
+ t.Fatal("expected no signal after unsubscribe")
+ default:
+ }
+
+ require.Empty(t, s.runtimeWatchers)
+}
diff --git a/backend/internal/services/environment_runtime_state.go b/backend/internal/services/environment_runtime_state.go
deleted file mode 100644
index 918cc5ff6..000000000
--- a/backend/internal/services/environment_runtime_state.go
+++ /dev/null
@@ -1,67 +0,0 @@
-package services
-
-import (
- "time"
-
- "github.com/getarcaneapp/arcane/backend/v2/internal/models"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
- environmenttypes "github.com/getarcaneapp/arcane/types/v2/environment"
-)
-
-// ApplyEnvironmentRuntimeState normalizes edge environment runtime status using
-// in-memory tunnel and poll registries without mutating persisted state.
-func ApplyEnvironmentRuntimeState(env *environmenttypes.Environment) {
- if env == nil || !env.IsEdge {
- return
- }
-
- connected := false
- env.Connected = &connected
- env.ConnectedAt = nil
- env.LastHeartbeat = nil
- env.LastPollAt = nil
- env.EdgeTransport = nil
- env.EdgeSecurityMode = nil
- env.EdgeSessionID = nil
- env.EdgeAgentInstance = nil
- env.EdgeCapabilities = nil
-
- if pollState, ok := edge.GetPollRuntimeRegistry().Get(env.ID, time.Now()).Get(); ok {
- env.LastPollAt = pollState.LastPollAt
- }
-
- if runtimeState, ok := edge.GetTunnelRuntimeState(env.ID).Get(); ok {
- connected = true
- env.Connected = &connected
- env.Status = string(models.EnvironmentStatusOnline)
- env.ConnectedAt = runtimeState.ConnectedAt
- env.LastHeartbeat = runtimeState.LastHeartbeat
- if runtimeState.SecurityMode != "" {
- env.EdgeSecurityMode = &runtimeState.SecurityMode
- }
- if runtimeState.SessionID != "" {
- env.EdgeSessionID = &runtimeState.SessionID
- }
- if runtimeState.AgentInstance != "" {
- env.EdgeAgentInstance = &runtimeState.AgentInstance
- }
- if len(runtimeState.Capabilities) > 0 {
- env.EdgeCapabilities = append([]string(nil), runtimeState.Capabilities...)
- }
- if transport, ok := edge.GetActiveTunnelTransport(env.ID).Get(); ok {
- env.EdgeTransport = &transport
- } else if runtimeState.Transport != "" {
- env.EdgeTransport = &runtimeState.Transport
- }
- return
- }
-
- if env.LastPollAt != nil {
- env.Status = string(models.EnvironmentStatusStandby)
- return
- }
-
- if env.Status != string(models.EnvironmentStatusPending) {
- env.Status = string(models.EnvironmentStatusOffline)
- }
-}
diff --git a/backend/internal/services/environment_service.go b/backend/internal/services/environment_service.go
index 9481d13b2..dc67f2b9d 100644
--- a/backend/internal/services/environment_service.go
+++ b/backend/internal/services/environment_service.go
@@ -63,6 +63,12 @@ type EnvironmentService struct {
// variableSyncer is injected post-construction via SetVariableSyncer
// (manager-only) to avoid a wire cycle with VariableService.
variableSyncer VariableSyncer
+
+ // runtimeWatchers receive a coalesced wake-up whenever an environment's
+ // liveness changes. See environment_runtime_notify.go.
+ runtimeWatchMu sync.Mutex
+ runtimeWatchers map[int]chan struct{}
+ runtimeWatcherSeq int
}
// VariableSyncer pushes the effective global-variable set to one environment.
@@ -1378,6 +1384,9 @@ func (s *EnvironmentService) updateEnvironmentStatusInternal(ctx context.Context
if err := s.db.WithContext(ctx).Model(&models.Environment{}).Where("id = ?", id).Updates(updates).Error; err != nil {
return errors.WrapIf(err, "failed to update environment status")
}
+ // Direct environments have no tunnel callback, so this health-check write is
+ // the only moment their liveness changes.
+ s.NotifyRuntimeStateChanged()
return nil
}
@@ -1397,6 +1406,12 @@ func (s *EnvironmentService) UpdateEnvironmentHeartbeat(ctx context.Context, id
return errors.WrapIf(result.Error, "failed to update environment heartbeat")
}
+ // The 30s throttle above doubles as the notify throttle: a no-op heartbeat
+ // changed nothing worth waking a stream for.
+ if result.RowsAffected > 0 {
+ s.NotifyRuntimeStateChanged()
+ }
+
return nil
}
@@ -1424,9 +1439,120 @@ func (s *EnvironmentService) UpdateEnvironmentConnectionState(ctx context.Contex
return errors.WrapIf(err, "failed to update environment connection state")
}
+ s.NotifyRuntimeStateChanged()
+
return nil
}
+// ApplyEnvironmentRuntimeState normalizes edge environment runtime status using
+// in-memory tunnel and poll registries without mutating persisted state.
+func ApplyEnvironmentRuntimeState(env *environment.Environment) {
+ if env == nil || !env.IsEdge {
+ return
+ }
+
+ connected := false
+ env.Connected = &connected
+ env.ConnectedAt = nil
+ env.LastHeartbeat = nil
+ env.LastPollAt = nil
+ env.EdgeTransport = nil
+ env.EdgeSecurityMode = nil
+ env.EdgeSessionID = nil
+ env.EdgeAgentInstance = nil
+ env.EdgeCapabilities = nil
+
+ if pollState, ok := edge.GetPollRuntimeRegistry().Get(env.ID, time.Now()).Get(); ok {
+ env.LastPollAt = pollState.LastPollAt
+ }
+
+ if runtimeState, ok := edge.GetTunnelRuntimeState(env.ID).Get(); ok {
+ connected = true
+ env.Connected = &connected
+ env.Status = string(models.EnvironmentStatusOnline)
+ env.ConnectedAt = runtimeState.ConnectedAt
+ env.LastHeartbeat = runtimeState.LastHeartbeat
+ if runtimeState.SecurityMode != "" {
+ env.EdgeSecurityMode = &runtimeState.SecurityMode
+ }
+ if runtimeState.SessionID != "" {
+ env.EdgeSessionID = &runtimeState.SessionID
+ }
+ if runtimeState.AgentInstance != "" {
+ env.EdgeAgentInstance = &runtimeState.AgentInstance
+ }
+ if len(runtimeState.Capabilities) > 0 {
+ env.EdgeCapabilities = append([]string(nil), runtimeState.Capabilities...)
+ }
+ if transport, ok := edge.GetActiveTunnelTransport(env.ID).Get(); ok {
+ env.EdgeTransport = &transport
+ } else if runtimeState.Transport != "" {
+ env.EdgeTransport = &runtimeState.Transport
+ }
+ return
+ }
+
+ if env.LastPollAt != nil {
+ env.Status = string(models.EnvironmentStatusStandby)
+ return
+ }
+
+ if env.Status != string(models.EnvironmentStatusPending) {
+ env.Status = string(models.EnvironmentStatusOffline)
+ }
+}
+
+// Environment liveness lives in process-local registries (the edge tunnel
+// registry and the poll registry), so nothing in the database changes when an
+// agent connects or drops. Open status streams therefore need a nudge rather
+// than a payload: they re-derive every environment they cover on wake, which is
+// why a bare signal is enough and why a dropped signal is harmless — the
+// watcher has one pending already and will read current state when it runs.
+
+// SubscribeRuntimeChanges returns a channel that receives a coalesced wake-up
+// whenever environment liveness may have changed, plus a function to release
+// it. The channel is never closed; callers select on it alongside their own
+// context.
+func (s *EnvironmentService) SubscribeRuntimeChanges() (<-chan struct{}, func()) {
+ s.runtimeWatchMu.Lock()
+ defer s.runtimeWatchMu.Unlock()
+
+ if s.runtimeWatchers == nil {
+ s.runtimeWatchers = make(map[int]chan struct{})
+ }
+
+ s.runtimeWatcherSeq++
+ id := s.runtimeWatcherSeq
+ // Capacity 1: a second signal arriving before the watcher wakes is
+ // redundant, because the watcher reads live state rather than a queue.
+ ch := make(chan struct{}, 1)
+ s.runtimeWatchers[id] = ch
+
+ return ch, func() {
+ s.runtimeWatchMu.Lock()
+ defer s.runtimeWatchMu.Unlock()
+ delete(s.runtimeWatchers, id)
+ }
+}
+
+// NotifyRuntimeStateChanged wakes every runtime watcher. It never blocks, so it
+// is safe to call from connection callbacks on the tunnel hot path.
+func (s *EnvironmentService) NotifyRuntimeStateChanged() {
+ if s == nil {
+ return
+ }
+
+ s.runtimeWatchMu.Lock()
+ defer s.runtimeWatchMu.Unlock()
+
+ for _, ch := range s.runtimeWatchers {
+ select {
+ case ch <- struct{}{}:
+ default:
+ }
+ }
+}
+
// ReconcileEdgeStatusesOnStartup resets edge environments to offline when the manager starts.
// Live edge tunnels are process-local runtime state, so persisted "online" flags can be stale
// after a restart until agents reconnect. Pending environments are left untouched.
diff --git a/backend/pkg/utils/httpx/aggregate_stream.go b/backend/pkg/utils/httpx/aggregate_stream.go
deleted file mode 100644
index c392f87e5..000000000
--- a/backend/pkg/utils/httpx/aggregate_stream.go
+++ /dev/null
@@ -1,57 +0,0 @@
-package httpx
-
-import (
- "context"
- "log/slog"
- "time"
-
- "emperror.dev/errors"
-
- "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
- "go.getarcane.app/streams/agg"
-)
-
-// streamDeadPeerTimeout is how long an aggregate stream's writes may stay
-// unacknowledged before the connection is torn down. A client that has
-// acknowledged nothing for this long is gone: healthy ones acknowledge the
-// heartbeat every 15 seconds.
-const streamDeadPeerTimeout = 45 * time.Second
-
-// RunAuthorizedAggregateStream selects local and remote producers from the
-// caller's effective permissions before starting an aggregate stream. Remote
-// producers must still filter individual environments with PermissionSet.Allows.
-func RunAuthorizedAggregateStream[T any](
- ctx context.Context,
- ps *authz.PermissionSet,
- permission string,
- config agg.Config[T],
- localProducer agg.Producer[T],
- remoteProducer agg.Producer[T],
-) error {
- config.Producers = make([]agg.Producer[T], 0, 2)
- if ps.Allows(permission, "0") {
- config.Producers = append(config.Producers, localProducer)
- }
- if ps.AllowsAny(permission) {
- config.Producers = append(config.Producers, remoteProducer)
- }
-
- releaseDeadPeerTimeout, timeoutErr := AcquireDeadPeerTimeout(ctx, streamDeadPeerTimeout)
- if timeoutErr != nil {
- slog.DebugContext(ctx, "could not bound stream dead-peer timeout", "permission", permission, "error", timeoutErr)
- }
- defer releaseDeadPeerTimeout()
-
- startedAt := time.Now()
- err := agg.Run(ctx, config)
-
- // Once the response headers are sent there is no way to report a failure to
- // the client, and the access log only records the connection's total
- // lifetime — so a client that vanishes mid-write is invisible without this.
- if err != nil && !errors.Is(err, context.Canceled) {
- slog.WarnContext(ctx, "aggregate stream ended with error", "permission", permission, "error", err, "duration", time.Since(startedAt))
- return err
- }
- slog.DebugContext(ctx, "aggregate stream closed", "permission", permission, "duration", time.Since(startedAt), "context_error", ctx.Err())
- return err
-}
diff --git a/frontend/src/lib/services/activity-service.ts b/frontend/src/lib/services/activity-service.ts
index 66a36dc41..dab81488f 100644
--- a/frontend/src/lib/services/activity-service.ts
+++ b/frontend/src/lib/services/activity-service.ts
@@ -1,9 +1,8 @@
-import BaseAPIService, { handleUnauthorizedResponseInternal } from './api-service';
+import BaseAPIService from './api-service';
import { environmentStore } from '#lib/stores/environment.store.svelte';
import type { Activity, ActivityClearHistoryResult, ActivityDetail } from '#lib/types/activity.type';
import type { Paginated, SearchPaginationSortRequest } from '#lib/types/shared';
import { transformPaginationParams } from '#lib/utils/tables';
-import { streamCacheBuster } from '#lib/utils/streaming';
class ActivityService extends BaseAPIService {
private async resolveEnvironmentId(environmentId?: string): Promise<string> {
@@ -34,39 +33,6 @@ class ActivityService extends BaseAPIService {
const envId = await this.resolveEnvironmentId(environmentId);
return this.handleResponse(this.api.delete(`/environments/${envId}/activities/history`));
}
-
- getActivityStreamUrl(limit = 50): string {
- const baseUrl = this.api.defaults.baseURL.replace(/\/+$/, '');
- const params = new URLSearchParams({ limit: String(limit), _: streamCacheBuster() });
- return `${baseUrl}/activities/stream?${params.toString()}`;
- }
-
- async openActivityStream(signal: AbortSignal, limit = 50, retry = false): Promise<Response> {
- const response = await fetch(this.getActivityStreamUrl(limit), {
- credentials: 'include',
- headers: { Accept: 'application/x-json-stream' },
- signal
- });
- if (response.status === 401) {
- // Nothing reads this body, so release its connection before deciding
- // what to do next.
- await response.body?.cancel().catch(() => {});
- const action = await handleUnauthorizedResponseInternal('/activities/stream', retry);
- if (action === 'retry') {
- return this.openActivityStream(signal, limit, true);
- }
- // The document is being replaced (login redirect, or the reload that
- // follows a backend self-update). Never settling stops the caller
- // from opening a fresh stream against a page that is going away.
- if (action === 'redirect' || action === 'reload') {
- return new Promise<Response>(() => {});
- }
- }
- if (!response.ok) {
- throw new Error(`Activity stream failed with status ${response.status}`);
- }
- return response;
- }
}
export const activityService = new ActivityService();
diff --git a/frontend/src/lib/services/auth-service.ts b/frontend/src/lib/services/auth-service.ts
index f2c721a63..81d0f6bdd 100644
--- a/frontend/src/lib/services/auth-service.ts
+++ b/frontend/src/lib/services/auth-service.ts
@@ -7,6 +7,7 @@ import type { OidcUserInfo, LoginCredentials, LoginResponseData, AutoLoginConfig
import type { QueryClient } from '@tanstack/svelte-query';
import { activityStore } from '#lib/stores/activity.store.svelte';
import { dashboardStore } from '#lib/stores/dashboard.store.svelte';
+import { environmentStatusStore } from '#lib/stores/environment-status.store.svelte';
import { getEffectiveLandingPage } from '#lib/utils/navigation';
const REFRESH_TOKEN_KEY = 'arcane_refresh_token';
@@ -208,11 +209,13 @@ class AuthService extends BaseAPIService {
queryClient.clear();
const restartActivityStore = activityStore.stop({ resetState: true });
const restartDashboardStore = dashboardStore.stop({ resetState: true });
+ const restartEnvironmentStatusStore = environmentStatusStore.stop();
userStore.clearUser();
if (options?.restartMountedStores) {
if (restartActivityStore) void activityStore.start();
if (restartDashboardStore) void dashboardStore.start();
+ if (restartEnvironmentStatusStore) void environmentStatusStore.start();
}
}
diff --git a/frontend/src/lib/services/dashboard-service.ts b/frontend/src/lib/services/dashboard-service.ts
index ad81e2c53..30ae7be4a 100644
--- a/frontend/src/lib/services/dashboard-service.ts
+++ b/frontend/src/lib/services/dashboard-service.ts
@@ -1,5 +1,4 @@
-import BaseAPIService, { handleUnauthorizedResponseInternal } from './api-service';
-import { streamCacheBuster } from '#lib/utils/streaming';
+import BaseAPIService from './api-service';
import type { DashboardSnapshot } from '#lib/types/shared';
interface GetDashboardOptions {
@@ -11,44 +10,6 @@ class DashboardService extends BaseAPIService {
const params = options?.debugAllGood ? { debugAllGood: 'true' } : undefined;
return this.handleResponse(this.api.get(`/environments/${environmentId}/dashboard`, { params }));
}
-
- getDashboardStreamUrl(debugAllGood = false): string {
- const baseUrl = this.api.defaults.baseURL.replace(/\/+$/, '');
- const params = new URLSearchParams();
- if (debugAllGood) {
- params.set('debugAllGood', 'true');
- }
- params.set('_', streamCacheBuster());
- const query = params.toString();
- return `${baseUrl}/dashboard/stream${query ? `?${query}` : ''}`;
- }
-
- async openDashboardStream(signal: AbortSignal, debugAllGood = false, retry = false): Promise<Response> {
- const response = await fetch(this.getDashboardStreamUrl(debugAllGood), {
- credentials: 'include',
- headers: { Accept: 'application/x-json-stream' },
- signal
- });
- if (response.status === 401) {
- // Nothing reads this body, so release its connection before deciding
- // what to do next.
- await response.body?.cancel().catch(() => {});
- const action = await handleUnauthorizedResponseInternal('/dashboard/stream', retry);
- if (action === 'retry') {
- return this.openDashboardStream(signal, debugAllGood, true);
- }
- // The document is being replaced (login redirect, or the reload that
- // follows a backend self-update). Never settling stops the caller
- // from opening a fresh stream against a page that is going away.
- if (action === 'redirect' || action === 'reload') {
- return new Promise<Response>(() => {});
- }
- }
- if (!response.ok) {
- throw new Error(`Dashboard stream failed with status ${response.status}`);
- }
- return response;
- }
}
export const dashboardService = new DashboardService();
diff --git a/frontend/src/lib/services/stream-service.ts b/frontend/src/lib/services/stream-service.ts
new file mode 100644
index 000000000..b7ac721a0
--- /dev/null
+++ b/frontend/src/lib/services/stream-service.ts
@@ -0,0 +1,50 @@
+import BaseAPIService, { handleUnauthorizedResponseInternal } from './api-service';
+import { streamCacheBuster } from '#lib/utils/streaming';
+
+/** Channels the multiplexed client stream can carry. Must match types/stream. */
+export const STREAM_CHANNEL_ENVIRONMENTS = 'environments';
+export const STREAM_CHANNEL_DASHBOARD = 'dashboard';
+export const STREAM_CHANNEL_ACTIVITIES = 'activities';
+
+class StreamService extends BaseAPIService {
+ getClientStreamUrl(channels: string[], params: Record<string, string> = {}): string {
+ const baseUrl = this.api.defaults.baseURL.replace(/\/+$/, '');
+ const search = new URLSearchParams({ ...params, channels: channels.join(',') });
+ search.set('_', streamCacheBuster());
+ return `${baseUrl}/stream?${search.toString()}`;
+ }
+
+ async openClientStream(
+ signal: AbortSignal,
+ channels: string[],
+ params: Record<string, string> = {},
+ retry = false
+ ): Promise<Response> {
+ const response = await fetch(this.getClientStreamUrl(channels, params), {
+ credentials: 'include',
+ headers: { Accept: 'application/x-json-stream' },
+ signal
+ });
+ if (response.status === 401) {
+ // Nothing reads this body, so release its connection before deciding
+ // what to do next.
+ await response.body?.cancel().catch(() => {});
+ const action = await handleUnauthorizedResponseInternal('/stream', retry);
+ if (action === 'retry') {
+ return this.openClientStream(signal, channels, params, true);
+ }
+ // The document is being replaced (login redirect, or the reload that
+ // follows a backend self-update). Never settling stops the caller
+ // from opening a fresh stream against a page that is going away.
+ if (action === 'redirect' || action === 'reload') {
+ return new Promise<Response>(() => {});
+ }
+ }
+ if (!response.ok) {
+ throw new Error(`Client stream failed with status ${response.status}`);
+ }
+ return response;
+ }
+}
+
+export const streamService = new StreamService();
diff --git a/frontend/src/lib/stores/activity.store.svelte.ts b/frontend/src/lib/stores/activity.store.svelte.ts
index 597826024..eb69b2819 100644
--- a/frontend/src/lib/stores/activity.store.svelte.ts
+++ b/frontend/src/lib/stores/activity.store.svelte.ts
@@ -1,4 +1,5 @@
import { activityService } from '#lib/services/activity-service';
+import { STREAM_CHANNEL_ACTIVITIES } from '#lib/services/stream-service';
import { LOCAL_DOCKER_ENVIRONMENT_ID } from '#lib/stores/environment.store.svelte';
import {
createEnvironmentStreamStore,
@@ -190,7 +191,7 @@ function createActivityStore() {
streamError: false
};
},
- openStream: (signal) => activityService.openActivityStream(signal, ACTIVITY_LIST_LIMIT),
+ channel: STREAM_CHANNEL_ACTIVITIES,
// REST-fetch history on start so the panel isn't stuck loading if the
// stream is slow or drops before delivering its first snapshot.
refreshOnStart: true,
diff --git a/frontend/src/lib/stores/client-stream.svelte.ts b/frontend/src/lib/stores/client-stream.svelte.ts
new file mode 100644
index 000000000..de9d48e72
--- /dev/null
+++ b/frontend/src/lib/stores/client-stream.svelte.ts
@@ -0,0 +1,134 @@
+import { streamService } from '#lib/services/stream-service';
+import { createJSONLineStream } from '#lib/stores/json-line-stream.svelte';
+
+type StreamEnvelope = {
+ type: string;
+ channel?: string;
+ dashboard?: unknown;
+ activity?: unknown;
+ environment?: unknown;
+};
+
+type ChannelSubscriber = {
+ channel: string;
+ onEvent(event: unknown): void;
+ onConnected?(): void;
+};
+
+/**
+ * The app's single server stream. Every live feed — environments, dashboard,
+ * activities — is multiplexed over one connection.
+ *
+ * Browsers allow only six concurrent HTTP/1.1 requests per origin, and each
+ * long-lived stream holds one for the life of the page. One connection also
+ * means one reconnect and one heartbeat to reason about instead of three that
+ * can disagree about whether the server is reachable.
+ *
+ * Channels are opened on demand rather than always-on because their costs are
+ * nothing alike: environments is a single local query, while dashboard proxies
+ * a request to every remote agent on every tick. Subscribing or unsubscribing
+ * changes the URL, so the connection is reopened — the same mechanism the
+ * dashboard already used when its debug flag changed.
+ */
+function createClientStreamInternal() {
+ const subscribers = new Set<ChannelSubscriber>();
+ let params: Record<string, string> = {};
+
+ function activeChannels(): string[] {
+ return [...new Set([...subscribers].map((subscriber) => subscriber.channel))].sort();
+ }
+
+ const transport = createJSONLineStream<StreamEnvelope>({
+ label: 'Client',
+ openStream: (signal) => streamService.openClientStream(signal, activeChannels(), params),
+ onConnected: () => {
+ for (const subscriber of subscribers) {
+ subscriber.onConnected?.();
+ }
+ },
+ onEvent: (envelope) => {
+ const payload = envelope.dashboard ?? envelope.activity ?? envelope.environment;
+ if (!envelope.channel || payload === undefined) {
+ return;
+ }
+ for (const subscriber of subscribers) {
+ if (subscriber.channel === envelope.channel) {
+ subscriber.onEvent(payload);
+ }
+ }
+ }
+ });
+
+ function reopen() {
+ if (!transport.isStarted) {
+ return;
+ }
+ if (activeChannels().length === 0) {
+ // Nothing left to carry; drop the connection rather than hold an
+ // idle one open with no channels.
+ transport.stop();
+ return;
+ }
+ transport.restart();
+ }
+
+ function ensureStarted() {
+ if (transport.isStarted || activeChannels().length === 0) {
+ return;
+ }
+ transport.markStarted();
+ transport.connect(transport.nextGeneration());
+ }
+
+ return {
+ get streamConnected(): boolean {
+ return transport.streamConnected;
+ },
+ get streamFailed(): boolean {
+ return transport.streamFailed;
+ },
+ get generation(): number {
+ return transport.generation;
+ },
+ get hasActiveStream(): boolean {
+ return transport.hasActiveStream;
+ },
+ isCurrentGeneration: transport.isCurrentGeneration,
+ /** Subscribes to a channel, (re)opening the connection to include it. */
+ subscribe(channel: string, handlers: { onEvent(event: unknown): void; onConnected?(): void }): () => void {
+ const subscriber: ChannelSubscriber = { channel, ...handlers };
+ const hadChannel = activeChannels().includes(channel);
+ subscribers.add(subscriber);
+
+ if (!transport.isStarted) {
+ ensureStarted();
+ } else if (!hadChannel) {
+ reopen();
+ }
+
+ return () => {
+ subscribers.delete(subscriber);
+ if (!activeChannels().includes(channel)) {
+ reopen();
+ }
+ };
+ },
+ /** Sets query params shared by the stream URL (e.g. the dashboard debug flag). */
+ setParams(next: Record<string, string>) {
+ const changed = JSON.stringify(next) !== JSON.stringify(params);
+ params = next;
+ if (changed) {
+ reopen();
+ }
+ },
+ /** Reopens the connection, clearing a give-up state so it retries. */
+ retry() {
+ transport.restart({ clearFailure: true });
+ },
+ restart() {
+ transport.restart();
+ }
+ };
+}
+
+export const clientStream = createClientStreamInternal();
diff --git a/frontend/src/lib/stores/dashboard.store.svelte.ts b/frontend/src/lib/stores/dashboard.store.svelte.ts
index 4d7e51ba8..82afc255c 100644
--- a/frontend/src/lib/stores/dashboard.store.svelte.ts
+++ b/frontend/src/lib/stores/dashboard.store.svelte.ts
@@ -1,5 +1,7 @@
import { browser } from '$app/env';
import { dashboardService } from '#lib/services/dashboard-service';
+import { STREAM_CHANNEL_DASHBOARD } from '#lib/services/stream-service';
+import { clientStream } from '#lib/stores/client-stream.svelte';
import { LOCAL_DOCKER_ENVIRONMENT_ID } from '#lib/stores/environment.store.svelte';
import {
createEnvironmentStreamStore,
@@ -38,7 +40,7 @@ function createDashboardStore() {
streamError: false
};
},
- openStream: (signal) => dashboardService.openDashboardStream(signal, debugAllGood),
+ channel: STREAM_CHANNEL_DASHBOARD,
applyEvent(environmentId, event) {
switch (event.type) {
case 'snapshot':
@@ -116,7 +118,7 @@ function createDashboardStore() {
}
// The flag is encoded in the stream URL; restart to apply it.
debugAllGood = nextDebugAllGood;
- core.restartStream();
+ clientStream.setParams(debugAllGood ? { debugAllGood: 'true' } : {});
return;
}
diff --git a/frontend/src/lib/stores/environment-status.store.svelte.ts b/frontend/src/lib/stores/environment-status.store.svelte.ts
new file mode 100644
index 000000000..874bae041
--- /dev/null
+++ b/frontend/src/lib/stores/environment-status.store.svelte.ts
@@ -0,0 +1,67 @@
+import { browser } from '$app/env';
+import { STREAM_CHANNEL_ENVIRONMENTS } from '#lib/services/stream-service';
+import { clientStream } from '#lib/stores/client-stream.svelte';
+import { environmentStore } from '#lib/stores/environment.store.svelte';
+import type { Environment } from '#lib/types/environment';
+
+type EnvironmentStreamEvent = {
+ type: string;
+ environments?: Environment[];
+};
+
+/**
+ * Keeps environmentStore's records live for as long as the app shell is mounted.
+ *
+ * Whether an edge environment is reachable lives only in the manager's
+ * in-memory tunnel and poll registries, so the browser can never work it out
+ * on its own — it has to be told. Without this stream the environment list is
+ * frozen at whatever the root layout load happened to see, which after a fleet
+ * update is "every agent offline".
+ */
+function createEnvironmentStatusStoreInternal() {
+ let started = false;
+ let lifecycleGeneration = 0;
+
+ let unsubscribeChannel: (() => void) | null = null;
+
+ return {
+ get streamConnected(): boolean {
+ return clientStream.streamConnected;
+ },
+ get streamFailed(): boolean {
+ return clientStream.streamFailed;
+ },
+ async start() {
+ if (!browser || started) {
+ return;
+ }
+ started = true;
+ const generation = ++lifecycleGeneration;
+ // The first snapshot replaces the list, so wait for the load-time
+ // list to land or the two would race on an empty store.
+ await environmentStore.ready;
+ if (generation !== lifecycleGeneration) {
+ return;
+ }
+ unsubscribeChannel = clientStream.subscribe(STREAM_CHANNEL_ENVIRONMENTS, {
+ onEvent: (payload) => {
+ const event = payload as EnvironmentStreamEvent;
+ if (event.type !== 'snapshot' || !event.environments) {
+ return;
+ }
+ environmentStore.applyLiveEnvironments(event.environments);
+ }
+ });
+ },
+ stop() {
+ const wasStarted = started;
+ started = false;
+ lifecycleGeneration += 1;
+ unsubscribeChannel?.();
+ unsubscribeChannel = null;
+ return wasStarted;
+ }
+ };
+}
+
+export const environmentStatusStore = createEnvironmentStatusStoreInternal();
diff --git a/frontend/src/lib/stores/environment-stream.svelte.ts b/frontend/src/lib/stores/environment-stream.svelte.ts
index b02845710..e6730dffb 100644
--- a/frontend/src/lib/stores/environment-stream.svelte.ts
+++ b/frontend/src/lib/stores/environment-stream.svelte.ts
@@ -1,10 +1,8 @@
import { browser } from '$app/env';
import { environmentStore, LOCAL_DOCKER_ENVIRONMENT_ID } from '#lib/stores/environment.store.svelte';
+import { clientStream } from '#lib/stores/client-stream.svelte';
import type { Environment } from '#lib/types/environment';
-const MAX_RECONNECT_DELAY = 15_000;
-const MAX_RECONNECT_ATTEMPTS = 20;
-
export type StreamEnvStateBase = {
id: string;
name: string;
@@ -36,7 +34,8 @@ export interface EnvStreamCoreConfig<TState extends StreamEnvStateBase, TEvent e
/** Used in console warnings, e.g. 'Dashboard' / 'Activity'. */
label: string;
createEnvironmentState(environment: Pick<Environment, 'id' | 'name'>): TState;
- openStream(signal: AbortSignal): Promise<Response>;
+ /** Channel to subscribe to on the shared client stream. */
+ channel: string;
/** Handles every event type except 'heartbeat' (core owns connection state). */
applyEvent(environmentId: string, event: TEvent): void;
/** Fully owns a per-environment REST refresh, including generation/removal guards via core helpers. */
@@ -61,18 +60,28 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,
let _environmentStates = $state<Record<string, TState>>({});
let started = false;
+ // REST snapshots belong to this store lifecycle, not the shared transport's reconnect lifecycle.
+ let lifecycleGeneration = 0;
let unsubscribeEnvironment: (() => void) | null = null;
let unsubscribeEnvironmentFilter: (() => void) | null = null;
- // A single aggregated stream carries every environment's events; per-env
- // connections would multiply requests and exhaust the browser's
- // 6-per-origin HTTP/1.1 limit.
- let streamAbortController: AbortController | null = null;
- let removePageLifecycleListeners: (() => void) | null = null;
- let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
- let reconnectAttempt = 0;
- let streamGeneration = 0;
- let _streamConnected = $state(false);
- let _streamFailed = $state(false);
+
+ let unsubscribeChannel: (() => void) | null = null;
+
+ const channelHandlers = {
+ // A fresh stream re-emits error events for environments that are still
+ // failing, so stale per-environment errors are cleared on every (re)connect.
+ onConnected: () => clearAllEnvironmentErrors(),
+ onEvent: (payload: unknown) => {
+ const event = payload as TEvent;
+ const environmentId = event.environmentId || LOCAL_DOCKER_ENVIRONMENT_ID;
+ // The aggregated stream can keep delivering events for an environment
+ // for a short while after it was removed locally; don't resurrect it.
+ if (!environmentState(environmentId)) {
+ return;
+ }
+ config.applyEvent(environmentId, event);
+ }
+ };
function environmentState(environmentId: string): TState | undefined {
return _environmentStates[environmentId];
@@ -118,53 +127,6 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,
}
}
- function nextGeneration(): number {
- streamGeneration += 1;
- return streamGeneration;
- }
-
- function isCurrentGeneration(generation: number): boolean {
- return streamGeneration === generation;
- }
-
- function clearReconnectTimer() {
- if (reconnectTimer) {
- clearTimeout(reconnectTimer);
- reconnectTimer = null;
- }
- }
-
- function abortStream() {
- clearReconnectTimer();
- streamAbortController?.abort();
- streamAbortController = null;
- _streamConnected = false;
- }
-
- // Closing a tab leaves teardown to whenever the browser gets around to
- // dropping the socket. Until it does, the server keeps writing heartbeats
- // into a connection nobody is reading — so tear down explicitly instead.
- function watchPageLifecycle() {
- if (!browser || removePageLifecycleListeners) {
- return;
- }
-
- const onPageHide = () => abortStream();
- // A bfcache restore comes back with the stream already torn down.
- const onPageShow = (event: PageTransitionEvent) => {
- if (event.persisted && started && !streamAbortController) {
- void connectStream(nextGeneration());
- }
- };
-
- window.addEventListener('pagehide', onPageHide);
- window.addEventListener('pageshow', onPageShow);
- removePageLifecycleListeners = () => {
- window.removeEventListener('pagehide', onPageHide);
- window.removeEventListener('pageshow', onPageShow);
- };
- }
-
function removeEnvironment(environmentId: string) {
const nextStates = { ..._environmentStates };
delete nextStates[environmentId];
@@ -172,136 +134,11 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,
config.onEnvironmentRemoved?.(environmentId);
}
- async function refresh(generation = streamGeneration) {
+ async function refresh(generation = lifecycleGeneration) {
reconcileEnvironments();
await Promise.all(Object.keys(_environmentStates).map((environmentId) => config.fetchSnapshot(environmentId, generation)));
}
- async function connectStream(generation: number) {
- if (!browser || !isCurrentGeneration(generation)) {
- return;
- }
-
- // Overlapping connects would otherwise strand the previous controller:
- // abortStream() only ever reaches the newest one, so the older request
- // would keep an open connection the server has no way to notice.
- streamAbortController?.abort();
-
- const controller = new AbortController();
- streamAbortController = controller;
- try {
- const response = await config.openStream(controller.signal);
- if (!isCurrentGeneration(generation) || !response.body) {
- // The response body is live even though nobody will read it;
- // dropping it on the floor leaves the server streaming into a
- // connection that stays open until its TCP timers expire.
- controller.abort();
- if (streamAbortController === controller) {
- streamAbortController = null;
- }
- return;
- }
-
- _streamConnected = true;
- _streamFailed = false;
- reconnectAttempt = 0;
- clearAllEnvironmentErrors();
- await readJSONLines(response.body, generation);
- } catch (error) {
- if (!controller.signal.aborted && isCurrentGeneration(generation)) {
- console.warn(`${config.label} stream disconnected:`, error);
- }
- } finally {
- if (streamAbortController === controller) {
- streamAbortController = null;
- }
- if (isCurrentGeneration(generation)) {
- _streamConnected = false;
- if (!controller.signal.aborted) {
- scheduleReconnect(generation);
- }
- } else if (!controller.signal.aborted) {
- // A superseded generation has no owner left to abort it.
- controller.abort();
- }
- }
- }
-
- async function readJSONLines(stream: ReadableStream<Uint8Array>, generation: number) {
- const reader = stream.getReader();
- const decoder = new TextDecoder();
- let buffer = '';
-
- try {
- while (isCurrentGeneration(generation)) {
- const { done, value } = await reader.read();
- if (done) {
- break;
- }
-
- buffer += decoder.decode(value, { stream: true });
- const lines = buffer.split('\n');
- buffer = lines.pop() ?? '';
- for (const line of lines) {
- handleStreamLine(line);
- }
- }
-
- buffer += decoder.decode();
- if (buffer.trim()) {
- handleStreamLine(buffer);
- }
- } finally {
- // The loop also exits when the generation advances, with the stream
- // still open. Cancelling is what actually closes the fetch and lets
- // the server see the disconnect; releaseLock alone does not.
- await reader.cancel().catch(() => {});
- reader.releaseLock();
- }
- }
-
- function handleStreamLine(line: string) {
- const trimmed = line.trim();
- if (!trimmed) {
- return;
- }
-
- try {
- const event = JSON.parse(trimmed) as TEvent;
- const environmentId = event.environmentId || LOCAL_DOCKER_ENVIRONMENT_ID;
- if (event.type === 'heartbeat') {
- _streamConnected = true;
- return;
- }
- // The aggregated stream can keep delivering events for an environment
- // for a short while after it was removed locally; don't resurrect it.
- if (!environmentState(environmentId)) {
- return;
- }
- config.applyEvent(environmentId, event);
- } catch (error) {
- console.warn(`Failed to parse ${config.label.toLowerCase()} stream line:`, error);
- }
- }
-
- function scheduleReconnect(generation: number) {
- if (!browser || !started || !isCurrentGeneration(generation)) {
- return;
- }
-
- if (reconnectAttempt >= MAX_RECONNECT_ATTEMPTS) {
- _streamFailed = true;
- return;
- }
-
- clearReconnectTimer();
- const delay = Math.min(1000 * 2 ** reconnectAttempt, MAX_RECONNECT_DELAY);
- reconnectAttempt += 1;
- reconnectTimer = setTimeout(() => {
- void connectStream(generation);
- }, delay);
- }
-
function reconcileEnvironments() {
if (!browser || !started) {
return;
@@ -335,8 +172,8 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,
// An already-open aggregated stream only picks new environments
// up on its server-side reconcile tick; fetch once so the first
// snapshot doesn't take up to that interval to appear.
- if (streamAbortController) {
- void config.fetchSnapshot(environmentId, streamGeneration);
+ if (clientStream.hasActiveStream) {
+ void config.fetchSnapshot(environmentId, lifecycleGeneration);
}
continue;
}
@@ -362,25 +199,22 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,
_environmentStates = value;
},
get streamConnected(): boolean {
- return _streamConnected;
- },
- set streamConnected(value: boolean) {
- _streamConnected = value;
+ return clientStream.streamConnected;
},
get streamFailed(): boolean {
- return _streamFailed;
+ return clientStream.streamFailed;
},
get generation(): number {
- return streamGeneration;
+ return lifecycleGeneration;
},
get hasActiveStream(): boolean {
- return streamAbortController !== null;
+ return clientStream.hasActiveStream;
},
environmentState,
updateEnvironmentState,
setEnvironmentError,
clearEnvironmentError,
- isCurrentGeneration,
+ isCurrentGeneration: (generation: number) => generation === lifecycleGeneration,
reconcileEnvironments,
refresh,
async start() {
@@ -389,18 +223,17 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,
}
started = true;
+ const generation = ++lifecycleGeneration;
await environmentStore.ready;
- if (!started) {
+ if (generation !== lifecycleGeneration) {
return;
}
config.onSelectedEnvironment?.(environmentStore.selected);
- watchPageLifecycle();
reconcileEnvironments();
- const generation = nextGeneration();
+ unsubscribeChannel = clientStream.subscribe(config.channel, channelHandlers);
if (config.refreshOnStart) {
void refresh(generation);
}
- void connectStream(generation);
unsubscribeEnvironment = environmentStore.subscribeSelected((environment) => {
config.onSelectedEnvironment?.(environment);
reconcileEnvironments();
@@ -410,32 +243,24 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,
stop(options?: { resetState?: boolean; resetStreamFailed?: boolean }) {
const wasStarted = started;
started = false;
+ lifecycleGeneration += 1;
unsubscribeEnvironment?.();
unsubscribeEnvironment = null;
unsubscribeEnvironmentFilter?.();
unsubscribeEnvironmentFilter = null;
- removePageLifecycleListeners?.();
- removePageLifecycleListeners = null;
- nextGeneration();
- abortStream();
- reconnectAttempt = 0;
+ unsubscribeChannel?.();
+ unsubscribeChannel = null;
if (options?.resetState) {
_environmentStates = {};
}
- if (options?.resetState || options?.resetStreamFailed) {
- _streamFailed = false;
- }
return wasStarted;
},
retryStream() {
- _streamFailed = false;
- reconnectAttempt = 0;
// Environments may have been added/removed while the stream was down;
// reconcile so the new stream's snapshots aren't dropped as unknown.
reconcileEnvironments();
clearAllEnvironmentErrors();
- abortStream();
- void connectStream(nextGeneration());
+ clientStream.retry();
},
// Tear down and reopen the stream without touching existing environment
// data (e.g. when a flag encoded in the stream URL changes).
@@ -443,8 +268,7 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,
// Same as retryStream: pick up environments added since the last
// reconcile so their first snapshots aren't discarded.
reconcileEnvironments();
- abortStream();
- void connectStream(nextGeneration());
+ clientStream.restart();
}
};
}
diff --git a/frontend/src/lib/stores/environment.store.svelte.ts b/frontend/src/lib/stores/environment.store.svelte.ts
index c4c5e69fc..e9c99e244 100644
--- a/frontend/src/lib/stores/environment.store.svelte.ts
+++ b/frontend/src/lib/stores/environment.store.svelte.ts
@@ -148,6 +148,25 @@ function createEnvironmentManagementStore() {
}
}
},
+ // Applies a live environment list pushed by the backend stream.
+ //
+ // Edge liveness is process-local state on the manager, so a list fetched
+ // once goes stale the moment an agent reconnects — after a fleet update
+ // that leaves every agent stuck rendering as offline. This replaces the
+ // records wholesale rather than merging runtime fields, which keeps
+ // renames and enable toggles live too. Unlike initialize() it never
+ // re-runs initial selection: switching the user's environment out from
+ // under them because of a transient blip would be worse than the blip.
+ applyLiveEnvironments: (environments: Environment[]) => {
+ const available = _updateAvailable(environments);
+ if (!_selectedEnvironment) {
+ return;
+ }
+ const updated = available.find((env) => env.id === _selectedEnvironment!.id);
+ if (updated) {
+ _assignSelectedEnvironment(updated);
+ }
+ },
setEnvironment: async (environment: Environment) => {
if (!environment.enabled) return;
if (_selectedEnvironment?.id !== environment.id) {
diff --git a/frontend/src/lib/stores/json-line-stream.svelte.ts b/frontend/src/lib/stores/json-line-stream.svelte.ts
new file mode 100644
index 000000000..f6b8d5471
--- /dev/null
+++ b/frontend/src/lib/stores/json-line-stream.svelte.ts
@@ -0,0 +1,258 @@
+import { browser } from '$app/env';
+
+const MAX_RECONNECT_DELAY = 15_000;
+const MAX_RECONNECT_ATTEMPTS = 20;
+
+export type JSONLineEventBase = {
+ type: string;
+};
+
+export interface JSONLineStreamConfig<TEvent extends JSONLineEventBase> {
+ /** Used in console warnings, e.g. 'Dashboard' / 'Activity' / 'Environment'. */
+ label: string;
+ openStream(signal: AbortSignal): Promise<Response>;
+ /** Receives every event except 'heartbeat', which is connection state the transport owns. */
+ onEvent(event: TEvent): void;
+ /** Runs on each successful (re)connect, before any event is delivered. */
+ onConnected?(): void;
+}
+
+/**
+ * The NDJSON transport shared by every aggregate stream: connect, read lines,
+ * reconnect with backoff, and tear down explicitly on page hide.
+ *
+ * It is deliberately free of domain state so the environment, dashboard and
+ * activity streams cannot drift apart in how they handle a dropped connection.
+ */
+export function createJSONLineStream<TEvent extends JSONLineEventBase>(config: JSONLineStreamConfig<TEvent>) {
+ let started = false;
+ // A single aggregated stream carries every environment's events; per-env
+ // connections would multiply requests and exhaust the browser's
+ // 6-per-origin HTTP/1.1 limit.
+ let streamAbortController: AbortController | null = null;
+ let removePageLifecycleListeners: (() => void) | null = null;
+ let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
+ let reconnectAttempt = 0;
+ let streamGeneration = 0;
+ let _streamConnected = $state(false);
+ let _streamFailed = $state(false);
+
+ function nextGeneration(): number {
+ streamGeneration += 1;
+ return streamGeneration;
+ }
+
+ function isCurrentGeneration(generation: number): boolean {
+ return streamGeneration === generation;
+ }
+
+ function clearReconnectTimer() {
+ if (reconnectTimer) {
+ clearTimeout(reconnectTimer);
+ reconnectTimer = null;
+ }
+ }
+
+ function abortStream() {
+ clearReconnectTimer();
+ streamAbortController?.abort();
+ streamAbortController = null;
+ _streamConnected = false;
+ }
+
+ // Closing a tab leaves teardown to whenever the browser gets around to
+ // dropping the socket. Until it does, the server keeps writing heartbeats
+ // into a connection nobody is reading — so tear down explicitly instead.
+ function watchPageLifecycle() {
+ if (!browser || removePageLifecycleListeners) {
+ return;
+ }
+
+ const onPageHide = () => abortStream();
+ // A bfcache restore comes back with the stream already torn down.
+ const onPageShow = (event: PageTransitionEvent) => {
+ if (event.persisted && started && !streamAbortController) {
+ void connectStream(nextGeneration());
+ }
+ };
+
+ window.addEventListener('pagehide', onPageHide);
+ window.addEventListener('pageshow', onPageShow);
+ removePageLifecycleListeners = () => {
+ window.removeEventListener('pagehide', onPageHide);
+ window.removeEventListener('pageshow', onPageShow);
+ };
+ }
+
+ async function connectStream(generation: number) {
+ if (!browser || !isCurrentGeneration(generation)) {
+ return;
+ }
+
+ // Overlapping connects would otherwise strand the previous controller:
+ // abortStream() only ever reaches the newest one, so the older request
+ // would keep an open connection the server has no way to notice.
+ streamAbortController?.abort();
+
+ const controller = new AbortController();
+ streamAbortController = controller;
+ try {
+ const response = await config.openStream(controller.signal);
+ if (!isCurrentGeneration(generation) || !response.body) {
+ // The response body is live even though nobody will read it;
+ // dropping it on the floor leaves the server streaming into a
+ // connection that stays open until its TCP timers expire.
+ controller.abort();
+ if (streamAbortController === controller) {
+ streamAbortController = null;
+ }
+ return;
+ }
+
+ _streamConnected = true;
+ _streamFailed = false;
+ reconnectAttempt = 0;
+ config.onConnected?.();
+ await readJSONLines(response.body, generation);
+ } catch (error) {
+ if (!controller.signal.aborted && isCurrentGeneration(generation)) {
+ console.warn(`${config.label} stream disconnected:`, error);
+ }
+ } finally {
+ if (streamAbortController === controller) {
+ streamAbortController = null;
+ }
+ if (isCurrentGeneration(generation)) {
+ _streamConnected = false;
+ if (!controller.signal.aborted) {
+ scheduleReconnect(generation);
+ }
+ } else if (!controller.signal.aborted) {
+ // A superseded generation has no owner left to abort it.
+ controller.abort();
+ }
+ }
+ }
+
+ async function readJSONLines(stream: ReadableStream<Uint8Array>, generation: number) {
+ const reader = stream.getReader();
+ const decoder = new TextDecoder();
+ let buffer = '';
+
+ try {
+ while (isCurrentGeneration(generation)) {
+ const { done, value } = await reader.read();
+ if (done) {
+ break;
+ }
+
+ buffer += decoder.decode(value, { stream: true });
+ const lines = buffer.split('\n');
+ buffer = lines.pop() ?? '';
+ for (const line of lines) {
+ handleStreamLine(line);
+ }
+ }
+
+ buffer += decoder.decode();
+ if (buffer.trim()) {
+ handleStreamLine(buffer);
+ }
+ } finally {
+ // The loop also exits when the generation advances, with the stream
+ // still open. Cancelling is what actually closes the fetch and lets
+ // the server see the disconnect; releaseLock alone does not.
+ await reader.cancel().catch(() => {});
+ reader.releaseLock();
+ }
+ }
+
+ function handleStreamLine(line: string) {
+ const trimmed = line.trim();
+ if (!trimmed) {
+ return;
+ }
+
+ try {
+ const event = JSON.parse(trimmed) as TEvent;
+ if (event.type === 'heartbeat') {
+ _streamConnected = true;
+ return;
+ }
+ config.onEvent(event);
+ } catch (error) {
+ console.warn(`Failed to parse ${config.label.toLowerCase()} stream line:`, error);
+ }
+ }
+
+ function scheduleReconnect(generation: number) {
+ if (!browser || !started || !isCurrentGeneration(generation)) {
+ return;
+ }
+
+ if (reconnectAttempt >= MAX_RECONNECT_ATTEMPTS) {
+ _streamFailed = true;
+ return;
+ }
+
+ clearReconnectTimer();
+ const delay = Math.min(1000 * 2 ** reconnectAttempt, MAX_RECONNECT_DELAY);
+ reconnectAttempt += 1;
+ reconnectTimer = setTimeout(() => {
+ void connectStream(generation);
+ }, delay);
+ }
+
+ return {
+ get streamConnected(): boolean {
+ return _streamConnected;
+ },
+ set streamConnected(value: boolean) {
+ _streamConnected = value;
+ },
+ get streamFailed(): boolean {
+ return _streamFailed;
+ },
+ get generation(): number {
+ return streamGeneration;
+ },
+ get hasActiveStream(): boolean {
+ return streamAbortController !== null;
+ },
+ get isStarted(): boolean {
+ return started;
+ },
+ isCurrentGeneration,
+ nextGeneration,
+ /** Opens the stream. Callers that need a generation for their own guards should call nextGeneration() first and pass it. */
+ connect(generation: number) {
+ void connectStream(generation);
+ },
+ markStarted() {
+ started = true;
+ watchPageLifecycle();
+ },
+ stop(options?: { resetStreamFailed?: boolean }) {
+ const wasStarted = started;
+ started = false;
+ removePageLifecycleListeners?.();
+ removePageLifecycleListeners = null;
+ nextGeneration();
+ abortStream();
+ reconnectAttempt = 0;
+ if (options?.resetStreamFailed) {
+ _streamFailed = false;
+ }
+ return wasStarted;
+ },
+ /** Tears the connection down and opens a fresh one under a new generation. */
+ restart({ clearFailure = false }: { clearFailure?: boolean } = {}) {
+ if (clearFailure) {
+ _streamFailed = false;
+ reconnectAttempt = 0;
+ }
+ abortStream();
+ void connectStream(nextGeneration());
+ }
+ };
+}
diff --git a/frontend/src/routes/(app)/+layout.svelte b/frontend/src/routes/(app)/+layout.svelte
index c5868c7b2..8f5bcd35f 100644
--- a/frontend/src/routes/(app)/+layout.svelte
+++ b/frontend/src/routes/(app)/+layout.svelte
@@ -12,6 +12,7 @@
import { getEffectiveLandingPage, getEffectiveNavigationSettings } from '#lib/utils/navigation';
import { browser } from '$app/env';
import { environmentStore } from '#lib/stores/environment.store.svelte';
+ import { environmentStatusStore } from '#lib/stores/environment-status.store.svelte';
import { navigationItems, getManagementItems, filterByPermissions, type NavigationItem } from '#lib/config/navigation-config';
import { isEditableTarget, matchesShortcutEvent } from '#lib/utils/navigation';
import { cn } from '#lib/utils';
@@ -49,6 +50,17 @@
return flattenNavigationItems(items).filter((item) => item.shortcut?.length);
});
+ // Environment liveness is pushed for as long as the app shell is mounted, so
+ // every surface that reads environmentStore — the board, the environments
+ // table, the switcher — sees an agent reconnect without a page load.
+ $effect(() => {
+ if (!user) {
+ return;
+ }
+ void environmentStatusStore.start();
+ return () => environmentStatusStore.stop();
+ });
+
$effect(() => {
const redirectPath = getAuthRedirectPath(
page.url.pathname,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index a2d9b48d4..aa7f12a2a 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -28,7 +28,7 @@ overrides:
ts-deepmerge: '>=8.0.0'
joi: '>=17.13.4'
-packageExtensionsChecksum: sha256-Yj0m2AD9TZO/cWup5bCg2l8l7VHtLu5doy9Oybhgv+w=
+packageExtensionsChecksum: sha256-apx16oGB3Irw6uCc9mGeizkosuzdg1ZfZQyBC/MQBeU=
importers:
@@ -4683,6 +4683,7 @@ snapshots:
dependencies:
+ '@codemirror/autocomplete': 6.20.3
'@codemirror/search': 6.7.1
codemirror: 6.0.2
svelte: 5.56.7
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 2d6f0ead4..a91a06c5e 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -6,6 +6,7 @@ packages:
packageExtensions:
dependencies:
+ "@codemirror/autocomplete": ^6.20.3
"@codemirror/search": ^6.7.0
overrides:
diff --git a/tests/spec/activity-center.spec.ts b/tests/spec/activity-center.spec.ts
index 886e4e16e..68191816b 100644
--- a/tests/spec/activity-center.spec.ts
+++ b/tests/spec/activity-center.spec.ts
@@ -140,11 +140,21 @@ function aggregatedActivityStreamBody(
const timestamp = new Date().toISOString();
const events: string[] = [];
for (const [environmentId, activities] of Object.entries(activitiesByEnvironment)) {
- events.push(JSON.stringify({ type: 'snapshot', environmentId, activities, timestamp }));
+ events.push(
+ JSON.stringify({
+ channel: 'activities',
+ activity: { type: 'snapshot', environmentId, activities, timestamp },
+ timestamp
+ })
+ );
}
for (const environmentId of failedEnvironmentIds) {
events.push(
- JSON.stringify({ type: 'error', environmentId, error: 'environment unavailable', timestamp })
+ JSON.stringify({
+ channel: 'activities',
+ activity: { type: 'error', environmentId, error: 'environment unavailable', timestamp },
+ timestamp
+ })
);
}
return events.join('\n') + '\n';
@@ -157,11 +167,14 @@ async function mockActivityReads(
failedActivityStatus = 503,
readEnvironmentIds?: string[]
) {
- await page.context().route(/\/api\/activities\/stream(?:\?.*)?$/, async (route: Route) => {
+ await page.context().route(/\/api\/stream(?:\?.*)?$/, async (route: Route) => {
+ const channels = new URL(route.request().url()).searchParams.get('channels')?.split(',') ?? [];
await route.fulfill({
status: 200,
contentType: 'application/x-json-stream',
- body: aggregatedActivityStreamBody(activitiesByEnvironment, failedEnvironmentIds)
+ body: channels.includes('activities')
+ ? aggregatedActivityStreamBody(activitiesByEnvironment, failedEnvironmentIds)
+ : ''
});
});
@@ -293,7 +306,12 @@ function activityRow(activityCenter: Locator, text: string) {
function waitForActivityStream(page: Page) {
return page.waitForResponse((response) => {
const url = new URL(response.url());
- return response.request().method() === 'GET' && url.pathname === '/api/activities/stream';
+ const channels = url.searchParams.get('channels')?.split(',') ?? [];
+ return (
+ response.request().method() === 'GET' &&
+ url.pathname === '/api/stream' &&
+ channels.includes('activities')
+ );
});
}
@@ -350,7 +368,7 @@ test.describe('Activity Center', () => {
});
test('does not mount the activity center without an effective read scope', async ({ page }) => {
- const streamRequests: string[] = [];
+ const streamChannels: string[][] = [];
await preserveLocalEnvironmentSelection(page);
await mockCurrentUser(page, () =>
@@ -361,16 +379,73 @@ test.describe('Activity Center', () => {
);
await mockEnvironmentList(page, [localEnvironment]);
page.on('request', (request) => {
- if (new URL(request.url()).pathname === '/api/activities/stream') {
- streamRequests.push(request.url());
+ const url = new URL(request.url());
+ if (url.pathname === '/api/stream') {
+ streamChannels.push(url.searchParams.get('channels')?.split(',') ?? []);
}
});
await page.goto('/dashboard');
await expect(page.getByRole('heading', { name: 'Environment Board' })).toBeVisible();
await expect(page.getByRole('button', { name: 'Open activity center' })).toHaveCount(0);
- await page.waitForTimeout(250);
- expect(streamRequests).toHaveLength(0);
+ await expect
+ .poll(() => streamChannels.some((channels) => channels.includes('dashboard')))
+ .toBe(true);
+ expect(streamChannels.some((channels) => channels.includes('activities'))).toBe(false);
+ });
+
+ test('keeps an in-flight activity snapshot when the dashboard channel restarts the stream', async ({
+ page
+ }) => {
+ const snapshotActivity = activity('delayed-activity', '0', 'Local', 'delayed-network', 1);
+ let markSnapshotRequested!: () => void;
+ const snapshotRequested = new Promise<void>((resolve) => {
+ markSnapshotRequested = resolve;
+ });
+ let releaseSnapshot!: () => void;
+ const snapshotReleased = new Promise<void>((resolve) => {
+ releaseSnapshot = resolve;
+ });
+
+ await preserveLocalEnvironmentSelection(page);
+ await mockCurrentUser(page, () => user('snapshot-generation-user', { global: ['*'] }));
+ await mockEnvironmentList(page, [localEnvironment]);
+ await page.context().route(/\/api\/stream(?:\?.*)?$/, async (route) => {
+ await route.fulfill({ status: 503, contentType: 'application/json', body: '{}' });
+ });
+ await page.context().route(/\/api\/environments\/0\/activities(?:\?.*)?$/, async (route) => {
+ markSnapshotRequested();
+ await snapshotReleased;
+ await route.fulfill({
+ status: 200,
+ contentType: 'application/json',
+ body: JSON.stringify(paginated([snapshotActivity]))
+ });
+ });
+
+ await page.goto('/containers');
+ await snapshotRequested;
+ const dashboardStream = page.waitForRequest((request) => {
+ const url = new URL(request.url());
+ const channels = url.searchParams.get('channels')?.split(',') ?? [];
+ return (
+ url.pathname === '/api/stream' &&
+ channels.includes('activities') &&
+ channels.includes('dashboard')
+ );
+ });
+ try {
+ await Promise.all([
+ dashboardStream,
+ page.getByRole('link', { name: 'Dashboard', exact: true }).click()
+ ]);
+ releaseSnapshot();
+
+ const activityCenter = await openActivityCenter(page);
+ await expect(activityRow(activityCenter, 'delayed-network')).toBeVisible();
+ } finally {
+ releaseSnapshot();
+ }
});
test('clears activity state when the authenticated user changes', async ({ page }) => {
@@ -381,17 +456,23 @@ test.describe('Activity Center', () => {
await preserveLocalEnvironmentSelection(page);
await mockCurrentUser(page, () => user(activeUserId, permissions));
await mockEnvironmentList(page, [localEnvironment]);
- await page.context().route(/\/api\/activities\/stream(?:\?.*)?$/, async (route) => {
+ await page.context().route(/\/api\/stream(?:\?.*)?$/, async (route) => {
+ const channels =
+ new URL(route.request().url()).searchParams.get('channels')?.split(',') ?? [];
const requestedBy = activeUserId;
- streamedUsers.push(requestedBy);
+ if (channels.includes('activities')) {
+ streamedUsers.push(requestedBy);
+ }
const resourceName = requestedBy === 'user-a' ? 'user-a-private-activity' : 'user-b-activity';
await route.fulfill({
status: 200,
contentType: 'application/x-json-stream',
- body: aggregatedActivityStreamBody(
- { '0': [activity(`${requestedBy}-activity`, '0', 'Local', resourceName, 1)] },
- new Set()
- )
+ body: channels.includes('activities')
+ ? aggregatedActivityStreamBody(
+ { '0': [activity(`${requestedBy}-activity`, '0', 'Local', resourceName, 1)] },
+ new Set()
+ )
+ : ''
});
});
await page.context().route(/\/api\/environments\/0\/activities(?:\?.*)?$/, async (route) => {
diff --git a/tests/spec/dashboard-system-stats.spec.ts b/tests/spec/dashboard-system-stats.spec.ts
index 1444306fc..885e8f645 100644
--- a/tests/spec/dashboard-system-stats.spec.ts
+++ b/tests/spec/dashboard-system-stats.spec.ts
@@ -136,7 +136,11 @@ function collectDashboardRequestPaths(page: Page): string[] {
page.on('request', (request) => {
const pathname = new URL(request.url()).pathname;
- if (pathname.startsWith('/api/environments/') || pathname.startsWith('/api/dashboard/')) {
+ if (
+ pathname === '/api/stream' ||
+ pathname.startsWith('/api/environments/') ||
+ pathname.startsWith('/api/dashboard/')
+ ) {
requestPaths.push(pathname);
}
});
@@ -173,7 +177,7 @@ test.describe('Dashboard system stats websocket', () => {
await page.waitForLoadState('load');
await expect(page.getByRole('heading', { name: 'Environment Board' })).toBeVisible();
- await expect.poll(() => requestPaths).toContain('/api/dashboard/stream');
+ await expect.poll(() => requestPaths).toContain('/api/stream');
expect(
countMatchingRequests(requestPaths, /\/api\/environments\/[^/]+\/system\/docker\/info$/)
diff --git a/tests/spec/environment-switch-isolation.spec.ts b/tests/spec/environment-switch-isolation.spec.ts
index a22a61a23..f6a46eb9a 100644
--- a/tests/spec/environment-switch-isolation.spec.ts
+++ b/tests/spec/environment-switch-isolation.spec.ts
@@ -90,6 +90,25 @@ async function mockEnvironmentCatalog(page: Page) {
body: JSON.stringify(paginated([localEnvironment, remoteEnvironment]))
});
});
+ await page.context().route(/\/api\/stream(?:\?.*)?$/, async (route) => {
+ const channels = new URL(route.request().url()).searchParams.get('channels')?.split(',') ?? [];
+ const timestamp = new Date().toISOString();
+ await route.fulfill({
+ status: 200,
+ contentType: 'application/x-json-stream',
+ body: channels.includes('environments')
+ ? `${JSON.stringify({
+ channel: 'environments',
+ environment: {
+ type: 'snapshot',
+ environments: [localEnvironment, remoteEnvironment],
+ timestamp
+ },
+ timestamp
+ })}\n`
+ : ''
+ });
+ });
await page
.context()
.route(/\/api\/environments\/(?:0|remote-switch-test)\/settings$/, async (route) => {
diff --git a/tests/spec/system-upgrade.spec.ts b/tests/spec/system-upgrade.spec.ts
index d10632e3e..165119701 100644
--- a/tests/spec/system-upgrade.spec.ts
+++ b/tests/spec/system-upgrade.spec.ts
@@ -156,16 +156,20 @@ test.describe('Manager self-update recovery', () => {
const snapshot = dashboardSnapshot();
let upgradeTriggered = false;
- await page.route(/\/api\/dashboard\/stream\?/, async (route) => {
+ await page.route(/\/api\/stream(?:\?.*)?$/, async (route) => {
+ const channels =
+ new URL(route.request().url()).searchParams.get('channels')?.split(',') ?? [];
+ const timestamp = new Date().toISOString();
await route.fulfill({
status: 200,
contentType: 'application/x-json-stream',
- body: `${JSON.stringify({
- type: 'snapshot',
- environmentId: '0',
- snapshot,
- timestamp: new Date().toISOString()
- })}\n`
+ body: channels.includes('dashboard')
+ ? `${JSON.stringify({
+ channel: 'dashboard',
+ dashboard: { type: 'snapshot', environmentId: '0', snapshot, timestamp },
+ timestamp
+ })}\n`
+ : ''
});
});
await page.route(/\/api\/environments\/0\/dashboard(?:\?.*)?$/, async (route) => {
@@ -201,8 +205,15 @@ test.describe('Manager self-update recovery', () => {
});
});
+ const dashboardUpgradeCheck = page.waitForResponse((response) => {
+ return (
+ new URL(response.url()).pathname === '/api/environments/0/system/upgrade/check' &&
+ response.status() === 200
+ );
+ });
await page.goto('/dashboard');
- const versionBadge = page.getByText('v1.0.0', { exact: true }).first();
+ await dashboardUpgradeCheck;
+ const versionBadge = page.locator('main').getByRole('button', { name: 'v1.0.0', exact: true });
await expect(versionBadge).toBeVisible();
await versionBadge.hover();
const upgradeButton = page.getByRole('button', { name: 'Update to 2.0.0', exact: true });
@@ -219,17 +230,32 @@ test.describe('Manager self-update recovery', () => {
await registerReloadCounter(page);
await registerTokenSeeding(page);
- let releaseActivity: () => void = () => {};
+ let releaseActivity!: () => void;
const activityRelease = new Promise<void>((resolve) => {
releaseActivity = resolve;
});
- await page.route(/\/api\/activities\/stream\?/, async (route) => {
+ await page.route(/\/api\/stream(?:\?.*)?$/, async (route) => {
+ const channels =
+ new URL(route.request().url()).searchParams.get('channels')?.split(',') ?? [];
+ if (!channels.includes('activities')) {
+ await route.fulfill({
+ status: 200,
+ contentType: 'application/x-json-stream',
+ body: ''
+ });
+ return;
+ }
await activityRelease;
if (route.request().headers()['cookie']?.includes(REFRESH_COOKIE)) {
+ const timestamp = new Date().toISOString();
await route.fulfill({
status: 200,
contentType: 'application/x-json-stream',
- body: `${JSON.stringify({ type: 'snapshot', activities: [] })}\n`
+ body: `${JSON.stringify({
+ channel: 'activities',
+ activity: { type: 'snapshot', environmentId: '0', activities: [], timestamp },
+ timestamp
+ })}\n`
});
return;
}
diff --git a/types/environment/environment.go b/types/environment/environment.go
index 82601a19b..363090cf8 100644
--- a/types/environment/environment.go
+++ b/types/environment/environment.go
@@ -212,6 +212,28 @@ type Environment struct {
ApiKey *string `json:"apiKey,omitempty"`
}
+// StreamEvent is one line of the environment stream. A "snapshot" event carries
+// the caller's full visible environment list with the manager's runtime overlay
+// already applied, so clients replace their cached list wholesale rather than
+// merging field by field — which also keeps renames and enable toggles live,
+// not just connectivity.
+type StreamEvent struct {
+ // Type is the event kind: "snapshot" or "heartbeat".
+ //
+ // Required: true
+ Type string `json:"type"`
+
+ // Environments is the caller's visible environment list for a "snapshot".
+ //
+ // Required: false
+ Environments []Environment `json:"environments,omitempty"`
+
+ // Timestamp is when the event was produced.
+ //
+ // Required: true
+ Timestamp time.Time `json:"timestamp"`
+}
+
// AgentPairRequest is the request body for pairing with an agent.
type AgentPairRequest struct {
// Rotate indicates if the token should be rotated.
diff --git a/types/stream/stream.go b/types/stream/stream.go
new file mode 100644
index 000000000..86c271931
--- /dev/null
+++ b/types/stream/stream.go
@@ -0,0 +1,61 @@
+// Package stream defines the multiplexed client stream: one connection that
+// carries every live channel the caller asked for.
+//
+// Browsers cap concurrent HTTP/1.1 requests per origin at six, and every
+// long-lived stream permanently consumes one of them. Multiplexing keeps that
+// budget for ordinary requests, and gives the client a single reconnect and
+// heartbeat to reason about instead of one per feature.
+package stream
+
+import (
+ "time"
+
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ dashboardtypes "github.com/getarcaneapp/arcane/types/v2/dashboard"
+ environmenttypes "github.com/getarcaneapp/arcane/types/v2/environment"
+)
+
+// Channel names a subscribable feed. Callers request channels explicitly
+// because they differ enormously in cost: environments is one local query,
+// while dashboard proxies a request to every remote agent on every tick.
+const (
+ ChannelEnvironments = "environments"
+ ChannelDashboard = "dashboard"
+ ChannelActivities = "activities"
+)
+
+// Event is one line of the multiplexed stream. Exactly one payload is set, and
+// each payload keeps its own channel's event type rather than being flattened
+// here, so a channel can add a field without this envelope changing.
+type Event struct {
+ // Channel identifies which feed produced this event. Empty on a heartbeat.
+ //
+ // Required: false
+ Channel string `json:"channel,omitempty"`
+
+ // Type is set to "heartbeat" for connection keepalives; otherwise the
+ // per-channel payload carries its own type.
+ //
+ // Required: false
+ Type string `json:"type,omitempty"`
+
+ // Dashboard carries a ChannelDashboard payload.
+ //
+ // Required: false
+ Dashboard *dashboardtypes.StreamEvent `json:"dashboard,omitempty"`
+
+ // Activity carries a ChannelActivities payload.
+ //
+ // Required: false
+ Activity *activitytypes.StreamEvent `json:"activity,omitempty"`
+
+ // Environment carries a ChannelEnvironments payload.
+ //
+ // Required: false
+ Environment *environmenttypes.StreamEvent `json:"environment,omitempty"`
+
+ // Timestamp is when the envelope was produced.
+ //
+ // Required: true
+ Timestamp time.Time `json:"timestamp"`
+}