Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: stream environment liveness over a multiplexed client stream (#3406)
Kyle Mendell1d054e1ac1f5
diff --git a/backend/api/api.go b/backend/api/api.goindex d86beec46..402f3857c 100644--- a/backend/api/api.go+++ b/backend/api/api.go@@ -397,4 +397,5 @@ func registerHandlersInternal(api huma.API, deps HandlerDeps, handlerAppCtx handhandlers.RegisterWebhooks(api, deps.Webhook)handlers.RegisterVulnerability(api, deps.Vulnerability, handlerAppCtx)handlers.RegisterDashboard(api, deps.Dashboard, deps.Environment)+ handlers.RegisterStream(api, deps.Dashboard, deps.Activity, deps.Environment)}diff --git a/backend/api/handlers/activities.go b/backend/api/handlers/activities.goindex 6624048cd..8cb3cbceb 100644--- a/backend/api/handlers/activities.go+++ b/backend/api/handlers/activities.go@@ -4,7 +4,6 @@ import ("context""fmt""hash/fnv"- "io""net/http""net/url""strconv"@@ -19,7 +18,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx""github.com/getarcaneapp/arcane/types/v2/activity""github.com/getarcaneapp/arcane/types/v2/base""go.getarcane.app/streams/agg"@@ -74,7 +72,6 @@ const (activityStreamRemotePollInterval = 5 * time.SecondactivityStreamEnvReconcileInterval = 30 * time.SecondactivityStreamRemotePollTimeout = 15 * time.Second- activityStreamEventBuffer = 256)type CancelActivityInput struct {@@ -87,6 +84,7 @@ type CancelActivityOutput struct {Body base.ApiResponse[activity.Activity]}+//nolint:dupl // Huma operation registrations are intentionally explicit.func RegisterActivities(api huma.API, activityService *services.ActivityService, environmentService *services.EnvironmentService) {h := &ActivityHandler{activityService: activityService,@@ -113,17 +111,6 @@ func RegisterActivities(api huma.API, activityService *services.ActivityService,Security: defaultOperationSecurityInternal(),}, authz.PermActivitiesRead, h.GetActivity)- huma.Register(api, huma.Operation{- OperationID: "stream-all-activities",- Method: http.MethodGet,- Path: "/activities/stream",- Summary: "Stream background activities across all environments",- Description: "Stream background activity updates for the local environment and all enabled remote environments as JSON lines",- Tags: []string{"Activities"},- Security: defaultOperationSecurityInternal(),- Middlewares: humamw.RequireAnyEnvironmentPermission(api, authz.PermActivitiesRead),- }, h.StreamAllActivities)-humamw.RegisterWithPermission(api, huma.Operation{OperationID: "cancel-activity",Method: http.MethodPost,@@ -277,47 +264,6 @@ func (h *ActivityHandler) cancelRequestedByInternal(ctx context.Context, forwardreturn strings.TrimSpace(forwarded)}-func (h *ActivityHandler) StreamAllActivities(ctx context.Context, input *StreamAllActivitiesInput) (*huma.StreamResponse, error) {- return &huma.StreamResponse{- Body: func(humaCtx huma.Context) { //nolint:contextcheck // streaming work must use humaCtx.Context()- httpx.SetJSONStreamHeaders(humaCtx)-- writer := humaCtx.BodyWriter()- flush := func() {- if f, ok := writer.(http.Flusher); ok {- f.Flush()- }- }-- ps, _ := humamw.PermissionsFromContext(humaCtx.Context())- h.streamAllActivitiesInternal(humaCtx.Context(), ps, input.Limit, writer, flush)- },- }, nil-}--// streamAllActivitiesInternal multiplexes activity events for the local-// environment and every enabled remote environment over a single response so-// the browser needs one connection regardless of environment count.-func (h *ActivityHandler) streamAllActivitiesInternal(ctx context.Context, ps *authz.PermissionSet, limit int, writer io.Writer, flush func()) {- // RunAuthorizedAggregateStream logs how the stream ended; there is nothing- // left to report to a client whose response headers were sent long ago.- _ = httpx.RunAuthorizedAggregateStream(ctx, ps, authz.PermActivitiesRead, agg.Config[activity.StreamEvent]{- Writer: writer,- Flush: flush,- Buffer: activityStreamEventBuffer,- HeartbeatInterval: activityStreamHeartbeatInterval,- MakeHeartbeat: func() activity.StreamEvent {- return activity.StreamEvent{Type: "heartbeat", Timestamp: time.Now()}- },- },- func(ctx context.Context, events chan<- activity.StreamEvent) {- h.runLocalActivityStreamProducerInternal(ctx, limit, events)- },- func(ctx context.Context, events chan<- activity.StreamEvent) {- h.runRemoteActivityStreamPollersInternal(ctx, ps, limit, events)- })-}-func (h *ActivityHandler) runLocalActivityStreamProducerInternal(ctx context.Context, limit int, events chan<- activity.StreamEvent) {sendSnapshot := func() bool {activities, _, err := h.activityService.ListActivitiesPaginated(ctx, "0", pagination.QueryParams{diff --git a/backend/api/handlers/activities_test.go b/backend/api/handlers/activities_test.goindex 68b4a6df4..93665c272 100644--- a/backend/api/handlers/activities_test.go+++ b/backend/api/handlers/activities_test.go@@ -21,6 +21,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/types/v2/activity"+ streamtypes "github.com/getarcaneapp/arcane/types/v2/stream")func setupActivityHandlerTestDBInternal(t *testing.T) *database.DB {@@ -146,10 +147,9 @@ func createStreamTestRemoteEnvironmentInternal(t *testing.T, db *database.DB, en}).Error)}-// runStreamAllInternal drives streamAllActivitiesInternal through a pipe and-// returns each decoded event to onEvent until it reports done or the stream-// ends; remaining output is drained so a blocked writer can always finish.-func runStreamAllInternal(t *testing.T, ctx context.Context, cancel context.CancelFunc, handler *ActivityHandler, ps *authz.PermissionSet, onEvent func(activity.StreamEvent) bool) {+// runActivityClientStreamInternal drives the activities channel through a pipe+// and returns each decoded activity event to onEvent until it reports done.+func runActivityClientStreamInternal(t *testing.T, ctx context.Context, cancel context.CancelFunc, handler *ActivityHandler, ps *authz.PermissionSet, onEvent func(activity.StreamEvent) bool) {t.Helper()pr, pw := io.Pipe()@@ -157,14 +157,18 @@ func runStreamAllInternal(t *testing.T, ctx context.Context, cancel context.Cancgo func() {defer close(done)defer func() { _ = pw.Close() }()- handler.streamAllActivitiesInternal(ctx, ps, 50, pw, func() {})+ streamHandler := &StreamHandler{activity: handler}+ streamHandler.streamClientInternal(ctx, ps, &StreamClientInput{+ Channels: streamtypes.ChannelActivities,+ Limit: 50,+ }, pw, func() {})}()scanner := bufio.NewScanner(pr)for scanner.Scan() {- var event activity.StreamEvent+ var event streamtypes.Eventrequire.NoError(t, json.Unmarshal(scanner.Bytes(), &event))- if onEvent(event) {+ if event.Activity != nil && onEvent(*event.Activity) {cancel()break}@@ -207,7 +211,7 @@ func TestActivityHandlerStreamAllEmitsEnvironmentScopedEventsInternal(t *testing}var localSnapshot, remoteSnapshot bool- runStreamAllInternal(t, ctx, cancel, handler, authz.SudoPermissionSet(), func(event activity.StreamEvent) bool {+ runActivityClientStreamInternal(t, ctx, cancel, handler, authz.SudoPermissionSet(), func(event activity.StreamEvent) bool {if event.Type == "snapshot" && event.EnvironmentID == "0" && len(event.Activities) == 1 {require.Equal(t, local.ID, event.Activities[0].ID)require.Equal(t, "0", event.Activities[0].SourceEnvironmentID)@@ -262,7 +266,7 @@ func TestActivityHandlerStreamAllReusesRemoteEnvironmentAfterInitialPollInternal}remoteSnapshotCount := 0- runStreamAllInternal(t, ctx, cancel, handler, authz.SudoPermissionSet(), func(event activity.StreamEvent) bool {+ runActivityClientStreamInternal(t, ctx, cancel, handler, authz.SudoPermissionSet(), func(event activity.StreamEvent) bool {if event.Type != "snapshot" || event.EnvironmentID != "remote-1" {return false}@@ -315,7 +319,7 @@ func TestActivityHandlerStreamAllRemoteFailureEmitsErrorAndKeepsStreamingInterna}var localSnapshot, remoteError bool- runStreamAllInternal(t, ctx, cancel, handler, authz.SudoPermissionSet(), func(event activity.StreamEvent) bool {+ runActivityClientStreamInternal(t, ctx, cancel, handler, authz.SudoPermissionSet(), func(event activity.StreamEvent) bool {if event.Type == "snapshot" && event.EnvironmentID == "0" {localSnapshot = true}@@ -366,7 +370,7 @@ func TestActivityHandlerStreamAllFiltersUnauthorizedEnvironmentsInternal(t *testps.AddEnv("remote-allowed", authz.PermActivitiesRead)seenEnvironments := make(map[string]struct{})- runStreamAllInternal(t, ctx, cancel, handler, ps, func(event activity.StreamEvent) bool {+ runActivityClientStreamInternal(t, ctx, cancel, handler, ps, func(event activity.StreamEvent) bool {if event.EnvironmentID != "" {seenEnvironments[event.EnvironmentID] = struct{}{}}diff --git a/backend/api/handlers/dashboard.go b/backend/api/handlers/dashboard.goindex 6825d3e59..70c971892 100644--- a/backend/api/handlers/dashboard.go+++ b/backend/api/handlers/dashboard.go@@ -3,7 +3,6 @@ package handlersimport ("context"stderrors "errors"- "io""net/http""time"@@ -16,7 +15,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx""github.com/getarcaneapp/arcane/types/v2/base"containertypes "github.com/getarcaneapp/arcane/types/v2/container"dashboardtypes "github.com/getarcaneapp/arcane/types/v2/dashboard"@@ -40,10 +38,6 @@ type GetDashboardOutput struct {Body base.ApiResponse[dashboardtypes.Snapshot]}-type StreamAllDashboardsInput struct {- DebugAllGood bool `query:"debugAllGood" default:"false" doc:"Debug mode: force an empty action item list"`-}-const (dashboardStreamHeartbeatInterval = 15 * time.SeconddashboardStreamLocalPollInterval = 15 * time.Second@@ -68,17 +62,6 @@ func RegisterDashboard(api huma.API, dashboardService *services.DashboardServiceTags: []string{"Dashboard"},Security: defaultOperationSecurityInternal(),}, authz.PermDashboardRead, h.GetDashboard)-- huma.Register(api, huma.Operation{- OperationID: "stream-all-dashboards",- Method: http.MethodGet,- Path: "/dashboard/stream",- Summary: "Stream dashboard snapshots across all environments",- Description: "Stream dashboard snapshot updates for the local environment and all enabled remote environments as JSON lines",- Tags: []string{"Dashboard"},- Security: defaultOperationSecurityInternal(),- Middlewares: humamw.RequireAnyEnvironmentPermission(api, authz.PermDashboardRead),- }, h.StreamAllDashboards)}func (h *DashboardHandler) GetDashboard(ctx context.Context, input *GetDashboardInput) (*GetDashboardOutput, error) {@@ -104,47 +87,6 @@ func (h *DashboardHandler) GetDashboard(ctx context.Context, input *GetDashboard}, nil}-func (h *DashboardHandler) StreamAllDashboards(ctx context.Context, input *StreamAllDashboardsInput) (*huma.StreamResponse, error) {- return &huma.StreamResponse{- Body: func(humaCtx huma.Context) { //nolint:contextcheck // streaming work must use humaCtx.Context()- httpx.SetJSONStreamHeaders(humaCtx)-- writer := humaCtx.BodyWriter()- flush := func() {- if f, ok := writer.(http.Flusher); ok {- f.Flush()- }- }-- ps, _ := humamw.PermissionsFromContext(humaCtx.Context())- h.streamAllDashboardsInternal(humaCtx.Context(), ps, input.DebugAllGood, writer, flush)- },- }, nil-}--// streamAllDashboardsInternal multiplexes dashboard snapshots for the local-// environment and every enabled remote environment over a single response so-// the browser needs one connection regardless of environment count.-func (h *DashboardHandler) streamAllDashboardsInternal(ctx context.Context, ps *authz.PermissionSet, debugAllGood bool, writer io.Writer, flush func()) {- // RunAuthorizedAggregateStream logs how the stream ended; there is nothing- // left to report to a client whose response headers were sent long ago.- _ = httpx.RunAuthorizedAggregateStream(ctx, ps, authz.PermDashboardRead, agg.Config[dashboardtypes.StreamEvent]{- Writer: writer,- Flush: flush,- Buffer: dashboardStreamEventBuffer,- HeartbeatInterval: dashboardStreamHeartbeatInterval,- MakeHeartbeat: func() dashboardtypes.StreamEvent {- return dashboardtypes.StreamEvent{Type: "heartbeat", Timestamp: time.Now()}- },- },- func(ctx context.Context, events chan<- dashboardtypes.StreamEvent) {- h.runLocalDashboardStreamProducerInternal(ctx, debugAllGood, events)- },- func(ctx context.Context, events chan<- dashboardtypes.StreamEvent) {- h.runRemoteDashboardStreamPollersInternal(ctx, ps, debugAllGood, events)- })-}-// trimDashboardStreamSnapshotInternal drops the first-page container/image// tables: the all-environments dashboard only reads the aggregate counters,// and re-sending table rows for every environment on every poll would bloatdiff --git a/backend/api/handlers/dashboard_test.go b/backend/api/handlers/dashboard_test.goindex 2822b7f78..9aca9e1b8 100644--- a/backend/api/handlers/dashboard_test.go+++ b/backend/api/handlers/dashboard_test.go@@ -18,6 +18,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/backend/v2/pkg/authz"dashboardtypes "github.com/getarcaneapp/arcane/types/v2/dashboard"+ streamtypes "github.com/getarcaneapp/arcane/types/v2/stream""github.com/libtnb/sqlite"dockercontainer "github.com/moby/moby/api/types/container"dockerimage "github.com/moby/moby/api/types/image"@@ -137,25 +138,33 @@ func TestDashboardHandlerGetDashboardReturnsSnapshot(t *testing.T) {}, snapshot.ActionItems.Items)}-// runDashboardStreamAllInternal drives streamAllDashboardsInternal through a-// pipe and returns each decoded event to onEvent until it reports done or the-// stream ends; remaining output is drained so a blocked writer can finish.+// runDashboardStreamAllInternal drives the dashboard channel of the multiplexed+// client stream through a pipe and returns each decoded event to onEvent until+// it reports done or the stream ends; remaining output is drained so a blocked+// writer can finish.func runDashboardStreamAllInternal(t *testing.T, ctx context.Context, cancel context.CancelFunc, handler *DashboardHandler, ps *authz.PermissionSet, onEvent func(dashboardtypes.StreamEvent) bool) {t.Helper()+ streamHandler := &StreamHandler{dashboard: handler}+ input := &StreamClientInput{Channels: streamtypes.ChannelDashboard}+pr, pw := io.Pipe()done := make(chan struct{})go func() {defer close(done)defer func() { _ = pw.Close() }()- handler.streamAllDashboardsInternal(ctx, ps, false, pw, func() {})+ streamHandler.streamClientInternal(ctx, ps, input, pw, func() {})}()scanner := bufio.NewScanner(pr)for scanner.Scan() {- var event dashboardtypes.StreamEvent- require.NoError(t, json.Unmarshal(scanner.Bytes(), &event))- if onEvent(event) {+ var envelope streamtypes.Event+ require.NoError(t, json.Unmarshal(scanner.Bytes(), &envelope))+ // Heartbeats are connection keepalives, not dashboard events.+ if envelope.Dashboard == nil {+ continue+ }+ if onEvent(*envelope.Dashboard) {cancel()break}diff --git a/backend/api/handlers/environments.go b/backend/api/handlers/environments.goindex 89b3e2b7b..e7b2f80c0 100644--- a/backend/api/handlers/environments.go+++ b/backend/api/handlers/environments.go@@ -29,10 +29,18 @@ import ("github.com/getarcaneapp/arcane/types/v2/base""github.com/getarcaneapp/arcane/types/v2/environment""github.com/getarcaneapp/arcane/types/v2/version"+ "go.getarcane.app/streams/agg")const localDockerEnvironmentID = "0"+const (+ // Only covers poll-mode TTL expiry; tunnel and health-check changes arrive+ // on the service's runtime-change signal instead.+ environmentStreamPollInterval = 5 * time.Second+ environmentStreamRefreshFloor = 30 * time.Second+)+// EnvironmentHandler handles environment management endpoints.type EnvironmentHandler struct {environmentService *services.EnvironmentService@@ -430,6 +438,93 @@ func accessibleEnvironmentIDsInternal(ps *authz.PermissionSet) []string {return ids}+// visibleEnvironmentsForInternal returns the environments the caller may see,+// with the manager's runtime overlay already applied. It reuses the same access+// rules as ListEnvironments so the stream and the REST list can never disagree+// about which environments a caller has.+func (h *EnvironmentHandler) visibleEnvironmentsForInternal(ctx context.Context, ps *authz.PermissionSet) ([]environment.Environment, error) {+ envs, err := h.environmentService.ListVisibleEnvironments(ctx)+ if err != nil {+ return nil, err+ }+ if environmentListerSeesAllInternal(ps) {+ return envs, nil+ }++ allowed := make(map[string]struct{}, len(ps.PerEnv))+ for _, envID := range accessibleEnvironmentIDsInternal(ps) {+ allowed[envID] = struct{}{}+ }++ filtered := envs[:0]+ for _, env := range envs {+ if _, ok := allowed[env.ID]; ok {+ filtered = append(filtered, env)+ }+ }+ return filtered, nil+}++func (h *EnvironmentHandler) runEnvironmentStreamProducerInternal(ctx context.Context, ps *authz.PermissionSet, events chan<- environment.StreamEvent) {+ changes, unsubscribe := h.environmentService.SubscribeRuntimeChanges()+ defer unsubscribe()++ var lastFingerprint []byte+ var lastSentAt time.Time++ send := func() bool {+ envs, err := h.visibleEnvironmentsForInternal(ctx, ps)+ if err != nil {+ // A failed read must not end the stream; the next tick retries.+ if ctx.Err() == nil {+ slog.WarnContext(ctx, "environment stream failed to list environments", "error", err)+ }+ return ctx.Err() == nil+ }++ fingerprint, err := json.Marshal(envs)+ if err != nil {+ slog.WarnContext(ctx, "environment stream failed to encode environments", "error", err)+ return ctx.Err() == nil+ }+ // Re-send unchanged state on a floor so relative timestamps in the UI+ // ("last seen 2 minutes ago") keep advancing.+ if bytes.Equal(fingerprint, lastFingerprint) && time.Since(lastSentAt) < environmentStreamRefreshFloor {+ return true+ }+ lastFingerprint = fingerprint+ lastSentAt = time.Now()++ return agg.Send(ctx, events, environment.StreamEvent{+ Type: "snapshot",+ Environments: envs,+ Timestamp: time.Now(),+ })+ }++ if !send() {+ return+ }++ ticker := time.NewTicker(environmentStreamPollInterval)+ defer ticker.Stop()++ for {+ select {+ case <-ctx.Done():+ return+ case <-changes:+ if !send() {+ return+ }+ case <-ticker.C:+ if !send() {+ return+ }+ }+ }+}+// CreateEnvironment creates a new environment.func (h *EnvironmentHandler) CreateEnvironment(ctx context.Context, input *CreateEnvironmentInput) (*CreateEnvironmentOutput, error) {user, err := requireUserInternal(ctx)diff --git a/backend/api/handlers/job_schedules.go b/backend/api/handlers/job_schedules.goindex 97ce19b53..f1e60d81f 100644--- a/backend/api/handlers/job_schedules.go+++ b/backend/api/handlers/job_schedules.go@@ -46,6 +46,7 @@ type RunJobOutput struct {Body jobschedule.JobRunResponse}+//nolint:dupl // Huma operation registrations are intentionally explicit.func RegisterJobSchedules(api huma.API, jobSvc *services.JobService, envSvc *services.EnvironmentService) {h := &JobSchedulesHandler{jobService: jobSvc,diff --git a/backend/api/handlers/stream.go b/backend/api/handlers/stream.gonew file mode 100644index 000000000..ee1f6f8bf--- /dev/null+++ b/backend/api/handlers/stream.go@@ -0,0 +1,220 @@+package handlers++import (+ "context"+ "io"+ "log/slog"+ "net/http"+ "strings"+ "time"++ "github.com/danielgtaylor/huma/v2"+ humamw "github.com/getarcaneapp/arcane/backend/v2/api/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/internal/services"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ httputils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ dashboardtypes "github.com/getarcaneapp/arcane/types/v2/dashboard"+ environmenttypes "github.com/getarcaneapp/arcane/types/v2/environment"+ streamtypes "github.com/getarcaneapp/arcane/types/v2/stream"+ "go.getarcane.app/streams/agg"+)++const (+ clientStreamHeartbeatInterval = 15 * time.Second+ clientStreamDeadPeerTimeout = 45 * time.Second+ clientStreamEventBuffer = 64+ // Per-channel handoff buffer between a channel's producer and the envelope+ // forwarder. Small on purpose: the shared writer is the real backpressure+ // point, and a deep per-channel buffer would only delay noticing a stalled+ // client.+ clientStreamChannelBuffer = 16+)++// StreamHandler multiplexes every live feed onto one connection.+type StreamHandler struct {+ dashboard *DashboardHandler+ activity *ActivityHandler+ environment *EnvironmentHandler+}++type StreamClientInput struct {+ Channels string `query:"channels" doc:"Comma-separated channels to subscribe to: environments, dashboard, activities"`+ DebugAllGood bool `query:"debugAllGood" default:"false" doc:"Debug mode for the dashboard channel: force an empty action item list"`+ Limit int `query:"limit" default:"0" doc:"Maximum activities to include in each activities-channel snapshot"`+}++// RegisterStream registers the multiplexed client stream. It builds its own+// per-domain handlers from the same services the individual handlers use, so+// each channel's producer stays the single implementation of that feed.+func RegisterStream(+ api huma.API,+ dashboardService *services.DashboardService,+ activityService *services.ActivityService,+ environmentService *services.EnvironmentService,+) {+ h := &StreamHandler{+ dashboard: &DashboardHandler{+ dashboardService: dashboardService,+ environmentService: environmentService,+ },+ activity: &ActivityHandler{+ activityService: activityService,+ environmentService: environmentService,+ },+ // Only the environment service is needed: the channel producer lists+ // environments and derives liveness, nothing more.+ environment: &EnvironmentHandler{environmentService: environmentService},+ }++ huma.Register(api, huma.Operation{+ OperationID: "streamClient",+ Method: http.MethodGet,+ Path: "/stream",+ Summary: "Multiplexed client stream",+ Description: "Streams the requested channels (environments, dashboard, activities) over a single JSON-lines connection",+ Tags: []string{"Stream"},+ Security: defaultOperationSecurityInternal(),+ // Ungated: each channel applies its own permission check below, and a+ // caller with access to none simply gets a heartbeat-only stream.+ }, h.StreamClient)+}++func (h *StreamHandler) StreamClient(ctx context.Context, input *StreamClientInput) (*huma.StreamResponse, error) {+ return &huma.StreamResponse{+ Body: func(humaCtx huma.Context) { //nolint:contextcheck // streaming work must use humaCtx.Context()+ httputils.SetJSONStreamHeaders(humaCtx)++ writer := humaCtx.BodyWriter()+ flush := func() {+ if f, ok := writer.(http.Flusher); ok {+ f.Flush()+ }+ }++ ps, _ := humamw.PermissionsFromContext(humaCtx.Context())+ h.streamClientInternal(humaCtx.Context(), ps, input, writer, flush)+ },+ }, nil+}++func (h *StreamHandler) streamClientInternal(ctx context.Context, ps *authz.PermissionSet, input *StreamClientInput, writer io.Writer, flush func()) {+ releaseDeadPeerTimeout, timeoutErr := httputils.AcquireDeadPeerTimeout(ctx, clientStreamDeadPeerTimeout)+ if timeoutErr != nil {+ slog.DebugContext(ctx, "could not bound client stream dead-peer timeout", "error", timeoutErr)+ }+ defer releaseDeadPeerTimeout()++ // Nothing can be reported to a client whose headers were sent long ago.+ _ = agg.Run(ctx, agg.Config[streamtypes.Event]{+ Writer: writer,+ Flush: flush,+ Buffer: clientStreamEventBuffer,+ HeartbeatInterval: clientStreamHeartbeatInterval,+ MakeHeartbeat: func() streamtypes.Event {+ return streamtypes.Event{Type: "heartbeat", Timestamp: time.Now()}+ },+ Producers: h.producersForInternal(ps, input),+ })+}++// producersForInternal selects the producers for the requested channels,+// dropping any the caller lacks permission for.+func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *StreamClientInput) []agg.Producer[streamtypes.Event] {+ requested := parseStreamChannelsInternal(input.Channels)+ producers := make([]agg.Producer[streamtypes.Event], 0, 4)++ if requested[streamtypes.ChannelEnvironments] {+ // Ungated like listEnvironments; the producer filters to the caller's+ // accessible environments.+ producers = append(producers, forwardStreamChannelInternal(+ streamtypes.ChannelEnvironments,+ func(channel string, event environmenttypes.StreamEvent) streamtypes.Event {+ return streamtypes.Event{Channel: channel, Environment: &event, Timestamp: event.Timestamp}+ },+ func(producerCtx context.Context, events chan<- environmenttypes.StreamEvent) {+ h.environment.runEnvironmentStreamProducerInternal(producerCtx, ps, events)+ },+ ))+ }++ if requested[streamtypes.ChannelDashboard] {+ wrapDashboard := func(channel string, event dashboardtypes.StreamEvent) streamtypes.Event {+ return streamtypes.Event{Channel: channel, Dashboard: &event, Timestamp: event.Timestamp}+ }+ if ps.Allows(authz.PermDashboardRead, localDockerEnvironmentID) {+ producers = append(producers, forwardStreamChannelInternal(+ streamtypes.ChannelDashboard, wrapDashboard,+ func(producerCtx context.Context, events chan<- dashboardtypes.StreamEvent) {+ h.dashboard.runLocalDashboardStreamProducerInternal(producerCtx, input.DebugAllGood, events)+ },+ ))+ }+ if ps.AllowsAny(authz.PermDashboardRead) {+ producers = append(producers, forwardStreamChannelInternal(+ streamtypes.ChannelDashboard, wrapDashboard,+ func(producerCtx context.Context, events chan<- dashboardtypes.StreamEvent) {+ h.dashboard.runRemoteDashboardStreamPollersInternal(producerCtx, ps, input.DebugAllGood, events)+ },+ ))+ }+ }++ if requested[streamtypes.ChannelActivities] {+ wrapActivity := func(channel string, event activitytypes.StreamEvent) streamtypes.Event {+ return streamtypes.Event{Channel: channel, Activity: &event, Timestamp: event.Timestamp}+ }+ if ps.Allows(authz.PermActivitiesRead, localDockerEnvironmentID) {+ producers = append(producers, forwardStreamChannelInternal(+ streamtypes.ChannelActivities, wrapActivity,+ func(producerCtx context.Context, events chan<- activitytypes.StreamEvent) {+ h.activity.runLocalActivityStreamProducerInternal(producerCtx, input.Limit, events)+ },+ ))+ }+ if ps.AllowsAny(authz.PermActivitiesRead) {+ producers = append(producers, forwardStreamChannelInternal(+ streamtypes.ChannelActivities, wrapActivity,+ func(producerCtx context.Context, events chan<- activitytypes.StreamEvent) {+ h.activity.runRemoteActivityStreamPollersInternal(producerCtx, ps, input.Limit, events)+ },+ ))+ }+ }++ return producers+}++func parseStreamChannelsInternal(raw string) map[string]bool {+ channels := make(map[string]bool, 3)+ for name := range strings.SplitSeq(raw, ",") {+ if trimmed := strings.TrimSpace(strings.ToLower(name)); trimmed != "" {+ channels[trimmed] = true+ }+ }+ return channels+}++// forwardStreamChannelInternal adapts a channel's own producer to the shared+// envelope. Each producer keeps emitting its native event type, so adding a+// channel never means rewriting its feed.+func forwardStreamChannelInternal[T any](+ channel string,+ wrap func(string, T) streamtypes.Event,+ producer func(context.Context, chan<- T),+) agg.Producer[streamtypes.Event] {+ return func(ctx context.Context, out chan<- streamtypes.Event) {+ inner := make(chan T, clientStreamChannelBuffer)++ go func() {+ defer close(inner)+ producer(ctx, inner)+ }()++ for event := range inner {+ if !agg.Send(ctx, out, wrap(channel, event)) {+ return+ }+ }+ }+}diff --git a/backend/api/handlers/stream_test.go b/backend/api/handlers/stream_test.gonew file mode 100644index 000000000..c9a87ba2a--- /dev/null+++ b/backend/api/handlers/stream_test.go@@ -0,0 +1,83 @@+package handlers++import (+ "bufio"+ "context"+ "encoding/json"+ "io"+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/internal/services"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ streamtypes "github.com/getarcaneapp/arcane/types/v2/stream"+ "github.com/stretchr/testify/require"+)++// The environments channel must report an environment the manager cannot reach.+//+// This is the property the whole live-status fix rests on, and the one neither+// of the other channels can provide: the dashboard channel only emits when it+// gets a snapshot back from an agent, so an environment that is down produces+// nothing. Without a pushed status an offline agent stays offline in the UI+// forever, which is exactly what happens after a fleet update.+func TestStreamHandlerEnvironmentsChannelEmitsUnreachableEnvironmentInternal(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()++ db := setupActivityHandlerTestDBInternal(t)+ limitStreamTestDBToSingleConnInternal(t, db)++ // An edge environment with no tunnel registered: unreachable by construction.+ createStreamTestRemoteEnvironmentInternal(t, db, "remote-1", "Remote", "edge://remote-1", "token")+ require.NoError(t, db.Exec(`UPDATE environments SET is_edge = ? WHERE id = ?`, true, "remote-1").Error)++ handler := &StreamHandler{+ environment: &EnvironmentHandler{+ environmentService: services.NewEnvironmentService(db, nil, nil, nil, nil, nil),+ },+ }+ ps := authz.NewPermissionSet()+ ps.Sudo = true++ pr, pw := io.Pipe()+ done := make(chan struct{})+ go func() {+ defer close(done)+ defer func() { _ = pw.Close() }()+ handler.streamClientInternal(ctx, ps, &StreamClientInput{Channels: streamtypes.ChannelEnvironments}, pw, func() {})+ }()++ var seen *streamtypes.Event+ scanner := bufio.NewScanner(pr)+ for scanner.Scan() {+ var envelope streamtypes.Event+ require.NoError(t, json.Unmarshal(scanner.Bytes(), &envelope))+ if envelope.Environment == nil {+ continue+ }+ seen = &envelope+ cancel()+ break+ }++ go func() { _, _ = io.Copy(io.Discard, pr) }()+ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("stream did not terminate after cancel")+ }++ require.NotNil(t, seen, "expected an environments snapshot for an unreachable environment")+ require.Equal(t, streamtypes.ChannelEnvironments, seen.Channel)++ var remote *string+ for _, env := range seen.Environment.Environments {+ if env.ID == "remote-1" {+ status := env.Status+ remote = &status+ }+ }+ require.NotNil(t, remote, "unreachable environment must still appear in the snapshot")+ require.Equal(t, "offline", *remote)+}diff --git a/backend/api/middleware/role.go b/backend/api/middleware/role.goindex d0d1b5c0e..8e2826e0a 100644--- a/backend/api/middleware/role.go+++ b/backend/api/middleware/role.go@@ -55,23 +55,6 @@ func RequirePermission(api huma.API, perm string) huma.Middlewares {}}}-// RequireAnyEnvironmentPermission protects aggregate operations that span-// environments but do not carry an environment ID in their path. The caller-// must hold perm globally or for at least one environment; handlers remain-// responsible for filtering aggregate output to the exact allowed scopes.-func RequireAnyEnvironmentPermission(api huma.API, perm string) huma.Middlewares {- return huma.Middlewares{func(ctx huma.Context, next func(huma.Context)) {- ps, _ := PermissionsFromContext(ctx.Context())- if !ps.AllowsAny(perm) {- if err := huma.WriteErr(api, ctx, http.StatusForbidden, "permission denied: "+perm); err != nil {- slog.WarnContext(ctx.Context(), "failed to write 403 response", "error", err)- }- return- }- next(ctx)- }}-}-// RequireGlobalAdmin returns a per-operation Huma middleware that rejects any// caller who is not a global admin (or sudo). Used for operations that are// intentionally not exposed as delegated permissions — role creation/edits,diff --git a/backend/api/middleware/role_test.go b/backend/api/middleware/role_test.goindex e9089729e..d9b1980cd 100644--- a/backend/api/middleware/role_test.go+++ b/backend/api/middleware/role_test.go@@ -219,58 +219,3 @@ func TestRequireSudoRejectsHumanAdminAndAllowsAgent(t *testing.T) {})}}--func TestRequireAnyEnvironmentPermission(t *testing.T) {- tests := []struct {- name string- permission *authz.PermissionSet- wantStatus int- wantRan bool- }{- {name: "missing", permission: authz.NewPermissionSet(), wantStatus: http.StatusForbidden},- {name: "other permission", permission: func() *authz.PermissionSet {- ps := authz.NewPermissionSet()- ps.AddEnv("env-1", authz.PermContainersList)- return ps- }(), wantStatus: http.StatusForbidden},- {name: "environment permission", permission: func() *authz.PermissionSet {- ps := authz.NewPermissionSet()- ps.AddEnv("env-1", authz.PermActivitiesRead)- return ps- }(), wantStatus: http.StatusNoContent, wantRan: true},- {name: "global permission", permission: func() *authz.PermissionSet {- ps := authz.NewPermissionSet()- ps.AddGlobal(authz.PermActivitiesRead)- return ps- }(), wantStatus: http.StatusNoContent, wantRan: true},- {name: "sudo", permission: authz.SudoPermissionSet(), wantStatus: http.StatusNoContent, wantRan: true},- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- router := echo.New()- api := humaecho.NewWithGroup(router, router.Group("/api"), huma.DefaultConfig("test", "1.0.0"))- api.UseMiddleware(func(ctx huma.Context, next func(huma.Context)) {- next(huma.WithContext(ctx, context.WithValue(ctx.Context(), ContextKeyUserPermissions, tt.permission)))- })-- handlerRan := false- huma.Register(api, huma.Operation{- OperationID: "aggregate-permission-" + tt.name,- Method: http.MethodGet,- Path: "/activities/stream",- Middlewares: RequireAnyEnvironmentPermission(api, authz.PermActivitiesRead),- }, func(_ context.Context, _ *struct{}) (*struct{}, error) {- handlerRan = true- return &struct{}{}, nil- })-- req := httptest.NewRequest(http.MethodGet, "/api/activities/stream", nil)- rec := httptest.NewRecorder()- router.ServeHTTP(rec, req)-- require.Equal(t, tt.wantStatus, rec.Code)- require.Equal(t, tt.wantRan, handlerRan)- })- }-}diff --git a/backend/internal/bootstrap/edge_bootstrap.go b/backend/internal/bootstrap/edge_bootstrap.goindex 6ec79cbba..f08131e91 100644--- a/backend/internal/bootstrap/edge_bootstrap.go+++ b/backend/internal/bootstrap/edge_bootstrap.go@@ -50,6 +50,11 @@ func registerEdgeTunnelRoutes(if err := createEdgeConnectionEvent(ctx, eventService, envID, envName, connected); err != nil {slog.WarnContext(ctx, "Failed to create edge connection event", "environment_id", envID, "connected", connected, "error", err)}++ // This is the only funnel for "an edge tunnel came up or went down"+ // (register, unregister and stale reaping all route through it), so it+ // is where open status streams learn about it without polling.+ environmentService.NotifyRuntimeStateChanged()}eventCallback := func(ctx context.Context, envID string, evt *edge.TunnelEvent) error {diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.goindex 542ab95bd..0ed0508ed 100644--- a/backend/internal/bootstrap/router_bootstrap.go+++ b/backend/internal/bootstrap/router_bootstrap.go@@ -44,8 +44,7 @@ var loggerSkipPatterns = []string{// ends, so it reports the full connection lifetime as request latency —// which reads as a multi-minute hung request. The handlers log their own// termination instead.- "GET /api/dashboard/stream",- "GET /api/activities/stream",+ "GET /api/stream","GET /_app/*","GET /img","GET /api/health",diff --git a/backend/internal/services/environment_runtime_notify_test.go b/backend/internal/services/environment_runtime_notify_test.gonew file mode 100644index 000000000..f34cdd260--- /dev/null+++ b/backend/internal/services/environment_runtime_notify_test.go@@ -0,0 +1,45 @@+package services++import (+ "testing"++ "github.com/stretchr/testify/require"+)++// A watcher must see a change signal, and a burst must not block the notifier —+// it runs on the edge tunnel's connect/disconnect path.+func TestEnvironmentServiceRuntimeChangeSignalCoalescesInternal(t *testing.T) {+ s := &EnvironmentService{}++ changes, unsubscribe := s.SubscribeRuntimeChanges()+ defer unsubscribe()++ // More notifications than the channel can hold: the extras are dropped+ // rather than blocking, because a watcher re-derives live state on wake.+ for range 5 {+ s.NotifyRuntimeStateChanged()+ }++ select {+ case <-changes:+ default:+ t.Fatal("expected a pending runtime change signal")+ }++ select {+ case <-changes:+ t.Fatal("expected coalesced signals, got a second wake-up")+ default:+ }++ // Unsubscribing must stop delivery so a closed stream cannot leak.+ unsubscribe()+ s.NotifyRuntimeStateChanged()+ select {+ case <-changes:+ t.Fatal("expected no signal after unsubscribe")+ default:+ }++ require.Empty(t, s.runtimeWatchers)+}diff --git a/backend/internal/services/environment_runtime_state.go b/backend/internal/services/environment_runtime_state.godeleted file mode 100644index 918cc5ff6..000000000--- a/backend/internal/services/environment_runtime_state.go+++ /dev/null@@ -1,67 +0,0 @@-package services--import (- "time"-- "github.com/getarcaneapp/arcane/backend/v2/internal/models"- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"- environmenttypes "github.com/getarcaneapp/arcane/types/v2/environment"-)--// ApplyEnvironmentRuntimeState normalizes edge environment runtime status using-// in-memory tunnel and poll registries without mutating persisted state.-func ApplyEnvironmentRuntimeState(env *environmenttypes.Environment) {- if env == nil || !env.IsEdge {- return- }-- connected := false- env.Connected = &connected- env.ConnectedAt = nil- env.LastHeartbeat = nil- env.LastPollAt = nil- env.EdgeTransport = nil- env.EdgeSecurityMode = nil- env.EdgeSessionID = nil- env.EdgeAgentInstance = nil- env.EdgeCapabilities = nil-- if pollState, ok := edge.GetPollRuntimeRegistry().Get(env.ID, time.Now()).Get(); ok {- env.LastPollAt = pollState.LastPollAt- }-- if runtimeState, ok := edge.GetTunnelRuntimeState(env.ID).Get(); ok {- connected = true- env.Connected = &connected- env.Status = string(models.EnvironmentStatusOnline)- env.ConnectedAt = runtimeState.ConnectedAt- env.LastHeartbeat = runtimeState.LastHeartbeat- if runtimeState.SecurityMode != "" {- env.EdgeSecurityMode = &runtimeState.SecurityMode- }- if runtimeState.SessionID != "" {- env.EdgeSessionID = &runtimeState.SessionID- }- if runtimeState.AgentInstance != "" {- env.EdgeAgentInstance = &runtimeState.AgentInstance- }- if len(runtimeState.Capabilities) > 0 {- env.EdgeCapabilities = append([]string(nil), runtimeState.Capabilities...)- }- if transport, ok := edge.GetActiveTunnelTransport(env.ID).Get(); ok {- env.EdgeTransport = &transport- } else if runtimeState.Transport != "" {- env.EdgeTransport = &runtimeState.Transport- }- return- }-- if env.LastPollAt != nil {- env.Status = string(models.EnvironmentStatusStandby)- return- }-- if env.Status != string(models.EnvironmentStatusPending) {- env.Status = string(models.EnvironmentStatusOffline)- }-}diff --git a/backend/internal/services/environment_service.go b/backend/internal/services/environment_service.goindex 9481d13b2..dc67f2b9d 100644--- a/backend/internal/services/environment_service.go+++ b/backend/internal/services/environment_service.go@@ -63,6 +63,12 @@ type EnvironmentService struct {// variableSyncer is injected post-construction via SetVariableSyncer// (manager-only) to avoid a wire cycle with VariableService.variableSyncer VariableSyncer++ // runtimeWatchers receive a coalesced wake-up whenever an environment's+ // liveness changes. See environment_runtime_notify.go.+ runtimeWatchMu sync.Mutex+ runtimeWatchers map[int]chan struct{}+ runtimeWatcherSeq int}// VariableSyncer pushes the effective global-variable set to one environment.@@ -1378,6 +1384,9 @@ func (s *EnvironmentService) updateEnvironmentStatusInternal(ctx context.Contextif err := s.db.WithContext(ctx).Model(&models.Environment{}).Where("id = ?", id).Updates(updates).Error; err != nil {return errors.WrapIf(err, "failed to update environment status")}+ // Direct environments have no tunnel callback, so this health-check write is+ // the only moment their liveness changes.+ s.NotifyRuntimeStateChanged()return nil}@@ -1397,6 +1406,12 @@ func (s *EnvironmentService) UpdateEnvironmentHeartbeat(ctx context.Context, idreturn errors.WrapIf(result.Error, "failed to update environment heartbeat")}+ // The 30s throttle above doubles as the notify throttle: a no-op heartbeat+ // changed nothing worth waking a stream for.+ if result.RowsAffected > 0 {+ s.NotifyRuntimeStateChanged()+ }+return nil}@@ -1424,9 +1439,120 @@ func (s *EnvironmentService) UpdateEnvironmentConnectionState(ctx context.Contexreturn errors.WrapIf(err, "failed to update environment connection state")}+ s.NotifyRuntimeStateChanged()+return nil}+// ApplyEnvironmentRuntimeState normalizes edge environment runtime status using+// in-memory tunnel and poll registries without mutating persisted state.+func ApplyEnvironmentRuntimeState(env *environment.Environment) {+ if env == nil || !env.IsEdge {+ return+ }++ connected := false+ env.Connected = &connected+ env.ConnectedAt = nil+ env.LastHeartbeat = nil+ env.LastPollAt = nil+ env.EdgeTransport = nil+ env.EdgeSecurityMode = nil+ env.EdgeSessionID = nil+ env.EdgeAgentInstance = nil+ env.EdgeCapabilities = nil++ if pollState, ok := edge.GetPollRuntimeRegistry().Get(env.ID, time.Now()).Get(); ok {+ env.LastPollAt = pollState.LastPollAt+ }++ if runtimeState, ok := edge.GetTunnelRuntimeState(env.ID).Get(); ok {+ connected = true+ env.Connected = &connected+ env.Status = string(models.EnvironmentStatusOnline)+ env.ConnectedAt = runtimeState.ConnectedAt+ env.LastHeartbeat = runtimeState.LastHeartbeat+ if runtimeState.SecurityMode != "" {+ env.EdgeSecurityMode = &runtimeState.SecurityMode+ }+ if runtimeState.SessionID != "" {+ env.EdgeSessionID = &runtimeState.SessionID+ }+ if runtimeState.AgentInstance != "" {+ env.EdgeAgentInstance = &runtimeState.AgentInstance+ }+ if len(runtimeState.Capabilities) > 0 {+ env.EdgeCapabilities = append([]string(nil), runtimeState.Capabilities...)+ }+ if transport, ok := edge.GetActiveTunnelTransport(env.ID).Get(); ok {+ env.EdgeTransport = &transport+ } else if runtimeState.Transport != "" {+ env.EdgeTransport = &runtimeState.Transport+ }+ return+ }++ if env.LastPollAt != nil {+ env.Status = string(models.EnvironmentStatusStandby)+ return+ }++ if env.Status != string(models.EnvironmentStatusPending) {+ env.Status = string(models.EnvironmentStatusOffline)+ }+}++// Environment liveness lives in process-local registries (the edge tunnel+// registry and the poll registry), so nothing in the database changes when an+// agent connects or drops. Open status streams therefore need a nudge rather+// than a payload: they re-derive every environment they cover on wake, which is+// why a bare signal is enough and why a dropped signal is harmless — the+// watcher has one pending already and will read current state when it runs.++// SubscribeRuntimeChanges returns a channel that receives a coalesced wake-up+// whenever environment liveness may have changed, plus a function to release+// it. The channel is never closed; callers select on it alongside their own+// context.+func (s *EnvironmentService) SubscribeRuntimeChanges() (<-chan struct{}, func()) {+ s.runtimeWatchMu.Lock()+ defer s.runtimeWatchMu.Unlock()++ if s.runtimeWatchers == nil {+ s.runtimeWatchers = make(map[int]chan struct{})+ }++ s.runtimeWatcherSeq+++ id := s.runtimeWatcherSeq+ // Capacity 1: a second signal arriving before the watcher wakes is+ // redundant, because the watcher reads live state rather than a queue.+ ch := make(chan struct{}, 1)+ s.runtimeWatchers[id] = ch++ return ch, func() {+ s.runtimeWatchMu.Lock()+ defer s.runtimeWatchMu.Unlock()+ delete(s.runtimeWatchers, id)+ }+}++// NotifyRuntimeStateChanged wakes every runtime watcher. It never blocks, so it+// is safe to call from connection callbacks on the tunnel hot path.+func (s *EnvironmentService) NotifyRuntimeStateChanged() {+ if s == nil {+ return+ }++ s.runtimeWatchMu.Lock()+ defer s.runtimeWatchMu.Unlock()++ for _, ch := range s.runtimeWatchers {+ select {+ case ch <- struct{}{}:+ default:+ }+ }+}+// ReconcileEdgeStatusesOnStartup resets edge environments to offline when the manager starts.// Live edge tunnels are process-local runtime state, so persisted "online" flags can be stale// after a restart until agents reconnect. Pending environments are left untouched.diff --git a/backend/pkg/utils/httpx/aggregate_stream.go b/backend/pkg/utils/httpx/aggregate_stream.godeleted file mode 100644index c392f87e5..000000000--- a/backend/pkg/utils/httpx/aggregate_stream.go+++ /dev/null@@ -1,57 +0,0 @@-package httpx--import (- "context"- "log/slog"- "time"-- "emperror.dev/errors"-- "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"- "go.getarcane.app/streams/agg"-)--// streamDeadPeerTimeout is how long an aggregate stream's writes may stay-// unacknowledged before the connection is torn down. A client that has-// acknowledged nothing for this long is gone: healthy ones acknowledge the-// heartbeat every 15 seconds.-const streamDeadPeerTimeout = 45 * time.Second--// RunAuthorizedAggregateStream selects local and remote producers from the-// caller's effective permissions before starting an aggregate stream. Remote-// producers must still filter individual environments with PermissionSet.Allows.-func RunAuthorizedAggregateStream[T any](- ctx context.Context,- ps *authz.PermissionSet,- permission string,- config agg.Config[T],- localProducer agg.Producer[T],- remoteProducer agg.Producer[T],-) error {- config.Producers = make([]agg.Producer[T], 0, 2)- if ps.Allows(permission, "0") {- config.Producers = append(config.Producers, localProducer)- }- if ps.AllowsAny(permission) {- config.Producers = append(config.Producers, remoteProducer)- }-- releaseDeadPeerTimeout, timeoutErr := AcquireDeadPeerTimeout(ctx, streamDeadPeerTimeout)- if timeoutErr != nil {- slog.DebugContext(ctx, "could not bound stream dead-peer timeout", "permission", permission, "error", timeoutErr)- }- defer releaseDeadPeerTimeout()-- startedAt := time.Now()- err := agg.Run(ctx, config)-- // Once the response headers are sent there is no way to report a failure to- // the client, and the access log only records the connection's total- // lifetime — so a client that vanishes mid-write is invisible without this.- if err != nil && !errors.Is(err, context.Canceled) {- slog.WarnContext(ctx, "aggregate stream ended with error", "permission", permission, "error", err, "duration", time.Since(startedAt))- return err- }- slog.DebugContext(ctx, "aggregate stream closed", "permission", permission, "duration", time.Since(startedAt), "context_error", ctx.Err())- return err-}diff --git a/frontend/src/lib/services/activity-service.ts b/frontend/src/lib/services/activity-service.tsindex 66a36dc41..dab81488f 100644--- a/frontend/src/lib/services/activity-service.ts+++ b/frontend/src/lib/services/activity-service.ts@@ -1,9 +1,8 @@-import BaseAPIService, { handleUnauthorizedResponseInternal } from './api-service';+import BaseAPIService from './api-service';import { environmentStore } from '#lib/stores/environment.store.svelte';import type { Activity, ActivityClearHistoryResult, ActivityDetail } from '#lib/types/activity.type';import type { Paginated, SearchPaginationSortRequest } from '#lib/types/shared';import { transformPaginationParams } from '#lib/utils/tables';-import { streamCacheBuster } from '#lib/utils/streaming';class ActivityService extends BaseAPIService {private async resolveEnvironmentId(environmentId?: string): Promise<string> {@@ -34,39 +33,6 @@ class ActivityService extends BaseAPIService {const envId = await this.resolveEnvironmentId(environmentId);return this.handleResponse(this.api.delete(`/environments/${envId}/activities/history`));}-- getActivityStreamUrl(limit = 50): string {- const baseUrl = this.api.defaults.baseURL.replace(/\/+$/, '');- const params = new URLSearchParams({ limit: String(limit), _: streamCacheBuster() });- return `${baseUrl}/activities/stream?${params.toString()}`;- }-- async openActivityStream(signal: AbortSignal, limit = 50, retry = false): Promise<Response> {- const response = await fetch(this.getActivityStreamUrl(limit), {- credentials: 'include',- headers: { Accept: 'application/x-json-stream' },- signal- });- if (response.status === 401) {- // Nothing reads this body, so release its connection before deciding- // what to do next.- await response.body?.cancel().catch(() => {});- const action = await handleUnauthorizedResponseInternal('/activities/stream', retry);- if (action === 'retry') {- return this.openActivityStream(signal, limit, true);- }- // The document is being replaced (login redirect, or the reload that- // follows a backend self-update). Never settling stops the caller- // from opening a fresh stream against a page that is going away.- if (action === 'redirect' || action === 'reload') {- return new Promise<Response>(() => {});- }- }- if (!response.ok) {- throw new Error(`Activity stream failed with status ${response.status}`);- }- return response;- }}export const activityService = new ActivityService();diff --git a/frontend/src/lib/services/auth-service.ts b/frontend/src/lib/services/auth-service.tsindex f2c721a63..81d0f6bdd 100644--- a/frontend/src/lib/services/auth-service.ts+++ b/frontend/src/lib/services/auth-service.ts@@ -7,6 +7,7 @@ import type { OidcUserInfo, LoginCredentials, LoginResponseData, AutoLoginConfigimport type { QueryClient } from '@tanstack/svelte-query';import { activityStore } from '#lib/stores/activity.store.svelte';import { dashboardStore } from '#lib/stores/dashboard.store.svelte';+import { environmentStatusStore } from '#lib/stores/environment-status.store.svelte';import { getEffectiveLandingPage } from '#lib/utils/navigation';const REFRESH_TOKEN_KEY = 'arcane_refresh_token';@@ -208,11 +209,13 @@ class AuthService extends BaseAPIService {queryClient.clear();const restartActivityStore = activityStore.stop({ resetState: true });const restartDashboardStore = dashboardStore.stop({ resetState: true });+ const restartEnvironmentStatusStore = environmentStatusStore.stop();userStore.clearUser();if (options?.restartMountedStores) {if (restartActivityStore) void activityStore.start();if (restartDashboardStore) void dashboardStore.start();+ if (restartEnvironmentStatusStore) void environmentStatusStore.start();}}diff --git a/frontend/src/lib/services/dashboard-service.ts b/frontend/src/lib/services/dashboard-service.tsindex ad81e2c53..30ae7be4a 100644--- a/frontend/src/lib/services/dashboard-service.ts+++ b/frontend/src/lib/services/dashboard-service.ts@@ -1,5 +1,4 @@-import BaseAPIService, { handleUnauthorizedResponseInternal } from './api-service';-import { streamCacheBuster } from '#lib/utils/streaming';+import BaseAPIService from './api-service';import type { DashboardSnapshot } from '#lib/types/shared';interface GetDashboardOptions {@@ -11,44 +10,6 @@ class DashboardService extends BaseAPIService {const params = options?.debugAllGood ? { debugAllGood: 'true' } : undefined;return this.handleResponse(this.api.get(`/environments/${environmentId}/dashboard`, { params }));}-- getDashboardStreamUrl(debugAllGood = false): string {- const baseUrl = this.api.defaults.baseURL.replace(/\/+$/, '');- const params = new URLSearchParams();- if (debugAllGood) {- params.set('debugAllGood', 'true');- }- params.set('_', streamCacheBuster());- const query = params.toString();- return `${baseUrl}/dashboard/stream${query ? `?${query}` : ''}`;- }-- async openDashboardStream(signal: AbortSignal, debugAllGood = false, retry = false): Promise<Response> {- const response = await fetch(this.getDashboardStreamUrl(debugAllGood), {- credentials: 'include',- headers: { Accept: 'application/x-json-stream' },- signal- });- if (response.status === 401) {- // Nothing reads this body, so release its connection before deciding- // what to do next.- await response.body?.cancel().catch(() => {});- const action = await handleUnauthorizedResponseInternal('/dashboard/stream', retry);- if (action === 'retry') {- return this.openDashboardStream(signal, debugAllGood, true);- }- // The document is being replaced (login redirect, or the reload that- // follows a backend self-update). Never settling stops the caller- // from opening a fresh stream against a page that is going away.- if (action === 'redirect' || action === 'reload') {- return new Promise<Response>(() => {});- }- }- if (!response.ok) {- throw new Error(`Dashboard stream failed with status ${response.status}`);- }- return response;- }}export const dashboardService = new DashboardService();diff --git a/frontend/src/lib/services/stream-service.ts b/frontend/src/lib/services/stream-service.tsnew file mode 100644index 000000000..b7ac721a0--- /dev/null+++ b/frontend/src/lib/services/stream-service.ts@@ -0,0 +1,50 @@+import BaseAPIService, { handleUnauthorizedResponseInternal } from './api-service';+import { streamCacheBuster } from '#lib/utils/streaming';++/** Channels the multiplexed client stream can carry. Must match types/stream. */+export const STREAM_CHANNEL_ENVIRONMENTS = 'environments';+export const STREAM_CHANNEL_DASHBOARD = 'dashboard';+export const STREAM_CHANNEL_ACTIVITIES = 'activities';++class StreamService extends BaseAPIService {+ getClientStreamUrl(channels: string[], params: Record<string, string> = {}): string {+ const baseUrl = this.api.defaults.baseURL.replace(/\/+$/, '');+ const search = new URLSearchParams({ ...params, channels: channels.join(',') });+ search.set('_', streamCacheBuster());+ return `${baseUrl}/stream?${search.toString()}`;+ }++ async openClientStream(+ signal: AbortSignal,+ channels: string[],+ params: Record<string, string> = {},+ retry = false+ ): Promise<Response> {+ const response = await fetch(this.getClientStreamUrl(channels, params), {+ credentials: 'include',+ headers: { Accept: 'application/x-json-stream' },+ signal+ });+ if (response.status === 401) {+ // Nothing reads this body, so release its connection before deciding+ // what to do next.+ await response.body?.cancel().catch(() => {});+ const action = await handleUnauthorizedResponseInternal('/stream', retry);+ if (action === 'retry') {+ return this.openClientStream(signal, channels, params, true);+ }+ // The document is being replaced (login redirect, or the reload that+ // follows a backend self-update). Never settling stops the caller+ // from opening a fresh stream against a page that is going away.+ if (action === 'redirect' || action === 'reload') {+ return new Promise<Response>(() => {});+ }+ }+ if (!response.ok) {+ throw new Error(`Client stream failed with status ${response.status}`);+ }+ return response;+ }+}++export const streamService = new StreamService();diff --git a/frontend/src/lib/stores/activity.store.svelte.ts b/frontend/src/lib/stores/activity.store.svelte.tsindex 597826024..eb69b2819 100644--- a/frontend/src/lib/stores/activity.store.svelte.ts+++ b/frontend/src/lib/stores/activity.store.svelte.ts@@ -1,4 +1,5 @@import { activityService } from '#lib/services/activity-service';+import { STREAM_CHANNEL_ACTIVITIES } from '#lib/services/stream-service';import { LOCAL_DOCKER_ENVIRONMENT_ID } from '#lib/stores/environment.store.svelte';import {createEnvironmentStreamStore,@@ -190,7 +191,7 @@ function createActivityStore() {streamError: false};},- openStream: (signal) => activityService.openActivityStream(signal, ACTIVITY_LIST_LIMIT),+ channel: STREAM_CHANNEL_ACTIVITIES,// REST-fetch history on start so the panel isn't stuck loading if the// stream is slow or drops before delivering its first snapshot.refreshOnStart: true,diff --git a/frontend/src/lib/stores/client-stream.svelte.ts b/frontend/src/lib/stores/client-stream.svelte.tsnew file mode 100644index 000000000..de9d48e72--- /dev/null+++ b/frontend/src/lib/stores/client-stream.svelte.ts@@ -0,0 +1,134 @@+import { streamService } from '#lib/services/stream-service';+import { createJSONLineStream } from '#lib/stores/json-line-stream.svelte';++type StreamEnvelope = {+ type: string;+ channel?: string;+ dashboard?: unknown;+ activity?: unknown;+ environment?: unknown;+};++type ChannelSubscriber = {+ channel: string;+ onEvent(event: unknown): void;+ onConnected?(): void;+};++/**+ * The app's single server stream. Every live feed — environments, dashboard,+ * activities — is multiplexed over one connection.+ *+ * Browsers allow only six concurrent HTTP/1.1 requests per origin, and each+ * long-lived stream holds one for the life of the page. One connection also+ * means one reconnect and one heartbeat to reason about instead of three that+ * can disagree about whether the server is reachable.+ *+ * Channels are opened on demand rather than always-on because their costs are+ * nothing alike: environments is a single local query, while dashboard proxies+ * a request to every remote agent on every tick. Subscribing or unsubscribing+ * changes the URL, so the connection is reopened — the same mechanism the+ * dashboard already used when its debug flag changed.+ */+function createClientStreamInternal() {+ const subscribers = new Set<ChannelSubscriber>();+ let params: Record<string, string> = {};++ function activeChannels(): string[] {+ return [...new Set([...subscribers].map((subscriber) => subscriber.channel))].sort();+ }++ const transport = createJSONLineStream<StreamEnvelope>({+ label: 'Client',+ openStream: (signal) => streamService.openClientStream(signal, activeChannels(), params),+ onConnected: () => {+ for (const subscriber of subscribers) {+ subscriber.onConnected?.();+ }+ },+ onEvent: (envelope) => {+ const payload = envelope.dashboard ?? envelope.activity ?? envelope.environment;+ if (!envelope.channel || payload === undefined) {+ return;+ }+ for (const subscriber of subscribers) {+ if (subscriber.channel === envelope.channel) {+ subscriber.onEvent(payload);+ }+ }+ }+ });++ function reopen() {+ if (!transport.isStarted) {+ return;+ }+ if (activeChannels().length === 0) {+ // Nothing left to carry; drop the connection rather than hold an+ // idle one open with no channels.+ transport.stop();+ return;+ }+ transport.restart();+ }++ function ensureStarted() {+ if (transport.isStarted || activeChannels().length === 0) {+ return;+ }+ transport.markStarted();+ transport.connect(transport.nextGeneration());+ }++ return {+ get streamConnected(): boolean {+ return transport.streamConnected;+ },+ get streamFailed(): boolean {+ return transport.streamFailed;+ },+ get generation(): number {+ return transport.generation;+ },+ get hasActiveStream(): boolean {+ return transport.hasActiveStream;+ },+ isCurrentGeneration: transport.isCurrentGeneration,+ /** Subscribes to a channel, (re)opening the connection to include it. */+ subscribe(channel: string, handlers: { onEvent(event: unknown): void; onConnected?(): void }): () => void {+ const subscriber: ChannelSubscriber = { channel, ...handlers };+ const hadChannel = activeChannels().includes(channel);+ subscribers.add(subscriber);++ if (!transport.isStarted) {+ ensureStarted();+ } else if (!hadChannel) {+ reopen();+ }++ return () => {+ subscribers.delete(subscriber);+ if (!activeChannels().includes(channel)) {+ reopen();+ }+ };+ },+ /** Sets query params shared by the stream URL (e.g. the dashboard debug flag). */+ setParams(next: Record<string, string>) {+ const changed = JSON.stringify(next) !== JSON.stringify(params);+ params = next;+ if (changed) {+ reopen();+ }+ },+ /** Reopens the connection, clearing a give-up state so it retries. */+ retry() {+ transport.restart({ clearFailure: true });+ },+ restart() {+ transport.restart();+ }+ };+}++export const clientStream = createClientStreamInternal();diff --git a/frontend/src/lib/stores/dashboard.store.svelte.ts b/frontend/src/lib/stores/dashboard.store.svelte.tsindex 4d7e51ba8..82afc255c 100644--- a/frontend/src/lib/stores/dashboard.store.svelte.ts+++ b/frontend/src/lib/stores/dashboard.store.svelte.ts@@ -1,5 +1,7 @@import { browser } from '$app/env';import { dashboardService } from '#lib/services/dashboard-service';+import { STREAM_CHANNEL_DASHBOARD } from '#lib/services/stream-service';+import { clientStream } from '#lib/stores/client-stream.svelte';import { LOCAL_DOCKER_ENVIRONMENT_ID } from '#lib/stores/environment.store.svelte';import {createEnvironmentStreamStore,@@ -38,7 +40,7 @@ function createDashboardStore() {streamError: false};},- openStream: (signal) => dashboardService.openDashboardStream(signal, debugAllGood),+ channel: STREAM_CHANNEL_DASHBOARD,applyEvent(environmentId, event) {switch (event.type) {case 'snapshot':@@ -116,7 +118,7 @@ function createDashboardStore() {}// The flag is encoded in the stream URL; restart to apply it.debugAllGood = nextDebugAllGood;- core.restartStream();+ clientStream.setParams(debugAllGood ? { debugAllGood: 'true' } : {});return;}diff --git a/frontend/src/lib/stores/environment-status.store.svelte.ts b/frontend/src/lib/stores/environment-status.store.svelte.tsnew file mode 100644index 000000000..874bae041--- /dev/null+++ b/frontend/src/lib/stores/environment-status.store.svelte.ts@@ -0,0 +1,67 @@+import { browser } from '$app/env';+import { STREAM_CHANNEL_ENVIRONMENTS } from '#lib/services/stream-service';+import { clientStream } from '#lib/stores/client-stream.svelte';+import { environmentStore } from '#lib/stores/environment.store.svelte';+import type { Environment } from '#lib/types/environment';++type EnvironmentStreamEvent = {+ type: string;+ environments?: Environment[];+};++/**+ * Keeps environmentStore's records live for as long as the app shell is mounted.+ *+ * Whether an edge environment is reachable lives only in the manager's+ * in-memory tunnel and poll registries, so the browser can never work it out+ * on its own — it has to be told. Without this stream the environment list is+ * frozen at whatever the root layout load happened to see, which after a fleet+ * update is "every agent offline".+ */+function createEnvironmentStatusStoreInternal() {+ let started = false;+ let lifecycleGeneration = 0;++ let unsubscribeChannel: (() => void) | null = null;++ return {+ get streamConnected(): boolean {+ return clientStream.streamConnected;+ },+ get streamFailed(): boolean {+ return clientStream.streamFailed;+ },+ async start() {+ if (!browser || started) {+ return;+ }+ started = true;+ const generation = ++lifecycleGeneration;+ // The first snapshot replaces the list, so wait for the load-time+ // list to land or the two would race on an empty store.+ await environmentStore.ready;+ if (generation !== lifecycleGeneration) {+ return;+ }+ unsubscribeChannel = clientStream.subscribe(STREAM_CHANNEL_ENVIRONMENTS, {+ onEvent: (payload) => {+ const event = payload as EnvironmentStreamEvent;+ if (event.type !== 'snapshot' || !event.environments) {+ return;+ }+ environmentStore.applyLiveEnvironments(event.environments);+ }+ });+ },+ stop() {+ const wasStarted = started;+ started = false;+ lifecycleGeneration += 1;+ unsubscribeChannel?.();+ unsubscribeChannel = null;+ return wasStarted;+ }+ };+}++export const environmentStatusStore = createEnvironmentStatusStoreInternal();diff --git a/frontend/src/lib/stores/environment-stream.svelte.ts b/frontend/src/lib/stores/environment-stream.svelte.tsindex b02845710..e6730dffb 100644--- a/frontend/src/lib/stores/environment-stream.svelte.ts+++ b/frontend/src/lib/stores/environment-stream.svelte.ts@@ -1,10 +1,8 @@import { browser } from '$app/env';import { environmentStore, LOCAL_DOCKER_ENVIRONMENT_ID } from '#lib/stores/environment.store.svelte';+import { clientStream } from '#lib/stores/client-stream.svelte';import type { Environment } from '#lib/types/environment';-const MAX_RECONNECT_DELAY = 15_000;-const MAX_RECONNECT_ATTEMPTS = 20;-export type StreamEnvStateBase = {id: string;name: string;@@ -36,7 +34,8 @@ export interface EnvStreamCoreConfig<TState extends StreamEnvStateBase, TEvent e/** Used in console warnings, e.g. 'Dashboard' / 'Activity'. */label: string;createEnvironmentState(environment: Pick<Environment, 'id' | 'name'>): TState;- openStream(signal: AbortSignal): Promise<Response>;+ /** Channel to subscribe to on the shared client stream. */+ channel: string;/** Handles every event type except 'heartbeat' (core owns connection state). */applyEvent(environmentId: string, event: TEvent): void;/** Fully owns a per-environment REST refresh, including generation/removal guards via core helpers. */@@ -61,18 +60,28 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,let _environmentStates = $state<Record<string, TState>>({});let started = false;+ // REST snapshots belong to this store lifecycle, not the shared transport's reconnect lifecycle.+ let lifecycleGeneration = 0;let unsubscribeEnvironment: (() => void) | null = null;let unsubscribeEnvironmentFilter: (() => void) | null = null;- // A single aggregated stream carries every environment's events; per-env- // connections would multiply requests and exhaust the browser's- // 6-per-origin HTTP/1.1 limit.- let streamAbortController: AbortController | null = null;- let removePageLifecycleListeners: (() => void) | null = null;- let reconnectTimer: ReturnType<typeof setTimeout> | null = null;- let reconnectAttempt = 0;- let streamGeneration = 0;- let _streamConnected = $state(false);- let _streamFailed = $state(false);++ let unsubscribeChannel: (() => void) | null = null;++ const channelHandlers = {+ // A fresh stream re-emits error events for environments that are still+ // failing, so stale per-environment errors are cleared on every (re)connect.+ onConnected: () => clearAllEnvironmentErrors(),+ onEvent: (payload: unknown) => {+ const event = payload as TEvent;+ const environmentId = event.environmentId || LOCAL_DOCKER_ENVIRONMENT_ID;+ // The aggregated stream can keep delivering events for an environment+ // for a short while after it was removed locally; don't resurrect it.+ if (!environmentState(environmentId)) {+ return;+ }+ config.applyEvent(environmentId, event);+ }+ };function environmentState(environmentId: string): TState | undefined {return _environmentStates[environmentId];@@ -118,53 +127,6 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,}}- function nextGeneration(): number {- streamGeneration += 1;- return streamGeneration;- }-- function isCurrentGeneration(generation: number): boolean {- return streamGeneration === generation;- }-- function clearReconnectTimer() {- if (reconnectTimer) {- clearTimeout(reconnectTimer);- reconnectTimer = null;- }- }-- function abortStream() {- clearReconnectTimer();- streamAbortController?.abort();- streamAbortController = null;- _streamConnected = false;- }-- // Closing a tab leaves teardown to whenever the browser gets around to- // dropping the socket. Until it does, the server keeps writing heartbeats- // into a connection nobody is reading — so tear down explicitly instead.- function watchPageLifecycle() {- if (!browser || removePageLifecycleListeners) {- return;- }-- const onPageHide = () => abortStream();- // A bfcache restore comes back with the stream already torn down.- const onPageShow = (event: PageTransitionEvent) => {- if (event.persisted && started && !streamAbortController) {- void connectStream(nextGeneration());- }- };-- window.addEventListener('pagehide', onPageHide);- window.addEventListener('pageshow', onPageShow);- removePageLifecycleListeners = () => {- window.removeEventListener('pagehide', onPageHide);- window.removeEventListener('pageshow', onPageShow);- };- }-function removeEnvironment(environmentId: string) {const nextStates = { ..._environmentStates };delete nextStates[environmentId];@@ -172,136 +134,11 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,config.onEnvironmentRemoved?.(environmentId);}- async function refresh(generation = streamGeneration) {+ async function refresh(generation = lifecycleGeneration) {reconcileEnvironments();await Promise.all(Object.keys(_environmentStates).map((environmentId) => config.fetchSnapshot(environmentId, generation)));}- async function connectStream(generation: number) {- if (!browser || !isCurrentGeneration(generation)) {- return;- }-- // Overlapping connects would otherwise strand the previous controller:- // abortStream() only ever reaches the newest one, so the older request- // would keep an open connection the server has no way to notice.- streamAbortController?.abort();-- const controller = new AbortController();- streamAbortController = controller;- try {- const response = await config.openStream(controller.signal);- if (!isCurrentGeneration(generation) || !response.body) {- // The response body is live even though nobody will read it;- // dropping it on the floor leaves the server streaming into a- // connection that stays open until its TCP timers expire.- controller.abort();- if (streamAbortController === controller) {- streamAbortController = null;- }- return;- }-- _streamConnected = true;- _streamFailed = false;- reconnectAttempt = 0;- clearAllEnvironmentErrors();- await readJSONLines(response.body, generation);- } catch (error) {- if (!controller.signal.aborted && isCurrentGeneration(generation)) {- console.warn(`${config.label} stream disconnected:`, error);- }- } finally {- if (streamAbortController === controller) {- streamAbortController = null;- }- if (isCurrentGeneration(generation)) {- _streamConnected = false;- if (!controller.signal.aborted) {- scheduleReconnect(generation);- }- } else if (!controller.signal.aborted) {- // A superseded generation has no owner left to abort it.- controller.abort();- }- }- }-- async function readJSONLines(stream: ReadableStream<Uint8Array>, generation: number) {- const reader = stream.getReader();- const decoder = new TextDecoder();- let buffer = '';-- try {- while (isCurrentGeneration(generation)) {- const { done, value } = await reader.read();- if (done) {- break;- }-- buffer += decoder.decode(value, { stream: true });- const lines = buffer.split('\n');- buffer = lines.pop() ?? '';- for (const line of lines) {- handleStreamLine(line);- }- }-- buffer += decoder.decode();- if (buffer.trim()) {- handleStreamLine(buffer);- }- } finally {- // The loop also exits when the generation advances, with the stream- // still open. Cancelling is what actually closes the fetch and lets- // the server see the disconnect; releaseLock alone does not.- await reader.cancel().catch(() => {});- reader.releaseLock();- }- }-- function handleStreamLine(line: string) {- const trimmed = line.trim();- if (!trimmed) {- return;- }-- try {- const event = JSON.parse(trimmed) as TEvent;- const environmentId = event.environmentId || LOCAL_DOCKER_ENVIRONMENT_ID;- if (event.type === 'heartbeat') {- _streamConnected = true;- return;- }- // The aggregated stream can keep delivering events for an environment- // for a short while after it was removed locally; don't resurrect it.- if (!environmentState(environmentId)) {- return;- }- config.applyEvent(environmentId, event);- } catch (error) {- console.warn(`Failed to parse ${config.label.toLowerCase()} stream line:`, error);- }- }-- function scheduleReconnect(generation: number) {- if (!browser || !started || !isCurrentGeneration(generation)) {- return;- }-- if (reconnectAttempt >= MAX_RECONNECT_ATTEMPTS) {- _streamFailed = true;- return;- }-- clearReconnectTimer();- const delay = Math.min(1000 * 2 ** reconnectAttempt, MAX_RECONNECT_DELAY);- reconnectAttempt += 1;- reconnectTimer = setTimeout(() => {- void connectStream(generation);- }, delay);- }-function reconcileEnvironments() {if (!browser || !started) {return;@@ -335,8 +172,8 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,// An already-open aggregated stream only picks new environments// up on its server-side reconcile tick; fetch once so the first// snapshot doesn't take up to that interval to appear.- if (streamAbortController) {- void config.fetchSnapshot(environmentId, streamGeneration);+ if (clientStream.hasActiveStream) {+ void config.fetchSnapshot(environmentId, lifecycleGeneration);}continue;}@@ -362,25 +199,22 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,_environmentStates = value;},get streamConnected(): boolean {- return _streamConnected;- },- set streamConnected(value: boolean) {- _streamConnected = value;+ return clientStream.streamConnected;},get streamFailed(): boolean {- return _streamFailed;+ return clientStream.streamFailed;},get generation(): number {- return streamGeneration;+ return lifecycleGeneration;},get hasActiveStream(): boolean {- return streamAbortController !== null;+ return clientStream.hasActiveStream;},environmentState,updateEnvironmentState,setEnvironmentError,clearEnvironmentError,- isCurrentGeneration,+ isCurrentGeneration: (generation: number) => generation === lifecycleGeneration,reconcileEnvironments,refresh,async start() {@@ -389,18 +223,17 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,}started = true;+ const generation = ++lifecycleGeneration;await environmentStore.ready;- if (!started) {+ if (generation !== lifecycleGeneration) {return;}config.onSelectedEnvironment?.(environmentStore.selected);- watchPageLifecycle();reconcileEnvironments();- const generation = nextGeneration();+ unsubscribeChannel = clientStream.subscribe(config.channel, channelHandlers);if (config.refreshOnStart) {void refresh(generation);}- void connectStream(generation);unsubscribeEnvironment = environmentStore.subscribeSelected((environment) => {config.onSelectedEnvironment?.(environment);reconcileEnvironments();@@ -410,32 +243,24 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,stop(options?: { resetState?: boolean; resetStreamFailed?: boolean }) {const wasStarted = started;started = false;+ lifecycleGeneration += 1;unsubscribeEnvironment?.();unsubscribeEnvironment = null;unsubscribeEnvironmentFilter?.();unsubscribeEnvironmentFilter = null;- removePageLifecycleListeners?.();- removePageLifecycleListeners = null;- nextGeneration();- abortStream();- reconnectAttempt = 0;+ unsubscribeChannel?.();+ unsubscribeChannel = null;if (options?.resetState) {_environmentStates = {};}- if (options?.resetState || options?.resetStreamFailed) {- _streamFailed = false;- }return wasStarted;},retryStream() {- _streamFailed = false;- reconnectAttempt = 0;// Environments may have been added/removed while the stream was down;// reconcile so the new stream's snapshots aren't dropped as unknown.reconcileEnvironments();clearAllEnvironmentErrors();- abortStream();- void connectStream(nextGeneration());+ clientStream.retry();},// Tear down and reopen the stream without touching existing environment// data (e.g. when a flag encoded in the stream URL changes).@@ -443,8 +268,7 @@ export function createEnvironmentStreamStore<TState extends StreamEnvStateBase,// Same as retryStream: pick up environments added since the last// reconcile so their first snapshots aren't discarded.reconcileEnvironments();- abortStream();- void connectStream(nextGeneration());+ clientStream.restart();}};}diff --git a/frontend/src/lib/stores/environment.store.svelte.ts b/frontend/src/lib/stores/environment.store.svelte.tsindex c4c5e69fc..e9c99e244 100644--- a/frontend/src/lib/stores/environment.store.svelte.ts+++ b/frontend/src/lib/stores/environment.store.svelte.ts@@ -148,6 +148,25 @@ function createEnvironmentManagementStore() {}}},+ // Applies a live environment list pushed by the backend stream.+ //+ // Edge liveness is process-local state on the manager, so a list fetched+ // once goes stale the moment an agent reconnects — after a fleet update+ // that leaves every agent stuck rendering as offline. This replaces the+ // records wholesale rather than merging runtime fields, which keeps+ // renames and enable toggles live too. Unlike initialize() it never+ // re-runs initial selection: switching the user's environment out from+ // under them because of a transient blip would be worse than the blip.+ applyLiveEnvironments: (environments: Environment[]) => {+ const available = _updateAvailable(environments);+ if (!_selectedEnvironment) {+ return;+ }+ const updated = available.find((env) => env.id === _selectedEnvironment!.id);+ if (updated) {+ _assignSelectedEnvironment(updated);+ }+ },setEnvironment: async (environment: Environment) => {if (!environment.enabled) return;if (_selectedEnvironment?.id !== environment.id) {diff --git a/frontend/src/lib/stores/json-line-stream.svelte.ts b/frontend/src/lib/stores/json-line-stream.svelte.tsnew file mode 100644index 000000000..f6b8d5471--- /dev/null+++ b/frontend/src/lib/stores/json-line-stream.svelte.ts@@ -0,0 +1,258 @@+import { browser } from '$app/env';++const MAX_RECONNECT_DELAY = 15_000;+const MAX_RECONNECT_ATTEMPTS = 20;++export type JSONLineEventBase = {+ type: string;+};++export interface JSONLineStreamConfig<TEvent extends JSONLineEventBase> {+ /** Used in console warnings, e.g. 'Dashboard' / 'Activity' / 'Environment'. */+ label: string;+ openStream(signal: AbortSignal): Promise<Response>;+ /** Receives every event except 'heartbeat', which is connection state the transport owns. */+ onEvent(event: TEvent): void;+ /** Runs on each successful (re)connect, before any event is delivered. */+ onConnected?(): void;+}++/**+ * The NDJSON transport shared by every aggregate stream: connect, read lines,+ * reconnect with backoff, and tear down explicitly on page hide.+ *+ * It is deliberately free of domain state so the environment, dashboard and+ * activity streams cannot drift apart in how they handle a dropped connection.+ */+export function createJSONLineStream<TEvent extends JSONLineEventBase>(config: JSONLineStreamConfig<TEvent>) {+ let started = false;+ // A single aggregated stream carries every environment's events; per-env+ // connections would multiply requests and exhaust the browser's+ // 6-per-origin HTTP/1.1 limit.+ let streamAbortController: AbortController | null = null;+ let removePageLifecycleListeners: (() => void) | null = null;+ let reconnectTimer: ReturnType<typeof setTimeout> | null = null;+ let reconnectAttempt = 0;+ let streamGeneration = 0;+ let _streamConnected = $state(false);+ let _streamFailed = $state(false);++ function nextGeneration(): number {+ streamGeneration += 1;+ return streamGeneration;+ }++ function isCurrentGeneration(generation: number): boolean {+ return streamGeneration === generation;+ }++ function clearReconnectTimer() {+ if (reconnectTimer) {+ clearTimeout(reconnectTimer);+ reconnectTimer = null;+ }+ }++ function abortStream() {+ clearReconnectTimer();+ streamAbortController?.abort();+ streamAbortController = null;+ _streamConnected = false;+ }++ // Closing a tab leaves teardown to whenever the browser gets around to+ // dropping the socket. Until it does, the server keeps writing heartbeats+ // into a connection nobody is reading — so tear down explicitly instead.+ function watchPageLifecycle() {+ if (!browser || removePageLifecycleListeners) {+ return;+ }++ const onPageHide = () => abortStream();+ // A bfcache restore comes back with the stream already torn down.+ const onPageShow = (event: PageTransitionEvent) => {+ if (event.persisted && started && !streamAbortController) {+ void connectStream(nextGeneration());+ }+ };++ window.addEventListener('pagehide', onPageHide);+ window.addEventListener('pageshow', onPageShow);+ removePageLifecycleListeners = () => {+ window.removeEventListener('pagehide', onPageHide);+ window.removeEventListener('pageshow', onPageShow);+ };+ }++ async function connectStream(generation: number) {+ if (!browser || !isCurrentGeneration(generation)) {+ return;+ }++ // Overlapping connects would otherwise strand the previous controller:+ // abortStream() only ever reaches the newest one, so the older request+ // would keep an open connection the server has no way to notice.+ streamAbortController?.abort();++ const controller = new AbortController();+ streamAbortController = controller;+ try {+ const response = await config.openStream(controller.signal);+ if (!isCurrentGeneration(generation) || !response.body) {+ // The response body is live even though nobody will read it;+ // dropping it on the floor leaves the server streaming into a+ // connection that stays open until its TCP timers expire.+ controller.abort();+ if (streamAbortController === controller) {+ streamAbortController = null;+ }+ return;+ }++ _streamConnected = true;+ _streamFailed = false;+ reconnectAttempt = 0;+ config.onConnected?.();+ await readJSONLines(response.body, generation);+ } catch (error) {+ if (!controller.signal.aborted && isCurrentGeneration(generation)) {+ console.warn(`${config.label} stream disconnected:`, error);+ }+ } finally {+ if (streamAbortController === controller) {+ streamAbortController = null;+ }+ if (isCurrentGeneration(generation)) {+ _streamConnected = false;+ if (!controller.signal.aborted) {+ scheduleReconnect(generation);+ }+ } else if (!controller.signal.aborted) {+ // A superseded generation has no owner left to abort it.+ controller.abort();+ }+ }+ }++ async function readJSONLines(stream: ReadableStream<Uint8Array>, generation: number) {+ const reader = stream.getReader();+ const decoder = new TextDecoder();+ let buffer = '';++ try {+ while (isCurrentGeneration(generation)) {+ const { done, value } = await reader.read();+ if (done) {+ break;+ }++ buffer += decoder.decode(value, { stream: true });+ const lines = buffer.split('\n');+ buffer = lines.pop() ?? '';+ for (const line of lines) {+ handleStreamLine(line);+ }+ }++ buffer += decoder.decode();+ if (buffer.trim()) {+ handleStreamLine(buffer);+ }+ } finally {+ // The loop also exits when the generation advances, with the stream+ // still open. Cancelling is what actually closes the fetch and lets+ // the server see the disconnect; releaseLock alone does not.+ await reader.cancel().catch(() => {});+ reader.releaseLock();+ }+ }++ function handleStreamLine(line: string) {+ const trimmed = line.trim();+ if (!trimmed) {+ return;+ }++ try {+ const event = JSON.parse(trimmed) as TEvent;+ if (event.type === 'heartbeat') {+ _streamConnected = true;+ return;+ }+ config.onEvent(event);+ } catch (error) {+ console.warn(`Failed to parse ${config.label.toLowerCase()} stream line:`, error);+ }+ }++ function scheduleReconnect(generation: number) {+ if (!browser || !started || !isCurrentGeneration(generation)) {+ return;+ }++ if (reconnectAttempt >= MAX_RECONNECT_ATTEMPTS) {+ _streamFailed = true;+ return;+ }++ clearReconnectTimer();+ const delay = Math.min(1000 * 2 ** reconnectAttempt, MAX_RECONNECT_DELAY);+ reconnectAttempt += 1;+ reconnectTimer = setTimeout(() => {+ void connectStream(generation);+ }, delay);+ }++ return {+ get streamConnected(): boolean {+ return _streamConnected;+ },+ set streamConnected(value: boolean) {+ _streamConnected = value;+ },+ get streamFailed(): boolean {+ return _streamFailed;+ },+ get generation(): number {+ return streamGeneration;+ },+ get hasActiveStream(): boolean {+ return streamAbortController !== null;+ },+ get isStarted(): boolean {+ return started;+ },+ isCurrentGeneration,+ nextGeneration,+ /** Opens the stream. Callers that need a generation for their own guards should call nextGeneration() first and pass it. */+ connect(generation: number) {+ void connectStream(generation);+ },+ markStarted() {+ started = true;+ watchPageLifecycle();+ },+ stop(options?: { resetStreamFailed?: boolean }) {+ const wasStarted = started;+ started = false;+ removePageLifecycleListeners?.();+ removePageLifecycleListeners = null;+ nextGeneration();+ abortStream();+ reconnectAttempt = 0;+ if (options?.resetStreamFailed) {+ _streamFailed = false;+ }+ return wasStarted;+ },+ /** Tears the connection down and opens a fresh one under a new generation. */+ restart({ clearFailure = false }: { clearFailure?: boolean } = {}) {+ if (clearFailure) {+ _streamFailed = false;+ reconnectAttempt = 0;+ }+ abortStream();+ void connectStream(nextGeneration());+ }+ };+}diff --git a/frontend/src/routes/(app)/+layout.svelte b/frontend/src/routes/(app)/+layout.svelteindex c5868c7b2..8f5bcd35f 100644--- a/frontend/src/routes/(app)/+layout.svelte+++ b/frontend/src/routes/(app)/+layout.svelte@@ -12,6 +12,7 @@import { getEffectiveLandingPage, getEffectiveNavigationSettings } from '#lib/utils/navigation';import { browser } from '$app/env';import { environmentStore } from '#lib/stores/environment.store.svelte';+ import { environmentStatusStore } from '#lib/stores/environment-status.store.svelte';import { navigationItems, getManagementItems, filterByPermissions, type NavigationItem } from '#lib/config/navigation-config';import { isEditableTarget, matchesShortcutEvent } from '#lib/utils/navigation';import { cn } from '#lib/utils';@@ -49,6 +50,17 @@return flattenNavigationItems(items).filter((item) => item.shortcut?.length);});+ // Environment liveness is pushed for as long as the app shell is mounted, so+ // every surface that reads environmentStore — the board, the environments+ // table, the switcher — sees an agent reconnect without a page load.+ $effect(() => {+ if (!user) {+ return;+ }+ void environmentStatusStore.start();+ return () => environmentStatusStore.stop();+ });+$effect(() => {const redirectPath = getAuthRedirectPath(page.url.pathname,diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex a2d9b48d4..aa7f12a2a 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -28,7 +28,7 @@ overrides:ts-deepmerge: '>=8.0.0'joi: '>=17.13.4'-packageExtensionsChecksum: sha256-Yj0m2AD9TZO/cWup5bCg2l8l7VHtLu5doy9Oybhgv+w=+packageExtensionsChecksum: sha256-apx16oGB3Irw6uCc9mGeizkosuzdg1ZfZQyBC/MQBeU=importers:@@ -4683,6 +4683,7 @@ snapshots:dependencies:+ '@codemirror/autocomplete': 6.20.3'@codemirror/search': 6.7.1codemirror: 6.0.2svelte: 5.56.7diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex 2d6f0ead4..a91a06c5e 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -6,6 +6,7 @@ packages:packageExtensions:dependencies:+ "@codemirror/autocomplete": ^6.20.3"@codemirror/search": ^6.7.0overrides:diff --git a/tests/spec/activity-center.spec.ts b/tests/spec/activity-center.spec.tsindex 886e4e16e..68191816b 100644--- a/tests/spec/activity-center.spec.ts+++ b/tests/spec/activity-center.spec.ts@@ -140,11 +140,21 @@ function aggregatedActivityStreamBody(const timestamp = new Date().toISOString();const events: string[] = [];for (const [environmentId, activities] of Object.entries(activitiesByEnvironment)) {- events.push(JSON.stringify({ type: 'snapshot', environmentId, activities, timestamp }));+ events.push(+ JSON.stringify({+ channel: 'activities',+ activity: { type: 'snapshot', environmentId, activities, timestamp },+ timestamp+ })+ );}for (const environmentId of failedEnvironmentIds) {events.push(- JSON.stringify({ type: 'error', environmentId, error: 'environment unavailable', timestamp })+ JSON.stringify({+ channel: 'activities',+ activity: { type: 'error', environmentId, error: 'environment unavailable', timestamp },+ timestamp+ }));}return events.join('\n') + '\n';@@ -157,11 +167,14 @@ async function mockActivityReads(failedActivityStatus = 503,readEnvironmentIds?: string[]) {- await page.context().route(/\/api\/activities\/stream(?:\?.*)?$/, async (route: Route) => {+ await page.context().route(/\/api\/stream(?:\?.*)?$/, async (route: Route) => {+ const channels = new URL(route.request().url()).searchParams.get('channels')?.split(',') ?? [];await route.fulfill({status: 200,contentType: 'application/x-json-stream',- body: aggregatedActivityStreamBody(activitiesByEnvironment, failedEnvironmentIds)+ body: channels.includes('activities')+ ? aggregatedActivityStreamBody(activitiesByEnvironment, failedEnvironmentIds)+ : ''});});@@ -293,7 +306,12 @@ function activityRow(activityCenter: Locator, text: string) {function waitForActivityStream(page: Page) {return page.waitForResponse((response) => {const url = new URL(response.url());- return response.request().method() === 'GET' && url.pathname === '/api/activities/stream';+ const channels = url.searchParams.get('channels')?.split(',') ?? [];+ return (+ response.request().method() === 'GET' &&+ url.pathname === '/api/stream' &&+ channels.includes('activities')+ );});}@@ -350,7 +368,7 @@ test.describe('Activity Center', () => {});test('does not mount the activity center without an effective read scope', async ({ page }) => {- const streamRequests: string[] = [];+ const streamChannels: string[][] = [];await preserveLocalEnvironmentSelection(page);await mockCurrentUser(page, () =>@@ -361,16 +379,73 @@ test.describe('Activity Center', () => {);await mockEnvironmentList(page, [localEnvironment]);page.on('request', (request) => {- if (new URL(request.url()).pathname === '/api/activities/stream') {- streamRequests.push(request.url());+ const url = new URL(request.url());+ if (url.pathname === '/api/stream') {+ streamChannels.push(url.searchParams.get('channels')?.split(',') ?? []);}});await page.goto('/dashboard');await expect(page.getByRole('heading', { name: 'Environment Board' })).toBeVisible();await expect(page.getByRole('button', { name: 'Open activity center' })).toHaveCount(0);- await page.waitForTimeout(250);- expect(streamRequests).toHaveLength(0);+ await expect+ .poll(() => streamChannels.some((channels) => channels.includes('dashboard')))+ .toBe(true);+ expect(streamChannels.some((channels) => channels.includes('activities'))).toBe(false);+ });++ test('keeps an in-flight activity snapshot when the dashboard channel restarts the stream', async ({+ page+ }) => {+ const snapshotActivity = activity('delayed-activity', '0', 'Local', 'delayed-network', 1);+ let markSnapshotRequested!: () => void;+ const snapshotRequested = new Promise<void>((resolve) => {+ markSnapshotRequested = resolve;+ });+ let releaseSnapshot!: () => void;+ const snapshotReleased = new Promise<void>((resolve) => {+ releaseSnapshot = resolve;+ });++ await preserveLocalEnvironmentSelection(page);+ await mockCurrentUser(page, () => user('snapshot-generation-user', { global: ['*'] }));+ await mockEnvironmentList(page, [localEnvironment]);+ await page.context().route(/\/api\/stream(?:\?.*)?$/, async (route) => {+ await route.fulfill({ status: 503, contentType: 'application/json', body: '{}' });+ });+ await page.context().route(/\/api\/environments\/0\/activities(?:\?.*)?$/, async (route) => {+ markSnapshotRequested();+ await snapshotReleased;+ await route.fulfill({+ status: 200,+ contentType: 'application/json',+ body: JSON.stringify(paginated([snapshotActivity]))+ });+ });++ await page.goto('/containers');+ await snapshotRequested;+ const dashboardStream = page.waitForRequest((request) => {+ const url = new URL(request.url());+ const channels = url.searchParams.get('channels')?.split(',') ?? [];+ return (+ url.pathname === '/api/stream' &&+ channels.includes('activities') &&+ channels.includes('dashboard')+ );+ });+ try {+ await Promise.all([+ dashboardStream,+ page.getByRole('link', { name: 'Dashboard', exact: true }).click()+ ]);+ releaseSnapshot();++ const activityCenter = await openActivityCenter(page);+ await expect(activityRow(activityCenter, 'delayed-network')).toBeVisible();+ } finally {+ releaseSnapshot();+ }});test('clears activity state when the authenticated user changes', async ({ page }) => {@@ -381,17 +456,23 @@ test.describe('Activity Center', () => {await preserveLocalEnvironmentSelection(page);await mockCurrentUser(page, () => user(activeUserId, permissions));await mockEnvironmentList(page, [localEnvironment]);- await page.context().route(/\/api\/activities\/stream(?:\?.*)?$/, async (route) => {+ await page.context().route(/\/api\/stream(?:\?.*)?$/, async (route) => {+ const channels =+ new URL(route.request().url()).searchParams.get('channels')?.split(',') ?? [];const requestedBy = activeUserId;- streamedUsers.push(requestedBy);+ if (channels.includes('activities')) {+ streamedUsers.push(requestedBy);+ }const resourceName = requestedBy === 'user-a' ? 'user-a-private-activity' : 'user-b-activity';await route.fulfill({status: 200,contentType: 'application/x-json-stream',- body: aggregatedActivityStreamBody(- { '0': [activity(`${requestedBy}-activity`, '0', 'Local', resourceName, 1)] },- new Set()- )+ body: channels.includes('activities')+ ? aggregatedActivityStreamBody(+ { '0': [activity(`${requestedBy}-activity`, '0', 'Local', resourceName, 1)] },+ new Set()+ )+ : ''});});await page.context().route(/\/api\/environments\/0\/activities(?:\?.*)?$/, async (route) => {diff --git a/tests/spec/dashboard-system-stats.spec.ts b/tests/spec/dashboard-system-stats.spec.tsindex 1444306fc..885e8f645 100644--- a/tests/spec/dashboard-system-stats.spec.ts+++ b/tests/spec/dashboard-system-stats.spec.ts@@ -136,7 +136,11 @@ function collectDashboardRequestPaths(page: Page): string[] {page.on('request', (request) => {const pathname = new URL(request.url()).pathname;- if (pathname.startsWith('/api/environments/') || pathname.startsWith('/api/dashboard/')) {+ if (+ pathname === '/api/stream' ||+ pathname.startsWith('/api/environments/') ||+ pathname.startsWith('/api/dashboard/')+ ) {requestPaths.push(pathname);}});@@ -173,7 +177,7 @@ test.describe('Dashboard system stats websocket', () => {await page.waitForLoadState('load');await expect(page.getByRole('heading', { name: 'Environment Board' })).toBeVisible();- await expect.poll(() => requestPaths).toContain('/api/dashboard/stream');+ await expect.poll(() => requestPaths).toContain('/api/stream');expect(countMatchingRequests(requestPaths, /\/api\/environments\/[^/]+\/system\/docker\/info$/)diff --git a/tests/spec/environment-switch-isolation.spec.ts b/tests/spec/environment-switch-isolation.spec.tsindex a22a61a23..f6a46eb9a 100644--- a/tests/spec/environment-switch-isolation.spec.ts+++ b/tests/spec/environment-switch-isolation.spec.ts@@ -90,6 +90,25 @@ async function mockEnvironmentCatalog(page: Page) {body: JSON.stringify(paginated([localEnvironment, remoteEnvironment]))});});+ await page.context().route(/\/api\/stream(?:\?.*)?$/, async (route) => {+ const channels = new URL(route.request().url()).searchParams.get('channels')?.split(',') ?? [];+ const timestamp = new Date().toISOString();+ await route.fulfill({+ status: 200,+ contentType: 'application/x-json-stream',+ body: channels.includes('environments')+ ? `${JSON.stringify({+ channel: 'environments',+ environment: {+ type: 'snapshot',+ environments: [localEnvironment, remoteEnvironment],+ timestamp+ },+ timestamp+ })}\n`+ : ''+ });+ });await page.context().route(/\/api\/environments\/(?:0|remote-switch-test)\/settings$/, async (route) => {diff --git a/tests/spec/system-upgrade.spec.ts b/tests/spec/system-upgrade.spec.tsindex d10632e3e..165119701 100644--- a/tests/spec/system-upgrade.spec.ts+++ b/tests/spec/system-upgrade.spec.ts@@ -156,16 +156,20 @@ test.describe('Manager self-update recovery', () => {const snapshot = dashboardSnapshot();let upgradeTriggered = false;- await page.route(/\/api\/dashboard\/stream\?/, async (route) => {+ await page.route(/\/api\/stream(?:\?.*)?$/, async (route) => {+ const channels =+ new URL(route.request().url()).searchParams.get('channels')?.split(',') ?? [];+ const timestamp = new Date().toISOString();await route.fulfill({status: 200,contentType: 'application/x-json-stream',- body: `${JSON.stringify({- type: 'snapshot',- environmentId: '0',- snapshot,- timestamp: new Date().toISOString()- })}\n`+ body: channels.includes('dashboard')+ ? `${JSON.stringify({+ channel: 'dashboard',+ dashboard: { type: 'snapshot', environmentId: '0', snapshot, timestamp },+ timestamp+ })}\n`+ : ''});});await page.route(/\/api\/environments\/0\/dashboard(?:\?.*)?$/, async (route) => {@@ -201,8 +205,15 @@ test.describe('Manager self-update recovery', () => {});});+ const dashboardUpgradeCheck = page.waitForResponse((response) => {+ return (+ new URL(response.url()).pathname === '/api/environments/0/system/upgrade/check' &&+ response.status() === 200+ );+ });await page.goto('/dashboard');- const versionBadge = page.getByText('v1.0.0', { exact: true }).first();+ await dashboardUpgradeCheck;+ const versionBadge = page.locator('main').getByRole('button', { name: 'v1.0.0', exact: true });await expect(versionBadge).toBeVisible();await versionBadge.hover();const upgradeButton = page.getByRole('button', { name: 'Update to 2.0.0', exact: true });@@ -219,17 +230,32 @@ test.describe('Manager self-update recovery', () => {await registerReloadCounter(page);await registerTokenSeeding(page);- let releaseActivity: () => void = () => {};+ let releaseActivity!: () => void;const activityRelease = new Promise<void>((resolve) => {releaseActivity = resolve;});- await page.route(/\/api\/activities\/stream\?/, async (route) => {+ await page.route(/\/api\/stream(?:\?.*)?$/, async (route) => {+ const channels =+ new URL(route.request().url()).searchParams.get('channels')?.split(',') ?? [];+ if (!channels.includes('activities')) {+ await route.fulfill({+ status: 200,+ contentType: 'application/x-json-stream',+ body: ''+ });+ return;+ }await activityRelease;if (route.request().headers()['cookie']?.includes(REFRESH_COOKIE)) {+ const timestamp = new Date().toISOString();await route.fulfill({status: 200,contentType: 'application/x-json-stream',- body: `${JSON.stringify({ type: 'snapshot', activities: [] })}\n`+ body: `${JSON.stringify({+ channel: 'activities',+ activity: { type: 'snapshot', environmentId: '0', activities: [], timestamp },+ timestamp+ })}\n`});return;}diff --git a/types/environment/environment.go b/types/environment/environment.goindex 82601a19b..363090cf8 100644--- a/types/environment/environment.go+++ b/types/environment/environment.go@@ -212,6 +212,28 @@ type Environment struct {ApiKey *string `json:"apiKey,omitempty"`}+// StreamEvent is one line of the environment stream. A "snapshot" event carries+// the caller's full visible environment list with the manager's runtime overlay+// already applied, so clients replace their cached list wholesale rather than+// merging field by field — which also keeps renames and enable toggles live,+// not just connectivity.+type StreamEvent struct {+ // Type is the event kind: "snapshot" or "heartbeat".+ //+ // Required: true+ Type string `json:"type"`++ // Environments is the caller's visible environment list for a "snapshot".+ //+ // Required: false+ Environments []Environment `json:"environments,omitempty"`++ // Timestamp is when the event was produced.+ //+ // Required: true+ Timestamp time.Time `json:"timestamp"`+}+// AgentPairRequest is the request body for pairing with an agent.type AgentPairRequest struct {// Rotate indicates if the token should be rotated.diff --git a/types/stream/stream.go b/types/stream/stream.gonew file mode 100644index 000000000..86c271931--- /dev/null+++ b/types/stream/stream.go@@ -0,0 +1,61 @@+// Package stream defines the multiplexed client stream: one connection that+// carries every live channel the caller asked for.+//+// Browsers cap concurrent HTTP/1.1 requests per origin at six, and every+// long-lived stream permanently consumes one of them. Multiplexing keeps that+// budget for ordinary requests, and gives the client a single reconnect and+// heartbeat to reason about instead of one per feature.+package stream++import (+ "time"++ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ dashboardtypes "github.com/getarcaneapp/arcane/types/v2/dashboard"+ environmenttypes "github.com/getarcaneapp/arcane/types/v2/environment"+)++// Channel names a subscribable feed. Callers request channels explicitly+// because they differ enormously in cost: environments is one local query,+// while dashboard proxies a request to every remote agent on every tick.+const (+ ChannelEnvironments = "environments"+ ChannelDashboard = "dashboard"+ ChannelActivities = "activities"+)++// Event is one line of the multiplexed stream. Exactly one payload is set, and+// each payload keeps its own channel's event type rather than being flattened+// here, so a channel can add a field without this envelope changing.+type Event struct {+ // Channel identifies which feed produced this event. Empty on a heartbeat.+ //+ // Required: false+ Channel string `json:"channel,omitempty"`++ // Type is set to "heartbeat" for connection keepalives; otherwise the+ // per-channel payload carries its own type.+ //+ // Required: false+ Type string `json:"type,omitempty"`++ // Dashboard carries a ChannelDashboard payload.+ //+ // Required: false+ Dashboard *dashboardtypes.StreamEvent `json:"dashboard,omitempty"`++ // Activity carries a ChannelActivities payload.+ //+ // Required: false+ Activity *activitytypes.StreamEvent `json:"activity,omitempty"`++ // Environment carries a ChannelEnvironments payload.+ //+ // Required: false+ Environment *environmenttypes.StreamEvent `json:"environment,omitempty"`++ // Timestamp is when the envelope was produced.+ //+ // Required: true+ Timestamp time.Time `json:"timestamp"`+}