Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: activities stream using main context causing app to hang at certain places (#2887)
Kyle Mendellb7d7c20b3810
diff --git a/backend/api/handlers/activities.go b/backend/api/handlers/activities.goindex 0010127eb..40381f319 100644--- a/backend/api/handlers/activities.go+++ b/backend/api/handlers/activities.go@@ -5,10 +5,12 @@ import ("encoding/json""errors""fmt"+ "log/slog""net/http""net/url""strconv""strings"+ "sync""time""github.com/danielgtaylor/huma/v2"@@ -61,11 +63,18 @@ type ClearActivityHistoryOutput struct {Body base.ApiResponse[activity.ClearHistoryResult]}-type StreamActivitiesInput struct {- EnvironmentID string `path:"id" doc:"Environment ID"`- Limit int `query:"limit" default:"50" doc:"Initial snapshot limit"`+type StreamAllActivitiesInput struct {+ Limit int `query:"limit" default:"50" doc:"Snapshot limit per environment"`}+const (+ activityStreamHeartbeatInterval = 15 * time.Second+ activityStreamRemotePollInterval = 5 * time.Second+ activityStreamEnvReconcileInterval = 30 * time.Second+ activityStreamRemotePollTimeout = 15 * time.Second+ activityStreamEventBuffer = 256+)+type CancelActivityInput struct {EnvironmentID string `path:"id" doc:"Environment ID"`ActivityID string `path:"activityId" doc:"Activity ID"`@@ -111,18 +120,18 @@ func RegisterActivities(api huma.API, activityService *services.ActivityService,}, h.GetActivity)huma.Register(api, huma.Operation{- OperationID: "stream-activities",+ OperationID: "stream-all-activities",Method: http.MethodGet,- Path: "/environments/{id}/activities/stream",- Summary: "Stream background activities",- Description: "Stream background activity updates as JSON lines",+ Path: "/activities/stream",+ Summary: "Stream background activities across all environments",+ Description: "Stream background activity updates for the local environment and all enabled remote environments as JSON lines",Tags: []string{"Activities"},Security: []map[string][]string{{"BearerAuth": {}},{"ApiKeyAuth": {}},},Middlewares: humamw.RequirePermission(api, authz.PermActivitiesRead),- }, h.StreamActivities)+ }, h.StreamAllActivities)huma.Register(api, huma.Operation{OperationID: "cancel-activity",@@ -221,95 +230,6 @@ func (h *ActivityHandler) GetActivity(ctx context.Context, input *GetActivityInp}, nil}-func (h *ActivityHandler) StreamActivities(ctx context.Context, input *StreamActivitiesInput) (*huma.StreamResponse, error) {- if h.activityService == nil {- return nil, huma.Error500InternalServerError("service not available")- }-- return &huma.StreamResponse{- Body: func(humaCtx huma.Context) { //nolint:contextcheck // streaming work must use humaCtx.Context()- httpx.SetJSONStreamHeaders(humaCtx)-- writer := humaCtx.BodyWriter()- encoder := json.NewEncoder(writer)- flush := func() {- if f, ok := writer.(http.Flusher); ok {- f.Flush()- }- }-- if input.EnvironmentID != "0" {- h.streamRemoteActivitySnapshotsInternal(humaCtx.Context(), input, encoder, flush)- return- }-- h.streamLocalActivitiesInternal(humaCtx.Context(), input, encoder, flush)- },- }, nil-}--func (h *ActivityHandler) streamLocalActivitiesInternal(- ctx context.Context,- input *StreamActivitiesInput,- encoder *json.Encoder,- flush func(),-) {- sendSnapshot := func() bool {- activities, _, err := h.activityService.ListActivitiesPaginated(ctx, input.EnvironmentID, pagination.QueryParams{- Params: pagination.Params{Limit: resolveActivityStreamLimitInternal(input.Limit)},- })- if err != nil {- return false- }- h.applyActivitySourceLabelsInternal(ctx, input.EnvironmentID, activities)- if err := encoder.Encode(activity.StreamEvent{- Type: "snapshot",- Activities: activities,- Timestamp: time.Now(),- }); err != nil {- return false- }- flush()- return true- }- if !sendSnapshot() {- return- }-- events, missedEvents, unsubscribe := h.activityService.Subscribe(input.EnvironmentID)- defer unsubscribe()-- ticker := time.NewTicker(15 * time.Second)- defer ticker.Stop()-- for {- select {- case <-ctx.Done():- return- case event, ok := <-events:- if !ok {- return- }- h.applyActivityStreamEventSourceLabelInternal(ctx, input.EnvironmentID, &event)- if err := encoder.Encode(event); err != nil {- return- }- flush()- case <-ticker.C:- if missedEvents() && !sendSnapshot() {- return- }- if err := encoder.Encode(activity.StreamEvent{- Type: "heartbeat",- Timestamp: time.Now(),- }); err != nil {- return- }- flush()- }- }-}-func (h *ActivityHandler) ClearHistory(ctx context.Context, input *ClearActivityHistoryInput) (*ClearActivityHistoryOutput, error) {if input.EnvironmentID != "0" {return h.proxyClearHistoryInternal(ctx, input)@@ -395,58 +315,246 @@ func (h *ActivityHandler) cancelRequestedByInternal(ctx context.Context, forwardreturn strings.TrimSpace(forwarded)}-func (h *ActivityHandler) streamRemoteActivitySnapshotsInternal(- ctx context.Context,- input *StreamActivitiesInput,- encoder *json.Encoder,- flush func(),-) {- pollTicker := time.NewTicker(5 * time.Second)- defer pollTicker.Stop()- heartbeatTicker := time.NewTicker(15 * time.Second)- defer heartbeatTicker.Stop()-- sendSnapshot := func(ctx context.Context) bool {- output, err := h.proxyListActivitiesInternal(ctx, &ListActivitiesInput{- EnvironmentID: input.EnvironmentID,- Start: 0,- Limit: resolveActivityStreamLimitInternal(input.Limit),- Order: "desc",+func (h *ActivityHandler) StreamAllActivities(ctx context.Context, input *StreamAllActivitiesInput) (*huma.StreamResponse, error) {+ if h.activityService == nil || h.environmentService == nil {+ return nil, huma.Error500InternalServerError("service not available")+ }++ return &huma.StreamResponse{+ Body: func(humaCtx huma.Context) { //nolint:contextcheck // streaming work must use humaCtx.Context()+ httpx.SetJSONStreamHeaders(humaCtx)++ writer := humaCtx.BodyWriter()+ encoder := json.NewEncoder(writer)+ flush := func() {+ if f, ok := writer.(http.Flusher); ok {+ f.Flush()+ }+ }++ h.streamAllActivitiesInternal(humaCtx.Context(), input.Limit, encoder, flush)+ },+ }, nil+}++// streamAllActivitiesInternal multiplexes activity events for the local+// environment and every enabled remote environment over a single response so+// the browser needs one connection regardless of environment count.+func (h *ActivityHandler) streamAllActivitiesInternal(ctx context.Context, limit int, encoder *json.Encoder, flush func()) {+ streamCtx, cancel := context.WithCancel(ctx)++ events := make(chan activity.StreamEvent, activityStreamEventBuffer)+ var wg sync.WaitGroup+ wg.Add(2)+ go func() {+ defer wg.Done()+ h.runLocalActivityStreamProducerInternal(streamCtx, limit, events)+ }()+ go func() {+ defer wg.Done()+ h.runRemoteActivityStreamPollersInternal(streamCtx, limit, events)+ }()+ defer wg.Wait()+ defer cancel()++ heartbeat := time.NewTicker(activityStreamHeartbeatInterval)+ defer heartbeat.Stop()++ for {+ select {+ case <-streamCtx.Done():+ return+ case event := <-events:+ if err := encoder.Encode(event); err != nil {+ return+ }+ flush()+ case <-heartbeat.C:+ if err := encoder.Encode(activity.StreamEvent{+ Type: "heartbeat",+ Timestamp: time.Now(),+ }); err != nil {+ return+ }+ flush()+ }+ }+}++// sendActivityStreamEventInternal forwards an event to the stream writer,+// giving up when the stream is shutting down so producers can never block.+func sendActivityStreamEventInternal(ctx context.Context, events chan<- activity.StreamEvent, event activity.StreamEvent) bool {+ select {+ case events <- event:+ return true+ case <-ctx.Done():+ return false+ }+}++func (h *ActivityHandler) runLocalActivityStreamProducerInternal(ctx context.Context, limit int, events chan<- activity.StreamEvent) {+ sendSnapshot := func() bool {+ activities, _, err := h.activityService.ListActivitiesPaginated(ctx, "0", pagination.QueryParams{+ Params: pagination.Params{Limit: resolveActivityStreamLimitInternal(limit)},})if err != nil {+ if ctx.Err() == nil {+ sendActivityStreamEventInternal(ctx, events, activity.StreamEvent{+ Type: "error",+ EnvironmentID: "0",+ Error: err.Error(),+ Timestamp: time.Now(),+ })+ }return false}- if err := encoder.Encode(activity.StreamEvent{- Type: "snapshot",- Activities: output.Body.Data,- Timestamp: time.Now(),- }); err != nil {- return false+ h.applyActivitySourceLabelsInternal(ctx, "0", activities)+ return sendActivityStreamEventInternal(ctx, events, activity.StreamEvent{+ Type: "snapshot",+ EnvironmentID: "0",+ Activities: activities,+ Timestamp: time.Now(),+ })+ }++ snapshotOK := sendSnapshot()++ eventCh, missedEvents, unsubscribe := h.activityService.Subscribe("0")+ defer unsubscribe()++ ticker := time.NewTicker(activityStreamHeartbeatInterval)+ defer ticker.Stop()++ for {+ select {+ case <-ctx.Done():+ return+ case event, ok := <-eventCh:+ if !ok {+ return+ }+ event.EnvironmentID = "0"+ h.applyActivityStreamEventSourceLabelInternal(ctx, "0", &event)+ if !sendActivityStreamEventInternal(ctx, events, event) {+ return+ }+ case <-ticker.C:+ if !snapshotOK || missedEvents() {+ snapshotOK = sendSnapshot()+ }}- flush()- return true}+}- if !sendSnapshot(ctx) {- return+// runRemoteActivityStreamPollersInternal keeps one poller goroutine per+// enabled remote environment, re-listing periodically so environments added+// or removed while the stream is open are picked up without a reconnect.+func (h *ActivityHandler) runRemoteActivityStreamPollersInternal(ctx context.Context, limit int, events chan<- activity.StreamEvent) {+ pollers := make(map[string]context.CancelFunc)+ var wg sync.WaitGroup+ defer wg.Wait()+ defer func() {+ for _, cancelPoll := range pollers {+ cancelPoll()+ }+ }()++ reconcile := func() {+ envs, err := h.environmentService.ListRemoteEnvironments(ctx)+ if err != nil {+ if ctx.Err() == nil {+ slog.WarnContext(ctx, "failed to list environments for activity stream", "error", err)+ }+ return+ }++ current := make(map[string]struct{}, len(envs))+ for _, env := range envs {+ current[env.ID] = struct{}{}+ if _, ok := pollers[env.ID]; ok {+ continue+ }+ pollCtx, cancelPoll := context.WithCancel(ctx) //nolint:gosec // cancel is retained in pollers and invoked on env removal or via the deferred cleanup.+ pollers[env.ID] = cancelPoll+ wg.Add(1)+ go func(environmentID string) {+ defer wg.Done()+ h.runRemoteActivityStreamPollerInternal(pollCtx, environmentID, limit, events)+ }(env.ID)+ }++ for id, cancelPoll := range pollers {+ if _, ok := current[id]; !ok {+ cancelPoll()+ delete(pollers, id)+ }+ }}+ reconcile()++ ticker := time.NewTicker(activityStreamEnvReconcileInterval)+ defer ticker.Stop()+for {select {case <-ctx.Done():return- case <-pollTicker.C:- if !sendSnapshot(ctx) {+ case <-ticker.C:+ reconcile()+ }+ }+}++func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Context, environmentID string, limit int, events chan<- activity.StreamEvent) {+ lastError := ""++ poll := func() {+ pollCtx, cancelPoll := context.WithTimeout(ctx, activityStreamRemotePollTimeout)+ defer cancelPoll()++ output, err := h.proxyListActivitiesInternal(pollCtx, &ListActivitiesInput{+ EnvironmentID: environmentID,+ Limit: resolveActivityStreamLimitInternal(limit),+ Order: "desc",+ })+ if err != nil {+ if ctx.Err() != nil {return}- case <-heartbeatTicker.C:- if err := encoder.Encode(activity.StreamEvent{- Type: "heartbeat",- Timestamp: time.Now(),- }); err != nil {- return+ // A failing environment must not end the stream; surface the error+ // once per distinct message and keep polling.+ if msg := err.Error(); msg != lastError {+ lastError = msg+ sendActivityStreamEventInternal(ctx, events, activity.StreamEvent{+ Type: "error",+ EnvironmentID: environmentID,+ Error: msg,+ Timestamp: time.Now(),+ })}- flush()+ return+ }+ lastError = ""+ sendActivityStreamEventInternal(ctx, events, activity.StreamEvent{+ Type: "snapshot",+ EnvironmentID: environmentID,+ Activities: output.Body.Data,+ Timestamp: time.Now(),+ })+ }++ poll()++ ticker := time.NewTicker(activityStreamRemotePollInterval)+ defer ticker.Stop()++ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ poll()}}}diff --git a/backend/api/handlers/activities_test.go b/backend/api/handlers/activities_test.goindex 1b042963a..295671b50 100644--- a/backend/api/handlers/activities_test.go+++ b/backend/api/handlers/activities_test.go@@ -1,7 +1,10 @@package handlersimport (+ "bufio""context"+ "encoding/json"+ "io""net/http""net/http/httptest""testing"@@ -14,6 +17,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/models""github.com/getarcaneapp/arcane/backend/v2/internal/services"+ "github.com/getarcaneapp/arcane/types/v2/activity")func setupActivityHandlerTestDBInternal(t *testing.T) *database.DB {@@ -97,3 +101,145 @@ func TestActivityHandlerClearHistoryProxiesRemoteEnvironmentInternal(t *testing.require.NoError(t, err)require.EqualValues(t, 7, out.Body.Data.Deleted)}++// limitStreamTestDBToSingleConnInternal serializes DB access: the aggregated+// stream queries from concurrent goroutines, and every extra pooled+// connection to a :memory: SQLite database is a fresh empty database.+func limitStreamTestDBToSingleConnInternal(t *testing.T, db *database.DB) {+ t.Helper()+ sqlDB, err := db.DB.DB()+ require.NoError(t, err)+ sqlDB.SetMaxOpenConns(1)+}++func createStreamTestRemoteEnvironmentInternal(t *testing.T, db *database.DB, apiURL, token string) {+ t.Helper()+ now := time.Now()+ require.NoError(t, db.Create(&models.Environment{+ BaseModel: models.BaseModel{+ ID: "remote-1",+ CreatedAt: now,+ UpdatedAt: &now,+ },+ Name: "Remote",+ ApiUrl: apiURL,+ Status: string(models.EnvironmentStatusOnline),+ Enabled: true,+ AccessToken: &token,+ }).Error)+}++// runStreamAllInternal drives streamAllActivitiesInternal through a pipe and+// returns each decoded event to onEvent until it reports done or the stream+// ends; remaining output is drained so a blocked encoder can always finish.+func runStreamAllInternal(t *testing.T, ctx context.Context, cancel context.CancelFunc, handler *ActivityHandler, onEvent func(activity.StreamEvent) bool) {+ t.Helper()++ pr, pw := io.Pipe()+ done := make(chan struct{})+ go func() {+ defer close(done)+ defer pw.Close()+ handler.streamAllActivitiesInternal(ctx, 50, json.NewEncoder(pw), func() {})+ }()++ scanner := bufio.NewScanner(pr)+ for scanner.Scan() {+ var event activity.StreamEvent+ require.NoError(t, json.Unmarshal(scanner.Bytes(), &event))+ if onEvent(event) {+ cancel()+ break+ }+ }++ go func() {+ _, _ = io.Copy(io.Discard, pr)+ }()+ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("stream did not terminate after cancel")+ }+}++func TestActivityHandlerStreamAllEmitsEnvironmentScopedEventsInternal(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()++ db := setupActivityHandlerTestDBInternal(t)+ limitStreamTestDBToSingleConnInternal(t, db)+ settingsService, err := services.NewSettingsService(ctx, db)+ require.NoError(t, err)+ activityService := services.NewActivityService(db)++ local, err := activityService.StartActivity(ctx, services.StartActivityRequest{EnvironmentID: "0", Type: models.ActivityTypeResourceAction})+ require.NoError(t, err)++ token := "remote-token"+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"success":true,"data":[{"id":"remote-activity-1"}],"pagination":{"totalPages":1,"totalItems":1,"currentPage":1,"itemsPerPage":50}}`))+ }))+ defer server.Close()+ createStreamTestRemoteEnvironmentInternal(t, db, server.URL, token)++ handler := &ActivityHandler{+ activityService: activityService,+ environmentService: services.NewEnvironmentService(db, server.Client(), nil, nil, settingsService, nil),+ }++ var localSnapshot, remoteSnapshot bool+ runStreamAllInternal(t, ctx, cancel, handler, func(event activity.StreamEvent) bool {+ if event.Type == "snapshot" && event.EnvironmentID == "0" && len(event.Activities) == 1 {+ require.Equal(t, local.ID, event.Activities[0].ID)+ require.Equal(t, "0", event.Activities[0].SourceEnvironmentID)+ localSnapshot = true+ }+ if event.Type == "snapshot" && event.EnvironmentID == "remote-1" && len(event.Activities) == 1 {+ require.Equal(t, "remote-activity-1", event.Activities[0].ID)+ remoteSnapshot = true+ }+ return localSnapshot && remoteSnapshot+ })++ require.True(t, localSnapshot)+ require.True(t, remoteSnapshot)+}++func TestActivityHandlerStreamAllRemoteFailureEmitsErrorAndKeepsStreamingInternal(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()++ db := setupActivityHandlerTestDBInternal(t)+ limitStreamTestDBToSingleConnInternal(t, db)+ settingsService, err := services.NewSettingsService(ctx, db)+ require.NoError(t, err)+ activityService := services.NewActivityService(db)++ token := "remote-token"+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ http.Error(w, "boom", http.StatusInternalServerError)+ }))+ defer server.Close()+ createStreamTestRemoteEnvironmentInternal(t, db, server.URL, token)++ handler := &ActivityHandler{+ activityService: activityService,+ environmentService: services.NewEnvironmentService(db, server.Client(), nil, nil, settingsService, nil),+ }++ var localSnapshot, remoteError bool+ runStreamAllInternal(t, ctx, cancel, handler, func(event activity.StreamEvent) bool {+ if event.Type == "snapshot" && event.EnvironmentID == "0" {+ localSnapshot = true+ }+ if event.Type == "error" && event.EnvironmentID == "remote-1" && event.Error != "" {+ remoteError = true+ }+ return localSnapshot && remoteError+ })++ require.True(t, localSnapshot)+ require.True(t, remoteError)+}diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.goindex 2374c2e5e..3c329346c 100644--- a/backend/internal/bootstrap/bootstrap.go+++ b/backend/internal/bootstrap/bootstrap.go@@ -6,6 +6,7 @@ import ("fmt""io""log/slog"+ "net""net/http""os""os/signal"@@ -377,7 +378,14 @@ func runServicesInternal(appCtx context.Context, cfg *config.Config, router httphttpHandler, grpcServer := configureTunnelServerInternal(appCtx, cfg, router, tunnelServer, listenAddr)httpHandler, protocols := configureHTTPProtocolsInternal(useTLS, httpHandler)- srv, err := newHTTPServerInternal(listenAddr, httpHandler, protocols, useTLS, edgeCfg)+ // Base context for all request contexts. Derived from Background, NOT+ // appCtx: appCtx carries the app-lifecycle marker value and inheriting it+ // would make every request context pass utils.IsAppLifecycleContext (see+ // pkg/projects/cmds.go).+ baseCtx, cancelBase := context.WithCancel(context.Background())+ defer cancelBase()++ srv, err := newHTTPServerInternal(baseCtx, listenAddr, httpHandler, protocols, useTLS, edgeCfg) //nolint:contextcheck // baseCtx is deliberately not derived from appCtx, see comment aboveif err != nil {return err}@@ -407,6 +415,12 @@ func runServicesInternal(appCtx context.Context, cfg *config.Config, router httpslog.InfoContext(appCtx, "Context canceled")}+ // http.Server.Shutdown waits for in-flight handlers but does not cancel+ // their request contexts; streaming handlers (activity streams, JSON-lines+ // progress) loop on ctx.Done() and would otherwise pin Shutdown until the+ // deadline below.+ cancelBase()+// Use background context for shutdown as appCtx is already canceledshutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)defer shutdownCancel()@@ -489,7 +503,7 @@ func configureHTTPProtocolsInternal(useTLS bool, handler http.Handler) (http.Hanreturn handler, &protocols}-func newHTTPServerInternal(listenAddr string, handler http.Handler, protocols *http.Protocols, useTLS bool, edgeCfg *edge.Config) (*http.Server, error) {+func newHTTPServerInternal(baseCtx context.Context, listenAddr string, handler http.Handler, protocols *http.Protocols, useTLS bool, edgeCfg *edge.Config) (*http.Server, error) {srv := &http.Server{Addr: listenAddr,Handler: handler,@@ -501,6 +515,10 @@ func newHTTPServerInternal(listenAddr string, handler http.Handler, protocols *h// streaming endpoints (deploy/pull/build progress) need long-lived// connections.IdleTimeout: 120 * time.Second,+ // BaseContext is canceled right before Shutdown so long-lived+ // streaming request contexts unblock and graceful shutdown can+ // complete within its deadline.+ BaseContext: func(net.Listener) context.Context { return baseCtx },}if !useTLS {return srv, nildiff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.goindex 277e1c32f..159fe3bcb 100644--- a/backend/internal/bootstrap/bootstrap_test.go+++ b/backend/internal/bootstrap/bootstrap_test.go@@ -212,6 +212,47 @@ func TestHTTP2APIResponsesDoNotUseAPIGzipInternal(t *testing.T) {}}+func TestShutdownCancelsStreamingRequestContextsInternal(t *testing.T) {+ baseCtx, cancelBase := context.WithCancel(context.Background())+ defer cancelBase()++ handlerEntered := make(chan struct{})+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/x-json-stream")+ w.WriteHeader(http.StatusOK)+ if f, ok := w.(http.Flusher); ok {+ f.Flush()+ }+ close(handlerEntered)+ // Streaming handlers block until their request context ends; Shutdown+ // alone never cancels it, so this models an open activity stream.+ <-r.Context().Done()+ })++ srv, err := newHTTPServerInternal(baseCtx, "127.0.0.1:0", handler, nil, false, nil)+ require.NoError(t, err)++ listener, err := net.Listen("tcp", "127.0.0.1:0")+ require.NoError(t, err)++ errCh := make(chan error, 1)+ go func() { errCh <- srv.Serve(listener) }()++ resp, err := http.Get("http://" + listener.Addr().String() + "/stream")+ require.NoError(t, err)+ defer func() { _ = resp.Body.Close() }()+ <-handlerEntered++ cancelBase()++ shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)+ defer cancel()+ start := time.Now()+ require.NoError(t, srv.Shutdown(shutdownCtx))+ require.Less(t, time.Since(start), time.Second)+ require.ErrorIs(t, <-errCh, http.ErrServerClosed)+}+func TestPrepareServerTLSInternal_AgentModeSkipsManagerMTLSValidation(t *testing.T) {cfg := &config.Config{AgentMode: true,diff --git a/backend/internal/middleware/environment_middleware_test.go b/backend/internal/middleware/environment_middleware_test.goindex a61bd689c..6a84730a8 100644--- a/backend/internal/middleware/environment_middleware_test.go+++ b/backend/internal/middleware/environment_middleware_test.go@@ -157,12 +157,6 @@ func TestEnvironmentMiddleware_KeepsActivityEndpointsLocal(t *testing.T) {route: "/environments/:id/activities",path: "/api/environments/env-edge/activities?limit=50",},- {- name: "stream activities",- method: http.MethodGet,- route: "/environments/:id/activities/stream",- path: "/api/environments/env-edge/activities/stream?limit=50",- },}for _, tt := range tests {diff --git a/backend/pkg/libarcane/edge/commands.go b/backend/pkg/libarcane/edge/commands.goindex 8a36ec882..0f2a0f130 100644--- a/backend/pkg/libarcane/edge/commands.go+++ b/backend/pkg/libarcane/edge/commands.go@@ -51,7 +51,6 @@ var commandRoutes = []commandRoute{{Method: http.MethodGet, PathPattern: "/api/environments/{id}/image-updates/summary", CommandName: "image_update.summary"},{Method: http.MethodGet, PathPattern: "/api/environments/{id}/activities", CommandName: "activity.list"},- {Method: http.MethodGet, PathPattern: "/api/environments/{id}/activities/stream", LocalOnly: true},{Method: http.MethodGet, PathPattern: "/api/environments/{id}/activities/{activityId}", CommandName: "activity.inspect"},{Method: http.MethodPost, PathPattern: "/api/environments/{id}/activities/{activityId}/cancel", CommandName: "activity.cancel"},{Method: http.MethodDelete, PathPattern: "/api/environments/{id}/activities/history", CommandName: "activity.history.clear"},diff --git a/backend/pkg/libarcane/edge/commands_test.go b/backend/pkg/libarcane/edge/commands_test.goindex 2ee27d822..bf02ef673 100644--- a/backend/pkg/libarcane/edge/commands_test.go+++ b/backend/pkg/libarcane/edge/commands_test.go@@ -29,7 +29,6 @@ func TestResolveEdgeCommandName(t *testing.T) {{name: "activity inspect", method: "GET", path: "/api/environments/0/activities/activity-1", command: "activity.inspect", shouldHit: true},{name: "activity cancel", method: "POST", path: "/api/environments/0/activities/activity-1/cancel", command: "activity.cancel", shouldHit: true},{name: "activity history clear", method: "DELETE", path: "/api/environments/0/activities/history", command: "activity.history.clear", shouldHit: true},- {name: "activity stream remains manager local", method: "GET", path: "/api/environments/0/activities/stream?limit=50", shouldHit: false},{name: "health", method: "HEAD", path: "/api/environments/0/system/health", command: "system.health", shouldHit: true},{name: "swarm node identity", method: "GET", path: "/api/swarm/node-identity", command: "swarm.node_identity", shouldHit: true},{name: "unknown", method: "PATCH", path: "/api/environments/0/containers", shouldHit: false},diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.goindex 452dc9fd6..b2a369efb 100644--- a/backend/pkg/scheduler/auto_heal_job.go+++ b/backend/pkg/scheduler/auto_heal_job.go@@ -34,10 +34,14 @@ type AutoHealJob struct {mu sync.Mutexrestarts map[string]*restartRecord- getDockerClient func() (*client.Client, error)- listContainers func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error)- inspectContainer func(ctx context.Context, dockerClient *client.Client, containerID string) (container.InspectResponse, error)- restartContainer func(ctx context.Context, dockerClient *client.Client, containerID string) error+ selfIDOnce sync.Once+ selfID string++ getDockerClient func() (*client.Client, error)+ listContainers func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error)+ inspectContainer func(ctx context.Context, dockerClient *client.Client, containerID string) (container.InspectResponse, error)+ restartContainer func(ctx context.Context, dockerClient *client.Client, containerID string) error+ getSelfContainerID func() (string, error)}func NewAutoHealJob(@@ -103,7 +107,8 @@ func (j *AutoHealJob) Run(ctx context.Context) {restartWindowMinutes := j.settingsService.GetIntSetting(ctx, "autoHealRestartWindow", 30)restartWindow := time.Duration(restartWindowMinutes) * time.Minute- candidates := j.filterCandidatesInternal(containers, excludedContainers)+ selfID := j.selfContainerIDInternal(ctx)+ candidates := j.filterCandidatesInternal(containers, excludedContainers, selfID)g, groupCtx := errgroup.WithContext(ctx)g.SetLimit(autoHealInspectConcurrency)@@ -118,9 +123,37 @@ func (j *AutoHealJob) Run(ctx context.Context) {_ = g.Wait()}-func (j *AutoHealJob) filterCandidatesInternal(containers []container.Summary, excludedContainers map[string]struct{}) []container.Summary {+// selfContainerIDInternal resolves and caches the ID (full 64-char or short+// prefix) of the container Arcane itself runs in. Returns "" when detection+// fails (e.g. binary running directly on the host), disabling the guard.+func (j *AutoHealJob) selfContainerIDInternal(ctx context.Context) string {+ j.selfIDOnce.Do(func() {+ detect := j.getSelfContainerID+ if detect == nil {+ detect = dockerutil.GetCurrentContainerID+ }+ id, err := detect()+ if err != nil {+ slog.DebugContext(ctx, "auto-heal: could not determine own container ID; self-protection disabled", "error", err)+ return+ }+ j.selfID = strings.ToLower(strings.TrimSpace(id))+ slog.InfoContext(ctx, "auto-heal: detected own container; it will never be auto-restarted", "container_id", j.selfID)+ })+ return j.selfID+}++func (j *AutoHealJob) filterCandidatesInternal(containers []container.Summary, excludedContainers map[string]struct{}, selfID string) []container.Summary {candidates := make([]container.Summary, 0, len(containers))for _, c := range containers {+ // Never restart the container Arcane itself runs in: a slow or+ // mid-startup manager that trips its own healthcheck would otherwise+ // be restarted by its own auto-heal, in a loop. Prefix match because+ // hostname-based detection yields the short 12-char ID.+ if selfID != "" && strings.HasPrefix(strings.ToLower(c.ID), selfID) {+ continue+ }+if libarcane.IsInternalContainer(c.Labels) {continue}diff --git a/backend/pkg/scheduler/auto_heal_job_test.go b/backend/pkg/scheduler/auto_heal_job_test.goindex e5658bbbd..012e852de 100644--- a/backend/pkg/scheduler/auto_heal_job_test.go+++ b/backend/pkg/scheduler/auto_heal_job_test.go@@ -22,6 +22,30 @@ func newTestAutoHealJob() *AutoHealJob {}}+func TestAutoHeal_FilterCandidates_SkipsSelfContainer(t *testing.T) {+ job := newTestAutoHealJob()++ selfFullID := "407163929c492b5c4b01a3981f5de4774c37aa8300bd214b4d62412a3dc56468"+ containers := []container.Summary{+ {ID: selfFullID, Names: []string{"/arcane"}},+ {ID: "aaaabbbbccccddddeeeeffff0000111122223333444455556666777788889999", Names: []string{"/other"}},+ }++ // Hostname-based detection yields the short 12-char ID.+ candidates := job.filterCandidatesInternal(containers, nil, selfFullID[:12])+ require.Len(t, candidates, 1)+ require.Equal(t, "/other", candidates[0].Names[0])++ // cgroup/mountinfo-based detection yields the full ID.+ candidates = job.filterCandidatesInternal(containers, nil, selfFullID)+ require.Len(t, candidates, 1)+ require.Equal(t, "/other", candidates[0].Names[0])++ // Outside Docker no self ID is detected and the guard is a no-op.+ candidates = job.filterCandidatesInternal(containers, nil, "")+ require.Len(t, candidates, 2)+}+func TestAutoHeal_CanRestart_UnderLimit(t *testing.T) {job := newTestAutoHealJob()diff --git a/frontend/src/lib/components/activity/activity-center.svelte b/frontend/src/lib/components/activity/activity-center.svelteindex b402f01d4..b1df36966 100644--- a/frontend/src/lib/components/activity/activity-center.svelte+++ b/frontend/src/lib/components/activity/activity-center.svelte@@ -19,6 +19,7 @@onMount(() => {void activityStore.start();+ return () => activityStore.stop();});function handleOpenChangeInternal(open: boolean) {diff --git a/frontend/src/lib/services/activity-service.ts b/frontend/src/lib/services/activity-service.tsindex b03f8eec2..5b2426b28 100644--- a/frontend/src/lib/services/activity-service.ts+++ b/frontend/src/lib/services/activity-service.ts@@ -31,25 +31,22 @@ class ActivityService extends BaseAPIService {return this.handleResponse(this.api.delete(`/environments/${envId}/activities/history`));}- getActivityStreamUrl(environmentId: string, limit = 50): string {+ getActivityStreamUrl(limit = 50): string {const baseUrl = this.api.defaults.baseURL.replace(/\/+$/, '');const params = new URLSearchParams({ limit: String(limit) });- return `${baseUrl}/environments/${encodeURIComponent(environmentId)}/activities/stream?${params.toString()}`;+ return `${baseUrl}/activities/stream?${params.toString()}`;}- async openActivityStream(environmentId: string, signal: AbortSignal, limit = 50, retry = false): Promise<Response> {- const response = await fetch(this.getActivityStreamUrl(environmentId, limit), {+ async openActivityStream(signal: AbortSignal, limit = 50, retry = false): Promise<Response> {+ const response = await fetch(this.getActivityStreamUrl(limit), {credentials: 'include',headers: { Accept: 'application/x-json-stream' },signal});if (response.status === 401) {- const action = await handleUnauthorizedResponseInternal(- `/environments/${encodeURIComponent(environmentId)}/activities/stream`,- retry- );+ const action = await handleUnauthorizedResponseInternal('/activities/stream', retry);if (action === 'retry') {- return this.openActivityStream(environmentId, signal, limit, true);+ return this.openActivityStream(signal, limit, true);}if (action === 'redirect') {return new Promise<Response>(() => {});diff --git a/frontend/src/lib/stores/activity.store.svelte.ts b/frontend/src/lib/stores/activity.store.svelte.tsindex 4d5c67a24..6389f7afa 100644--- a/frontend/src/lib/stores/activity.store.svelte.ts+++ b/frontend/src/lib/stores/activity.store.svelte.ts@@ -23,7 +23,6 @@ type ActivityEnvironmentState = {name: string;activities: Activity[];loading: boolean;- connected: boolean;streamError: boolean;errorMessage?: string;};@@ -92,10 +91,14 @@ function createActivityStore() {let started = false;let unsubscribeEnvironment: (() => void) | null = null;- const streamAbortControllers = new Map<string, AbortController>();- const reconnectTimers = new Map<string, ReturnType<typeof setTimeout>>();- const reconnectAttempts = new Map<string, number>();- const streamGenerations = new Map<string, number>();+ // A single aggregated stream carries every environment's events; per-env+ // connections would exhaust the browser's 6-per-origin HTTP/1.1 limit.+ let streamAbortController: AbortController | null = null;+ let reconnectTimer: ReturnType<typeof setTimeout> | null = null;+ let reconnectAttempt = 0;+ let streamGeneration = 0;+ let _streamConnected = $state(false);+ let _streamFailed = $state(false);function createEnvironmentStateInternal(environment: Pick<Environment, 'id' | 'name'>): ActivityEnvironmentState {return {@@ -103,7 +106,6 @@ function createActivityStore() {name: environmentNameInternal(environment),activities: [],loading: true,- connected: false,streamError: false};}@@ -128,7 +130,6 @@ function createActivityStore() {updateEnvironmentStateInternal(environmentId, (state) => ({...state,loading: false,- connected: false,streamError: true,errorMessage: errorMessageInternal(error)}));@@ -142,44 +143,40 @@ function createActivityStore() {}));}- function generationInternal(environmentId: string): number {- return streamGenerations.get(environmentId) ?? 0;+ // A fresh stream re-emits error events for environments that are still+ // failing, so stale per-environment errors are cleared on every (re)connect.+ function clearAllEnvironmentErrorsInternal() {+ for (const environmentId of Object.keys(_environmentStates)) {+ if (environmentStateInternal(environmentId)?.streamError) {+ clearEnvironmentErrorInternal(environmentId);+ }+ }}- function nextGenerationInternal(environmentId: string): number {- const generation = generationInternal(environmentId) + 1;- streamGenerations.set(environmentId, generation);- return generation;+ function nextGenerationInternal(): number {+ streamGeneration += 1;+ return streamGeneration;}- function isCurrentGenerationInternal(environmentId: string, generation: number): boolean {- return generationInternal(environmentId) === generation;+ function isCurrentGenerationInternal(generation: number): boolean {+ return streamGeneration === generation;}- function clearReconnectTimerInternal(environmentId: string) {- const timer = reconnectTimers.get(environmentId);- if (timer) {- clearTimeout(timer);- reconnectTimers.delete(environmentId);+ function clearReconnectTimerInternal() {+ if (reconnectTimer) {+ clearTimeout(reconnectTimer);+ reconnectTimer = null;}}- function abortEnvironmentStreamInternal(environmentId: string) {- clearReconnectTimerInternal(environmentId);- streamAbortControllers.get(environmentId)?.abort();- streamAbortControllers.delete(environmentId);- updateEnvironmentStateInternal(environmentId, (state) => ({- ...state,- connected: false- }));+ function abortStreamInternal() {+ clearReconnectTimerInternal();+ streamAbortController?.abort();+ streamAbortController = null;+ _streamConnected = false;}- function stopEnvironmentInternal(environmentId: string) {- nextGenerationInternal(environmentId);- abortEnvironmentStreamInternal(environmentId);- reconnectAttempts.delete(environmentId);- streamGenerations.delete(environmentId);-+ function removeEnvironmentInternal(environmentId: string) {const nextStates = { ..._environmentStates };delete nextStates[environmentId];_environmentStates = nextStates;@@ -199,6 +196,11 @@ function createActivityStore() {}function replaceEnvironmentSnapshotInternal(environmentId: string, activities: Activity[]) {+ // Snapshots can still arrive (stream or in-flight REST) after the+ // environment was removed locally; don't resurrect it.+ if (!environmentStateInternal(environmentId)) {+ return;+ }const normalizedActivities = sortActivitiesInternal(activities.map((activity) => normalizeActivityInternal(activity, environmentId)));@@ -283,6 +285,11 @@ function createActivityStore() {}function applyStreamEventInternal(environmentId: string, event: ActivityStreamEvent) {+ // The aggregated stream can keep delivering events for an environment+ // for a short while after it was removed locally; don't resurrect it.+ if (event.type !== 'heartbeat' && !environmentStateInternal(environmentId)) {+ return;+ }switch (event.type) {case 'snapshot':replaceEnvironmentSnapshotInternal(environmentId, event.activities ?? []);@@ -298,27 +305,28 @@ function createActivityStore() {}break;case 'heartbeat':- updateEnvironmentStateInternal(environmentId, (state) => ({- ...state,- connected: true- }));+ _streamConnected = true;+ break;+ case 'error':+ setEnvironmentErrorInternal(environmentId, new Error(event.error || 'Activity stream error'));break;}}- async function refreshEnvironmentInternal(environmentId: string, generation = generationInternal(environmentId)) {+ async function refreshEnvironmentInternal(environmentId: string, generation = streamGeneration) {updateEnvironmentStateInternal(environmentId, (state) => ({...state,loading: true}));try {const result = await activityService.getActivities({ pagination: { page: 1, limit: ACTIVITY_LIST_LIMIT } }, environmentId);- if (!isCurrentGenerationInternal(environmentId, generation)) {+ // The environment can be removed while the fetch is in-flight; don't resurrect it.+ if (!isCurrentGenerationInternal(generation) || !environmentStateInternal(environmentId)) {return;}replaceEnvironmentSnapshotInternal(environmentId, result.data ?? []);} catch (error) {- if (isCurrentGenerationInternal(environmentId, generation)) {+ if (isCurrentGenerationInternal(generation) && environmentStateInternal(environmentId)) {console.warn('Failed to refresh activities:', error);setEnvironmentErrorInternal(environmentId, error);}@@ -330,57 +338,51 @@ function createActivityStore() {await Promise.all(Object.keys(_environmentStates).map((environmentId) => refreshEnvironmentInternal(environmentId)));}- async function connectStreamInternal(environmentId: string, generation: number) {- if (!browser || !isCurrentGenerationInternal(environmentId, generation)) {+ async function connectStreamInternal(generation: number) {+ if (!browser || !isCurrentGenerationInternal(generation)) {return;}const controller = new AbortController();- streamAbortControllers.set(environmentId, controller);+ streamAbortController = controller;try {- const response = await activityService.openActivityStream(environmentId, controller.signal, ACTIVITY_LIST_LIMIT);- if (!isCurrentGenerationInternal(environmentId, generation) || !response.body) {- if (streamAbortControllers.get(environmentId) === controller) {- streamAbortControllers.delete(environmentId);+ const response = await activityService.openActivityStream(controller.signal, ACTIVITY_LIST_LIMIT);+ if (!isCurrentGenerationInternal(generation) || !response.body) {+ if (streamAbortController === controller) {+ streamAbortController = null;}return;}- updateEnvironmentStateInternal(environmentId, (state) => ({- ...state,- connected: true,- streamError: false,- errorMessage: undefined- }));- reconnectAttempts.set(environmentId, 0);- await readJSONLinesInternal(environmentId, response.body, generation);+ _streamConnected = true;+ _streamFailed = false;+ reconnectAttempt = 0;+ clearAllEnvironmentErrorsInternal();+ await readJSONLinesInternal(response.body, generation);} catch (error) {- if (!controller.signal.aborted && isCurrentGenerationInternal(environmentId, generation)) {+ if (!controller.signal.aborted && isCurrentGenerationInternal(generation)) {console.warn('Activity stream disconnected:', error);}} finally {- if (streamAbortControllers.get(environmentId) === controller) {- streamAbortControllers.delete(environmentId);+ if (streamAbortController === controller) {+ streamAbortController = null;}- if (isCurrentGenerationInternal(environmentId, generation)) {- updateEnvironmentStateInternal(environmentId, (state) => ({- ...state,- connected: false- }));+ if (isCurrentGenerationInternal(generation)) {+ _streamConnected = false;if (!controller.signal.aborted) {- scheduleReconnectInternal(environmentId, generation);+ scheduleReconnectInternal(generation);}}}}- async function readJSONLinesInternal(environmentId: string, stream: ReadableStream<Uint8Array>, generation: number) {+ async function readJSONLinesInternal(stream: ReadableStream<Uint8Array>, generation: number) {const reader = stream.getReader();const decoder = new TextDecoder();let buffer = '';try {- while (isCurrentGenerationInternal(environmentId, generation)) {+ while (isCurrentGenerationInternal(generation)) {const { done, value } = await reader.read();if (done) {break;@@ -390,77 +392,49 @@ function createActivityStore() {const lines = buffer.split('\n');buffer = lines.pop() ?? '';for (const line of lines) {- handleStreamLineInternal(environmentId, line);+ handleStreamLineInternal(line);}}buffer += decoder.decode();if (buffer.trim()) {- handleStreamLineInternal(environmentId, buffer);+ handleStreamLineInternal(buffer);}} finally {reader.releaseLock();}}- function handleStreamLineInternal(environmentId: string, line: string) {+ function handleStreamLineInternal(line: string) {const trimmed = line.trim();if (!trimmed) {return;}try {- applyStreamEventInternal(environmentId, JSON.parse(trimmed) as ActivityStreamEvent);+ const event = JSON.parse(trimmed) as ActivityStreamEvent;+ applyStreamEventInternal(event.environmentId || LOCAL_DOCKER_ENVIRONMENT_ID, event);} catch (error) {console.warn('Failed to parse activity stream line:', error);}}- function scheduleReconnectInternal(environmentId: string, generation: number) {- if (!browser || !started || !isCurrentGenerationInternal(environmentId, generation)) {- return;- }-- const attempt = reconnectAttempts.get(environmentId) ?? 0;- if (attempt >= MAX_RECONNECT_ATTEMPTS) {- setEnvironmentErrorInternal(environmentId, new Error('Activity stream reconnect attempts exhausted'));+ function scheduleReconnectInternal(generation: number) {+ if (!browser || !started || !isCurrentGenerationInternal(generation)) {return;}- clearReconnectTimerInternal(environmentId);- const delay = Math.min(1000 * 2 ** attempt, MAX_RECONNECT_DELAY);- reconnectAttempts.set(environmentId, attempt + 1);- reconnectTimers.set(- environmentId,- setTimeout(() => {- void connectStreamInternal(environmentId, generation);- }, delay)- );- }-- function startEnvironmentInternal(environment: Pick<Environment, 'id' | 'name'>) {- const environmentId = environment.id || LOCAL_DOCKER_ENVIRONMENT_ID;- updateEnvironmentStateInternal(environmentId, (state) => ({- ...state,- id: environmentId,- name: environmentNameInternal(environment)- }));-- const generation = nextGenerationInternal(environmentId);- reconnectAttempts.set(environmentId, 0);- abortEnvironmentStreamInternal(environmentId);- void refreshEnvironmentInternal(environmentId, generation);- void connectStreamInternal(environmentId, generation);- }-- function restartEnvironmentInternal(environmentId: string) {- const state = environmentStateInternal(environmentId);- if (!state) {+ if (reconnectAttempt >= MAX_RECONNECT_ATTEMPTS) {+ _streamFailed = true;return;}- clearEnvironmentErrorInternal(environmentId);- startEnvironmentInternal({ id: state.id, name: state.name });+ clearReconnectTimerInternal();+ const delay = Math.min(1000 * 2 ** reconnectAttempt, MAX_RECONNECT_DELAY);+ reconnectAttempt += 1;+ reconnectTimer = setTimeout(() => {+ void connectStreamInternal(generation);+ }, delay);}function reconcileEnvironmentsInternal() {@@ -468,7 +442,9 @@ function createActivityStore() {return;}- const available = environmentStore.available;+ // Track only enabled environments — they are the ones the aggregated+ // stream serves; a disabled environment would never leave "loading".+ const available = environmentStore.available.filter((environment) => environment.enabled);const environments =available.length > 0? available@@ -482,7 +458,7 @@ function createActivityStore() {for (const environmentId of Object.keys(_environmentStates)) {if (!targetIds.has(environmentId)) {- stopEnvironmentInternal(environmentId);+ removeEnvironmentInternal(environmentId);}}@@ -494,7 +470,12 @@ function createActivityStore() {..._environmentStates,[environmentId]: createEnvironmentStateInternal(environment)};- startEnvironmentInternal(environment);+ // An already-open aggregated stream only picks new environments+ // up on its server-side reconcile tick; fetch once so the first+ // snapshot doesn't take up to that interval to appear.+ if (streamAbortController) {+ void refreshEnvironmentInternal(environmentId);+ }continue;}@@ -612,10 +593,10 @@ function createActivityStore() {return Object.values(_environmentStates).some((state) => state.loading);},get connected(): boolean {- return Object.values(_environmentStates).some((state) => state.connected);+ return _streamConnected;},get streamError(): boolean {- return Object.values(_environmentStates).some((state) => state.streamError);+ return _streamFailed || Object.values(_environmentStates).some((state) => state.streamError);},get environmentFailures(): ActivityEnvironmentFailure[] {return environmentFailuresInternal();@@ -654,6 +635,7 @@ function createActivityStore() {await environmentStore.ready;_currentEnvironmentId = environmentStore.selected?.id ?? LOCAL_DOCKER_ENVIRONMENT_ID;reconcileEnvironmentsInternal();+ void connectStreamInternal(nextGenerationInternal());unsubscribeEnvironment = environmentStore.subscribeSelected((environment) => {_currentEnvironmentId = environment?.id ?? LOCAL_DOCKER_ENVIRONMENT_ID;reconcileEnvironmentsInternal();@@ -663,11 +645,9 @@ function createActivityStore() {started = false;unsubscribeEnvironment?.();unsubscribeEnvironment = null;-- for (const environmentId of Object.keys(_environmentStates)) {- nextGenerationInternal(environmentId);- abortEnvironmentStreamInternal(environmentId);- }+ nextGenerationInternal();+ abortStreamInternal();+ reconnectAttempt = 0;},refresh: () => refreshInternal(),cancelActivity: async (activityId: string) => {@@ -742,12 +722,11 @@ function createActivityStore() {void loadDetailInternal(activityId);},retryStream: () => {- for (const environmentId of Object.keys(_environmentStates)) {- const state = environmentStateInternal(environmentId);- if (state?.streamError) {- restartEnvironmentInternal(environmentId);- }- }+ _streamFailed = false;+ reconnectAttempt = 0;+ clearAllEnvironmentErrorsInternal();+ abortStreamInternal();+ void connectStreamInternal(nextGenerationInternal());},setActivityExpanded,toggleActivitydiff --git a/frontend/src/lib/types/activity.type.ts b/frontend/src/lib/types/activity.type.tsindex 3fd0b69ef..8c158bfbc 100644--- a/frontend/src/lib/types/activity.type.ts+++ b/frontend/src/lib/types/activity.type.ts@@ -85,10 +85,12 @@ export interface ActivityClearHistorySummary {}export interface ActivityStreamEvent {- type: 'snapshot' | 'activity' | 'message' | 'heartbeat';+ type: 'snapshot' | 'activity' | 'message' | 'heartbeat' | 'error';+ environmentId?: string;activityId?: string;activity?: Activity;activities?: Activity[];message?: ActivityMessage;+ error?: string;timestamp: string;}diff --git a/tests/spec/activity-center.spec.ts b/tests/spec/activity-center.spec.tsindex 4fd42aeee..c1223fec3 100644--- a/tests/spec/activity-center.spec.ts+++ b/tests/spec/activity-center.spec.ts@@ -39,7 +39,7 @@ const remoteEnvironment: MockEnvironment = {name: 'Remote Lab',apiUrl: 'https://remote.example.invalid',status: 'offline',- enabled: false,+ enabled: true,isEdge: false};@@ -83,7 +83,7 @@ function activity(}function activityEnvironmentIdFromPath(pathname: string): string | null {- const match = pathname.match(/^\/api\/environments\/([^/]+)\/activities(?:\/stream)?$/);+ const match = pathname.match(/^\/api\/environments\/([^/]+)\/activities$/);return match ? decodeURIComponent(match[1]) : null;}@@ -103,54 +103,61 @@ async function mockEnvironmentList(page: Page, environments: MockEnvironment[])});}+function aggregatedActivityStreamBody(+ activitiesByEnvironment: Record<string, MockActivity[]>,+ failedEnvironmentIds: Set<string>+): string {+ const timestamp = new Date().toISOString();+ const events: string[] = [];+ for (const [environmentId, activities] of Object.entries(activitiesByEnvironment)) {+ events.push(JSON.stringify({ type: 'snapshot', environmentId, activities, timestamp }));+ }+ for (const environmentId of failedEnvironmentIds) {+ events.push(+ JSON.stringify({ type: 'error', environmentId, error: 'environment unavailable', timestamp })+ );+ }+ return events.join('\n') + '\n';+}+async function mockActivityReads(page: Page,activitiesByEnvironment: Record<string, MockActivity[]>,failedEnvironmentIds = new Set<string>()) {+ await page.context().route(/\/api\/activities\/stream(?:\?.*)?$/, async (route: Route) => {+ await route.fulfill({+ status: 200,+ contentType: 'application/x-json-stream',+ body: aggregatedActivityStreamBody(activitiesByEnvironment, failedEnvironmentIds)+ });+ });+await page.context()- .route(- /\/api\/environments\/[^/]+\/activities(?:\/stream)?(?:\?.*)?$/,- async (route: Route) => {- const url = new URL(route.request().url());- const environmentId = activityEnvironmentIdFromPath(url.pathname);- if (!environmentId) {- await route.continue();- return;- }-- if (failedEnvironmentIds.has(environmentId)) {- await route.fulfill({- status: 503,- contentType: 'application/json',- body: JSON.stringify({ success: false, message: 'environment unavailable' })- });- return;- }-- const activities = activitiesByEnvironment[environmentId] ?? [];- if (url.pathname.endsWith('/stream')) {- await route.fulfill({- status: 200,- contentType: 'application/x-json-stream',- body:- JSON.stringify({- type: 'snapshot',- activities,- timestamp: new Date().toISOString()- }) + '\n'- });- return;- }+ .route(/\/api\/environments\/[^/]+\/activities(?:\?.*)?$/, async (route: Route) => {+ const url = new URL(route.request().url());+ const environmentId = activityEnvironmentIdFromPath(url.pathname);+ if (!environmentId) {+ await route.continue();+ return;+ }+ if (failedEnvironmentIds.has(environmentId)) {await route.fulfill({- status: 200,+ status: 503,contentType: 'application/json',- body: JSON.stringify(paginated(activities))+ body: JSON.stringify({ success: false, message: 'environment unavailable' })});+ return;}- );++ await route.fulfill({+ status: 200,+ contentType: 'application/json',+ body: JSON.stringify(paginated(activitiesByEnvironment[environmentId] ?? []))+ });+ });}function extractActivityId(value: unknown): string | undefined {@@ -236,13 +243,10 @@ function activityRow(activityCenter: Locator, text: string) {.first();}-function waitForActivityList(page: Page, environmentId: string) {+function waitForActivityStream(page: Page) {return page.waitForResponse((response) => {const url = new URL(response.url());- return (- response.request().method() === 'GET' &&- url.pathname === `/api/environments/${encodeURIComponent(environmentId)}/activities`- );+ return response.request().method() === 'GET' && url.pathname === '/api/activities/stream';});}@@ -257,12 +261,10 @@ test.describe('Activity Center', () => {]});- const localActivityList = waitForActivityList(page, '0');- const remoteActivityList = waitForActivityList(page, 'remote-activity-test');+ const activityStream = waitForActivityStream(page);await page.goto('/dashboard');- await Promise.all([localActivityList, remoteActivityList]);+ await activityStream;await page.waitForLoadState('load');- await page.waitForLoadState('networkidle');const activityCenter = await openActivityCenter(page);await activityCenter.getByRole('button', { name: 'Completed' }).click();@@ -285,12 +287,10 @@ test.describe('Activity Center', () => {new Set(['remote-activity-test']));- const localActivityList = waitForActivityList(page, '0');- const remoteActivityList = waitForActivityList(page, 'remote-activity-test');+ const activityStream = waitForActivityStream(page);await page.goto('/dashboard');- await Promise.all([localActivityList, remoteActivityList]);+ await activityStream;await page.waitForLoadState('load');- await page.waitForLoadState('networkidle');const activityCenter = await openActivityCenter(page);await activityCenter.getByRole('button', { name: 'Completed' }).click();@@ -334,10 +334,9 @@ test.describe('Activity Center', () => {});});- const localActivityList = waitForActivityList(page, '0');- const remoteActivityList = waitForActivityList(page, 'remote-activity-test');+ const activityStream = waitForActivityStream(page);await page.goto('/dashboard');- await Promise.all([localActivityList, remoteActivityList]);+ await activityStream;await page.waitForLoadState('load');const activityCenter = await openActivityCenter(page);diff --git a/types/activity/activity.go b/types/activity/activity.goindex ff7f32b86..48f1a3016 100644--- a/types/activity/activity.go+++ b/types/activity/activity.go@@ -89,12 +89,14 @@ type Detail struct {}type StreamEvent struct {- Type string `json:"type"`- ActivityID string `json:"activityId,omitempty"`- Activity *Activity `json:"activity,omitempty"`- Activities []Activity `json:"activities,omitempty"`- Message *Message `json:"message,omitempty"`- Timestamp time.Time `json:"timestamp"`+ Type string `json:"type"`+ EnvironmentID string `json:"environmentId,omitempty"`+ ActivityID string `json:"activityId,omitempty"`+ Activity *Activity `json:"activity,omitempty"`+ Activities []Activity `json:"activities,omitempty"`+ Message *Message `json:"message,omitempty"`+ Error string `json:"error,omitempty"`+ Timestamp time.Time `json:"timestamp"`}type ClearHistoryResult struct {