Skip to content

Modern Docker Management, Designed for Everyone

fix: activities stream using main context causing app to hang at certain places (#2887)

Kyle Mendellb7d7c20b3810
diff --git a/backend/api/handlers/activities.go b/backend/api/handlers/activities.go
index 0010127eb..40381f319 100644
--- a/backend/api/handlers/activities.go
+++ b/backend/api/handlers/activities.go
@@ -5,10 +5,12 @@ import (
"encoding/json"
"errors"
"fmt"
+ "log/slog"
"net/http"
"net/url"
"strconv"
"strings"
+ "sync"
"time"
"github.com/danielgtaylor/huma/v2"
@@ -61,11 +63,18 @@ type ClearActivityHistoryOutput struct {
Body base.ApiResponse[activity.ClearHistoryResult]
}
-type StreamActivitiesInput struct {
- EnvironmentID string `path:"id" doc:"Environment ID"`
- Limit int `query:"limit" default:"50" doc:"Initial snapshot limit"`
+type StreamAllActivitiesInput struct {
+ Limit int `query:"limit" default:"50" doc:"Snapshot limit per environment"`
}
+const (
+ activityStreamHeartbeatInterval = 15 * time.Second
+ activityStreamRemotePollInterval = 5 * time.Second
+ activityStreamEnvReconcileInterval = 30 * time.Second
+ activityStreamRemotePollTimeout = 15 * time.Second
+ activityStreamEventBuffer = 256
+)
+
type CancelActivityInput struct {
EnvironmentID string `path:"id" doc:"Environment ID"`
ActivityID string `path:"activityId" doc:"Activity ID"`
@@ -111,18 +120,18 @@ func RegisterActivities(api huma.API, activityService *services.ActivityService,
}, h.GetActivity)
huma.Register(api, huma.Operation{
- OperationID: "stream-activities",
+ OperationID: "stream-all-activities",
Method: http.MethodGet,
- Path: "/environments/{id}/activities/stream",
- Summary: "Stream background activities",
- Description: "Stream background activity updates as JSON lines",
+ Path: "/activities/stream",
+ Summary: "Stream background activities across all environments",
+ Description: "Stream background activity updates for the local environment and all enabled remote environments as JSON lines",
Tags: []string{"Activities"},
Security: []map[string][]string{
{"BearerAuth": {}},
{"ApiKeyAuth": {}},
},
Middlewares: humamw.RequirePermission(api, authz.PermActivitiesRead),
- }, h.StreamActivities)
+ }, h.StreamAllActivities)
huma.Register(api, huma.Operation{
OperationID: "cancel-activity",
@@ -221,95 +230,6 @@ func (h *ActivityHandler) GetActivity(ctx context.Context, input *GetActivityInp
}, nil
}
-func (h *ActivityHandler) StreamActivities(ctx context.Context, input *StreamActivitiesInput) (*huma.StreamResponse, error) {
- if h.activityService == nil {
- return nil, huma.Error500InternalServerError("service not available")
- }
-
- return &huma.StreamResponse{
- Body: func(humaCtx huma.Context) { //nolint:contextcheck // streaming work must use humaCtx.Context()
- httpx.SetJSONStreamHeaders(humaCtx)
-
- writer := humaCtx.BodyWriter()
- encoder := json.NewEncoder(writer)
- flush := func() {
- if f, ok := writer.(http.Flusher); ok {
- f.Flush()
- }
- }
-
- if input.EnvironmentID != "0" {
- h.streamRemoteActivitySnapshotsInternal(humaCtx.Context(), input, encoder, flush)
- return
- }
-
- h.streamLocalActivitiesInternal(humaCtx.Context(), input, encoder, flush)
- },
- }, nil
-}
-
-func (h *ActivityHandler) streamLocalActivitiesInternal(
- ctx context.Context,
- input *StreamActivitiesInput,
- encoder *json.Encoder,
- flush func(),
-) {
- sendSnapshot := func() bool {
- activities, _, err := h.activityService.ListActivitiesPaginated(ctx, input.EnvironmentID, pagination.QueryParams{
- Params: pagination.Params{Limit: resolveActivityStreamLimitInternal(input.Limit)},
- })
- if err != nil {
- return false
- }
- h.applyActivitySourceLabelsInternal(ctx, input.EnvironmentID, activities)
- if err := encoder.Encode(activity.StreamEvent{
- Type: "snapshot",
- Activities: activities,
- Timestamp: time.Now(),
- }); err != nil {
- return false
- }
- flush()
- return true
- }
- if !sendSnapshot() {
- return
- }
-
- events, missedEvents, unsubscribe := h.activityService.Subscribe(input.EnvironmentID)
- defer unsubscribe()
-
- ticker := time.NewTicker(15 * time.Second)
- defer ticker.Stop()
-
- for {
- select {
- case <-ctx.Done():
- return
- case event, ok := <-events:
- if !ok {
- return
- }
- h.applyActivityStreamEventSourceLabelInternal(ctx, input.EnvironmentID, &event)
- if err := encoder.Encode(event); err != nil {
- return
- }
- flush()
- case <-ticker.C:
- if missedEvents() && !sendSnapshot() {
- return
- }
- if err := encoder.Encode(activity.StreamEvent{
- Type: "heartbeat",
- Timestamp: time.Now(),
- }); err != nil {
- return
- }
- flush()
- }
- }
-}
-
func (h *ActivityHandler) ClearHistory(ctx context.Context, input *ClearActivityHistoryInput) (*ClearActivityHistoryOutput, error) {
if input.EnvironmentID != "0" {
return h.proxyClearHistoryInternal(ctx, input)
@@ -395,58 +315,246 @@ func (h *ActivityHandler) cancelRequestedByInternal(ctx context.Context, forward
return strings.TrimSpace(forwarded)
}
-func (h *ActivityHandler) streamRemoteActivitySnapshotsInternal(
- ctx context.Context,
- input *StreamActivitiesInput,
- encoder *json.Encoder,
- flush func(),
-) {
- pollTicker := time.NewTicker(5 * time.Second)
- defer pollTicker.Stop()
- heartbeatTicker := time.NewTicker(15 * time.Second)
- defer heartbeatTicker.Stop()
-
- sendSnapshot := func(ctx context.Context) bool {
- output, err := h.proxyListActivitiesInternal(ctx, &ListActivitiesInput{
- EnvironmentID: input.EnvironmentID,
- Start: 0,
- Limit: resolveActivityStreamLimitInternal(input.Limit),
- Order: "desc",
+func (h *ActivityHandler) StreamAllActivities(ctx context.Context, input *StreamAllActivitiesInput) (*huma.StreamResponse, error) {
+ if h.activityService == nil || h.environmentService == nil {
+ return nil, huma.Error500InternalServerError("service not available")
+ }
+
+ return &huma.StreamResponse{
+ Body: func(humaCtx huma.Context) { //nolint:contextcheck // streaming work must use humaCtx.Context()
+ httpx.SetJSONStreamHeaders(humaCtx)
+
+ writer := humaCtx.BodyWriter()
+ encoder := json.NewEncoder(writer)
+ flush := func() {
+ if f, ok := writer.(http.Flusher); ok {
+ f.Flush()
+ }
+ }
+
+ h.streamAllActivitiesInternal(humaCtx.Context(), input.Limit, encoder, flush)
+ },
+ }, nil
+}
+
+// streamAllActivitiesInternal multiplexes activity events for the local
+// environment and every enabled remote environment over a single response so
+// the browser needs one connection regardless of environment count.
+func (h *ActivityHandler) streamAllActivitiesInternal(ctx context.Context, limit int, encoder *json.Encoder, flush func()) {
+ streamCtx, cancel := context.WithCancel(ctx)
+
+ events := make(chan activity.StreamEvent, activityStreamEventBuffer)
+ var wg sync.WaitGroup
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ h.runLocalActivityStreamProducerInternal(streamCtx, limit, events)
+ }()
+ go func() {
+ defer wg.Done()
+ h.runRemoteActivityStreamPollersInternal(streamCtx, limit, events)
+ }()
+ defer wg.Wait()
+ defer cancel()
+
+ heartbeat := time.NewTicker(activityStreamHeartbeatInterval)
+ defer heartbeat.Stop()
+
+ for {
+ select {
+ case <-streamCtx.Done():
+ return
+ case event := <-events:
+ if err := encoder.Encode(event); err != nil {
+ return
+ }
+ flush()
+ case <-heartbeat.C:
+ if err := encoder.Encode(activity.StreamEvent{
+ Type: "heartbeat",
+ Timestamp: time.Now(),
+ }); err != nil {
+ return
+ }
+ flush()
+ }
+ }
+}
+
+// sendActivityStreamEventInternal forwards an event to the stream writer,
+// giving up when the stream is shutting down so producers can never block.
+func sendActivityStreamEventInternal(ctx context.Context, events chan<- activity.StreamEvent, event activity.StreamEvent) bool {
+ select {
+ case events <- event:
+ return true
+ case <-ctx.Done():
+ return false
+ }
+}
+
+func (h *ActivityHandler) runLocalActivityStreamProducerInternal(ctx context.Context, limit int, events chan<- activity.StreamEvent) {
+ sendSnapshot := func() bool {
+ activities, _, err := h.activityService.ListActivitiesPaginated(ctx, "0", pagination.QueryParams{
+ Params: pagination.Params{Limit: resolveActivityStreamLimitInternal(limit)},
})
if err != nil {
+ if ctx.Err() == nil {
+ sendActivityStreamEventInternal(ctx, events, activity.StreamEvent{
+ Type: "error",
+ EnvironmentID: "0",
+ Error: err.Error(),
+ Timestamp: time.Now(),
+ })
+ }
return false
}
- if err := encoder.Encode(activity.StreamEvent{
- Type: "snapshot",
- Activities: output.Body.Data,
- Timestamp: time.Now(),
- }); err != nil {
- return false
+ h.applyActivitySourceLabelsInternal(ctx, "0", activities)
+ return sendActivityStreamEventInternal(ctx, events, activity.StreamEvent{
+ Type: "snapshot",
+ EnvironmentID: "0",
+ Activities: activities,
+ Timestamp: time.Now(),
+ })
+ }
+
+ snapshotOK := sendSnapshot()
+
+ eventCh, missedEvents, unsubscribe := h.activityService.Subscribe("0")
+ defer unsubscribe()
+
+ ticker := time.NewTicker(activityStreamHeartbeatInterval)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case event, ok := <-eventCh:
+ if !ok {
+ return
+ }
+ event.EnvironmentID = "0"
+ h.applyActivityStreamEventSourceLabelInternal(ctx, "0", &event)
+ if !sendActivityStreamEventInternal(ctx, events, event) {
+ return
+ }
+ case <-ticker.C:
+ if !snapshotOK || missedEvents() {
+ snapshotOK = sendSnapshot()
+ }
}
- flush()
- return true
}
+}
- if !sendSnapshot(ctx) {
- return
+// runRemoteActivityStreamPollersInternal keeps one poller goroutine per
+// enabled remote environment, re-listing periodically so environments added
+// or removed while the stream is open are picked up without a reconnect.
+func (h *ActivityHandler) runRemoteActivityStreamPollersInternal(ctx context.Context, limit int, events chan<- activity.StreamEvent) {
+ pollers := make(map[string]context.CancelFunc)
+ var wg sync.WaitGroup
+ defer wg.Wait()
+ defer func() {
+ for _, cancelPoll := range pollers {
+ cancelPoll()
+ }
+ }()
+
+ reconcile := func() {
+ envs, err := h.environmentService.ListRemoteEnvironments(ctx)
+ if err != nil {
+ if ctx.Err() == nil {
+ slog.WarnContext(ctx, "failed to list environments for activity stream", "error", err)
+ }
+ return
+ }
+
+ current := make(map[string]struct{}, len(envs))
+ for _, env := range envs {
+ current[env.ID] = struct{}{}
+ if _, ok := pollers[env.ID]; ok {
+ continue
+ }
+ pollCtx, cancelPoll := context.WithCancel(ctx) //nolint:gosec // cancel is retained in pollers and invoked on env removal or via the deferred cleanup.
+ pollers[env.ID] = cancelPoll
+ wg.Add(1)
+ go func(environmentID string) {
+ defer wg.Done()
+ h.runRemoteActivityStreamPollerInternal(pollCtx, environmentID, limit, events)
+ }(env.ID)
+ }
+
+ for id, cancelPoll := range pollers {
+ if _, ok := current[id]; !ok {
+ cancelPoll()
+ delete(pollers, id)
+ }
+ }
}
+ reconcile()
+
+ ticker := time.NewTicker(activityStreamEnvReconcileInterval)
+ defer ticker.Stop()
+
for {
select {
case <-ctx.Done():
return
- case <-pollTicker.C:
- if !sendSnapshot(ctx) {
+ case <-ticker.C:
+ reconcile()
+ }
+ }
+}
+
+func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Context, environmentID string, limit int, events chan<- activity.StreamEvent) {
+ lastError := ""
+
+ poll := func() {
+ pollCtx, cancelPoll := context.WithTimeout(ctx, activityStreamRemotePollTimeout)
+ defer cancelPoll()
+
+ output, err := h.proxyListActivitiesInternal(pollCtx, &ListActivitiesInput{
+ EnvironmentID: environmentID,
+ Limit: resolveActivityStreamLimitInternal(limit),
+ Order: "desc",
+ })
+ if err != nil {
+ if ctx.Err() != nil {
return
}
- case <-heartbeatTicker.C:
- if err := encoder.Encode(activity.StreamEvent{
- Type: "heartbeat",
- Timestamp: time.Now(),
- }); err != nil {
- return
+ // A failing environment must not end the stream; surface the error
+ // once per distinct message and keep polling.
+ if msg := err.Error(); msg != lastError {
+ lastError = msg
+ sendActivityStreamEventInternal(ctx, events, activity.StreamEvent{
+ Type: "error",
+ EnvironmentID: environmentID,
+ Error: msg,
+ Timestamp: time.Now(),
+ })
}
- flush()
+ return
+ }
+ lastError = ""
+ sendActivityStreamEventInternal(ctx, events, activity.StreamEvent{
+ Type: "snapshot",
+ EnvironmentID: environmentID,
+ Activities: output.Body.Data,
+ Timestamp: time.Now(),
+ })
+ }
+
+ poll()
+
+ ticker := time.NewTicker(activityStreamRemotePollInterval)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ poll()
}
}
}
diff --git a/backend/api/handlers/activities_test.go b/backend/api/handlers/activities_test.go
index 1b042963a..295671b50 100644
--- a/backend/api/handlers/activities_test.go
+++ b/backend/api/handlers/activities_test.go
@@ -1,7 +1,10 @@
package handlers
import (
+ "bufio"
"context"
+ "encoding/json"
+ "io"
"net/http"
"net/http/httptest"
"testing"
@@ -14,6 +17,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
+ "github.com/getarcaneapp/arcane/types/v2/activity"
)
func setupActivityHandlerTestDBInternal(t *testing.T) *database.DB {
@@ -97,3 +101,145 @@ func TestActivityHandlerClearHistoryProxiesRemoteEnvironmentInternal(t *testing.
require.NoError(t, err)
require.EqualValues(t, 7, out.Body.Data.Deleted)
}
+
+// limitStreamTestDBToSingleConnInternal serializes DB access: the aggregated
+// stream queries from concurrent goroutines, and every extra pooled
+// connection to a :memory: SQLite database is a fresh empty database.
+func limitStreamTestDBToSingleConnInternal(t *testing.T, db *database.DB) {
+ t.Helper()
+ sqlDB, err := db.DB.DB()
+ require.NoError(t, err)
+ sqlDB.SetMaxOpenConns(1)
+}
+
+func createStreamTestRemoteEnvironmentInternal(t *testing.T, db *database.DB, apiURL, token string) {
+ t.Helper()
+ now := time.Now()
+ require.NoError(t, db.Create(&models.Environment{
+ BaseModel: models.BaseModel{
+ ID: "remote-1",
+ CreatedAt: now,
+ UpdatedAt: &now,
+ },
+ Name: "Remote",
+ ApiUrl: apiURL,
+ Status: string(models.EnvironmentStatusOnline),
+ Enabled: true,
+ AccessToken: &token,
+ }).Error)
+}
+
+// runStreamAllInternal drives streamAllActivitiesInternal through a pipe and
+// returns each decoded event to onEvent until it reports done or the stream
+// ends; remaining output is drained so a blocked encoder can always finish.
+func runStreamAllInternal(t *testing.T, ctx context.Context, cancel context.CancelFunc, handler *ActivityHandler, onEvent func(activity.StreamEvent) bool) {
+ t.Helper()
+
+ pr, pw := io.Pipe()
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ defer pw.Close()
+ handler.streamAllActivitiesInternal(ctx, 50, json.NewEncoder(pw), func() {})
+ }()
+
+ scanner := bufio.NewScanner(pr)
+ for scanner.Scan() {
+ var event activity.StreamEvent
+ require.NoError(t, json.Unmarshal(scanner.Bytes(), &event))
+ if onEvent(event) {
+ cancel()
+ break
+ }
+ }
+
+ go func() {
+ _, _ = io.Copy(io.Discard, pr)
+ }()
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("stream did not terminate after cancel")
+ }
+}
+
+func TestActivityHandlerStreamAllEmitsEnvironmentScopedEventsInternal(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ db := setupActivityHandlerTestDBInternal(t)
+ limitStreamTestDBToSingleConnInternal(t, db)
+ settingsService, err := services.NewSettingsService(ctx, db)
+ require.NoError(t, err)
+ activityService := services.NewActivityService(db)
+
+ local, err := activityService.StartActivity(ctx, services.StartActivityRequest{EnvironmentID: "0", Type: models.ActivityTypeResourceAction})
+ require.NoError(t, err)
+
+ token := "remote-token"
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"success":true,"data":[{"id":"remote-activity-1"}],"pagination":{"totalPages":1,"totalItems":1,"currentPage":1,"itemsPerPage":50}}`))
+ }))
+ defer server.Close()
+ createStreamTestRemoteEnvironmentInternal(t, db, server.URL, token)
+
+ handler := &ActivityHandler{
+ activityService: activityService,
+ environmentService: services.NewEnvironmentService(db, server.Client(), nil, nil, settingsService, nil),
+ }
+
+ var localSnapshot, remoteSnapshot bool
+ runStreamAllInternal(t, ctx, cancel, handler, func(event activity.StreamEvent) bool {
+ if event.Type == "snapshot" && event.EnvironmentID == "0" && len(event.Activities) == 1 {
+ require.Equal(t, local.ID, event.Activities[0].ID)
+ require.Equal(t, "0", event.Activities[0].SourceEnvironmentID)
+ localSnapshot = true
+ }
+ if event.Type == "snapshot" && event.EnvironmentID == "remote-1" && len(event.Activities) == 1 {
+ require.Equal(t, "remote-activity-1", event.Activities[0].ID)
+ remoteSnapshot = true
+ }
+ return localSnapshot && remoteSnapshot
+ })
+
+ require.True(t, localSnapshot)
+ require.True(t, remoteSnapshot)
+}
+
+func TestActivityHandlerStreamAllRemoteFailureEmitsErrorAndKeepsStreamingInternal(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ db := setupActivityHandlerTestDBInternal(t)
+ limitStreamTestDBToSingleConnInternal(t, db)
+ settingsService, err := services.NewSettingsService(ctx, db)
+ require.NoError(t, err)
+ activityService := services.NewActivityService(db)
+
+ token := "remote-token"
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "boom", http.StatusInternalServerError)
+ }))
+ defer server.Close()
+ createStreamTestRemoteEnvironmentInternal(t, db, server.URL, token)
+
+ handler := &ActivityHandler{
+ activityService: activityService,
+ environmentService: services.NewEnvironmentService(db, server.Client(), nil, nil, settingsService, nil),
+ }
+
+ var localSnapshot, remoteError bool
+ runStreamAllInternal(t, ctx, cancel, handler, func(event activity.StreamEvent) bool {
+ if event.Type == "snapshot" && event.EnvironmentID == "0" {
+ localSnapshot = true
+ }
+ if event.Type == "error" && event.EnvironmentID == "remote-1" && event.Error != "" {
+ remoteError = true
+ }
+ return localSnapshot && remoteError
+ })
+
+ require.True(t, localSnapshot)
+ require.True(t, remoteError)
+}
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go
index 2374c2e5e..3c329346c 100644
--- a/backend/internal/bootstrap/bootstrap.go
+++ b/backend/internal/bootstrap/bootstrap.go
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"log/slog"
+ "net"
"net/http"
"os"
"os/signal"
@@ -377,7 +378,14 @@ func runServicesInternal(appCtx context.Context, cfg *config.Config, router http
httpHandler, grpcServer := configureTunnelServerInternal(appCtx, cfg, router, tunnelServer, listenAddr)
httpHandler, protocols := configureHTTPProtocolsInternal(useTLS, httpHandler)
- srv, err := newHTTPServerInternal(listenAddr, httpHandler, protocols, useTLS, edgeCfg)
+ // Base context for all request contexts. Derived from Background, NOT
+ // appCtx: appCtx carries the app-lifecycle marker value and inheriting it
+ // would make every request context pass utils.IsAppLifecycleContext (see
+ // pkg/projects/cmds.go).
+ baseCtx, cancelBase := context.WithCancel(context.Background())
+ defer cancelBase()
+
+ srv, err := newHTTPServerInternal(baseCtx, listenAddr, httpHandler, protocols, useTLS, edgeCfg) //nolint:contextcheck // baseCtx is deliberately not derived from appCtx, see comment above
if err != nil {
return err
}
@@ -407,6 +415,12 @@ func runServicesInternal(appCtx context.Context, cfg *config.Config, router http
slog.InfoContext(appCtx, "Context canceled")
}
+ // http.Server.Shutdown waits for in-flight handlers but does not cancel
+ // their request contexts; streaming handlers (activity streams, JSON-lines
+ // progress) loop on ctx.Done() and would otherwise pin Shutdown until the
+ // deadline below.
+ cancelBase()
+
// Use background context for shutdown as appCtx is already canceled
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer shutdownCancel()
@@ -489,7 +503,7 @@ func configureHTTPProtocolsInternal(useTLS bool, handler http.Handler) (http.Han
return handler, &protocols
}
-func newHTTPServerInternal(listenAddr string, handler http.Handler, protocols *http.Protocols, useTLS bool, edgeCfg *edge.Config) (*http.Server, error) {
+func newHTTPServerInternal(baseCtx context.Context, listenAddr string, handler http.Handler, protocols *http.Protocols, useTLS bool, edgeCfg *edge.Config) (*http.Server, error) {
srv := &http.Server{
Addr: listenAddr,
Handler: handler,
@@ -501,6 +515,10 @@ func newHTTPServerInternal(listenAddr string, handler http.Handler, protocols *h
// streaming endpoints (deploy/pull/build progress) need long-lived
// connections.
IdleTimeout: 120 * time.Second,
+ // BaseContext is canceled right before Shutdown so long-lived
+ // streaming request contexts unblock and graceful shutdown can
+ // complete within its deadline.
+ BaseContext: func(net.Listener) context.Context { return baseCtx },
}
if !useTLS {
return srv, nil
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index 277e1c32f..159fe3bcb 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -212,6 +212,47 @@ func TestHTTP2APIResponsesDoNotUseAPIGzipInternal(t *testing.T) {
}
}
+func TestShutdownCancelsStreamingRequestContextsInternal(t *testing.T) {
+ baseCtx, cancelBase := context.WithCancel(context.Background())
+ defer cancelBase()
+
+ handlerEntered := make(chan struct{})
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/x-json-stream")
+ w.WriteHeader(http.StatusOK)
+ if f, ok := w.(http.Flusher); ok {
+ f.Flush()
+ }
+ close(handlerEntered)
+ // Streaming handlers block until their request context ends; Shutdown
+ // alone never cancels it, so this models an open activity stream.
+ <-r.Context().Done()
+ })
+
+ srv, err := newHTTPServerInternal(baseCtx, "127.0.0.1:0", handler, nil, false, nil)
+ require.NoError(t, err)
+
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+
+ errCh := make(chan error, 1)
+ go func() { errCh <- srv.Serve(listener) }()
+
+ resp, err := http.Get("http://" + listener.Addr().String() + "/stream")
+ require.NoError(t, err)
+ defer func() { _ = resp.Body.Close() }()
+ <-handlerEntered
+
+ cancelBase()
+
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+ start := time.Now()
+ require.NoError(t, srv.Shutdown(shutdownCtx))
+ require.Less(t, time.Since(start), time.Second)
+ require.ErrorIs(t, <-errCh, http.ErrServerClosed)
+}
+
func TestPrepareServerTLSInternal_AgentModeSkipsManagerMTLSValidation(t *testing.T) {
cfg := &config.Config{
AgentMode: true,
diff --git a/backend/internal/middleware/environment_middleware_test.go b/backend/internal/middleware/environment_middleware_test.go
index a61bd689c..6a84730a8 100644
--- a/backend/internal/middleware/environment_middleware_test.go
+++ b/backend/internal/middleware/environment_middleware_test.go
@@ -157,12 +157,6 @@ func TestEnvironmentMiddleware_KeepsActivityEndpointsLocal(t *testing.T) {
route: "/environments/:id/activities",
path: "/api/environments/env-edge/activities?limit=50",
},
- {
- name: "stream activities",
- method: http.MethodGet,
- route: "/environments/:id/activities/stream",
- path: "/api/environments/env-edge/activities/stream?limit=50",
- },
}
for _, tt := range tests {
diff --git a/backend/pkg/libarcane/edge/commands.go b/backend/pkg/libarcane/edge/commands.go
index 8a36ec882..0f2a0f130 100644
--- a/backend/pkg/libarcane/edge/commands.go
+++ b/backend/pkg/libarcane/edge/commands.go
@@ -51,7 +51,6 @@ var commandRoutes = []commandRoute{
{Method: http.MethodGet, PathPattern: "/api/environments/{id}/image-updates/summary", CommandName: "image_update.summary"},
{Method: http.MethodGet, PathPattern: "/api/environments/{id}/activities", CommandName: "activity.list"},
- {Method: http.MethodGet, PathPattern: "/api/environments/{id}/activities/stream", LocalOnly: true},
{Method: http.MethodGet, PathPattern: "/api/environments/{id}/activities/{activityId}", CommandName: "activity.inspect"},
{Method: http.MethodPost, PathPattern: "/api/environments/{id}/activities/{activityId}/cancel", CommandName: "activity.cancel"},
{Method: http.MethodDelete, PathPattern: "/api/environments/{id}/activities/history", CommandName: "activity.history.clear"},
diff --git a/backend/pkg/libarcane/edge/commands_test.go b/backend/pkg/libarcane/edge/commands_test.go
index 2ee27d822..bf02ef673 100644
--- a/backend/pkg/libarcane/edge/commands_test.go
+++ b/backend/pkg/libarcane/edge/commands_test.go
@@ -29,7 +29,6 @@ func TestResolveEdgeCommandName(t *testing.T) {
{name: "activity inspect", method: "GET", path: "/api/environments/0/activities/activity-1", command: "activity.inspect", shouldHit: true},
{name: "activity cancel", method: "POST", path: "/api/environments/0/activities/activity-1/cancel", command: "activity.cancel", shouldHit: true},
{name: "activity history clear", method: "DELETE", path: "/api/environments/0/activities/history", command: "activity.history.clear", shouldHit: true},
- {name: "activity stream remains manager local", method: "GET", path: "/api/environments/0/activities/stream?limit=50", shouldHit: false},
{name: "health", method: "HEAD", path: "/api/environments/0/system/health", command: "system.health", shouldHit: true},
{name: "swarm node identity", method: "GET", path: "/api/swarm/node-identity", command: "swarm.node_identity", shouldHit: true},
{name: "unknown", method: "PATCH", path: "/api/environments/0/containers", shouldHit: false},
diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.go
index 452dc9fd6..b2a369efb 100644
--- a/backend/pkg/scheduler/auto_heal_job.go
+++ b/backend/pkg/scheduler/auto_heal_job.go
@@ -34,10 +34,14 @@ type AutoHealJob struct {
mu sync.Mutex
restarts map[string]*restartRecord
- getDockerClient func() (*client.Client, error)
- listContainers func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error)
- inspectContainer func(ctx context.Context, dockerClient *client.Client, containerID string) (container.InspectResponse, error)
- restartContainer func(ctx context.Context, dockerClient *client.Client, containerID string) error
+ selfIDOnce sync.Once
+ selfID string
+
+ getDockerClient func() (*client.Client, error)
+ listContainers func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error)
+ inspectContainer func(ctx context.Context, dockerClient *client.Client, containerID string) (container.InspectResponse, error)
+ restartContainer func(ctx context.Context, dockerClient *client.Client, containerID string) error
+ getSelfContainerID func() (string, error)
}
func NewAutoHealJob(
@@ -103,7 +107,8 @@ func (j *AutoHealJob) Run(ctx context.Context) {
restartWindowMinutes := j.settingsService.GetIntSetting(ctx, "autoHealRestartWindow", 30)
restartWindow := time.Duration(restartWindowMinutes) * time.Minute
- candidates := j.filterCandidatesInternal(containers, excludedContainers)
+ selfID := j.selfContainerIDInternal(ctx)
+ candidates := j.filterCandidatesInternal(containers, excludedContainers, selfID)
g, groupCtx := errgroup.WithContext(ctx)
g.SetLimit(autoHealInspectConcurrency)
@@ -118,9 +123,37 @@ func (j *AutoHealJob) Run(ctx context.Context) {
_ = g.Wait()
}
-func (j *AutoHealJob) filterCandidatesInternal(containers []container.Summary, excludedContainers map[string]struct{}) []container.Summary {
+// selfContainerIDInternal resolves and caches the ID (full 64-char or short
+// prefix) of the container Arcane itself runs in. Returns "" when detection
+// fails (e.g. binary running directly on the host), disabling the guard.
+func (j *AutoHealJob) selfContainerIDInternal(ctx context.Context) string {
+ j.selfIDOnce.Do(func() {
+ detect := j.getSelfContainerID
+ if detect == nil {
+ detect = dockerutil.GetCurrentContainerID
+ }
+ id, err := detect()
+ if err != nil {
+ slog.DebugContext(ctx, "auto-heal: could not determine own container ID; self-protection disabled", "error", err)
+ return
+ }
+ j.selfID = strings.ToLower(strings.TrimSpace(id))
+ slog.InfoContext(ctx, "auto-heal: detected own container; it will never be auto-restarted", "container_id", j.selfID)
+ })
+ return j.selfID
+}
+
+func (j *AutoHealJob) filterCandidatesInternal(containers []container.Summary, excludedContainers map[string]struct{}, selfID string) []container.Summary {
candidates := make([]container.Summary, 0, len(containers))
for _, c := range containers {
+ // Never restart the container Arcane itself runs in: a slow or
+ // mid-startup manager that trips its own healthcheck would otherwise
+ // be restarted by its own auto-heal, in a loop. Prefix match because
+ // hostname-based detection yields the short 12-char ID.
+ if selfID != "" && strings.HasPrefix(strings.ToLower(c.ID), selfID) {
+ continue
+ }
+
if libarcane.IsInternalContainer(c.Labels) {
continue
}
diff --git a/backend/pkg/scheduler/auto_heal_job_test.go b/backend/pkg/scheduler/auto_heal_job_test.go
index e5658bbbd..012e852de 100644
--- a/backend/pkg/scheduler/auto_heal_job_test.go
+++ b/backend/pkg/scheduler/auto_heal_job_test.go
@@ -22,6 +22,30 @@ func newTestAutoHealJob() *AutoHealJob {
}
}
+func TestAutoHeal_FilterCandidates_SkipsSelfContainer(t *testing.T) {
+ job := newTestAutoHealJob()
+
+ selfFullID := "407163929c492b5c4b01a3981f5de4774c37aa8300bd214b4d62412a3dc56468"
+ containers := []container.Summary{
+ {ID: selfFullID, Names: []string{"/arcane"}},
+ {ID: "aaaabbbbccccddddeeeeffff0000111122223333444455556666777788889999", Names: []string{"/other"}},
+ }
+
+ // Hostname-based detection yields the short 12-char ID.
+ candidates := job.filterCandidatesInternal(containers, nil, selfFullID[:12])
+ require.Len(t, candidates, 1)
+ require.Equal(t, "/other", candidates[0].Names[0])
+
+ // cgroup/mountinfo-based detection yields the full ID.
+ candidates = job.filterCandidatesInternal(containers, nil, selfFullID)
+ require.Len(t, candidates, 1)
+ require.Equal(t, "/other", candidates[0].Names[0])
+
+ // Outside Docker no self ID is detected and the guard is a no-op.
+ candidates = job.filterCandidatesInternal(containers, nil, "")
+ require.Len(t, candidates, 2)
+}
+
func TestAutoHeal_CanRestart_UnderLimit(t *testing.T) {
job := newTestAutoHealJob()
diff --git a/frontend/src/lib/components/activity/activity-center.svelte b/frontend/src/lib/components/activity/activity-center.svelte
index b402f01d4..b1df36966 100644
--- a/frontend/src/lib/components/activity/activity-center.svelte
+++ b/frontend/src/lib/components/activity/activity-center.svelte
@@ -19,6 +19,7 @@
onMount(() => {
void activityStore.start();
+ return () => activityStore.stop();
});
function handleOpenChangeInternal(open: boolean) {
diff --git a/frontend/src/lib/services/activity-service.ts b/frontend/src/lib/services/activity-service.ts
index b03f8eec2..5b2426b28 100644
--- a/frontend/src/lib/services/activity-service.ts
+++ b/frontend/src/lib/services/activity-service.ts
@@ -31,25 +31,22 @@ class ActivityService extends BaseAPIService {
return this.handleResponse(this.api.delete(`/environments/${envId}/activities/history`));
}
- getActivityStreamUrl(environmentId: string, limit = 50): string {
+ getActivityStreamUrl(limit = 50): string {
const baseUrl = this.api.defaults.baseURL.replace(/\/+$/, '');
const params = new URLSearchParams({ limit: String(limit) });
- return `${baseUrl}/environments/${encodeURIComponent(environmentId)}/activities/stream?${params.toString()}`;
+ return `${baseUrl}/activities/stream?${params.toString()}`;
}
- async openActivityStream(environmentId: string, signal: AbortSignal, limit = 50, retry = false): Promise<Response> {
- const response = await fetch(this.getActivityStreamUrl(environmentId, limit), {
+ async openActivityStream(signal: AbortSignal, limit = 50, retry = false): Promise<Response> {
+ const response = await fetch(this.getActivityStreamUrl(limit), {
credentials: 'include',
headers: { Accept: 'application/x-json-stream' },
signal
});
if (response.status === 401) {
- const action = await handleUnauthorizedResponseInternal(
- `/environments/${encodeURIComponent(environmentId)}/activities/stream`,
- retry
- );
+ const action = await handleUnauthorizedResponseInternal('/activities/stream', retry);
if (action === 'retry') {
- return this.openActivityStream(environmentId, signal, limit, true);
+ return this.openActivityStream(signal, limit, true);
}
if (action === 'redirect') {
return new Promise<Response>(() => {});
diff --git a/frontend/src/lib/stores/activity.store.svelte.ts b/frontend/src/lib/stores/activity.store.svelte.ts
index 4d5c67a24..6389f7afa 100644
--- a/frontend/src/lib/stores/activity.store.svelte.ts
+++ b/frontend/src/lib/stores/activity.store.svelte.ts
@@ -23,7 +23,6 @@ type ActivityEnvironmentState = {
name: string;
activities: Activity[];
loading: boolean;
- connected: boolean;
streamError: boolean;
errorMessage?: string;
};
@@ -92,10 +91,14 @@ function createActivityStore() {
let started = false;
let unsubscribeEnvironment: (() => void) | null = null;
- const streamAbortControllers = new Map<string, AbortController>();
- const reconnectTimers = new Map<string, ReturnType<typeof setTimeout>>();
- const reconnectAttempts = new Map<string, number>();
- const streamGenerations = new Map<string, number>();
+ // A single aggregated stream carries every environment's events; per-env
+ // connections would exhaust the browser's 6-per-origin HTTP/1.1 limit.
+ let streamAbortController: AbortController | null = null;
+ let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
+ let reconnectAttempt = 0;
+ let streamGeneration = 0;
+ let _streamConnected = $state(false);
+ let _streamFailed = $state(false);
function createEnvironmentStateInternal(environment: Pick<Environment, 'id' | 'name'>): ActivityEnvironmentState {
return {
@@ -103,7 +106,6 @@ function createActivityStore() {
name: environmentNameInternal(environment),
activities: [],
loading: true,
- connected: false,
streamError: false
};
}
@@ -128,7 +130,6 @@ function createActivityStore() {
updateEnvironmentStateInternal(environmentId, (state) => ({
...state,
loading: false,
- connected: false,
streamError: true,
errorMessage: errorMessageInternal(error)
}));
@@ -142,44 +143,40 @@ function createActivityStore() {
}));
}
- function generationInternal(environmentId: string): number {
- return streamGenerations.get(environmentId) ?? 0;
+ // A fresh stream re-emits error events for environments that are still
+ // failing, so stale per-environment errors are cleared on every (re)connect.
+ function clearAllEnvironmentErrorsInternal() {
+ for (const environmentId of Object.keys(_environmentStates)) {
+ if (environmentStateInternal(environmentId)?.streamError) {
+ clearEnvironmentErrorInternal(environmentId);
+ }
+ }
}
- function nextGenerationInternal(environmentId: string): number {
- const generation = generationInternal(environmentId) + 1;
- streamGenerations.set(environmentId, generation);
- return generation;
+ function nextGenerationInternal(): number {
+ streamGeneration += 1;
+ return streamGeneration;
}
- function isCurrentGenerationInternal(environmentId: string, generation: number): boolean {
- return generationInternal(environmentId) === generation;
+ function isCurrentGenerationInternal(generation: number): boolean {
+ return streamGeneration === generation;
}
- function clearReconnectTimerInternal(environmentId: string) {
- const timer = reconnectTimers.get(environmentId);
- if (timer) {
- clearTimeout(timer);
- reconnectTimers.delete(environmentId);
+ function clearReconnectTimerInternal() {
+ if (reconnectTimer) {
+ clearTimeout(reconnectTimer);
+ reconnectTimer = null;
}
}
- function abortEnvironmentStreamInternal(environmentId: string) {
- clearReconnectTimerInternal(environmentId);
- streamAbortControllers.get(environmentId)?.abort();
- streamAbortControllers.delete(environmentId);
- updateEnvironmentStateInternal(environmentId, (state) => ({
- ...state,
- connected: false
- }));
+ function abortStreamInternal() {
+ clearReconnectTimerInternal();
+ streamAbortController?.abort();
+ streamAbortController = null;
+ _streamConnected = false;
}
- function stopEnvironmentInternal(environmentId: string) {
- nextGenerationInternal(environmentId);
- abortEnvironmentStreamInternal(environmentId);
- reconnectAttempts.delete(environmentId);
- streamGenerations.delete(environmentId);
-
+ function removeEnvironmentInternal(environmentId: string) {
const nextStates = { ..._environmentStates };
delete nextStates[environmentId];
_environmentStates = nextStates;
@@ -199,6 +196,11 @@ function createActivityStore() {
}
function replaceEnvironmentSnapshotInternal(environmentId: string, activities: Activity[]) {
+ // Snapshots can still arrive (stream or in-flight REST) after the
+ // environment was removed locally; don't resurrect it.
+ if (!environmentStateInternal(environmentId)) {
+ return;
+ }
const normalizedActivities = sortActivitiesInternal(
activities.map((activity) => normalizeActivityInternal(activity, environmentId))
);
@@ -283,6 +285,11 @@ function createActivityStore() {
}
function applyStreamEventInternal(environmentId: string, event: ActivityStreamEvent) {
+ // The aggregated stream can keep delivering events for an environment
+ // for a short while after it was removed locally; don't resurrect it.
+ if (event.type !== 'heartbeat' && !environmentStateInternal(environmentId)) {
+ return;
+ }
switch (event.type) {
case 'snapshot':
replaceEnvironmentSnapshotInternal(environmentId, event.activities ?? []);
@@ -298,27 +305,28 @@ function createActivityStore() {
}
break;
case 'heartbeat':
- updateEnvironmentStateInternal(environmentId, (state) => ({
- ...state,
- connected: true
- }));
+ _streamConnected = true;
+ break;
+ case 'error':
+ setEnvironmentErrorInternal(environmentId, new Error(event.error || 'Activity stream error'));
break;
}
}
- async function refreshEnvironmentInternal(environmentId: string, generation = generationInternal(environmentId)) {
+ async function refreshEnvironmentInternal(environmentId: string, generation = streamGeneration) {
updateEnvironmentStateInternal(environmentId, (state) => ({
...state,
loading: true
}));
try {
const result = await activityService.getActivities({ pagination: { page: 1, limit: ACTIVITY_LIST_LIMIT } }, environmentId);
- if (!isCurrentGenerationInternal(environmentId, generation)) {
+ // The environment can be removed while the fetch is in-flight; don't resurrect it.
+ if (!isCurrentGenerationInternal(generation) || !environmentStateInternal(environmentId)) {
return;
}
replaceEnvironmentSnapshotInternal(environmentId, result.data ?? []);
} catch (error) {
- if (isCurrentGenerationInternal(environmentId, generation)) {
+ if (isCurrentGenerationInternal(generation) && environmentStateInternal(environmentId)) {
console.warn('Failed to refresh activities:', error);
setEnvironmentErrorInternal(environmentId, error);
}
@@ -330,57 +338,51 @@ function createActivityStore() {
await Promise.all(Object.keys(_environmentStates).map((environmentId) => refreshEnvironmentInternal(environmentId)));
}
- async function connectStreamInternal(environmentId: string, generation: number) {
- if (!browser || !isCurrentGenerationInternal(environmentId, generation)) {
+ async function connectStreamInternal(generation: number) {
+ if (!browser || !isCurrentGenerationInternal(generation)) {
return;
}
const controller = new AbortController();
- streamAbortControllers.set(environmentId, controller);
+ streamAbortController = controller;
try {
- const response = await activityService.openActivityStream(environmentId, controller.signal, ACTIVITY_LIST_LIMIT);
- if (!isCurrentGenerationInternal(environmentId, generation) || !response.body) {
- if (streamAbortControllers.get(environmentId) === controller) {
- streamAbortControllers.delete(environmentId);
+ const response = await activityService.openActivityStream(controller.signal, ACTIVITY_LIST_LIMIT);
+ if (!isCurrentGenerationInternal(generation) || !response.body) {
+ if (streamAbortController === controller) {
+ streamAbortController = null;
}
return;
}
- updateEnvironmentStateInternal(environmentId, (state) => ({
- ...state,
- connected: true,
- streamError: false,
- errorMessage: undefined
- }));
- reconnectAttempts.set(environmentId, 0);
- await readJSONLinesInternal(environmentId, response.body, generation);
+ _streamConnected = true;
+ _streamFailed = false;
+ reconnectAttempt = 0;
+ clearAllEnvironmentErrorsInternal();
+ await readJSONLinesInternal(response.body, generation);
} catch (error) {
- if (!controller.signal.aborted && isCurrentGenerationInternal(environmentId, generation)) {
+ if (!controller.signal.aborted && isCurrentGenerationInternal(generation)) {
console.warn('Activity stream disconnected:', error);
}
} finally {
- if (streamAbortControllers.get(environmentId) === controller) {
- streamAbortControllers.delete(environmentId);
+ if (streamAbortController === controller) {
+ streamAbortController = null;
}
- if (isCurrentGenerationInternal(environmentId, generation)) {
- updateEnvironmentStateInternal(environmentId, (state) => ({
- ...state,
- connected: false
- }));
+ if (isCurrentGenerationInternal(generation)) {
+ _streamConnected = false;
if (!controller.signal.aborted) {
- scheduleReconnectInternal(environmentId, generation);
+ scheduleReconnectInternal(generation);
}
}
}
}
- async function readJSONLinesInternal(environmentId: string, stream: ReadableStream<Uint8Array>, generation: number) {
+ async function readJSONLinesInternal(stream: ReadableStream<Uint8Array>, generation: number) {
const reader = stream.getReader();
const decoder = new TextDecoder();
let buffer = '';
try {
- while (isCurrentGenerationInternal(environmentId, generation)) {
+ while (isCurrentGenerationInternal(generation)) {
const { done, value } = await reader.read();
if (done) {
break;
@@ -390,77 +392,49 @@ function createActivityStore() {
const lines = buffer.split('\n');
buffer = lines.pop() ?? '';
for (const line of lines) {
- handleStreamLineInternal(environmentId, line);
+ handleStreamLineInternal(line);
}
}
buffer += decoder.decode();
if (buffer.trim()) {
- handleStreamLineInternal(environmentId, buffer);
+ handleStreamLineInternal(buffer);
}
} finally {
reader.releaseLock();
}
}
- function handleStreamLineInternal(environmentId: string, line: string) {
+ function handleStreamLineInternal(line: string) {
const trimmed = line.trim();
if (!trimmed) {
return;
}
try {
- applyStreamEventInternal(environmentId, JSON.parse(trimmed) as ActivityStreamEvent);
+ const event = JSON.parse(trimmed) as ActivityStreamEvent;
+ applyStreamEventInternal(event.environmentId || LOCAL_DOCKER_ENVIRONMENT_ID, event);
} catch (error) {
console.warn('Failed to parse activity stream line:', error);
}
}
- function scheduleReconnectInternal(environmentId: string, generation: number) {
- if (!browser || !started || !isCurrentGenerationInternal(environmentId, generation)) {
- return;
- }
-
- const attempt = reconnectAttempts.get(environmentId) ?? 0;
- if (attempt >= MAX_RECONNECT_ATTEMPTS) {
- setEnvironmentErrorInternal(environmentId, new Error('Activity stream reconnect attempts exhausted'));
+ function scheduleReconnectInternal(generation: number) {
+ if (!browser || !started || !isCurrentGenerationInternal(generation)) {
return;
}
- clearReconnectTimerInternal(environmentId);
- const delay = Math.min(1000 * 2 ** attempt, MAX_RECONNECT_DELAY);
- reconnectAttempts.set(environmentId, attempt + 1);
- reconnectTimers.set(
- environmentId,
- setTimeout(() => {
- void connectStreamInternal(environmentId, generation);
- }, delay)
- );
- }
-
- function startEnvironmentInternal(environment: Pick<Environment, 'id' | 'name'>) {
- const environmentId = environment.id || LOCAL_DOCKER_ENVIRONMENT_ID;
- updateEnvironmentStateInternal(environmentId, (state) => ({
- ...state,
- id: environmentId,
- name: environmentNameInternal(environment)
- }));
-
- const generation = nextGenerationInternal(environmentId);
- reconnectAttempts.set(environmentId, 0);
- abortEnvironmentStreamInternal(environmentId);
- void refreshEnvironmentInternal(environmentId, generation);
- void connectStreamInternal(environmentId, generation);
- }
-
- function restartEnvironmentInternal(environmentId: string) {
- const state = environmentStateInternal(environmentId);
- if (!state) {
+ if (reconnectAttempt >= MAX_RECONNECT_ATTEMPTS) {
+ _streamFailed = true;
return;
}
- clearEnvironmentErrorInternal(environmentId);
- startEnvironmentInternal({ id: state.id, name: state.name });
+ clearReconnectTimerInternal();
+ const delay = Math.min(1000 * 2 ** reconnectAttempt, MAX_RECONNECT_DELAY);
+ reconnectAttempt += 1;
+ reconnectTimer = setTimeout(() => {
+ void connectStreamInternal(generation);
+ }, delay);
}
function reconcileEnvironmentsInternal() {
@@ -468,7 +442,9 @@ function createActivityStore() {
return;
}
- const available = environmentStore.available;
+ // Track only enabled environments — they are the ones the aggregated
+ // stream serves; a disabled environment would never leave "loading".
+ const available = environmentStore.available.filter((environment) => environment.enabled);
const environments =
available.length > 0
? available
@@ -482,7 +458,7 @@ function createActivityStore() {
for (const environmentId of Object.keys(_environmentStates)) {
if (!targetIds.has(environmentId)) {
- stopEnvironmentInternal(environmentId);
+ removeEnvironmentInternal(environmentId);
}
}
@@ -494,7 +470,12 @@ function createActivityStore() {
..._environmentStates,
[environmentId]: createEnvironmentStateInternal(environment)
};
- startEnvironmentInternal(environment);
+ // An already-open aggregated stream only picks new environments
+ // up on its server-side reconcile tick; fetch once so the first
+ // snapshot doesn't take up to that interval to appear.
+ if (streamAbortController) {
+ void refreshEnvironmentInternal(environmentId);
+ }
continue;
}
@@ -612,10 +593,10 @@ function createActivityStore() {
return Object.values(_environmentStates).some((state) => state.loading);
},
get connected(): boolean {
- return Object.values(_environmentStates).some((state) => state.connected);
+ return _streamConnected;
},
get streamError(): boolean {
- return Object.values(_environmentStates).some((state) => state.streamError);
+ return _streamFailed || Object.values(_environmentStates).some((state) => state.streamError);
},
get environmentFailures(): ActivityEnvironmentFailure[] {
return environmentFailuresInternal();
@@ -654,6 +635,7 @@ function createActivityStore() {
await environmentStore.ready;
_currentEnvironmentId = environmentStore.selected?.id ?? LOCAL_DOCKER_ENVIRONMENT_ID;
reconcileEnvironmentsInternal();
+ void connectStreamInternal(nextGenerationInternal());
unsubscribeEnvironment = environmentStore.subscribeSelected((environment) => {
_currentEnvironmentId = environment?.id ?? LOCAL_DOCKER_ENVIRONMENT_ID;
reconcileEnvironmentsInternal();
@@ -663,11 +645,9 @@ function createActivityStore() {
started = false;
unsubscribeEnvironment?.();
unsubscribeEnvironment = null;
-
- for (const environmentId of Object.keys(_environmentStates)) {
- nextGenerationInternal(environmentId);
- abortEnvironmentStreamInternal(environmentId);
- }
+ nextGenerationInternal();
+ abortStreamInternal();
+ reconnectAttempt = 0;
},
refresh: () => refreshInternal(),
cancelActivity: async (activityId: string) => {
@@ -742,12 +722,11 @@ function createActivityStore() {
void loadDetailInternal(activityId);
},
retryStream: () => {
- for (const environmentId of Object.keys(_environmentStates)) {
- const state = environmentStateInternal(environmentId);
- if (state?.streamError) {
- restartEnvironmentInternal(environmentId);
- }
- }
+ _streamFailed = false;
+ reconnectAttempt = 0;
+ clearAllEnvironmentErrorsInternal();
+ abortStreamInternal();
+ void connectStreamInternal(nextGenerationInternal());
},
setActivityExpanded,
toggleActivity
diff --git a/frontend/src/lib/types/activity.type.ts b/frontend/src/lib/types/activity.type.ts
index 3fd0b69ef..8c158bfbc 100644
--- a/frontend/src/lib/types/activity.type.ts
+++ b/frontend/src/lib/types/activity.type.ts
@@ -85,10 +85,12 @@ export interface ActivityClearHistorySummary {
}
export interface ActivityStreamEvent {
- type: 'snapshot' | 'activity' | 'message' | 'heartbeat';
+ type: 'snapshot' | 'activity' | 'message' | 'heartbeat' | 'error';
+ environmentId?: string;
activityId?: string;
activity?: Activity;
activities?: Activity[];
message?: ActivityMessage;
+ error?: string;
timestamp: string;
}
diff --git a/tests/spec/activity-center.spec.ts b/tests/spec/activity-center.spec.ts
index 4fd42aeee..c1223fec3 100644
--- a/tests/spec/activity-center.spec.ts
+++ b/tests/spec/activity-center.spec.ts
@@ -39,7 +39,7 @@ const remoteEnvironment: MockEnvironment = {
name: 'Remote Lab',
apiUrl: 'https://remote.example.invalid',
status: 'offline',
- enabled: false,
+ enabled: true,
isEdge: false
};
@@ -83,7 +83,7 @@ function activity(
}
function activityEnvironmentIdFromPath(pathname: string): string | null {
- const match = pathname.match(/^\/api\/environments\/([^/]+)\/activities(?:\/stream)?$/);
+ const match = pathname.match(/^\/api\/environments\/([^/]+)\/activities$/);
return match ? decodeURIComponent(match[1]) : null;
}
@@ -103,54 +103,61 @@ async function mockEnvironmentList(page: Page, environments: MockEnvironment[])
});
}
+function aggregatedActivityStreamBody(
+ activitiesByEnvironment: Record<string, MockActivity[]>,
+ failedEnvironmentIds: Set<string>
+): string {
+ const timestamp = new Date().toISOString();
+ const events: string[] = [];
+ for (const [environmentId, activities] of Object.entries(activitiesByEnvironment)) {
+ events.push(JSON.stringify({ type: 'snapshot', environmentId, activities, timestamp }));
+ }
+ for (const environmentId of failedEnvironmentIds) {
+ events.push(
+ JSON.stringify({ type: 'error', environmentId, error: 'environment unavailable', timestamp })
+ );
+ }
+ return events.join('\n') + '\n';
+}
+
async function mockActivityReads(
page: Page,
activitiesByEnvironment: Record<string, MockActivity[]>,
failedEnvironmentIds = new Set<string>()
) {
+ await page.context().route(/\/api\/activities\/stream(?:\?.*)?$/, async (route: Route) => {
+ await route.fulfill({
+ status: 200,
+ contentType: 'application/x-json-stream',
+ body: aggregatedActivityStreamBody(activitiesByEnvironment, failedEnvironmentIds)
+ });
+ });
+
await page
.context()
- .route(
- /\/api\/environments\/[^/]+\/activities(?:\/stream)?(?:\?.*)?$/,
- async (route: Route) => {
- const url = new URL(route.request().url());
- const environmentId = activityEnvironmentIdFromPath(url.pathname);
- if (!environmentId) {
- await route.continue();
- return;
- }
-
- if (failedEnvironmentIds.has(environmentId)) {
- await route.fulfill({
- status: 503,
- contentType: 'application/json',
- body: JSON.stringify({ success: false, message: 'environment unavailable' })
- });
- return;
- }
-
- const activities = activitiesByEnvironment[environmentId] ?? [];
- if (url.pathname.endsWith('/stream')) {
- await route.fulfill({
- status: 200,
- contentType: 'application/x-json-stream',
- body:
- JSON.stringify({
- type: 'snapshot',
- activities,
- timestamp: new Date().toISOString()
- }) + '\n'
- });
- return;
- }
+ .route(/\/api\/environments\/[^/]+\/activities(?:\?.*)?$/, async (route: Route) => {
+ const url = new URL(route.request().url());
+ const environmentId = activityEnvironmentIdFromPath(url.pathname);
+ if (!environmentId) {
+ await route.continue();
+ return;
+ }
+ if (failedEnvironmentIds.has(environmentId)) {
await route.fulfill({
- status: 200,
+ status: 503,
contentType: 'application/json',
- body: JSON.stringify(paginated(activities))
+ body: JSON.stringify({ success: false, message: 'environment unavailable' })
});
+ return;
}
- );
+
+ await route.fulfill({
+ status: 200,
+ contentType: 'application/json',
+ body: JSON.stringify(paginated(activitiesByEnvironment[environmentId] ?? []))
+ });
+ });
}
function extractActivityId(value: unknown): string | undefined {
@@ -236,13 +243,10 @@ function activityRow(activityCenter: Locator, text: string) {
.first();
}
-function waitForActivityList(page: Page, environmentId: string) {
+function waitForActivityStream(page: Page) {
return page.waitForResponse((response) => {
const url = new URL(response.url());
- return (
- response.request().method() === 'GET' &&
- url.pathname === `/api/environments/${encodeURIComponent(environmentId)}/activities`
- );
+ return response.request().method() === 'GET' && url.pathname === '/api/activities/stream';
});
}
@@ -257,12 +261,10 @@ test.describe('Activity Center', () => {
]
});
- const localActivityList = waitForActivityList(page, '0');
- const remoteActivityList = waitForActivityList(page, 'remote-activity-test');
+ const activityStream = waitForActivityStream(page);
await page.goto('/dashboard');
- await Promise.all([localActivityList, remoteActivityList]);
+ await activityStream;
await page.waitForLoadState('load');
- await page.waitForLoadState('networkidle');
const activityCenter = await openActivityCenter(page);
await activityCenter.getByRole('button', { name: 'Completed' }).click();
@@ -285,12 +287,10 @@ test.describe('Activity Center', () => {
new Set(['remote-activity-test'])
);
- const localActivityList = waitForActivityList(page, '0');
- const remoteActivityList = waitForActivityList(page, 'remote-activity-test');
+ const activityStream = waitForActivityStream(page);
await page.goto('/dashboard');
- await Promise.all([localActivityList, remoteActivityList]);
+ await activityStream;
await page.waitForLoadState('load');
- await page.waitForLoadState('networkidle');
const activityCenter = await openActivityCenter(page);
await activityCenter.getByRole('button', { name: 'Completed' }).click();
@@ -334,10 +334,9 @@ test.describe('Activity Center', () => {
});
});
- const localActivityList = waitForActivityList(page, '0');
- const remoteActivityList = waitForActivityList(page, 'remote-activity-test');
+ const activityStream = waitForActivityStream(page);
await page.goto('/dashboard');
- await Promise.all([localActivityList, remoteActivityList]);
+ await activityStream;
await page.waitForLoadState('load');
const activityCenter = await openActivityCenter(page);
diff --git a/types/activity/activity.go b/types/activity/activity.go
index ff7f32b86..48f1a3016 100644
--- a/types/activity/activity.go
+++ b/types/activity/activity.go
@@ -89,12 +89,14 @@ type Detail struct {
}
type StreamEvent struct {
- Type string `json:"type"`
- ActivityID string `json:"activityId,omitempty"`
- Activity *Activity `json:"activity,omitempty"`
- Activities []Activity `json:"activities,omitempty"`
- Message *Message `json:"message,omitempty"`
- Timestamp time.Time `json:"timestamp"`
+ Type string `json:"type"`
+ EnvironmentID string `json:"environmentId,omitempty"`
+ ActivityID string `json:"activityId,omitempty"`
+ Activity *Activity `json:"activity,omitempty"`
+ Activities []Activity `json:"activities,omitempty"`
+ Message *Message `json:"message,omitempty"`
+ Error string `json:"error,omitempty"`
+ Timestamp time.Time `json:"timestamp"`
}
type ClearHistoryResult struct {