Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: repetitive database calls for envID (#3042)
Kyle Mendellf72838bf95d2
diff --git a/backend/api/handlers/activities.go b/backend/api/handlers/activities.goindex 9f6f7669e..695058542 100644--- a/backend/api/handlers/activities.go+++ b/backend/api/handlers/activities.go@@ -13,6 +13,7 @@ import ("github.com/danielgtaylor/huma/v2"humamw "github.com/getarcaneapp/arcane/backend/v2/api/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/internal/models""github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/aggstream"@@ -401,20 +402,44 @@ func (h *ActivityHandler) runLocalActivityStreamProducerInternal(ctx context.Con// enabled remote environment, re-listing periodically so environments added// or removed while the stream is open are picked up without a reconnect.func (h *ActivityHandler) runRemoteActivityStreamPollersInternal(ctx context.Context, limit int, events chan<- activity.StreamEvent) {- aggstream.ReconcileEnvironmentPollers(ctx, h.environmentService, activityStreamEnvReconcileInterval, "activity stream",- func(pollCtx context.Context, environmentID string) {- h.runRemoteActivityStreamPollerInternal(pollCtx, environmentID, limit, events)+ aggstream.ReconcilePollersByKey(ctx,+ h.environmentService.ListRemoteEnvironments,+ func(environment models.Environment) string {+ return environment.ID+ },+ activityStreamEnvironmentVersionInternal,+ activityStreamEnvReconcileInterval,+ "activity stream",+ func(pollCtx context.Context, environment models.Environment) {+ h.runRemoteActivityStreamPollerInternal(pollCtx, environment, limit, events)})}-func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Context, environmentID string, limit int, events chan<- activity.StreamEvent) {+func activityStreamEnvironmentVersionInternal(environment models.Environment) string {+ if environment.UpdatedAt == nil {+ return environment.ID+ }+ return environment.ID + ":" + environment.UpdatedAt.UTC().Format(time.RFC3339Nano)+}++func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Context, environment models.Environment, limit int, events chan<- activity.StreamEvent) {+ environmentID := environment.IDlastError := ""poll := func() {pollCtx, cancelPoll := context.WithTimeout(ctx, activityStreamRemotePollTimeout)defer cancelPoll()- output, err := h.proxyListActivitiesInternal(pollCtx, &ListActivitiesInput{+ currentEnvironment := environment+ if h.environmentService != nil {+ var ok bool+ currentEnvironment, ok = h.environmentService.GetActiveRemoteEnvironmentSnapshot(environmentID)+ if !ok {+ return+ }+ }++ output, err := h.proxyListActivitiesForEnvironmentInternal(pollCtx, currentEnvironment, &ListActivitiesInput{EnvironmentID: environmentID,Limit: resolveActivityStreamLimitInternal(limit),Order: "desc",@@ -473,6 +498,19 @@ func (h *ActivityHandler) proxyListActivitiesInternal(ctx context.Context, inputreturn &ListActivitiesOutput{Body: *out}, nil}+func (h *ActivityHandler) proxyListActivitiesForEnvironmentInternal(ctx context.Context, environment models.Environment, input *ListActivitiesInput) (*ListActivitiesOutput, error) {+ if h.environmentService == nil {+ return nil, huma.Error500InternalServerError("environment service not available")+ }+ path := "/api/environments/0/activities?" + activityListQueryInternal(input).Encode()+ var out base.Paginated[activity.Activity]+ if err := h.environmentService.ProxyJSONRequestForEnvironment(ctx, environment, http.MethodGet, path, nil, &out); err != nil {+ return nil, translateRemoteProxyErrorInternal(err)+ }+ applyActivitySourceLabelsForEnvironmentInternal(environment, out.Data)+ return &ListActivitiesOutput{Body: out}, nil+}+func (h *ActivityHandler) proxyGetActivityInternal(ctx context.Context, input *GetActivityInput) (*GetActivityOutput, error) {if h.environmentService == nil {return nil, huma.Error500InternalServerError("environment service not available")@@ -522,6 +560,29 @@ func (h *ActivityHandler) applyActivityStreamEventSourceLabelInternal(ctx contex}}+func applyActivitySourceLabelsForEnvironmentInternal(environment models.Environment, activities []activity.Activity) {+ sourceID, sourceName := activitySourceFromEnvironmentInternal(environment)+ for i := range activities {+ applyActivitySourceInternal(&activities[i], sourceID, sourceName)+ }+}++func activitySourceFromEnvironmentInternal(environment models.Environment) (string, string) {+ environmentID := environment.ID+ if environmentID == "" {+ environmentID = "0"+ }+ environmentName := environment.Name+ if environmentName == "" {+ if environmentID == "0" {+ environmentName = "Local"+ } else {+ environmentName = environmentID+ }+ }+ return environmentID, environmentName+}+func (h *ActivityHandler) resolveActivitySourceInternal(ctx context.Context, environmentID string) (string, string) {if environmentID == "" {environmentID = "0"diff --git a/backend/api/handlers/activities_test.go b/backend/api/handlers/activities_test.goindex 3383597a5..420f3ba66 100644--- a/backend/api/handlers/activities_test.go+++ b/backend/api/handlers/activities_test.go@@ -7,6 +7,7 @@ import ("io""net/http""net/http/httptest"+ "sync/atomic""testing""time"@@ -198,6 +199,8 @@ func TestActivityHandlerStreamAllEmitsEnvironmentScopedEventsInternal(t *testing}if event.Type == "snapshot" && event.EnvironmentID == "remote-1" && len(event.Activities) == 1 {require.Equal(t, "remote-activity-1", event.Activities[0].ID)+ require.Equal(t, "remote-1", event.Activities[0].SourceEnvironmentID)+ require.Equal(t, "Remote", event.Activities[0].SourceEnvironmentName)remoteSnapshot = true}return localSnapshot && remoteSnapshot@@ -207,6 +210,68 @@ func TestActivityHandlerStreamAllEmitsEnvironmentScopedEventsInternal(t *testingrequire.True(t, remoteSnapshot)}+func TestActivityHandlerStreamAllReusesRemoteEnvironmentAfterInitialPollInternal(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)+ defer cancel()++ db := setupActivityHandlerTestDBInternal(t)+ limitStreamTestDBToSingleConnInternal(t, db)+ settingsService, err := services.NewSettingsService(ctx, db)+ require.NoError(t, err)+ activityService := services.NewActivityService(db)++ var countEnvironmentQueries atomic.Bool+ var environmentQueryCount atomic.Int64+ require.NoError(t, db.Callback().Query().After("gorm:query").Register("arcane_test_count_environment_queries", func(tx *gorm.DB) {+ if countEnvironmentQueries.Load() && activityTestQueryLoadsEnvironmentIDInternal(tx, "remote-1") {+ environmentQueryCount.Add(1)+ }+ }))++ token := "remote-token"+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"success":true,"data":[{"id":"remote-activity-1"}],"pagination":{"totalPages":1,"totalItems":1,"currentPage":1,"itemsPerPage":50}}`))+ }))+ defer server.Close()+ createStreamTestRemoteEnvironmentInternal(t, db, server.URL, token)++ handler := &ActivityHandler{+ activityService: activityService,+ environmentService: services.NewEnvironmentService(db, server.Client(), nil, nil, settingsService, nil),+ }++ remoteSnapshotCount := 0+ runStreamAllInternal(t, ctx, cancel, handler, func(event activity.StreamEvent) bool {+ if event.Type != "snapshot" || event.EnvironmentID != "remote-1" {+ return false+ }++ remoteSnapshotCount+++ if remoteSnapshotCount == 1 {+ countEnvironmentQueries.Store(true)+ return false+ }++ require.Zero(t, environmentQueryCount.Load(), "steady-state remote activity poll should not reload environment rows")+ return true+ })++ require.GreaterOrEqual(t, remoteSnapshotCount, 2)+}++func activityTestQueryLoadsEnvironmentIDInternal(tx *gorm.DB, environmentID string) bool {+ if tx.Statement == nil || tx.Statement.Table != "environments" {+ return false+ }+ for _, arg := range tx.Statement.Vars {+ if arg == environmentID {+ return true+ }+ }+ return false+}+func TestActivityHandlerStreamAllRemoteFailureEmitsErrorAndKeepsStreamingInternal(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)defer cancel()diff --git a/backend/internal/services/environment_service.go b/backend/internal/services/environment_service.goindex 31ce95e6e..42ec3340f 100644--- a/backend/internal/services/environment_service.go+++ b/backend/internal/services/environment_service.go@@ -41,6 +41,8 @@ type EnvironmentService struct {tokenCacheMu sync.RWMutextokenCache map[string]edgeTokenCacheEntrytokenByEnvID map[string]string+ remoteEnvMu sync.RWMutex+ remoteEnvs map[string]models.Environment// scheduler and lifecycleCtx are injected post-construction via SetScheduler// (manager-only). Each enabled environment gets its own health-check job; this@@ -86,6 +88,7 @@ func NewEnvironmentService(db *database.DB, httpClient *http.Client, dockerServi}),tokenCache: make(map[string]edgeTokenCacheEntry),tokenByEnvID: make(map[string]string),+ remoteEnvs: make(map[string]models.Environment),}}@@ -396,6 +399,88 @@ func (s *EnvironmentService) syncEnvironmentTokenCacheInternal(envID string, tok}}+// GetActiveRemoteEnvironmentSnapshot returns the latest in-process snapshot for+// an enabled, visible, non-local remote environment.+func (s *EnvironmentService) GetActiveRemoteEnvironmentSnapshot(environmentID string) (models.Environment, bool) {+ if s == nil || environmentID == "" {+ return models.Environment{}, false+ }++ s.remoteEnvMu.RLock()+ environment, ok := s.remoteEnvs[environmentID]+ s.remoteEnvMu.RUnlock()+ if !ok || !isActiveRemoteEnvironmentInternal(environment) {+ return models.Environment{}, false+ }+ return environment, true+}++func isActiveRemoteEnvironmentInternal(environment models.Environment) bool {+ return environment.ID != "" && environment.ID != "0" && environment.Enabled && !environment.Hidden+}++func (s *EnvironmentService) syncRemoteEnvironmentSnapshotsInternal(environments []models.Environment) {+ if s == nil {+ return+ }++ next := make(map[string]models.Environment, len(environments))+ for _, environment := range environments {+ if isActiveRemoteEnvironmentInternal(environment) {+ next[environment.ID] = environment+ }+ }++ s.remoteEnvMu.Lock()+ s.remoteEnvs = next+ s.remoteEnvMu.Unlock()+}++func (s *EnvironmentService) cacheRemoteEnvironmentSnapshotInternal(environment models.Environment) {+ if s == nil {+ return+ }++ if !isActiveRemoteEnvironmentInternal(environment) {+ s.removeRemoteEnvironmentSnapshotInternal(environment.ID)+ return+ }++ s.remoteEnvMu.Lock()+ s.remoteEnvs[environment.ID] = environment+ s.remoteEnvMu.Unlock()+}++func (s *EnvironmentService) removeRemoteEnvironmentSnapshotInternal(environmentID string) {+ if s == nil || environmentID == "" {+ return+ }++ s.remoteEnvMu.Lock()+ delete(s.remoteEnvs, environmentID)+ s.remoteEnvMu.Unlock()+}++func (s *EnvironmentService) updateRemoteEnvironmentSnapshotInternal(environmentID string, update func(*models.Environment)) {+ if s == nil || environmentID == "" || update == nil {+ return+ }++ s.remoteEnvMu.Lock()+ defer s.remoteEnvMu.Unlock()++ environment, ok := s.remoteEnvs[environmentID]+ if !ok {+ return+ }+ update(&environment)+ if isActiveRemoteEnvironmentInternal(environment) {+ s.remoteEnvs[environmentID] = environment+ } else {+ delete(s.remoteEnvs, environmentID)+ }+}+func (s *EnvironmentService) EnsureLocalEnvironment(ctx context.Context, appUrl string) error {const localEnvID = "0"@@ -461,6 +546,7 @@ func (s *EnvironmentService) CreateEnvironment(ctx context.Context, environmentif environment.Enabled {s.registerHealthJobInternal(ctx, environment.ID)}+ s.cacheRemoteEnvironmentSnapshotInternal(*environment)return environment, nil}@@ -832,6 +918,7 @@ func (s *EnvironmentService) ListRemoteEnvironments(ctx context.Context) ([]modeif err != nil {return nil, fmt.Errorf("failed to list remote environments: %w", err)}+ s.syncRemoteEnvironmentSnapshotsInternal(envs)return envs, nil}@@ -883,6 +970,7 @@ func (s *EnvironmentService) UpdateEnvironment(ctx context.Context, id string, uif err != nil {return nil, err}+ s.cacheRemoteEnvironmentSnapshotInternal(*updated)if rawAccessToken, ok := updates["access_token"]; ok {accessToken, _ := rawAccessToken.(string)@@ -926,6 +1014,7 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, u}s.invalidateEnvironmentTokenInternal(id)+ s.removeRemoteEnvironmentSnapshotInternal(id)// Create event in backgroundgo s.createEnvironmentEvent(context.WithoutCancel(ctx), id, env.Name, models.EventTypeEnvironmentDelete, "Environment Deleted", fmt.Sprintf("Environment '%s' was deleted", env.Name), models.EventSeverityWarning, userID, username)@@ -1179,6 +1268,14 @@ func (s *EnvironmentService) RegenerateEnvironmentApiKey(ctx context.Context, en}s.syncEnvironmentTokenCacheInternal(envID, encryptedKey)+ now := time.Now()+ s.updateRemoteEnvironmentSnapshotInternal(envID, func(environment *models.Environment) {+ environment.ApiKeyID = &newApiKeyID+ environment.AccessToken = &encryptedKey+ environment.Status = string(models.EnvironmentStatusPending)+ environment.LastSeen = nil+ environment.UpdatedAt = &now+ })// Create event log in backgroundgo s.createEnvironmentEvent(context.WithoutCancel(ctx), envID, envName, models.EventTypeEnvironmentApiKeyRegenerated, "API Key Regenerated", "Environment API key was regenerated and status set to pending", models.EventSeverityInfo, new(userID), new(username))@@ -1483,7 +1580,11 @@ func (s *EnvironmentService) resolveRemoteEnvironmentTargetInternal(ctx context.return nil, fmt.Errorf("failed to get environment: %w", err)}- if envID == "0" {+ return s.remoteEnvironmentTargetFromModelInternal(*environment)+}++func (s *EnvironmentService) remoteEnvironmentTargetFromModelInternal(environment models.Environment) (*remoteEnvironmentTargetInternal, error) {+ if environment.ID == "0" {return nil, errors.New("cannot proxy request to local environment")}@@ -1605,6 +1706,20 @@ func (s *EnvironmentService) ProxyJSONRequest(ctx context.Context, envID string,return s.proxyJSONRequestForTargetInternal(proxyCtx, target, method, path, body, out)}+// ProxyJSONRequestForEnvironment sends a JSON request using an already-loaded+// environment row, avoiding an extra environment lookup on hot stream paths.+func (s *EnvironmentService) ProxyJSONRequestForEnvironment(ctx context.Context, environment models.Environment, method string, path string, body []byte, out any) error {+ proxyCtx, cancel := s.getProxyRequestContextInternal(ctx)+ defer cancel()++ target, err := s.remoteEnvironmentTargetFromModelInternal(environment)+ if err != nil {+ return err+ }++ return s.proxyJSONRequestForTargetInternal(proxyCtx, target, method, path, body, out)+}+func (s *EnvironmentService) proxyJSONRequestForTargetInternal(ctx context.Context,target *remoteEnvironmentTargetInternal,diff --git a/backend/pkg/libarcane/aggstream/aggstream.go b/backend/pkg/libarcane/aggstream/aggstream.goindex f7b6b4fcd..906891267 100644--- a/backend/pkg/libarcane/aggstream/aggstream.go+++ b/backend/pkg/libarcane/aggstream/aggstream.go@@ -88,17 +88,48 @@ func ReconcileEnvironmentPollers(streamLabel string,runPoller func(ctx context.Context, environmentID string),) {- pollers := make(map[string]context.CancelFunc)+ ReconcilePollersByKey(ctx,+ func(ctx context.Context) ([]string, error) {+ return lister.ListRemoteEnvironmentIDs(ctx)+ },+ func(environmentID string) string {+ return environmentID+ },+ nil,+ reconcileInterval,+ streamLabel,+ runPoller,+ )+}++// ReconcilePollersByKey keeps one poller goroutine per listed item, keyed by a+// stable item ID. If the same key is later returned with a different version,+// the old poller is replaced so callers can pass refreshed item metadata.+func ReconcilePollersByKey[T any](+ ctx context.Context,+ listItems func(context.Context) ([]T, error),+ keyFunc func(T) string,+ versionFunc func(T) string,+ reconcileInterval time.Duration,+ streamLabel string,+ runPoller func(ctx context.Context, item T),+) {+ type poller struct {+ cancel context.CancelFunc+ version string+ }++ pollers := make(map[string]poller)var wg sync.WaitGroupdefer wg.Wait()defer func() {- for _, cancelPoll := range pollers {- cancelPoll()+ for _, activePoller := range pollers {+ activePoller.cancel()}}()reconcile := func() {- environmentIDs, err := lister.ListRemoteEnvironmentIDs(ctx)+ items, err := listItems(ctx)if err != nil {if ctx.Err() == nil {slog.WarnContext(ctx, "failed to list environments for "+streamLabel, "error", err)@@ -106,24 +137,39 @@ func ReconcileEnvironmentPollers(return}- current := make(map[string]struct{}, len(environmentIDs))- for _, environmentID := range environmentIDs {- current[environmentID] = struct{}{}- if _, ok := pollers[environmentID]; ok {+ current := make(map[string]struct{}, len(items))+ for _, item := range items {+ key := keyFunc(item)+ if key == "" {continue}- pollCtx, cancelPoll := context.WithCancel(ctx) //nolint:gosec // cancel is retained in pollers and invoked on env removal or via the deferred cleanup.- pollers[environmentID] = cancelPoll+ version := ""+ if versionFunc != nil {+ version = versionFunc(item)+ }+ current[key] = struct{}{}++ if existingPoller, ok := pollers[key]; ok && existingPoller.version == version {+ continue+ }++ if existingPoller, ok := pollers[key]; ok {+ existingPoller.cancel()+ delete(pollers, key)+ }++ pollCtx, cancelPoll := context.WithCancel(ctx)+ pollers[key] = poller{cancel: cancelPoll, version: version}wg.Add(1)- go func(environmentID string) {+ go func(pollCtx context.Context, item T) {defer wg.Done()- runPoller(pollCtx, environmentID)- }(environmentID)+ runPoller(pollCtx, item)+ }(pollCtx, item)}- for id, cancelPoll := range pollers {+ for id, activePoller := range pollers {if _, ok := current[id]; !ok {- cancelPoll()+ activePoller.cancel()delete(pollers, id)}}