Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: load project services concurrently (#758)
Kyle Mendelld6241356520b
diff --git a/backend/internal/services/project_service.go b/backend/internal/services/project_service.goindex 13ed34cf8..6ce544459 100644--- a/backend/internal/services/project_service.go+++ b/backend/internal/services/project_service.go@@ -402,29 +402,73 @@ func (s *ProjectService) GetProjectStatusCounts(ctx context.Context) (folderCounreturn 0, 0, 0, 0, fmt.Errorf("unable to access projects directory %s: %w", projectsDir, statErr)}- // Get all projects and calculate live status- var projects []models.Project- if err := s.db.WithContext(ctx).Find(&projects).Error; err != nil {+ var projectsList []models.Project+ if err := s.db.WithContext(ctx).Find(&projectsList).Error; err != nil {return folderCount, 0, 0, 0, fmt.Errorf("failed to list projects: %w", err)}- totalProjects = len(projects)+ totalProjects = len(projectsList)++ // Fetch live status concurrently+ type statusResult struct {+ status models.ProjectStatus+ err error+ }++ resultChan := make(chan statusResult, len(projectsList))+ semaphore := make(chan struct{}, 10)++ queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second)+ defer cancel()++ for _, proj := range projectsList {+ go func(p models.Project) {+ semaphore <- struct{}{}+ defer func() { <-semaphore }()++ projectCtx, projectCancel := context.WithTimeout(queryCtx, 2*time.Second)+ defer projectCancel()++ services, serr := s.getProjectServicesWithTimeout(projectCtx, p.ID)+ if serr != nil {+ // On error, use cached status+ resultChan <- statusResult{status: p.Status, err: serr}+ return+ }++ status := s.calculateProjectStatus(services)+ resultChan <- statusResult{status: status, err: nil}+ }(proj)+ }++ // Collect resultsrunningProjects = 0stoppedProjects = 0- for _, proj := range projects {- services, serr := s.GetProjectServices(ctx, proj.ID)- if serr != nil {- continue- }-- status := s.calculateProjectStatus(services)- switch status {- case models.ProjectStatusRunning, models.ProjectStatusPartiallyRunning, models.ProjectStatusDeploying, models.ProjectStatusRestarting:- runningProjects++- case models.ProjectStatusStopped, models.ProjectStatusStopping:- stoppedProjects++- case models.ProjectStatusUnknown:+ for i := 0; i < len(projectsList); i++ {+ select {+ case res := <-resultChan:+ switch res.status {+ case models.ProjectStatusRunning, models.ProjectStatusPartiallyRunning, models.ProjectStatusDeploying, models.ProjectStatusRestarting:+ runningProjects+++ case models.ProjectStatusStopped, models.ProjectStatusStopping:+ stoppedProjects+++ case models.ProjectStatusUnknown:+ // Don't count unknown+ }+ case <-queryCtx.Done():+ // Timeout - use cached data for remaining+ for j := i; j < len(projectsList); j++ {+ switch projectsList[j].Status {+ case models.ProjectStatusRunning, models.ProjectStatusPartiallyRunning, models.ProjectStatusDeploying, models.ProjectStatusRestarting:+ runningProjects+++ case models.ProjectStatusStopped, models.ProjectStatusStopping:+ stoppedProjects+++ case models.ProjectStatusUnknown:+ // Don't count unknown+ }+ }+ return folderCount, runningProjects, stoppedProjects, totalProjects, nil}}@@ -824,69 +868,152 @@ func (s *ProjectService) ListProjects(ctx context.Context, params pagination.Quereturn nil, pagination.Response{}, fmt.Errorf("failed to paginate projects: %w", err)}- var result []dto.ProjectDetailsDto- for _, project := range projectsArray {- displayServiceCount := project.ServiceCount- displayRunningCount := project.RunningCount- var displayStatus string- var statusReason *string-- // Get live status from Docker- if services, serr := s.GetProjectServices(ctx, project.ID); serr == nil {- displayServiceCount = len(services)- _, displayRunningCount = s.getServiceCounts(services)- displayStatus = string(s.calculateProjectStatus(services))-- // If status is unknown, set reason- if displayStatus == string(models.ProjectStatusUnknown) && len(services) == 0 {- reason := "No services found in project"- statusReason = &reason- slog.WarnContext(ctx, "Project has unknown status",- "project_id", project.ID,- "project_name", project.Name,- "reason", reason)- }- } else {- // Error getting services - set status to unknown with reason- displayStatus = string(models.ProjectStatusUnknown)- reason := fmt.Sprintf("Failed to load project services: %v", serr)- statusReason = &reason- slog.WarnContext(ctx, "Project has unknown status due to error",- "project_id", project.ID,- "project_name", project.Name,- "reason", reason,- "error", serr)-- // Fallback: try to detect service count from compose file- if displayServiceCount == 0 {- if _, derr := projects.DetectComposeFile(project.Path); derr == nil {- // Get configured projects directory from settings- projectsDirSetting := s.settingsService.GetStringSetting(ctx, "projectsDirectory", "data/projects")- projectsDirectory, pdErr := fs.GetProjectsDirectory(ctx, strings.TrimSpace(projectsDirSetting))- if pdErr != nil {- projectsDirectory = "data/projects"- }- if proj, _, perr := projects.LoadComposeProjectFromDir(ctx, project.Path, project.Name, projectsDirectory); perr == nil {- displayServiceCount = len(proj.Services)+ slog.DebugContext(ctx, "Retrieved projects from database",+ "count", len(projectsArray))++ // Fetch live status concurrently for all projects+ result := s.fetchProjectStatusConcurrently(ctx, projectsArray)++ slog.DebugContext(ctx, "Completed ListProjects request",+ "result_count", len(result))++ return result, paginationResp, nil+}++// fetchProjectStatusConcurrently fetches live Docker status for multiple projects in parallel+func (s *ProjectService) fetchProjectStatusConcurrently(ctx context.Context, projects []models.Project) []dto.ProjectDetailsDto {+ type projectResult struct {+ index int+ dto dto.ProjectDetailsDto+ }++ resultChan := make(chan projectResult, len(projects))++ const (+ concurrency = 10+ perProjectTimeout = 3 * time.Second+ minGlobalTimeout = 30 * time.Second+ )++ // Calculate dynamic timeout: (projects / concurrency) * per-project-timeout * 1.5 buffer+ projectBatches := (len(projects) + concurrency - 1) / concurrency // Ceiling division+ calculatedTimeout := time.Duration(projectBatches) * perProjectTimeout * 3 / 2+ globalTimeout := minGlobalTimeout+ if calculatedTimeout > globalTimeout {+ globalTimeout = calculatedTimeout+ }++ semaphore := make(chan struct{}, concurrency)++ queryCtx, cancel := context.WithTimeout(ctx, globalTimeout)+ defer cancel()++ slog.DebugContext(ctx, "Starting concurrent project status queries",+ "project_count", len(projects),+ "concurrency", concurrency,+ "per_project_timeout_sec", perProjectTimeout.Seconds(),+ "global_timeout_sec", globalTimeout.Seconds())++ // Launch goroutine for each project+ for i, project := range projects {+ go func(idx int, proj models.Project) {+ semaphore <- struct{}{} // Acquire semaphore+ defer func() { <-semaphore }() // Release semaphore++ // Create per-project timeout+ projectCtx, projectCancel := context.WithTimeout(queryCtx, perProjectTimeout)+ defer projectCancel()++ displayServiceCount := proj.ServiceCount+ displayRunningCount := proj.RunningCount+ displayStatus := string(proj.Status)+ var statusReason *string++ // Try to get live status from Docker+ if services, serr := s.getProjectServicesWithTimeout(projectCtx, proj.ID); serr == nil {+ displayServiceCount = len(services)+ _, displayRunningCount = s.getServiceCounts(services)+ displayStatus = string(s.calculateProjectStatus(services))++ if displayStatus == string(models.ProjectStatusUnknown) && len(services) == 0 {+ reason := "No services found in project"+ statusReason = &reason+ }+ } else {+ // On timeout or error, use cached values+ if errors.Is(serr, context.DeadlineExceeded) {+ reason := "Status query timed out, showing cached status"+ statusReason = &reason+ displayStatus = string(proj.Status)+ } else if !errors.Is(serr, context.Canceled) {+ displayStatus = string(proj.Status)+ if proj.StatusReason != nil {+ statusReason = proj.StatusReason}+ slog.WarnContext(projectCtx, "Failed to get live project status, using cached",+ "project_id", proj.ID,+ "project_name", proj.Name,+ "error", serr)}}- }- result = append(result, dto.ProjectDetailsDto{- ID: project.ID,- Name: project.Name,- DirName: utils.DerefString(project.DirName),- Path: project.Path,- Status: displayStatus,- StatusReason: statusReason,- ServiceCount: displayServiceCount,- RunningCount: displayRunningCount,- CreatedAt: project.CreatedAt.Format(time.RFC3339),- UpdatedAt: project.UpdatedAt.Format(time.RFC3339),- })+ resultChan <- projectResult{+ index: idx,+ dto: dto.ProjectDetailsDto{+ ID: proj.ID,+ Name: proj.Name,+ DirName: utils.DerefString(proj.DirName),+ Path: proj.Path,+ Status: displayStatus,+ StatusReason: statusReason,+ ServiceCount: displayServiceCount,+ RunningCount: displayRunningCount,+ CreatedAt: proj.CreatedAt.Format(time.RFC3339),+ UpdatedAt: proj.UpdatedAt.Format(time.RFC3339),+ },+ }+ }(i, project)+ }++ // Collect results+ results := make([]dto.ProjectDetailsDto, len(projects))+ for i := 0; i < len(projects); i++ {+ select {+ case res := <-resultChan:+ results[res.index] = res.dto+ case <-queryCtx.Done():+ // Timeout - fill remaining with cached data+ remainingCount := len(projects) - i+ slog.WarnContext(ctx, "Project list query timed out, using cached data for remaining projects",+ "completed_count", i,+ "remaining_count", remainingCount,+ "total_projects", len(projects))++ for j := i; j < len(projects); j++ {+ proj := projects[j]+ results[j] = dto.ProjectDetailsDto{+ ID: proj.ID,+ Name: proj.Name,+ DirName: utils.DerefString(proj.DirName),+ Path: proj.Path,+ Status: string(proj.Status),+ StatusReason: proj.StatusReason,+ ServiceCount: proj.ServiceCount,+ RunningCount: proj.RunningCount,+ CreatedAt: proj.CreatedAt.Format(time.RFC3339),+ UpdatedAt: proj.UpdatedAt.Format(time.RFC3339),+ }+ }+ return results+ }}- return result, paginationResp, nil++ return results+}++// getProjectServicesWithTimeout is a wrapper that respects context timeout+func (s *ProjectService) getProjectServicesWithTimeout(ctx context.Context, projectID string) ([]ProjectServiceInfo, error) {+ return s.GetProjectServices(ctx, projectID)}// End Table Functions