Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: make compose up wait timeout configurable so long depends_on conditions don't abort deploys (#3557)
Kyle Mendell5d59f0e30c69
diff --git a/backend/internal/config/schema/schema_test.go b/backend/internal/config/schema/schema_test.goindex a7cd33c14..3739af693 100644--- a/backend/internal/config/schema/schema_test.go+++ b/backend/internal/config/schema/schema_test.go@@ -291,6 +291,7 @@ var expectedSettingOverrideKeys = []string{"buildsDirectory","defaultDeployPullPolicy","defaultShell",+ "deployWaitTimeout","depotProjectId","depotToken","diskUsagePath",diff --git a/backend/internal/models/settings.go b/backend/internal/models/settings.goindex 6ff169411..b153ba23a 100644--- a/backend/internal/models/settings.go+++ b/backend/internal/models/settings.go@@ -187,6 +187,7 @@ type Settings struct {HTTPClientTimeout SettingVariable `key:"httpClientTimeout,envOverride" meta:"label=HTTP Client Timeout;type=number;keywords=http,client,timeout,seconds,api,request;category=timeouts;description=Default timeout for HTTP requests in seconds (default: 30)"`RegistryTimeout SettingVariable `key:"registryTimeout,envOverride" meta:"label=Registry Timeout;type=number;keywords=registry,timeout,seconds,docker,auth;category=timeouts;description=Timeout for container registry operations in seconds (default: 30)"`ProxyRequestTimeout SettingVariable `key:"proxyRequestTimeout,envOverride" meta:"label=Proxy Request Timeout;type=number;keywords=proxy,request,timeout,seconds,forward;category=timeouts;description=Timeout for proxied requests in seconds (default: 60)"`+ DeployWaitTimeout SettingVariable `key:"deployWaitTimeout,envOverride" meta:"label=Deploy Wait Timeout;type=number;keywords=deploy,compose,up,wait,healthy,depends_on,timeout,seconds;category=timeouts;description=Timeout waiting for services to become healthy or complete during a deploy in seconds (default: 600 = 10 minutes)"`}func (SettingVariable) TableName() string {diff --git a/backend/internal/project/project_lifecycle.go b/backend/internal/project/project_lifecycle.goindex 5ef1aa2a7..465c2eed8 100644--- a/backend/internal/project/project_lifecycle.go+++ b/backend/internal/project/project_lifecycle.go@@ -148,7 +148,7 @@ func (s *ProjectService) UpdateProjectServices(ctx context.Context, projectID st}// 5. Up specific services- if err := composeUpProjectServicesInternal(ctx, compProj, servicesToUpdate, false, true, false, s.composeRegistryAuthConfigsInternal(ctx)); err != nil {+ if err := composeUpProjectServicesInternal(ctx, compProj, servicesToUpdate, false, true, false, s.composeRegistryAuthConfigsInternal(ctx), s.deployWaitTimeoutInternal()); err != nil {s.restoreProjectStatusAfterFailedDeployInternal(ctx, projectID)return errors.WrapIf(err, "failed to up services")}@@ -244,6 +244,12 @@ func (s *ProjectService) UnarchiveProject(ctx context.Context, projectID string,// resolveRemoveOrphansInternal decides whether compose up should remove orphan containers.// GitOps-managed projects always remove orphans so the running stack matches the// tracked compose file. Non-GitOps callers may opt in per request via DeployOptions.+// deployWaitTimeoutInternal resolves how long compose up waits for depends_on+// health/completion conditions, from the deployWaitTimeout setting.+func (s *ProjectService) deployWaitTimeoutInternal() time.Duration {+ return timeouts.GetDuration(s.settingsService.GetSettingsConfig().DeployWaitTimeout.AsInt(), timeouts.DefaultDeployWait)+}+func resolveRemoveOrphansInternal(gitOpsManaged bool, options *project.DeployOptions) bool {return gitOpsManaged || (options != nil && options.RemoveOrphans)}@@ -310,7 +316,7 @@ func (s *ProjectService) DeployProject(ctx context.Context, projectID string, usslog.Info("starting compose up with health check support", "projectID", projectID, "projectName", projectModel.Name, "services", len(projectModel.Services), "removeOrphans", removeOrphans)// Health/progress streaming (if any) is handled inside projects.ComposeUp via ctx.- if err := projects.ComposeUp(ctx, projectModel, nil, removeOrphans, forceRecreate, recreateVolumes, s.composeRegistryAuthConfigsInternal(ctx)); err != nil {+ if err := projects.ComposeUp(ctx, projectModel, nil, removeOrphans, forceRecreate, recreateVolumes, s.composeRegistryAuthConfigsInternal(ctx), s.deployWaitTimeoutInternal()); err != nil {slog.Error("compose up failed", "projectName", projectModel.Name, "projectID", projectID, "error", err)if containers, psErr := s.GetProjectServices(ctx, projectID); psErr == nil {slog.Info("containers after failed deploy", "projectID", projectID, "containers", containers)@@ -320,7 +326,7 @@ func (s *ProjectService) DeployProject(ctx context.Context, projectID string, us// Provide more helpful error messageserrMsg := err.Error()if strings.Contains(errMsg, "timeout") || strings.Contains(errMsg, "context deadline exceeded") {- return errors.WrapIf(err, "deployment timed out - check if services with 'condition: service_healthy' have healthchecks defined")+ return errors.WrapIf(err, "deployment timed out waiting for services - long-running 'service_healthy'/'service_completed_successfully' dependencies may need a higher Deploy Wait Timeout setting, and 'service_healthy' requires a healthcheck")}return errors.WrapIf(err, "failed to deploy project")}diff --git a/backend/internal/project/service_test.go b/backend/internal/project/service_test.goindex 52d4c843d..46bda4f21 100644--- a/backend/internal/project/service_test.go+++ b/backend/internal/project/service_test.go@@ -1049,7 +1049,7 @@ func TestProjectService_UpdateProjectServicesHardFailsWhenPullFailsInternal(t *tcomposeUpProjectServicesInternal = originalComposeUp})upCalled := false- composeUpProjectServicesInternal = func(context.Context, *composetypes.Project, []string, bool, bool, bool, map[string]dockerregistry.AuthConfig) error {+ composeUpProjectServicesInternal = func(context.Context, *composetypes.Project, []string, bool, bool, bool, map[string]dockerregistry.AuthConfig, time.Duration) error {upCalled = truereturn errors.New("compose up should not run")}@@ -1119,7 +1119,7 @@ func TestProjectService_UpdateProjectServicesForcesRecreateInternal(t *testing.T}upCalled := falseforceRecreate := false- composeUpProjectServicesInternal = func(_ context.Context, _ *composetypes.Project, services []string, removeOrphans bool, force bool, _ bool, _ map[string]dockerregistry.AuthConfig) error {+ composeUpProjectServicesInternal = func(_ context.Context, _ *composetypes.Project, services []string, removeOrphans bool, force bool, _ bool, _ map[string]dockerregistry.AuthConfig, _ time.Duration) error {upCalled = trueforceRecreate = forceassert.Equal(t, []string{"app"}, services)diff --git a/backend/internal/settings/service.go b/backend/internal/settings/service.goindex fe15592f7..27ccd6922 100644--- a/backend/internal/settings/service.go+++ b/backend/internal/settings/service.go@@ -276,6 +276,7 @@ func DefaultSettingsConfig() *models.Settings {HTTPClientTimeout: models.SettingVariable{Value: "30"},RegistryTimeout: models.SettingVariable{Value: "30"},ProxyRequestTimeout: models.SettingVariable{Value: "60"},+ DeployWaitTimeout: models.SettingVariable{Value: "600"},BuildProvider: models.SettingVariable{Value: "local"},BuildTimeout: models.SettingVariable{Value: "1800"},DepotProjectId: models.SettingVariable{Value: ""},diff --git a/backend/pkg/libarcane/settings.go b/backend/pkg/libarcane/settings.goindex 7eff3522a..dcc7294d9 100644--- a/backend/pkg/libarcane/settings.go+++ b/backend/pkg/libarcane/settings.go@@ -21,6 +21,7 @@ var timeoutSettingKeys = []string{"httpClientTimeout","registryTimeout","proxyRequestTimeout",+ "deployWaitTimeout","trivyResourceLimitsEnabled","trivyCpuLimit","trivyMemoryLimitMb",diff --git a/backend/pkg/libarcane/timeouts/timeouts.go b/backend/pkg/libarcane/timeouts/timeouts.goindex aa7289119..39aec6bdd 100644--- a/backend/pkg/libarcane/timeouts/timeouts.go+++ b/backend/pkg/libarcane/timeouts/timeouts.go@@ -13,6 +13,9 @@ const (DefaultRegistry = 30 * time.SecondDefaultProxyRequest = 60 * time.SecondDefaultBuildTimeout = 30 * time.Minute+ // DefaultDeployWait bounds how long compose up waits for services to become+ // healthy or complete (depends_on conditions) before the deploy fails.+ DefaultDeployWait = 10 * time.Minute// DefaultImageUpdateScan bounds an entire batch image update check// end-to-end; individual registry RPCs are separately bounded, but the// aggregate scan needs its own ceiling so a wedged batch cannot hold itsdiff --git a/backend/pkg/projects/cmds.go b/backend/pkg/projects/cmds.goindex f2f0cbb14..b58f76438 100644--- a/backend/pkg/projects/cmds.go+++ b/backend/pkg/projects/cmds.go@@ -12,6 +12,7 @@ import ("github.com/moby/moby/api/types/registry""github.com/moby/moby/client"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/utils")@@ -26,16 +27,16 @@ const defaultComposeTimeout = 30 * time.Minute// deadline-bounded by the parent. This allows compose operations to survive// HTTP request timeouts and proxy deadline cancellations. A standalone timeout// is applied so the operation cannot run forever. See #1209.-func detachFromHTTPContextInternal(parent context.Context) (context.Context, context.CancelFunc) {+func detachFromHTTPContextInternal(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {if utils.IsAppLifecycleContext(parent) {- return context.WithTimeout(parent, defaultComposeTimeout)+ return context.WithTimeout(parent, timeout)}ctx := context.WithoutCancel(parent)- return context.WithTimeout(ctx, defaultComposeTimeout)+ return context.WithTimeout(ctx, timeout)}func ComposeRestart(ctx context.Context, proj *types.Project, services []string) error {- restartCtx, cancel := detachFromHTTPContextInternal(ctx)+ restartCtx, cancel := detachFromHTTPContextInternal(ctx, defaultComposeTimeout)defer cancel()c, err := NewClient(restartCtx, nil, nil)@@ -51,7 +52,7 @@ func ComposeStop(ctx context.Context, proj *types.Project, services []string) erif len(services) == 0 {return nil}- stopCtx, cancel := detachFromHTTPContextInternal(ctx)+ stopCtx, cancel := detachFromHTTPContextInternal(ctx, defaultComposeTimeout)defer cancel()c, err := NewClient(stopCtx, nil, nil)@@ -63,10 +64,18 @@ func ComposeStop(ctx context.Context, proj *types.Project, services []string) erreturn c.svc.Stop(stopCtx, proj.Name, api.StopOptions{Services: services})}-func ComposeUp(ctx context.Context, proj *types.Project, services []string, removeOrphans bool, forceRecreate bool, recreateVolumes bool, authConfigs map[string]registry.AuthConfig) error {+func ComposeUp(ctx context.Context, proj *types.Project, services []string, removeOrphans bool, forceRecreate bool, recreateVolumes bool, authConfigs map[string]registry.AuthConfig, waitTimeout time.Duration) error {+ if waitTimeout <= 0 {+ waitTimeout = timeouts.DefaultDeployWait+ }+// Detach from the HTTP request context so that proxy timeouts and client- // disconnects do not cancel a long-running compose up. See #1209.- composeCtx, cancel := detachFromHTTPContextInternal(ctx)+ // disconnects do not cancel a long-running compose up. See #1209. The+ // operation deadline must exceed the configured wait, otherwise a Deploy+ // Wait Timeout above defaultComposeTimeout would be cut short; the extra+ // defaultComposeTimeout leaves room for image pulls and container creation+ // that happen before the wait starts.+ composeCtx, cancel := detachFromHTTPContextInternal(ctx, waitTimeout+defaultComposeTimeout)defer cancel()// Compose prompts before recreating a volume whose config diverged (data@@ -83,12 +92,12 @@ func ComposeUp(ctx context.Context, proj *types.Project, services []string, remo}defer func() { _ = c.Close() }()- upOptions, startOptions := composeUpOptions(proj, services, removeOrphans, forceRecreate)+ upOptions, startOptions := composeUpOptions(proj, services, removeOrphans, forceRecreate, waitTimeout)return c.svc.Up(composeCtx, proj, api.UpOptions{Create: upOptions, Start: startOptions})}-func composeUpOptions(proj *types.Project, services []string, removeOrphans bool, forceRecreate bool) (api.CreateOptions, api.StartOptions) {+func composeUpOptions(proj *types.Project, services []string, removeOrphans bool, forceRecreate bool, waitTimeout time.Duration) (api.CreateOptions, api.StartOptions) {recreatePolicy := api.RecreateDivergedif forceRecreate {recreatePolicy = api.RecreateForce@@ -102,12 +111,10 @@ func composeUpOptions(proj *types.Project, services []string, removeOrphans bool}startOptions := api.StartOptions{- Project: proj,- Services: services,- Wait: true,- // Reduced from 10 minutes to 2 minutes - if a service can't become healthy- // in 2 minutes, there's likely a configuration issue (missing healthcheck, etc.)- WaitTimeout: 2 * time.Minute,+ Project: proj,+ Services: services,+ Wait: true,+ WaitTimeout: waitTimeout,// CascadeFail ensures that if a dependency fails its health check,// the error propagates correctly instead of being ignoredOnExit: api.CascadeFail,@@ -127,7 +134,7 @@ func ComposePs(ctx context.Context, proj *types.Project, services []string, all}func ComposeDown(ctx context.Context, proj *types.Project, removeVolumes bool) error {- downCtx, cancel := detachFromHTTPContextInternal(ctx)+ downCtx, cancel := detachFromHTTPContextInternal(ctx, defaultComposeTimeout)defer cancel()c, err := NewClient(downCtx, nil, nil)diff --git a/backend/pkg/projects/cmds_test.go b/backend/pkg/projects/cmds_test.goindex a0ff5f8ec..b0a2540d2 100644--- a/backend/pkg/projects/cmds_test.go+++ b/backend/pkg/projects/cmds_test.go@@ -14,7 +14,7 @@ import (func TestDetachFromHTTPContextInternal(t *testing.T) {t.Run("survives parent cancellation", func(t *testing.T) {parent, parentCancel := context.WithCancel(context.Background())- detached, detachedCancel := detachFromHTTPContextInternal(parent)+ detached, detachedCancel := detachFromHTTPContextInternal(parent, defaultComposeTimeout)defer detachedCancel()// Cancel the parent (simulates HTTP request ending).@@ -31,14 +31,14 @@ func TestDetachFromHTTPContextInternal(t *testing.T) {t.Run("preserves context values", func(t *testing.T) {type testKey struct{}parent := context.WithValue(context.Background(), testKey{}, "hello")- detached, cancel := detachFromHTTPContextInternal(parent)+ detached, cancel := detachFromHTTPContextInternal(parent, defaultComposeTimeout)defer cancel()require.Equal(t, "hello", detached.Value(testKey{}))})t.Run("has its own deadline", func(t *testing.T) {- detached, cancel := detachFromHTTPContextInternal(context.Background())+ detached, cancel := detachFromHTTPContextInternal(context.Background(), defaultComposeTimeout)defer cancel()deadline, ok := detached.Deadline()@@ -52,7 +52,7 @@ func TestDetachFromHTTPContextInternal(t *testing.T) {time.Sleep(5 * time.Millisecond) // ensure parent deadline has passed- detached, detachedCancel := detachFromHTTPContextInternal(parent)+ detached, detachedCancel := detachFromHTTPContextInternal(parent, defaultComposeTimeout)defer detachedCancel()require.NoError(t, detached.Err())@@ -62,9 +62,18 @@ func TestDetachFromHTTPContextInternal(t *testing.T) {require.InDelta(t, float64(defaultComposeTimeout), float64(time.Until(deadline)), float64(5*time.Second))})+ t.Run("deadline scales with requested timeout", func(t *testing.T) {+ detached, cancel := detachFromHTTPContextInternal(context.Background(), 2*time.Hour)+ defer cancel()++ deadline, ok := detached.Deadline()+ require.True(t, ok)+ require.InDelta(t, float64(2*time.Hour), float64(time.Until(deadline)), float64(5*time.Second))+ })+t.Run("app lifecycle context cancels detached work on shutdown", func(t *testing.T) {appCtx, cancelApp := context.WithCancel(utils.WithAppLifecycleContext(context.Background()))- detached, detachedCancel := detachFromHTTPContextInternal(appCtx)+ detached, detachedCancel := detachFromHTTPContextInternal(appCtx, defaultComposeTimeout)defer detachedCancel()cancelApp()@@ -87,22 +96,22 @@ func TestComposeUpOptions_RemoveOrphans(t *testing.T) {proj := &composetypes.Project{Name: "test"}t.Run("removeOrphans true propagates to CreateOptions", func(t *testing.T) {- upOptions, _ := composeUpOptions(proj, nil, true, false)+ upOptions, _ := composeUpOptions(proj, nil, true, false, 0)require.True(t, upOptions.RemoveOrphans)})t.Run("removeOrphans false leaves CreateOptions disabled", func(t *testing.T) {- upOptions, _ := composeUpOptions(proj, nil, false, false)+ upOptions, _ := composeUpOptions(proj, nil, false, false, 0)require.False(t, upOptions.RemoveOrphans)})t.Run("removeOrphans is independent of forceRecreate", func(t *testing.T) {// forceRecreate drives the Recreate policy, not RemoveOrphans.- upOptions, _ := composeUpOptions(proj, nil, true, true)+ upOptions, _ := composeUpOptions(proj, nil, true, true, 0)require.True(t, upOptions.RemoveOrphans)require.Equal(t, api.RecreateForce, upOptions.Recreate)- upOptions, _ = composeUpOptions(proj, nil, false, true)+ upOptions, _ = composeUpOptions(proj, nil, false, true, 0)require.False(t, upOptions.RemoveOrphans)require.Equal(t, api.RecreateForce, upOptions.Recreate)})diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex 6ceaccc5b..976a8daff 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -1625,6 +1625,8 @@"docker_api_timeout_description": "Timeout for Docker list operations in seconds (default: 30)","docker_image_pull_timeout": "Docker Image Pull Timeout","docker_image_pull_timeout_description": "Timeout for Docker image pulls in seconds (default: 600 = 10 minutes)",+ "deploy_wait_timeout": "Deploy Wait Timeout",+ "deploy_wait_timeout_description": "How long a deploy waits for services to become healthy or complete (depends_on conditions) in seconds (default: 600 = 10 minutes)","trivy_scan_timeout": "Trivy Scan Timeout","trivy_scan_timeout_description": "Timeout for Trivy vulnerability scans on Docker images.","git_operation_timeout": "Git Operation Timeout",diff --git a/frontend/src/lib/types/settings.ts b/frontend/src/lib/types/settings.tsindex 69420dcbd..2058bf770 100644--- a/frontend/src/lib/types/settings.ts+++ b/frontend/src/lib/types/settings.ts@@ -95,6 +95,7 @@ export type Settings = {dockerApiTimeout: number;dockerImagePullTimeout: number;+ deployWaitTimeout: number;trivyScanTimeout: number;gitOperationTimeout: number;httpClientTimeout: number;diff --git a/frontend/src/routes/(app)/settings/timeouts/+page.svelte b/frontend/src/routes/(app)/settings/timeouts/+page.svelteindex 8a60565de..61bc3c309 100644--- a/frontend/src/routes/(app)/settings/timeouts/+page.svelte+++ b/frontend/src/routes/(app)/settings/timeouts/+page.svelte@@ -15,6 +15,7 @@const formSchema = z.object({dockerApiTimeout: z.coerce.number().int().min(1).max(3600),dockerImagePullTimeout: z.coerce.number().int().min(30).max(7200),+ deployWaitTimeout: z.coerce.number().int().min(30).max(14400),trivyScanTimeout: z.coerce.number().int().min(60).max(14400),gitOperationTimeout: z.coerce.number().int().min(30).max(3600),httpClientTimeout: z.coerce.number().int().min(5).max(300),@@ -27,6 +28,7 @@return {dockerApiTimeout: settings.dockerApiTimeout,dockerImagePullTimeout: settings.dockerImagePullTimeout,+ deployWaitTimeout: settings.deployWaitTimeout,trivyScanTimeout: settings.trivyScanTimeout,gitOperationTimeout: settings.gitOperationTimeout,httpClientTimeout: settings.httpClientTimeout,@@ -76,6 +78,15 @@helpText="Timeout in seconds (30-7200)"type="number"/>+ <TextInputWithLabel+ bind:value={$formInputs.deployWaitTimeout.value}+ error={$formInputs.deployWaitTimeout.error}+ label={m.deploy_wait_timeout()}+ description={m.deploy_wait_timeout_description()}+ placeholder="600"+ helpText="Timeout in seconds (30-14400)"+ type="number"+ /><TextInputWithLabelbind:value={$formInputs.trivyScanTimeout.value}error={$formInputs.trivyScanTimeout.error}diff --git a/types/settings/settings.go b/types/settings/settings.goindex a0205ea3b..28a170b47 100644--- a/types/settings/settings.go+++ b/types/settings/settings.go@@ -428,6 +428,11 @@ type Update struct {// Required: falseProxyRequestTimeout *string `json:"proxyRequestTimeout,omitempty"`+ // DeployWaitTimeout is the timeout waiting for services to become healthy or complete during a deploy in seconds.+ //+ // Required: false+ DeployWaitTimeout *string `json:"deployWaitTimeout,omitempty"`+// AutoUpdateExcludedContainers is a comma-separated list of container names to exclude from auto-update.//// Required: false