Skip to content

Modern Docker Management, Designed for Everyone

fix: make compose up wait timeout configurable so long depends_on conditions don't abort deploys (#3557)

Kyle Mendell5d59f0e30c69
diff --git a/backend/internal/config/schema/schema_test.go b/backend/internal/config/schema/schema_test.go
index a7cd33c14..3739af693 100644
--- a/backend/internal/config/schema/schema_test.go
+++ b/backend/internal/config/schema/schema_test.go
@@ -291,6 +291,7 @@ var expectedSettingOverrideKeys = []string{
"buildsDirectory",
"defaultDeployPullPolicy",
"defaultShell",
+ "deployWaitTimeout",
"depotProjectId",
"depotToken",
"diskUsagePath",
diff --git a/backend/internal/models/settings.go b/backend/internal/models/settings.go
index 6ff169411..b153ba23a 100644
--- a/backend/internal/models/settings.go
+++ b/backend/internal/models/settings.go
@@ -187,6 +187,7 @@ type Settings struct {
HTTPClientTimeout SettingVariable `key:"httpClientTimeout,envOverride" meta:"label=HTTP Client Timeout;type=number;keywords=http,client,timeout,seconds,api,request;category=timeouts;description=Default timeout for HTTP requests in seconds (default: 30)"`
RegistryTimeout SettingVariable `key:"registryTimeout,envOverride" meta:"label=Registry Timeout;type=number;keywords=registry,timeout,seconds,docker,auth;category=timeouts;description=Timeout for container registry operations in seconds (default: 30)"`
ProxyRequestTimeout SettingVariable `key:"proxyRequestTimeout,envOverride" meta:"label=Proxy Request Timeout;type=number;keywords=proxy,request,timeout,seconds,forward;category=timeouts;description=Timeout for proxied requests in seconds (default: 60)"`
+ DeployWaitTimeout SettingVariable `key:"deployWaitTimeout,envOverride" meta:"label=Deploy Wait Timeout;type=number;keywords=deploy,compose,up,wait,healthy,depends_on,timeout,seconds;category=timeouts;description=Timeout waiting for services to become healthy or complete during a deploy in seconds (default: 600 = 10 minutes)"`
}
func (SettingVariable) TableName() string {
diff --git a/backend/internal/project/project_lifecycle.go b/backend/internal/project/project_lifecycle.go
index 5ef1aa2a7..465c2eed8 100644
--- a/backend/internal/project/project_lifecycle.go
+++ b/backend/internal/project/project_lifecycle.go
@@ -148,7 +148,7 @@ func (s *ProjectService) UpdateProjectServices(ctx context.Context, projectID st
}
// 5. Up specific services
- if err := composeUpProjectServicesInternal(ctx, compProj, servicesToUpdate, false, true, false, s.composeRegistryAuthConfigsInternal(ctx)); err != nil {
+ if err := composeUpProjectServicesInternal(ctx, compProj, servicesToUpdate, false, true, false, s.composeRegistryAuthConfigsInternal(ctx), s.deployWaitTimeoutInternal()); err != nil {
s.restoreProjectStatusAfterFailedDeployInternal(ctx, projectID)
return errors.WrapIf(err, "failed to up services")
}
@@ -244,6 +244,12 @@ func (s *ProjectService) UnarchiveProject(ctx context.Context, projectID string,
// resolveRemoveOrphansInternal decides whether compose up should remove orphan containers.
// GitOps-managed projects always remove orphans so the running stack matches the
// tracked compose file. Non-GitOps callers may opt in per request via DeployOptions.
+// deployWaitTimeoutInternal resolves how long compose up waits for depends_on
+// health/completion conditions, from the deployWaitTimeout setting.
+func (s *ProjectService) deployWaitTimeoutInternal() time.Duration {
+ return timeouts.GetDuration(s.settingsService.GetSettingsConfig().DeployWaitTimeout.AsInt(), timeouts.DefaultDeployWait)
+}
+
func resolveRemoveOrphansInternal(gitOpsManaged bool, options *project.DeployOptions) bool {
return gitOpsManaged || (options != nil && options.RemoveOrphans)
}
@@ -310,7 +316,7 @@ func (s *ProjectService) DeployProject(ctx context.Context, projectID string, us
slog.Info("starting compose up with health check support", "projectID", projectID, "projectName", projectModel.Name, "services", len(projectModel.Services), "removeOrphans", removeOrphans)
// Health/progress streaming (if any) is handled inside projects.ComposeUp via ctx.
- if err := projects.ComposeUp(ctx, projectModel, nil, removeOrphans, forceRecreate, recreateVolumes, s.composeRegistryAuthConfigsInternal(ctx)); err != nil {
+ if err := projects.ComposeUp(ctx, projectModel, nil, removeOrphans, forceRecreate, recreateVolumes, s.composeRegistryAuthConfigsInternal(ctx), s.deployWaitTimeoutInternal()); err != nil {
slog.Error("compose up failed", "projectName", projectModel.Name, "projectID", projectID, "error", err)
if containers, psErr := s.GetProjectServices(ctx, projectID); psErr == nil {
slog.Info("containers after failed deploy", "projectID", projectID, "containers", containers)
@@ -320,7 +326,7 @@ func (s *ProjectService) DeployProject(ctx context.Context, projectID string, us
// Provide more helpful error messages
errMsg := err.Error()
if strings.Contains(errMsg, "timeout") || strings.Contains(errMsg, "context deadline exceeded") {
- return errors.WrapIf(err, "deployment timed out - check if services with 'condition: service_healthy' have healthchecks defined")
+ return errors.WrapIf(err, "deployment timed out waiting for services - long-running 'service_healthy'/'service_completed_successfully' dependencies may need a higher Deploy Wait Timeout setting, and 'service_healthy' requires a healthcheck")
}
return errors.WrapIf(err, "failed to deploy project")
}
diff --git a/backend/internal/project/service_test.go b/backend/internal/project/service_test.go
index 52d4c843d..46bda4f21 100644
--- a/backend/internal/project/service_test.go
+++ b/backend/internal/project/service_test.go
@@ -1049,7 +1049,7 @@ func TestProjectService_UpdateProjectServicesHardFailsWhenPullFailsInternal(t *t
composeUpProjectServicesInternal = originalComposeUp
})
upCalled := false
- composeUpProjectServicesInternal = func(context.Context, *composetypes.Project, []string, bool, bool, bool, map[string]dockerregistry.AuthConfig) error {
+ composeUpProjectServicesInternal = func(context.Context, *composetypes.Project, []string, bool, bool, bool, map[string]dockerregistry.AuthConfig, time.Duration) error {
upCalled = true
return errors.New("compose up should not run")
}
@@ -1119,7 +1119,7 @@ func TestProjectService_UpdateProjectServicesForcesRecreateInternal(t *testing.T
}
upCalled := false
forceRecreate := false
- composeUpProjectServicesInternal = func(_ context.Context, _ *composetypes.Project, services []string, removeOrphans bool, force bool, _ bool, _ map[string]dockerregistry.AuthConfig) error {
+ composeUpProjectServicesInternal = func(_ context.Context, _ *composetypes.Project, services []string, removeOrphans bool, force bool, _ bool, _ map[string]dockerregistry.AuthConfig, _ time.Duration) error {
upCalled = true
forceRecreate = force
assert.Equal(t, []string{"app"}, services)
diff --git a/backend/internal/settings/service.go b/backend/internal/settings/service.go
index fe15592f7..27ccd6922 100644
--- a/backend/internal/settings/service.go
+++ b/backend/internal/settings/service.go
@@ -276,6 +276,7 @@ func DefaultSettingsConfig() *models.Settings {
HTTPClientTimeout: models.SettingVariable{Value: "30"},
RegistryTimeout: models.SettingVariable{Value: "30"},
ProxyRequestTimeout: models.SettingVariable{Value: "60"},
+ DeployWaitTimeout: models.SettingVariable{Value: "600"},
BuildProvider: models.SettingVariable{Value: "local"},
BuildTimeout: models.SettingVariable{Value: "1800"},
DepotProjectId: models.SettingVariable{Value: ""},
diff --git a/backend/pkg/libarcane/settings.go b/backend/pkg/libarcane/settings.go
index 7eff3522a..dcc7294d9 100644
--- a/backend/pkg/libarcane/settings.go
+++ b/backend/pkg/libarcane/settings.go
@@ -21,6 +21,7 @@ var timeoutSettingKeys = []string{
"httpClientTimeout",
"registryTimeout",
"proxyRequestTimeout",
+ "deployWaitTimeout",
"trivyResourceLimitsEnabled",
"trivyCpuLimit",
"trivyMemoryLimitMb",
diff --git a/backend/pkg/libarcane/timeouts/timeouts.go b/backend/pkg/libarcane/timeouts/timeouts.go
index aa7289119..39aec6bdd 100644
--- a/backend/pkg/libarcane/timeouts/timeouts.go
+++ b/backend/pkg/libarcane/timeouts/timeouts.go
@@ -13,6 +13,9 @@ const (
DefaultRegistry = 30 * time.Second
DefaultProxyRequest = 60 * time.Second
DefaultBuildTimeout = 30 * time.Minute
+ // DefaultDeployWait bounds how long compose up waits for services to become
+ // healthy or complete (depends_on conditions) before the deploy fails.
+ DefaultDeployWait = 10 * time.Minute
// DefaultImageUpdateScan bounds an entire batch image update check
// end-to-end; individual registry RPCs are separately bounded, but the
// aggregate scan needs its own ceiling so a wedged batch cannot hold its
diff --git a/backend/pkg/projects/cmds.go b/backend/pkg/projects/cmds.go
index f2f0cbb14..b58f76438 100644
--- a/backend/pkg/projects/cmds.go
+++ b/backend/pkg/projects/cmds.go
@@ -12,6 +12,7 @@ import (
"github.com/moby/moby/api/types/registry"
"github.com/moby/moby/client"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)
@@ -26,16 +27,16 @@ const defaultComposeTimeout = 30 * time.Minute
// deadline-bounded by the parent. This allows compose operations to survive
// HTTP request timeouts and proxy deadline cancellations. A standalone timeout
// is applied so the operation cannot run forever. See #1209.
-func detachFromHTTPContextInternal(parent context.Context) (context.Context, context.CancelFunc) {
+func detachFromHTTPContextInternal(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
if utils.IsAppLifecycleContext(parent) {
- return context.WithTimeout(parent, defaultComposeTimeout)
+ return context.WithTimeout(parent, timeout)
}
ctx := context.WithoutCancel(parent)
- return context.WithTimeout(ctx, defaultComposeTimeout)
+ return context.WithTimeout(ctx, timeout)
}
func ComposeRestart(ctx context.Context, proj *types.Project, services []string) error {
- restartCtx, cancel := detachFromHTTPContextInternal(ctx)
+ restartCtx, cancel := detachFromHTTPContextInternal(ctx, defaultComposeTimeout)
defer cancel()
c, err := NewClient(restartCtx, nil, nil)
@@ -51,7 +52,7 @@ func ComposeStop(ctx context.Context, proj *types.Project, services []string) er
if len(services) == 0 {
return nil
}
- stopCtx, cancel := detachFromHTTPContextInternal(ctx)
+ stopCtx, cancel := detachFromHTTPContextInternal(ctx, defaultComposeTimeout)
defer cancel()
c, err := NewClient(stopCtx, nil, nil)
@@ -63,10 +64,18 @@ func ComposeStop(ctx context.Context, proj *types.Project, services []string) er
return c.svc.Stop(stopCtx, proj.Name, api.StopOptions{Services: services})
}
-func ComposeUp(ctx context.Context, proj *types.Project, services []string, removeOrphans bool, forceRecreate bool, recreateVolumes bool, authConfigs map[string]registry.AuthConfig) error {
+func ComposeUp(ctx context.Context, proj *types.Project, services []string, removeOrphans bool, forceRecreate bool, recreateVolumes bool, authConfigs map[string]registry.AuthConfig, waitTimeout time.Duration) error {
+ if waitTimeout <= 0 {
+ waitTimeout = timeouts.DefaultDeployWait
+ }
+
// Detach from the HTTP request context so that proxy timeouts and client
- // disconnects do not cancel a long-running compose up. See #1209.
- composeCtx, cancel := detachFromHTTPContextInternal(ctx)
+ // disconnects do not cancel a long-running compose up. See #1209. The
+ // operation deadline must exceed the configured wait, otherwise a Deploy
+ // Wait Timeout above defaultComposeTimeout would be cut short; the extra
+ // defaultComposeTimeout leaves room for image pulls and container creation
+ // that happen before the wait starts.
+ composeCtx, cancel := detachFromHTTPContextInternal(ctx, waitTimeout+defaultComposeTimeout)
defer cancel()
// Compose prompts before recreating a volume whose config diverged (data
@@ -83,12 +92,12 @@ func ComposeUp(ctx context.Context, proj *types.Project, services []string, remo
}
defer func() { _ = c.Close() }()
- upOptions, startOptions := composeUpOptions(proj, services, removeOrphans, forceRecreate)
+ upOptions, startOptions := composeUpOptions(proj, services, removeOrphans, forceRecreate, waitTimeout)
return c.svc.Up(composeCtx, proj, api.UpOptions{Create: upOptions, Start: startOptions})
}
-func composeUpOptions(proj *types.Project, services []string, removeOrphans bool, forceRecreate bool) (api.CreateOptions, api.StartOptions) {
+func composeUpOptions(proj *types.Project, services []string, removeOrphans bool, forceRecreate bool, waitTimeout time.Duration) (api.CreateOptions, api.StartOptions) {
recreatePolicy := api.RecreateDiverged
if forceRecreate {
recreatePolicy = api.RecreateForce
@@ -102,12 +111,10 @@ func composeUpOptions(proj *types.Project, services []string, removeOrphans bool
}
startOptions := api.StartOptions{
- Project: proj,
- Services: services,
- Wait: true,
- // Reduced from 10 minutes to 2 minutes - if a service can't become healthy
- // in 2 minutes, there's likely a configuration issue (missing healthcheck, etc.)
- WaitTimeout: 2 * time.Minute,
+ Project: proj,
+ Services: services,
+ Wait: true,
+ WaitTimeout: waitTimeout,
// CascadeFail ensures that if a dependency fails its health check,
// the error propagates correctly instead of being ignored
OnExit: api.CascadeFail,
@@ -127,7 +134,7 @@ func ComposePs(ctx context.Context, proj *types.Project, services []string, all
}
func ComposeDown(ctx context.Context, proj *types.Project, removeVolumes bool) error {
- downCtx, cancel := detachFromHTTPContextInternal(ctx)
+ downCtx, cancel := detachFromHTTPContextInternal(ctx, defaultComposeTimeout)
defer cancel()
c, err := NewClient(downCtx, nil, nil)
diff --git a/backend/pkg/projects/cmds_test.go b/backend/pkg/projects/cmds_test.go
index a0ff5f8ec..b0a2540d2 100644
--- a/backend/pkg/projects/cmds_test.go
+++ b/backend/pkg/projects/cmds_test.go
@@ -14,7 +14,7 @@ import (
func TestDetachFromHTTPContextInternal(t *testing.T) {
t.Run("survives parent cancellation", func(t *testing.T) {
parent, parentCancel := context.WithCancel(context.Background())
- detached, detachedCancel := detachFromHTTPContextInternal(parent)
+ detached, detachedCancel := detachFromHTTPContextInternal(parent, defaultComposeTimeout)
defer detachedCancel()
// Cancel the parent (simulates HTTP request ending).
@@ -31,14 +31,14 @@ func TestDetachFromHTTPContextInternal(t *testing.T) {
t.Run("preserves context values", func(t *testing.T) {
type testKey struct{}
parent := context.WithValue(context.Background(), testKey{}, "hello")
- detached, cancel := detachFromHTTPContextInternal(parent)
+ detached, cancel := detachFromHTTPContextInternal(parent, defaultComposeTimeout)
defer cancel()
require.Equal(t, "hello", detached.Value(testKey{}))
})
t.Run("has its own deadline", func(t *testing.T) {
- detached, cancel := detachFromHTTPContextInternal(context.Background())
+ detached, cancel := detachFromHTTPContextInternal(context.Background(), defaultComposeTimeout)
defer cancel()
deadline, ok := detached.Deadline()
@@ -52,7 +52,7 @@ func TestDetachFromHTTPContextInternal(t *testing.T) {
time.Sleep(5 * time.Millisecond) // ensure parent deadline has passed
- detached, detachedCancel := detachFromHTTPContextInternal(parent)
+ detached, detachedCancel := detachFromHTTPContextInternal(parent, defaultComposeTimeout)
defer detachedCancel()
require.NoError(t, detached.Err())
@@ -62,9 +62,18 @@ func TestDetachFromHTTPContextInternal(t *testing.T) {
require.InDelta(t, float64(defaultComposeTimeout), float64(time.Until(deadline)), float64(5*time.Second))
})
+ t.Run("deadline scales with requested timeout", func(t *testing.T) {
+ detached, cancel := detachFromHTTPContextInternal(context.Background(), 2*time.Hour)
+ defer cancel()
+
+ deadline, ok := detached.Deadline()
+ require.True(t, ok)
+ require.InDelta(t, float64(2*time.Hour), float64(time.Until(deadline)), float64(5*time.Second))
+ })
+
t.Run("app lifecycle context cancels detached work on shutdown", func(t *testing.T) {
appCtx, cancelApp := context.WithCancel(utils.WithAppLifecycleContext(context.Background()))
- detached, detachedCancel := detachFromHTTPContextInternal(appCtx)
+ detached, detachedCancel := detachFromHTTPContextInternal(appCtx, defaultComposeTimeout)
defer detachedCancel()
cancelApp()
@@ -87,22 +96,22 @@ func TestComposeUpOptions_RemoveOrphans(t *testing.T) {
proj := &composetypes.Project{Name: "test"}
t.Run("removeOrphans true propagates to CreateOptions", func(t *testing.T) {
- upOptions, _ := composeUpOptions(proj, nil, true, false)
+ upOptions, _ := composeUpOptions(proj, nil, true, false, 0)
require.True(t, upOptions.RemoveOrphans)
})
t.Run("removeOrphans false leaves CreateOptions disabled", func(t *testing.T) {
- upOptions, _ := composeUpOptions(proj, nil, false, false)
+ upOptions, _ := composeUpOptions(proj, nil, false, false, 0)
require.False(t, upOptions.RemoveOrphans)
})
t.Run("removeOrphans is independent of forceRecreate", func(t *testing.T) {
// forceRecreate drives the Recreate policy, not RemoveOrphans.
- upOptions, _ := composeUpOptions(proj, nil, true, true)
+ upOptions, _ := composeUpOptions(proj, nil, true, true, 0)
require.True(t, upOptions.RemoveOrphans)
require.Equal(t, api.RecreateForce, upOptions.Recreate)
- upOptions, _ = composeUpOptions(proj, nil, false, true)
+ upOptions, _ = composeUpOptions(proj, nil, false, true, 0)
require.False(t, upOptions.RemoveOrphans)
require.Equal(t, api.RecreateForce, upOptions.Recreate)
})
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 6ceaccc5b..976a8daff 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -1625,6 +1625,8 @@
"docker_api_timeout_description": "Timeout for Docker list operations in seconds (default: 30)",
"docker_image_pull_timeout": "Docker Image Pull Timeout",
"docker_image_pull_timeout_description": "Timeout for Docker image pulls in seconds (default: 600 = 10 minutes)",
+ "deploy_wait_timeout": "Deploy Wait Timeout",
+ "deploy_wait_timeout_description": "How long a deploy waits for services to become healthy or complete (depends_on conditions) in seconds (default: 600 = 10 minutes)",
"trivy_scan_timeout": "Trivy Scan Timeout",
"trivy_scan_timeout_description": "Timeout for Trivy vulnerability scans on Docker images.",
"git_operation_timeout": "Git Operation Timeout",
diff --git a/frontend/src/lib/types/settings.ts b/frontend/src/lib/types/settings.ts
index 69420dcbd..2058bf770 100644
--- a/frontend/src/lib/types/settings.ts
+++ b/frontend/src/lib/types/settings.ts
@@ -95,6 +95,7 @@ export type Settings = {
dockerApiTimeout: number;
dockerImagePullTimeout: number;
+ deployWaitTimeout: number;
trivyScanTimeout: number;
gitOperationTimeout: number;
httpClientTimeout: number;
diff --git a/frontend/src/routes/(app)/settings/timeouts/+page.svelte b/frontend/src/routes/(app)/settings/timeouts/+page.svelte
index 8a60565de..61bc3c309 100644
--- a/frontend/src/routes/(app)/settings/timeouts/+page.svelte
+++ b/frontend/src/routes/(app)/settings/timeouts/+page.svelte
@@ -15,6 +15,7 @@
const formSchema = z.object({
dockerApiTimeout: z.coerce.number().int().min(1).max(3600),
dockerImagePullTimeout: z.coerce.number().int().min(30).max(7200),
+ deployWaitTimeout: z.coerce.number().int().min(30).max(14400),
trivyScanTimeout: z.coerce.number().int().min(60).max(14400),
gitOperationTimeout: z.coerce.number().int().min(30).max(3600),
httpClientTimeout: z.coerce.number().int().min(5).max(300),
@@ -27,6 +28,7 @@
return {
dockerApiTimeout: settings.dockerApiTimeout,
dockerImagePullTimeout: settings.dockerImagePullTimeout,
+ deployWaitTimeout: settings.deployWaitTimeout,
trivyScanTimeout: settings.trivyScanTimeout,
gitOperationTimeout: settings.gitOperationTimeout,
httpClientTimeout: settings.httpClientTimeout,
@@ -76,6 +78,15 @@
helpText="Timeout in seconds (30-7200)"
type="number"
/>
+ <TextInputWithLabel
+ bind:value={$formInputs.deployWaitTimeout.value}
+ error={$formInputs.deployWaitTimeout.error}
+ label={m.deploy_wait_timeout()}
+ description={m.deploy_wait_timeout_description()}
+ placeholder="600"
+ helpText="Timeout in seconds (30-14400)"
+ type="number"
+ />
<TextInputWithLabel
bind:value={$formInputs.trivyScanTimeout.value}
error={$formInputs.trivyScanTimeout.error}
diff --git a/types/settings/settings.go b/types/settings/settings.go
index a0205ea3b..28a170b47 100644
--- a/types/settings/settings.go
+++ b/types/settings/settings.go
@@ -428,6 +428,11 @@ type Update struct {
// Required: false
ProxyRequestTimeout *string `json:"proxyRequestTimeout,omitempty"`
+ // DeployWaitTimeout is the timeout waiting for services to become healthy or complete during a deploy in seconds.
+ //
+ // Required: false
+ DeployWaitTimeout *string `json:"deployWaitTimeout,omitempty"`
+
// AutoUpdateExcludedContainers is a comma-separated list of container names to exclude from auto-update.
//
// Required: false