Skip to content

Modern Docker Management, Designed for Everyone

fix(health): report down when the francis host exits and extend its health check deadline (#4402)

Kyle Mendelld74ea47e8cfb
diff --git a/backend/api/api.go b/backend/api/api.go
index 2ab300eac..74124264e 100644
--- a/backend/api/api.go
+++ b/backend/api/api.go
@@ -61,6 +61,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
"github.com/getarcaneapp/arcane/backend/v2/internal/webhook"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/userctx"
)
@@ -226,6 +227,7 @@ type HandlerDeps struct {
Role *role.Module
Variable *variable.Module
Upload *upload.Module
+ Actors *francis.Runtime
}
// SetupAPI creates and configures the Huma API attached to the Echo router.
@@ -272,7 +274,7 @@ func SetupAPI(e *echo.Echo, apiGroup *echo.Group, appCtx handlerutil.ActivityApp
api.UseMiddleware(middleware.NewActivityBatchID())
registerNormalization(api)
- health.RegisterRoutes(api)
+ health.RegisterRoutes(api, deps.Actors)
deps.Auth.RegisterRoutes(api)
passkey.RegisterPasskeys(api, deps.Passkey, deps.Auth.Service(), deps.User.Service())
deps.ApiKey.RegisterRoutes(api)
diff --git a/backend/internal/health/handler.go b/backend/internal/health/handler.go
index 5a21177c0..0e8045e5e 100644
--- a/backend/internal/health/handler.go
+++ b/backend/internal/health/handler.go
@@ -2,24 +2,48 @@ package health
import (
"context"
+ "net/http"
"github.com/getarcaneapp/arcane/types/v2/system"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
)
-type healthHandler struct{}
+type healthHandler struct {
+ actors *francis.Runtime
+}
type healthOutput struct {
- Body system.HealthResponse
+ Status int
+ Body system.HealthResponse
+}
+
+type headHealthOutput struct {
+ Status int
}
-func (h *healthHandler) getHealth(context.Context, *struct{}) (*healthOutput, error) {
- return &healthOutput{
- Body: system.HealthResponse{
- Status: "UP",
- },
- }, nil
+type healthOutputs interface {
+ healthOutput | headHealthOutput
}
-func (h *healthHandler) headHealth(context.Context, *struct{}) (*struct{}, error) {
- return nil, nil
+// health reports DOWN once the Francis host has exited; startup stays UP.
+func health[T healthOutputs](h *healthHandler) func(context.Context, *struct{}) (*T, error) {
+ return func(context.Context, *struct{}) (*T, error) {
+ status, body := http.StatusOK, system.HealthResponse{Status: "UP"}
+ if h.actors != nil {
+ select {
+ case <-h.actors.Done():
+ status, body = http.StatusServiceUnavailable, system.HealthResponse{Status: "DOWN"}
+ default:
+ }
+ }
+ var output T
+ switch typed := any(&output).(type) {
+ case *healthOutput:
+ typed.Status, typed.Body = status, body
+ case *headHealthOutput:
+ typed.Status = status
+ }
+ return &output, nil
+ }
}
diff --git a/backend/internal/health/module.go b/backend/internal/health/module.go
index bf148b542..50fe624ef 100644
--- a/backend/internal/health/module.go
+++ b/backend/internal/health/module.go
@@ -4,11 +4,13 @@ import (
"net/http"
"github.com/danielgtaylor/huma/v2"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
)
// RegisterRoutes registers the health check routes.
-func RegisterRoutes(api huma.API) {
- handler := &healthHandler{}
+func RegisterRoutes(api huma.API, actors *francis.Runtime) {
+ handler := &healthHandler{actors: actors}
huma.Register(api, huma.Operation{
OperationID: "health-check",
@@ -18,7 +20,7 @@ func RegisterRoutes(api huma.API) {
Description: "Check if the API is healthy",
Tags: []string{"Health"},
Security: []map[string][]string{},
- }, handler.getHealth)
+ }, health[healthOutput](handler))
huma.Register(api, huma.Operation{
OperationID: "health-check-head",
@@ -28,5 +30,5 @@ func RegisterRoutes(api huma.API) {
Description: "Check if the API is healthy (HEAD request)",
Tags: []string{"Health"},
Security: []map[string][]string{},
- }, handler.headHealth)
+ }, health[headHealthOutput](handler))
}
diff --git a/backend/pkg/utils/francis/runtime.go b/backend/pkg/utils/francis/runtime.go
index 61ace1238..68dbb498b 100644
--- a/backend/pkg/utils/francis/runtime.go
+++ b/backend/pkg/utils/francis/runtime.go
@@ -79,7 +79,7 @@ func New(databaseURL, encryptionKey, instanceID, port string, options ...local.H
service: &actor.Service{}, ready: make(chan struct{}), done: make(chan struct{}), address: address, storeURL: storeURL,
options: []local.HostOption{
local.WithAddress(address), providerOption(storeURL),
- local.WithMaxHosts(1), local.WithHostHealthCheckDeadline(90 * time.Second),
+ local.WithMaxHosts(1), local.WithHostHealthCheckDeadline(150 * time.Second),
local.WithShutdownGracePeriod(10 * time.Second), local.WithAlarmsPollInterval(time.Second),
local.WithAlarmsFetchAheadInterval(30 * time.Second), local.WithAlarmsLeaseDuration(180 * time.Second),
},
@@ -117,6 +117,7 @@ func (r *Runtime) ConfigureIdentity(encryptionKey, instanceID string) error {
func (r *Runtime) Service() *actor.Service { return r.service }
func (r *Runtime) Ready() <-chan struct{} { return r.ready }
+func (r *Runtime) Done() <-chan struct{} { return r.done }
// Store lazily opens a small pool on Francis storage for diagnostics and backups.
func (r *Runtime) Store() (*gorm.DB, error) {