Skip to content

Modern Docker Management, Designed for Everyone

feat: add OpenTelemetry metrics, traces, and logs (#4367)

Kyle Mendellf3bfc284b1f0
diff --git a/backend/api/api.go b/backend/api/api.go
index 40a250a31..2ab300eac 100644
--- a/backend/api/api.go
+++ b/backend/api/api.go
@@ -1,11 +1,13 @@
package api
import (
+ "cmp"
"context"
"encoding/json/v2"
"io"
"maps"
"net/http"
+ "path"
"reflect"
"strings"
@@ -60,13 +62,13 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
"github.com/getarcaneapp/arcane/backend/v2/internal/webhook"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/userctx"
)
const (
arcaneTypesPrefix = "github.com/getarcaneapp/arcane/types/v2/"
dockerSDKPrefix = "github.com/moby/moby"
- // scalarDocsHTML returns the HTML template for Scalar API documentation.
scalarDocsHTML = `<!doctype html>
<html>
<head>
@@ -115,93 +117,46 @@ var jsonV2Format = huma.Format{
},
}
-// customSchemaNamer creates unique schema names using package prefix for types
-// from github.com/getarcaneapp/arcane/types/v2 to avoid conflicts between packages that have
-// types with the same name (e.g., image.Summary vs env.Summary).
+// customSchemaNamer prefixes Arcane and Docker type schemas with their package name
+// so same-named types (e.g. image.Summary vs env.Summary) do not collide.
func customSchemaNamer(t reflect.Type, hint string) string {
name := huma.DefaultSchemaNamer(t, hint)
typeStr := t.String()
- pkgPath := packagePathForType(t)
- shortPkg := shortPackageFromTypeString(typeStr)
+ elem := t
+ for elem.Kind() == reflect.Pointer {
+ elem = elem.Elem()
+ }
+ pkgPath := elem.PkgPath()
+ pkgName := path.Base(pkgPath)
+ shortPkg, _, _ := strings.Cut(typeStr, ".")
+ isArcaneType := strings.HasPrefix(pkgPath, arcaneTypesPrefix)
- if pkgName, ok := arcanePackageName(pkgPath); ok {
- name = pkgName + name
- } else if dockerPrefix, localOk := dockerSchemaPrefix(pkgPath, shortPkg); localOk {
- name = dockerPrefix + name
+ if isArcaneType {
+ name = kit.Capitalize(pkgName) + name
+ } else {
+ prefix := dockerSchemaPrefixes[shortPkg]
+ if strings.Contains(pkgPath, dockerSDKPrefix) {
+ prefix = cmp.Or(dockerSchemaPrefixes[pkgName], prefix)
+ }
+ name = prefix + name
}
// Preserve the published name after moving the volume backup response to types.
if pkgPath == arcaneTypesPrefix+"volume" && strings.TrimLeft(typeStr, "*") == "volume.Backup" {
name = "DockerVolumeVolumeBackup"
}
- return qualifyGenericArcaneArgumentsInternal(pkgPath, typeStr, name)
-}
-
-func packagePathForType(t reflect.Type) string {
- for t.Kind() == reflect.Pointer {
- t = t.Elem()
- }
-
- return t.PkgPath()
-}
-
-func shortPackageFromTypeString(typeStr string) string {
- before, _, ok := strings.Cut(typeStr, ".")
- return kit.Ternary(!ok, "", before)
-}
-
-func arcanePackageName(pkgPath string) (string, bool) {
- if !strings.HasPrefix(pkgPath, arcaneTypesPrefix) {
- return "", false
- }
-
- parts := strings.Split(pkgPath, "/")
- if len(parts) == 0 {
- return "", false
- }
-
- pkg := parts[len(parts)-1]
- if pkg == "" {
- return "", false
- }
-
- return kit.Capitalize(pkg), true
-}
-
-func dockerSchemaPrefix(pkgPath, shortPkg string) (string, bool) {
- if strings.Contains(pkgPath, dockerSDKPrefix) {
- parts := strings.Split(pkgPath, "/")
- last := parts[len(parts)-1]
- if prefix, ok := dockerSchemaPrefixes[last]; ok {
- return prefix, true
- }
- }
-
- prefix, ok := dockerSchemaPrefixes[shortPkg]
- if !ok {
- return "", false
- }
-
- return prefix, true
-}
-
-func qualifyGenericArcaneArgumentsInternal(pkgPath, typeName, schemaName string) string {
- openBracket := strings.IndexByte(typeName, '[')
- if !strings.HasPrefix(pkgPath, arcaneTypesPrefix) || openBracket < 0 {
- return schemaName
- }
- outerPackage := strings.TrimPrefix(pkgPath, arcaneTypesPrefix)
- if _, after, found := strings.CutLast(outerPackage, "/"); found {
- outerPackage = after
+ // Qualify Arcane generic arguments with their package unless it matches the outer type's package.
+ openBracket := strings.IndexByte(typeStr, '[')
+ if !isArcaneType || openBracket < 0 {
+ return name
}
-
- plainSchemaName := huma.DefaultSchemaNamer(reflect.TypeFor[struct{}](), typeName)
- schemaPrefix, ok := strings.CutSuffix(schemaName, plainSchemaName)
+ anonymous := reflect.TypeFor[struct{}]()
+ schemaPrefix, ok := strings.CutSuffix(name, huma.DefaultSchemaNamer(anonymous, typeStr))
if !ok {
- return schemaName
+ return name
}
- rewrittenTypeName := typeName
+ rewrittenTypeName := typeStr
searchOffset := openBracket + 1
for {
prefixIndex := strings.Index(rewrittenTypeName[searchOffset:], arcaneTypesPrefix)
@@ -216,28 +171,18 @@ func qualifyGenericArcaneArgumentsInternal(pkgPath, typeName, schemaName string)
break
}
- innerPackage := afterPrefix[:separator]
- if _, after, found := strings.CutLast(innerPackage, "/"); found {
- innerPackage = after
- }
-
- replacement := ""
- if innerPackage != outerPackage {
- replacement = kit.Capitalize(innerPackage)
- }
-
+ innerPackage := path.Base(afterPrefix[:separator])
+ replacement := kit.Ternary(innerPackage != pkgName, kit.Capitalize(innerPackage), "")
argumentTypeIndex := prefixIndex + len(arcaneTypesPrefix) + separator + 1
rewrittenTypeName = rewrittenTypeName[:prefixIndex] + replacement + rewrittenTypeName[argumentTypeIndex:]
searchOffset = prefixIndex + len(replacement)
}
- return schemaPrefix + huma.DefaultSchemaNamer(reflect.TypeFor[struct{}](), rewrittenTypeName)
+ return schemaPrefix + huma.DefaultSchemaNamer(anonymous, rewrittenTypeName)
}
// HandlerDeps contains the services required to register HTTP API handlers.
-//
-// It intentionally contains only dependencies consumed by SetupAPI,
-// registerHandlersInternal, and the authentication middleware.
+// It contains only dependencies consumed by SetupAPI and the authentication middleware.
type HandlerDeps struct {
fx.In
@@ -292,22 +237,9 @@ func SetupAPI(e *echo.Echo, apiGroup *echo.Group, appCtx handlerutil.ActivityApp
humaConfig.Formats["application/json"] = jsonV2Format
humaConfig.Formats["json"] = jsonV2Format
humaConfig.Info.Description = "Modern Docker Management, Designed for Everyone"
-
// Disable default docs path - we'll use Scalar instead
humaConfig.DocsPath = ""
-
- // Configure servers for OpenAPI spec
- if cfg.AppUrl != "" {
- humaConfig.Servers = []*huma.Server{
- {URL: cfg.AppUrl + "/api"},
- }
- } else {
- humaConfig.Servers = []*huma.Server{
- {URL: "/api"},
- }
- }
-
- // Configure security schemes
+ humaConfig.Servers = []*huma.Server{{URL: cfg.AppUrl + "/api"}}
humaConfig.Components.SecuritySchemes = map[string]*huma.SecurityScheme{
"BearerAuth": {
Type: "http",
@@ -326,82 +258,20 @@ func SetupAPI(e *echo.Echo, apiGroup *echo.Group, appCtx handlerutil.ActivityApp
{"BearerAuth": {}},
{"ApiKeyAuth": {}},
}
-
- // Use custom schema namer to avoid conflicts between types with same name
- // from different packages (e.g., image.Summary vs env.Summary)
humaConfig.Components.Schemas = huma.NewMapRegistry("#/components/schemas/", customSchemaNamer)
- // Create Huma API wrapping the Echo router group
api := humaecho.NewWithGroup(e, apiGroup, humaConfig)
- // Add authentication middleware
api.UseMiddleware(auth.NewHumaMiddleware(api, deps.Auth.Service(), deps.ApiKey.Service(), deps.Role.Service(), deps.Environment.Service(), cfg))
- api.UseMiddleware(middleware.NewActivityBatchID())
- registerNormalizationInternal(api)
-
- // Register all Huma handlers
- registerHandlersInternal(api, deps, appCtx, cfg)
-
- // Register Scalar API docs endpoint with dark mode
- registerScalarDocs(apiGroup)
-
- return api
-}
-
-// registerScalarDocs adds the Scalar API documentation endpoint.
-func registerScalarDocs(apiGroup *echo.Group) {
- apiGroup.GET("/docs", func(c *echo.Context) error {
- return c.HTML(http.StatusOK, scalarDocsHTML)
+ api.UseMiddleware(func(ctx huma.Context, next func(huma.Context)) {
+ if actor, ok := userctx.CurrentUserFromContext(ctx.Context()); ok {
+ middleware.RecordAuthenticatedRequest(ctx.Context(), humaecho.Unwrap(ctx).Request().Header, actor)
+ }
+ next(ctx)
})
-}
-
-// SetupAPIForSpec creates a Huma API instance for OpenAPI spec generation only.
-// No services are required - this is purely for schema generation.
-func SetupAPIForSpec() huma.API {
- e := echo.New()
- apiGroup := e.Group("/api")
-
- humaConfig := huma.DefaultConfig("Arcane API", config.Version)
- humaConfig.Formats = maps.Clone(humaConfig.Formats)
- humaConfig.Formats["application/json"] = jsonV2Format
- humaConfig.Formats["json"] = jsonV2Format
- humaConfig.Info.Description = "Modern Docker Management, Designed for Everyone"
- humaConfig.Servers = []*huma.Server{
- {URL: "/api"},
- }
- humaConfig.Components.SecuritySchemes = map[string]*huma.SecurityScheme{
- "BearerAuth": {
- Type: "http",
- Scheme: "bearer",
- BearerFormat: "JWT",
- Description: "JWT Bearer token authentication",
- },
- "ApiKeyAuth": {
- Type: "apiKey",
- In: "header",
- Name: "X-API-Key",
- Description: "API Key authentication",
- },
- }
- humaConfig.Security = []map[string][]string{
- {"BearerAuth": {}},
- {"ApiKeyAuth": {}},
- }
-
- // Use custom schema namer to avoid conflicts between types with same name
- humaConfig.Components.Schemas = huma.NewMapRegistry("#/components/schemas/", customSchemaNamer)
-
- api := humaecho.NewWithGroup(e, apiGroup, humaConfig)
-
- // Register handlers with zero-value dependencies for schema discovery only.
- registerNormalizationInternal(api)
- registerHandlersInternal(api, HandlerDeps{}, handlerutil.NewActivityAppContext(context.Background()), nil) //nolint:forbidigo // Schema discovery has no running request or application lifecycle.
-
- return api
-}
+ api.UseMiddleware(middleware.NewActivityBatchID())
+ registerNormalization(api)
-// registerHandlersInternal registers all Huma-based API handlers.
-func registerHandlersInternal(api huma.API, deps HandlerDeps, handlerAppCtx handlerutil.ActivityAppContext, cfg *config.Config) {
health.RegisterRoutes(api)
deps.Auth.RegisterRoutes(api)
passkey.RegisterPasskeys(api, deps.Passkey, deps.Auth.Service(), deps.User.Service())
@@ -410,37 +280,51 @@ func registerHandlersInternal(api huma.API, deps HandlerDeps, handlerAppCtx hand
deps.Role.RegisterRoutes(api)
appimages.RegisterAppImages(api, deps.AppImages)
deps.User.RegisterRoutes(api)
- deps.Project.RegisterRoutes(api, handlerAppCtx)
+ deps.Project.RegisterRoutes(api, appCtx)
version.RegisterVersion(api, deps.Version)
deps.Event.RegisterRoutes(api)
deps.Activity.RegisterRoutes(api)
oidc.RegisterOidc(api, deps.Auth.Service(), deps.Passkey, deps.Oidc, deps.Role.Service(), deps.User.Service(), cfg)
- deps.Environment.RegisterRoutes(api, handlerAppCtx)
+ deps.Environment.RegisterRoutes(api, appCtx, deps.Version.GetAppVersionInfo)
deps.ContainerRegistry.RegisterRoutes(api)
deps.Template.RegisterRoutes(api)
deps.Variable.RegisterRoutes(api, cfg)
- deps.Image.RegisterRoutes(api, handlerAppCtx)
+ deps.Image.RegisterRoutes(api, appCtx)
deps.Upload.RegisterRoutes(api)
build.RegisterBuildWorkspaces(api, deps.Build, deps.Upload.Service())
- deps.ImageUpdate.RegisterRoutes(api, handlerAppCtx)
+ deps.ImageUpdate.RegisterRoutes(api, appCtx)
deps.Settings.RegisterRoutes(api)
deps.S3Destination.RegisterRoutes(api)
deps.JobSchedule.RegisterRoutes(api)
- deps.Volume.RegisterRoutes(api, handlerAppCtx)
- deps.Container.RegisterRoutes(api, handlerAppCtx)
+ deps.Volume.RegisterRoutes(api, appCtx)
+ deps.Container.RegisterRoutes(api, appCtx)
port.RegisterPorts(api, deps.Port)
- network.RegisterNetworks(api, deps.Network, deps.Docker, deps.Activity.Service(), handlerAppCtx)
+ network.RegisterNetworks(api, deps.Network, deps.Docker, deps.Activity.Service(), appCtx)
deps.Swarm.RegisterRoutes(api)
deps.Notification.RegisterRoutes(api)
deps.Apns.RegisterRoutes(api)
- deps.Updater.RegisterRoutes(api, handlerAppCtx)
+ deps.Updater.RegisterRoutes(api, appCtx)
deps.Search.RegisterRoutes(api)
- deps.System.RegisterRoutes(api, handlerAppCtx)
+ deps.System.RegisterRoutes(api, appCtx)
handlers.RegisterDiagnostics(api, deps.Diagnostics)
deps.GitRepository.RegisterRoutes(api)
deps.GitOpsSync.RegisterRoutes(api)
deps.Webhook.RegisterRoutes(api)
- deps.Vulnerability.RegisterRoutes(api, handlerAppCtx)
+ deps.Vulnerability.RegisterRoutes(api, appCtx)
deps.Dashboard.RegisterRoutes(api)
handlers.RegisterStream(api, deps.Dashboard.Handler(), deps.Activity.Handler(), deps.Environment.Handler(), deps.Event.Service(), deps.Version)
+
+ apiGroup.GET("/docs", func(c *echo.Context) error {
+ return c.HTML(http.StatusOK, scalarDocsHTML)
+ })
+
+ return api
+}
+
+// SetupAPIForSpec creates a Huma API instance for OpenAPI spec generation only.
+// No services are required - this is purely for schema generation.
+func SetupAPIForSpec() huma.API {
+ e := echo.New()
+ appCtx := handlerutil.NewActivityAppContext(context.Background()) //nolint:forbidigo // Schema discovery has no running request or application lifecycle.
+ return SetupAPI(e, e.Group("/api"), appCtx, &config.Config{}, HandlerDeps{})
}
diff --git a/backend/api/api_test.go b/backend/api/api_test.go
index 202c61f23..194782de8 100644
--- a/backend/api/api_test.go
+++ b/backend/api/api_test.go
@@ -385,9 +385,8 @@ func TestEasyJoinRoutesDeclareSwarmJoinPermission(t *testing.T) {
}
}
-// TestEnvScopedOperationsDeclarePermission guards the remote-environment proxy
-// authorization model. Every authenticated environment-scoped operation must
-// declare the permission that the proxy enforces before forwarding it.
+// TestEnvScopedOperationsDeclarePermission ensures every authenticated env-scoped operation
+// declares the permission the remote-environment proxy enforces before forwarding.
func TestEnvScopedOperationsDeclarePermission(t *testing.T) {
api := SetupAPIForSpec()
oapi := api.OpenAPI()
@@ -400,14 +399,20 @@ func TestEnvScopedOperationsDeclarePermission(t *testing.T) {
if !strings.HasPrefix(path, "/environments/{id}/") {
continue
}
- // Upload-session routes derive their permission from the {kind} path
- // segment at request time (authz.UploadKindPermission); both the local
- // handlers and the proxy's uploadSessionKindInternal special case
- // enforce it, so no static mapping exists to declare here.
+ // Upload-session routes derive their permission from the {kind} segment at request time.
if strings.HasPrefix(path, "/environments/{id}/uploads/{kind}") {
continue
}
- for method, op := range envScopedTestOperationsInternal(item) {
+ operations := map[string]*humav2.Operation{
+ http.MethodGet: item.Get,
+ http.MethodPost: item.Post,
+ http.MethodPut: item.Put,
+ http.MethodDelete: item.Delete,
+ http.MethodPatch: item.Patch,
+ http.MethodHead: item.Head,
+ http.MethodOptions: item.Options,
+ }
+ for method, op := range operations {
if op == nil {
continue
}
@@ -427,15 +432,3 @@ func TestEnvScopedOperationsDeclarePermission(t *testing.T) {
len(missing), strings.Join(missing, "\n "))
}
}
-
-func envScopedTestOperationsInternal(item *humav2.PathItem) map[string]*humav2.Operation {
- return map[string]*humav2.Operation{
- http.MethodGet: item.Get,
- http.MethodPost: item.Post,
- http.MethodPut: item.Put,
- http.MethodDelete: item.Delete,
- http.MethodPatch: item.Patch,
- http.MethodHead: item.Head,
- http.MethodOptions: item.Options,
- }
-}
diff --git a/backend/api/handlers/stream.go b/backend/api/handlers/stream.go
index 1de71eb44..74b726cb2 100644
--- a/backend/api/handlers/stream.go
+++ b/backend/api/handlers/stream.go
@@ -17,6 +17,11 @@ import (
streamtypes "github.com/getarcaneapp/arcane/types/v2/stream"
versiontypes "github.com/getarcaneapp/arcane/types/v2/version"
"go.getarcane.app/streams/agg"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
+ "go.opentelemetry.io/otel/propagation"
+ "go.opentelemetry.io/otel/trace"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/dashboard"
@@ -27,16 +32,14 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
const (
clientStreamHeartbeatInterval = 15 * time.Second
clientStreamDeadPeerTimeout = 45 * time.Second
clientStreamEventBuffer = 64
- // Per-channel handoff buffer between a channel's producer and the envelope
- // forwarder. Small on purpose: the shared writer is the real backpressure
- // point, and a deep per-channel buffer would only delay noticing a stalled
- // client.
+ // Per-channel handoff buffer; small because the shared writer is the real backpressure point.
clientStreamChannelBuffer = 16
)
@@ -55,9 +58,7 @@ type StreamClientInput struct {
Limit int `query:"limit" default:"0" doc:"Maximum activities to include in each activities-channel snapshot"`
}
-// RegisterStream registers the multiplexed client stream. It builds its own
-// per-domain handlers from the same services the individual handlers use, so
-// each channel's producer stays the single implementation of that feed.
+// RegisterStream registers the multiplexed client stream, reusing each domain's producer for its channel.
func RegisterStream(
api huma.API,
dashboardHandler *dashboard.DashboardHandler,
@@ -74,6 +75,13 @@ func RegisterStream(
version: versionService,
}
+ meter := otel.Meter(tracing.InstrumentationName)
+ connections, err := meter.Int64UpDownCounter("arcane.sse.connections",
+ metric.WithDescription("Open client SSE stream connections"), metric.WithUnit("{connection}"))
+ if err != nil {
+ otel.Handle(err)
+ }
+
sse.Register(api, huma.Operation{
OperationID: "streamClient",
Method: http.MethodGet,
@@ -84,68 +92,53 @@ func RegisterStream(
Security: handlerutil.DefaultOperationSecurity(),
Middlewares: huma.Middlewares{func(ctx huma.Context, next func(huma.Context)) {
httpx.SetStreamHeaders(ctx)
+ // The HTTP tracing middleware skips streams, so the lifetime span is a root unless the client sent a traceparent.
+ carrier := propagation.MapCarrier{}
+ for _, field := range otel.GetTextMapPropagator().Fields() {
+ if value := ctx.Header(field); value != "" {
+ carrier[field] = value
+ }
+ }
+ parent := otel.GetTextMapPropagator().Extract(ctx.Context(), carrier)
+ _, span := otel.Tracer(tracing.InstrumentationName).Start(parent, "sse "+ctx.Operation().Path,
+ trace.WithSpanKind(trace.SpanKindServer),
+ trace.WithAttributes(
+ attribute.String("http.route", ctx.Operation().Path),
+ attribute.String("arcane.sse.channels", ctx.Query("channels")),
+ attribute.String("client.address", ctx.RemoteAddr()),
+ ))
+ metricCtx := context.WithoutCancel(ctx.Context())
+ connections.Add(metricCtx, 1)
+ defer func() {
+ connections.Add(metricCtx, -1)
+ span.End()
+ }()
next(ctx)
}},
- // Ungated: each channel applies its own permission check below, and a
- // caller with access to none simply gets a heartbeat-only stream.
+ // Ungated: each channel checks its own permission; a caller with none gets a heartbeat-only stream.
}, map[string]any{"message": streamtypes.Event{}}, h.StreamClient)
}
func (h *StreamHandler) StreamClient(ctx context.Context, input *StreamClientInput, send sse.Sender) {
ps, _ := middleware.PermissionsFromContext(ctx)
- h.streamClientInternal(ctx, ps, input, send)
-}
-
-func (h *StreamHandler) streamClientInternal(ctx context.Context, ps *authz.PermissionSet, input *StreamClientInput, send sse.Sender) {
releaseDeadPeerTimeout, timeoutErr := httpx.AcquireDeadPeerTimeout(ctx, clientStreamDeadPeerTimeout)
if timeoutErr != nil {
slog.DebugContext(ctx, "could not bound client stream dead-peer timeout", "error", timeoutErr)
}
defer releaseDeadPeerTimeout()
- streamCtx, cancel := context.WithCancel(ctx)
- events := make(chan streamtypes.Event, clientStreamEventBuffer)
- var producers sync.WaitGroup
- for _, producer := range h.producersForInternal(ps, input) {
- producers.Go(func() {
- producer(streamCtx, events)
- })
- }
- defer producers.Wait()
- defer cancel()
-
- heartbeat := time.NewTicker(clientStreamHeartbeatInterval)
- defer heartbeat.Stop()
-
- for {
- var envelope streamtypes.Event
- select {
- case <-streamCtx.Done():
- return
- case envelope = <-events:
- case <-heartbeat.C:
- envelope = streamtypes.Event{Type: "heartbeat", Timestamp: time.Now()}
- }
- if streamCtx.Err() != nil {
- return
- }
- if err := send.Data(envelope); err != nil {
- slog.DebugContext(ctx, "client stream disconnected", "error", err)
- return
+ // Select the producers for the requested channels, dropping any the caller lacks permission for.
+ requested := make(map[string]bool, 5)
+ for name := range strings.SplitSeq(input.Channels, ",") {
+ if trimmed := strings.TrimSpace(strings.ToLower(name)); trimmed != "" {
+ requested[trimmed] = true
}
}
-}
-
-// producersForInternal selects the producers for the requested channels,
-// dropping any the caller lacks permission for.
-func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *StreamClientInput) []agg.Producer[streamtypes.Event] {
- requested := parseStreamChannelsInternal(input.Channels)
producers := make([]agg.Producer[streamtypes.Event], 0, 5)
if requested[streamtypes.ChannelEnvironments] {
- // Ungated like listEnvironments; the producer filters to the caller's
- // accessible environments.
- producers = append(producers, forwardStreamChannelInternal(
+ // Ungated like listEnvironments; the producer filters to the caller's accessible environments.
+ producers = append(producers, forwardStreamChannel(
streamtypes.ChannelEnvironments,
func(channel string, event environmenttypes.StreamEvent) streamtypes.Event {
return streamtypes.Event{Channel: channel, Environment: &event, Timestamp: event.Timestamp}
@@ -161,7 +154,7 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Str
return streamtypes.Event{Channel: channel, Dashboard: &event, Timestamp: event.Timestamp}
}
if ps.Allows(authz.PermDashboardRead, environment.LocalEnvironmentID) {
- producers = append(producers, forwardStreamChannelInternal(
+ producers = append(producers, forwardStreamChannel(
streamtypes.ChannelDashboard, wrapDashboard,
func(producerCtx context.Context, events chan<- dashboardtypes.StreamEvent) {
h.dashboard.RunLocalStreamProducer(producerCtx, input.DebugAllGood, events)
@@ -169,7 +162,7 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Str
))
}
if ps.AllowsAny(authz.PermDashboardRead) {
- producers = append(producers, forwardStreamChannelInternal(
+ producers = append(producers, forwardStreamChannel(
streamtypes.ChannelDashboard, wrapDashboard,
func(producerCtx context.Context, events chan<- dashboardtypes.StreamEvent) {
h.dashboard.RunRemoteStreamPollers(producerCtx, ps, input.DebugAllGood, events)
@@ -183,7 +176,7 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Str
return streamtypes.Event{Channel: channel, Activity: &event, Timestamp: event.Timestamp}
}
if ps.Allows(authz.PermActivitiesRead, environment.LocalEnvironmentID) {
- producers = append(producers, forwardStreamChannelInternal(
+ producers = append(producers, forwardStreamChannel(
streamtypes.ChannelActivities, wrapActivity,
func(producerCtx context.Context, events chan<- activitytypes.StreamEvent) {
h.activity.RunLocalStreamProducer(producerCtx, input.Limit, events)
@@ -191,7 +184,7 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Str
))
}
if ps.AllowsAny(authz.PermActivitiesRead) {
- producers = append(producers, forwardStreamChannelInternal(
+ producers = append(producers, forwardStreamChannel(
streamtypes.ChannelActivities, wrapActivity,
func(producerCtx context.Context, events chan<- activitytypes.StreamEvent) {
h.activity.RunRemoteStreamPollers(producerCtx, ps, input.Limit, events)
@@ -201,7 +194,7 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Str
}
if requested[streamtypes.ChannelEvents] && ps.Allows(authz.PermEventsRead, "") {
- producers = append(producers, forwardStreamChannelInternal(
+ producers = append(producers, forwardStreamChannel(
streamtypes.ChannelEvents,
func(channel string, event eventtypes.StreamEvent) streamtypes.Event {
return streamtypes.Event{Channel: channel, EventLog: &event, Timestamp: event.Timestamp}
@@ -212,7 +205,7 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Str
if requested[streamtypes.ChannelVersion] {
// Ungated like GET /app-version.
- producers = append(producers, forwardStreamChannelInternal(
+ producers = append(producers, forwardStreamChannel(
streamtypes.ChannelVersion,
func(channel string, event versiontypes.StreamEvent) streamtypes.Event {
return streamtypes.Event{Channel: channel, Version: &event, Timestamp: event.Timestamp}
@@ -220,23 +213,42 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Str
h.version.RunStreamProducer,
))
}
- return producers
-}
-func parseStreamChannelsInternal(raw string) map[string]bool {
- channels := make(map[string]bool, 5)
- for name := range strings.SplitSeq(raw, ",") {
- if trimmed := strings.TrimSpace(strings.ToLower(name)); trimmed != "" {
- channels[trimmed] = true
+ streamCtx, cancel := context.WithCancel(ctx)
+ events := make(chan streamtypes.Event, clientStreamEventBuffer)
+ var running sync.WaitGroup
+ for _, producer := range producers {
+ running.Go(func() {
+ producer(streamCtx, events)
+ })
+ }
+ defer running.Wait()
+ defer cancel()
+
+ heartbeat := time.NewTicker(clientStreamHeartbeatInterval)
+ defer heartbeat.Stop()
+
+ for {
+ var envelope streamtypes.Event
+ select {
+ case <-streamCtx.Done():
+ return
+ case envelope = <-events:
+ case <-heartbeat.C:
+ envelope = streamtypes.Event{Type: "heartbeat", Timestamp: time.Now()}
+ }
+ if streamCtx.Err() != nil {
+ return
+ }
+ if err := send.Data(envelope); err != nil {
+ slog.DebugContext(ctx, "client stream disconnected", "error", err)
+ return
}
}
- return channels
}
-// forwardStreamChannelInternal adapts a channel's own producer to the shared
-// envelope. Each producer keeps emitting its native event type, so adding a
-// channel never means rewriting its feed.
-func forwardStreamChannelInternal[T any](
+// forwardStreamChannel adapts a channel's producer, which keeps emitting its native event type, to the shared envelope.
+func forwardStreamChannel[T any](
channel string,
wrap func(string, T) streamtypes.Event,
producer func(context.Context, chan<- T),
diff --git a/backend/api/normalization.go b/backend/api/normalization.go
index f3ef0c3d0..4e6615f7f 100644
--- a/backend/api/normalization.go
+++ b/backend/api/normalization.go
@@ -19,7 +19,7 @@ import (
)
// Install before routes so Huma validates normalized input.
-func registerNormalizationInternal(api huma.API) {
+func registerNormalization(api huma.API) {
api.OpenAPI().OnAddOperation = append(api.OpenAPI().OnAddOperation, func(_ *huma.OpenAPI, op *huma.Operation) {
typ := normalizationBodyTypeInternal(api, op)
if typ == nil {
diff --git a/backend/api/normalization_test.go b/backend/api/normalization_test.go
index d7ba463ee..b88d33b17 100644
--- a/backend/api/normalization_test.go
+++ b/backend/api/normalization_test.go
@@ -49,7 +49,7 @@ func TestNormalizationBeforeHumaValidation(t *testing.T) {
cfg := huma.DefaultConfig("test", "1")
cfg.Formats = map[string]huma.Format{"application/json": jsonV2Format, "json": jsonV2Format}
api := humaecho.New(router, cfg)
- registerNormalizationInternal(api)
+ registerNormalization(api)
called := false
var received normalizationTestBody
huma.Register(api, huma.Operation{
@@ -88,7 +88,7 @@ func TestNormalizationBeforeHumaValidation(t *testing.T) {
t.Run("identity fields pass through unchanged", func(t *testing.T) {
router := echo.New()
api := humaecho.New(router, huma.DefaultConfig("test", "1"))
- registerNormalizationInternal(api)
+ registerNormalization(api)
var received usertypes.CreateUser
huma.Register(api, huma.Operation{
OperationID: "create",
@@ -133,7 +133,7 @@ func TestNormalizationBeforeHumaValidation(t *testing.T) {
func TestNormalizationBodyLimits(t *testing.T) {
router := echo.New()
api := humaecho.New(router, huma.DefaultConfig("test", "1"))
- registerNormalizationInternal(api)
+ registerNormalization(api)
huma.Register(api, huma.Operation{
OperationID: "limit",
Method: http.MethodPost,
@@ -218,7 +218,7 @@ func TestNormalizationReadTimeout(t *testing.T) {
logs.Reset()
router := echo.New()
api := humaecho.New(router, huma.DefaultConfig("test", "1"))
- registerNormalizationInternal(api)
+ registerNormalization(api)
called := false
huma.Register(api, huma.Operation{
OperationID: "timeout",
@@ -254,7 +254,7 @@ func TestNormalizationReadTimeout(t *testing.T) {
func TestNormalizationRejectsInvalidTagsAtRegistration(t *testing.T) {
router := echo.New()
api := humaecho.New(router, huma.DefaultConfig("test", "1"))
- registerNormalizationInternal(api)
+ registerNormalization(api)
require.Panics(t, func() {
huma.Register(api, huma.Operation{OperationID: "invalid", Method: http.MethodPost, Path: "/invalid"}, func(_ context.Context, _ *struct {
Body struct {
diff --git a/backend/api/ws/container_stats.go b/backend/api/ws/container_stats.go
index 3c3032a5c..8ddd86e78 100644
--- a/backend/api/ws/container_stats.go
+++ b/backend/api/ws/container_stats.go
@@ -34,7 +34,7 @@ func (h *WebSocketHandler) ContainerStats(c *echo.Context) error {
return c.JSON(http.StatusBadRequest, map[string]any{"success": false, "error": "Container ID is required"})
}
- conn, onRemove, ok := h.acceptWSInternal(c, systemtypes.WSKindContainerStats, containerID)
+ conn, onRemove, ok := h.acceptWS(c, systemtypes.WSKindContainerStats, containerID)
if !ok {
return nil
}
diff --git a/backend/api/ws/diagnostics.go b/backend/api/ws/diagnostics.go
index 743ebd37c..d9ecc2be6 100644
--- a/backend/api/ws/diagnostics.go
+++ b/backend/api/ws/diagnostics.go
@@ -132,7 +132,7 @@ func (h *WebSocketHandler) ActorDiagnosticsStream(c *echo.Context) error {
// streamSnapshotsInternal writes build's result on connect and every interval until
// the client disconnects. A non-nil handle answers client commands concurrently.
func (h *WebSocketHandler) streamSnapshotsInternal(c *echo.Context, interval time.Duration, build func(context.Context) any, handle func(context.Context, systemtypes.DiagnosticsCommand) any) error {
- conn, unregister, accepted := h.acceptWSInternal(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)
+ conn, unregister, accepted := h.acceptWS(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)
if !accepted {
return nil
}
@@ -190,7 +190,7 @@ func (h *WebSocketHandler) streamSnapshotsInternal(c *echo.Context, interval tim
// ServerLogsStream replays the recent backend log backlog then streams new entries live.
func (h *WebSocketHandler) ServerLogsStream(c *echo.Context) error {
- conn, unregister, accepted := h.acceptWSInternal(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)
+ conn, unregister, accepted := h.acceptWS(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)
if !accepted {
return nil
}
diff --git a/backend/api/ws/exec.go b/backend/api/ws/exec.go
index 8b2476c69..4b20554cc 100644
--- a/backend/api/ws/exec.go
+++ b/backend/api/ws/exec.go
@@ -40,7 +40,7 @@ func (h *WebSocketHandler) ContainerExec(c *echo.Context) error {
shell := cmp.Or(c.QueryParam("shell"), "/bin/sh")
- conn, unregister, ok := h.acceptWSInternal(c, systemtypes.WSKindContainerExec, containerID)
+ conn, unregister, ok := h.acceptWS(c, systemtypes.WSKindContainerExec, containerID)
if !ok {
return nil
}
@@ -54,7 +54,7 @@ func (h *WebSocketHandler) ContainerExec(c *echo.Context) error {
defer cancel(nil)
// The pong is serviced by the concurrent stdin reader.
- go keepWSConnAliveInternal(ctx, func() { cancel(errors.New("websocket ping failed")) }, conn, 54*time.Second)
+ go keepWSConnAlive(ctx, func() { cancel(errors.New("websocket ping failed")) }, conn, 54*time.Second)
h.runContainerExecInternal(ctx, cancel, conn, containerID, shell)
return nil
diff --git a/backend/api/ws/handler.go b/backend/api/ws/handler.go
index 936510b47..b39a94612 100644
--- a/backend/api/ws/handler.go
+++ b/backend/api/ws/handler.go
@@ -13,6 +13,10 @@ import (
"github.com/labstack/echo/v5"
"github.com/samber/hot"
"go.getarcane.app/sys/cgroup"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/propagation"
+ "go.opentelemetry.io/otel/trace"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -26,9 +30,13 @@ import (
wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
-var defaultWebSocketMetrics = wshub.NewWebSocketMetrics()
+var (
+ defaultWebSocketMetrics = wshub.NewWebSocketMetrics()
+ observeWebSocketMetricsOnce sync.Once
+)
// WebSocketHandler consolidates all WebSocket and streaming endpoints.
// REST endpoints are handled by Huma handlers.
@@ -39,7 +47,6 @@ type WebSocketHandler struct {
systemService *system.SystemService
diagnosticsService *diagnostics.DiagnosticsService
checkWSOrigin func(*http.Request) bool
- wsMetrics *wshub.WebSocketMetrics
activeConnectionsMu sync.Mutex
activeConnections map[string]int
logStreamsMu sync.Mutex
@@ -88,7 +95,6 @@ func NewWebSocketHandler(
swarmService: swarmService,
systemService: systemService,
diagnosticsService: diagnosticsService,
- wsMetrics: defaultWebSocketMetrics,
logStreams: make(map[string]*wsLogStream),
cgroupCache: cgroup.NewCache(cgroupCacheTTL),
gpuMonitor: systemlib.NewGPUMonitor(cfg.GPUMonitoringEnabled, cfg.GPUType),
@@ -97,6 +103,11 @@ func NewWebSocketHandler(
Build(),
checkWSOrigin: httpx.ValidateWebSocketOrigin(cfg.GetAppURL()),
}
+ observeWebSocketMetricsOnce.Do(func() {
+ if err := defaultWebSocketMetrics.ObserveConnections(otel.Meter(tracing.InstrumentationName)); err != nil {
+ otel.Handle(err)
+ }
+ })
wsGroup := group.Group("/environments/:id/ws", authMiddleware.WithAdminNotRequired().Add())
for _, r := range handler.proxiedRoutes() {
wsGroup.GET(r.path, r.handler, middleware.RequireEchoPermission(r.perm))
@@ -104,37 +115,48 @@ func NewWebSocketHandler(
handler.registerDiagnosticsRoutesInternal(group, authMiddleware)
}
-func buildWSConnectionInfoInternal(c *echo.Context, kind, resourceID string) systemtypes.WebSocketConnectionInfo {
+// acceptWS upgrades the request to a WebSocket, registers it with the metrics tracker, and starts its lifetime span.
+// The returned unregister func must be called exactly once when the connection ends.
+func (h *WebSocketHandler) acceptWS(c *echo.Context, kind, resourceID string) (*websocket.Conn, func(), bool) {
+ req := c.Request()
+ conn, err := wshub.Accept(c.Response(), req, h.checkWSOrigin)
+ if err != nil {
+ slog.DebugContext(req.Context(), "websocket accept failed", "kind", kind, "resourceId", resourceID, "error", err)
+ return nil, nil, false
+ }
userID, _ := c.Get("userID").(string)
- return systemtypes.WebSocketConnectionInfo{
+ info := systemtypes.WebSocketConnectionInfo{
Kind: kind,
EnvID: c.Param("id"),
ResourceID: resourceID,
ClientIP: c.RealIP(),
UserID: userID,
- UserAgent: c.Request().Header.Get("User-Agent"),
- }
-}
-
-// acceptWSInternal upgrades the request to a WebSocket and registers it with
-// the metrics tracker. The returned unregister func must be called exactly
-// once when the connection ends.
-func (h *WebSocketHandler) acceptWSInternal(c *echo.Context, kind, resourceID string) (*websocket.Conn, func(), bool) {
- conn, err := wshub.Accept(c.Response(), c.Request(), h.checkWSOrigin)
- if err != nil {
- slog.DebugContext(c.Request().Context(), "websocket accept failed", "kind", kind, "resourceId", resourceID, "error", err)
- return nil, nil, false
+ UserAgent: req.Header.Get("User-Agent"),
}
- connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, kind, resourceID))
- return conn, func() { h.wsMetrics.UnregisterConnection(connID) }, true
+ connID := defaultWebSocketMetrics.RegisterConnection(info)
+ // The HTTP tracing middleware skips upgrades, so this is a root span unless the client sent a traceparent.
+ parent := otel.GetTextMapPropagator().Extract(req.Context(), propagation.HeaderCarrier(req.Header))
+ _, span := otel.Tracer(tracing.InstrumentationName).Start(parent, "websocket "+kind,
+ trace.WithSpanKind(trace.SpanKindServer),
+ trace.WithAttributes(
+ attribute.String("arcane.websocket.kind", kind),
+ attribute.String("arcane.websocket.connection_id", connID),
+ attribute.String("arcane.environment.id", info.EnvID),
+ attribute.String("arcane.websocket.resource_id", resourceID),
+ attribute.String("http.route", c.Path()),
+ attribute.String("client.address", info.ClientIP),
+ attribute.String("user.id", userID),
+ attribute.String("user_agent.original", info.UserAgent),
+ ))
+ return conn, func() {
+ defaultWebSocketMetrics.UnregisterConnection(connID)
+ span.End()
+ }, true
}
-// keepWSConnAliveInternal pings the peer every period. Ping round-trips (the
-// pong must be serviced by the connection's concurrent reader) and is safe
-// alongside a concurrent writer. A failed ping means the client is gone, so
-// it cancels the connection's context — a silently-dead client would
-// otherwise keep the session open until a write fails.
-func keepWSConnAliveInternal(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn, period time.Duration) {
+// keepWSConnAlive pings the peer every period and cancels the connection when a ping fails.
+// The pong must be serviced by the connection's concurrent reader.
+func keepWSConnAlive(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn, period time.Duration) {
ticker := time.NewTicker(period)
defer ticker.Stop()
for {
diff --git a/backend/api/ws/log_streams.go b/backend/api/ws/log_streams.go
index 6c4f275e6..8975492c6 100644
--- a/backend/api/ws/log_streams.go
+++ b/backend/api/ws/log_streams.go
@@ -142,7 +142,7 @@ func (h *WebSocketHandler) serveLogStreamInternal(
params logStreamParams,
hubBuilder func(streamKey string, onEmpty func(*wsLogStream)) *wsLogStream,
) {
- conn, unregister, ok := h.acceptWSInternal(c, kind, resourceID)
+ conn, unregister, ok := h.acceptWS(c, kind, resourceID)
if !ok {
return
}
diff --git a/backend/api/ws/system_stats.go b/backend/api/ws/system_stats.go
index 3446c9005..290519b0b 100644
--- a/backend/api/ws/system_stats.go
+++ b/backend/api/ws/system_stats.go
@@ -428,7 +428,7 @@ func (h *WebSocketHandler) SystemStats(c *echo.Context) error {
}
defer h.releaseRateLimitInternal(clientIP)
- conn, unregister, ok := h.acceptWSInternal(c, systemtypes.WSKindSystemStats, "")
+ conn, unregister, ok := h.acceptWS(c, systemtypes.WSKindSystemStats, "")
if !ok {
return nil
}
@@ -460,7 +460,7 @@ func (h *WebSocketHandler) SystemStats(c *echo.Context) error {
go h.readSystemStatsPumpInternal(ctx, cancel, conn)
// The pong is serviced by readSystemStatsPumpInternal.
- go keepWSConnAliveInternal(ctx, cancel, conn, 54*time.Second)
+ go keepWSConnAlive(ctx, cancel, conn, 54*time.Second)
send := func() error {
stats, _ := h.systemStatsSampler.latest.Load()
diff --git a/backend/go.mod b/backend/go.mod
index 2da257e5d..0f36d4037 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -38,6 +38,7 @@ require (
github.com/italypaleale/francis v0.1.0
github.com/jwx-go/jwkfetch/v4 v4.0.4
github.com/klauspost/compress v1.20.1
+ github.com/labstack/echo-otel/v5 v5.0.0
github.com/labstack/echo/v5 v5.4.0
github.com/lestrrat-go/httprc/v3 v3.0.6
github.com/lestrrat-go/jwx/v4 v4.5.0
@@ -72,6 +73,17 @@ require (
go.getarcane.app/sys/cgroup v0.2.4
go.getarcane.app/sys/crypto v0.3.0
go.getarcane.app/updater v0.11.6
+ go.opentelemetry.io/contrib/bridges/otelslog v0.20.1
+ go.opentelemetry.io/contrib/exporters/autoexport v0.71.0
+ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0
+ go.opentelemetry.io/contrib/instrumentation/runtime v0.71.0
+ go.opentelemetry.io/otel v1.46.0
+ go.opentelemetry.io/otel/log v0.22.0
+ go.opentelemetry.io/otel/metric v1.46.0
+ go.opentelemetry.io/otel/sdk v1.46.0
+ go.opentelemetry.io/otel/sdk/log v0.22.0
+ go.opentelemetry.io/otel/sdk/metric v1.46.0
+ go.opentelemetry.io/otel/trace v1.46.0
go.uber.org/fx v1.24.0
go.yaml.in/yaml/v4 v4.0.0-rc.6
golang.org/x/crypto v0.57.0
@@ -87,6 +99,7 @@ require (
google.golang.org/protobuf v1.36.12
gorm.io/driver/postgres v1.6.3
gorm.io/gorm v1.31.2
+ gorm.io/plugin/opentelemetry v0.1.16
modernc.org/sqlite v1.60.1
)
@@ -247,6 +260,7 @@ require (
github.com/prometheus/client_golang v1.24.1 // indirect
github.com/prometheus/client_model v0.6.3 // indirect
github.com/prometheus/common v0.71.0 // indirect
+ github.com/prometheus/otlptranslator v1.0.0 // indirect
github.com/prometheus/procfs v0.22.0 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/quic-go/quic-go v0.63.0 // indirect
@@ -285,19 +299,20 @@ require (
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zmb3/spotify/v2 v2.3.1 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+ go.opentelemetry.io/contrib/bridges/prometheus v0.71.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.71.0 // indirect
- go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect
- go.opentelemetry.io/otel v1.46.0 // indirect
+ go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 // indirect
+ go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 // indirect
- go.opentelemetry.io/otel/metric v1.46.0 // indirect
- go.opentelemetry.io/otel/sdk v1.46.0 // indirect
- go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect
- go.opentelemetry.io/otel/trace v1.46.0 // indirect
+ go.opentelemetry.io/otel/exporters/prometheus v0.68.0 // indirect
+ go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0 // indirect
+ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 // indirect
+ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
go.uber.org/dig v1.19.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index c509cb8bb..a15a1d128 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -454,6 +454,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/labstack/echo-otel/v5 v5.0.0 h1:/mcXe6HyOSoGJe8z4WuhWBfK3uFOkBi2oFEvaok1M0Q=
+github.com/labstack/echo-otel/v5 v5.0.0/go.mod h1:rMC5ekJzX2voZtHAW7SYMmxWBuTQBobCB+JC2gqfdxE=
github.com/labstack/echo/v4 v4.15.4 h1:DL45vVYa+BWE+XuW+zZNd9H0YEdZ80UAWJGcTVW4EVs=
github.com/labstack/echo/v4 v4.15.4/go.mod h1:CuMetKIRwsuO/qlAgMq+KTAalwGoB/h4tC+yPdrTj1g=
github.com/labstack/echo/v5 v5.4.0 h1:iY674460IvSmUcj7MziL3YrgyTClghz5BpdZoKiGxR4=
@@ -603,6 +607,8 @@ github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPT
github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8=
github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
+github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos=
+github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM=
github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0=
@@ -772,14 +778,28 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/contrib/bridges/otelslog v0.20.1 h1:5sHc4ToTFjfSZCtGAAM6jPunICAmJX73htv372T4ipc=
+go.opentelemetry.io/contrib/bridges/otelslog v0.20.1/go.mod h1:oa6kgvyz/3GYW04dohd0++xJIH4xdQY8PAbpeCMaM8M=
+go.opentelemetry.io/contrib/bridges/prometheus v0.71.0 h1:9qgxsFLskbDMXl8WMqThoF6w8yGJgCumn9qRc67OmnI=
+go.opentelemetry.io/contrib/bridges/prometheus v0.71.0/go.mod h1:2rCjF4F2siiTeLCzJsaGZ3CK0XIoimCSKXEBPdv+Je0=
+go.opentelemetry.io/contrib/exporters/autoexport v0.71.0 h1:VCsJbp0YLyPtx2tu5Vgv2a2/qLoaMCj8hT2uZ34+Mx0=
+go.opentelemetry.io/contrib/exporters/autoexport v0.71.0/go.mod h1:qxZqn7e10f6ajmMCkg/47rMS7qQYfaOl2nj/4aytHUQ=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw=
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.71.0 h1:oFNJW32h2SXnET7XXstgT7pVh4vN+jW+GfiIaBguIZE=
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.71.0/go.mod h1:+H3sPOFwag14eMHTPMElZtV0e4YfVZ/85KgrKUCB5FI=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.71.0 h1:v4KkRLVvE1cWqTJDfZZkTCG+Z4aolsa6RVos0FX7vqE=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.71.0/go.mod h1:g/xbuPC0XbgwMdKuyF5sKOUUEsorSkN6APydyFP/H9E=
+go.opentelemetry.io/contrib/propagators/b3 v1.46.0 h1:OFVqWObn7xLIbOjE/koO0LS9fZJNgAyBD0msA+UQAoc=
+go.opentelemetry.io/contrib/propagators/b3 v1.46.0/go.mod h1:t/d64xy7xuuEDJN/4ThqohLgRhIuQxL9y7P1v02bYuM=
go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 h1:Bu39F5tzJct+f2IZbB8989fwyTps3c8e7EsUQsz+vs8=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0/go.mod h1:dJUwod88EsFgYCqrDHaSPzhiY9pBUpt0d85/qSfua7k=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 h1:lYk7RmxdLK865qLwibroNGldHa1U7SWKYYvNjlK7PIo=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0/go.mod h1:6GvlND0H0xdUJanOtIAn0xfwLkauh1tmsYEEVSMDdqY=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 h1:qkDYCAFiZXLcs1L4aY+tP2wguQ4kURANqHOQMA2et2s=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0/go.mod h1:tkipS4DRzmpAmvg+Gw4++O1IdDq6TVDnvnYU6cmbQVs=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 h1:AP23h/mFgb/lc7tdck1Kfn9qxsM8TAeNPCU5C3pzaps=
@@ -790,12 +810,26 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CD
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 h1:KrC1YrQeSt46ITMWAbgQx1M1eV1/1TKzttrBzymPmss=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0/go.mod h1:zDSEzoEqsOrgBeGvH66KRgxh90VonFyJqBHA0Pk3+rM=
+go.opentelemetry.io/otel/exporters/prometheus v0.68.0 h1:QOf2IftqQwITVRJpnn0M7M9ZCbgWfxz4P7i9C9yc2N4=
+go.opentelemetry.io/otel/exporters/prometheus v0.68.0/go.mod h1:bgSvqu2TWGXiz7yr5UTMfObH8oqxJWHTnubQ3ef9BO4=
+go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0 h1:kvMAiLEudKmk+CSG+iYbU8vTUGNNDaf/V09OO5lrTwI=
+go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0/go.mod h1:L9Dlksri+MdT1cb2gIiA1cJJYW3Y92ipvDjNxYEyaDI=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 h1:PR9eAf7o0dQs3hshZNZpE9aW2dXWX/KdDf6pJilVD3U=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0/go.mod h1:2Z4KyNdH1uuzivdinyfGsxzNNT/Rl45pwtVwfYVI0xk=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 h1:KdRxPiAoMptR3vfWzvjjvutTsSiwbC2uG0496rzZNfo=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0/go.mod h1:K/qSA+3G7Eovxi4K09wzrAgkWRnosS0DAOZeEpve7sM=
+go.opentelemetry.io/otel/log v0.22.0 h1:5DBNnfvaJ6CVdkJ+Jle8Tzs50aSSv49TXGj9XRsEYw0=
+go.opentelemetry.io/otel/log v0.22.0/go.mod h1:gzOt/R67vF2GniAqWu8Qv0SXy89f71muHcrkz76PCdc=
go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=
go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o=
go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU=
go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4=
go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI=
go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM=
+go.opentelemetry.io/otel/sdk/log v0.22.0 h1:PRL+s6P63XT4E/bheEflopPUpVxuvANqZwtt89yhoGk=
+go.opentelemetry.io/otel/sdk/log v0.22.0/go.mod h1:JNp0sBELrjCTcu5W3GzABVypeU6vDJjBS+X0JISuz+g=
+go.opentelemetry.io/otel/sdk/log/logtest v0.22.0 h1:infPnfNrhCNgOUZRs3gWUg8vhoBUHihq02gwK05gzlg=
+go.opentelemetry.io/otel/sdk/log/logtest v0.22.0/go.mod h1:gkQZA3z15Bv3KU9vigBTi8dFechSozRP7v94X4VZv+s=
go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE=
go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4=
go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
@@ -1121,6 +1155,8 @@ gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo=
gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
+gorm.io/plugin/opentelemetry v0.1.16 h1:Kypj2YYAliJqkIczDZDde6P6sFMhKSlG5IpngMFQGpc=
+gorm.io/plugin/opentelemetry v0.1.16/go.mod h1:P3RmTeZXT+9n0F1ccUqR5uuTvEXDxF8k2UpO7mTIB2Y=
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go
index 295704c20..09eb82b3c 100644
--- a/backend/internal/bootstrap/bootstrap.go
+++ b/backend/internal/bootstrap/bootstrap.go
@@ -16,12 +16,11 @@ import (
"github.com/labstack/echo/v5"
"github.com/moby/moby/client"
"github.com/subosito/gotenv"
- "go.getarcane.app/streams/logs"
"go.getarcane.app/sys/crypto"
+ "go.opentelemetry.io/contrib/bridges/otelslog"
"go.uber.org/fx"
"go.uber.org/fx/fxevent"
- "github.com/getarcaneapp/arcane/backend/v2/api/ws"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -41,6 +40,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/startup"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
func Bootstrap(ctx context.Context) error {
@@ -61,9 +61,7 @@ func Bootstrap(ctx context.Context) error {
}
cfg.DockerConfig = runtimeIdentityCfg.DockerConfig
- SetupSlogLogger(cfg)
- // Tee all slog output into the in-memory ring buffer that powers the diagnostics live log tail.
- slog.SetDefault(slog.New(logs.NewSlogHandler(slog.Default().Handler(), ws.LogBroadcaster())))
+ SetupSlogLogger(cfg, nil)
database.SetGormLogger(BuildGormLogger(cfg))
slog.InfoContext(ctx, "Arcane is starting...", "version", config.Version)
startup.ApplyMemoryLimit(ctx)
@@ -72,11 +70,12 @@ func Bootstrap(ctx context.Context) error {
appCtx, cancelApp := context.WithCancel(ctx)
appCtx = utils.WithAppLifecycleContext(appCtx)
- db, err := initializeDBAndMigrateInternal(appCtx, cfg)
+ db, err := database.Initialize(appCtx, cfg.DatabaseURL, database.MigrationOptions{AllowDowngrade: cfg.AllowDowngrade})
if err != nil {
cancelApp()
return fmt.Errorf("failed to initialize database: %w", err)
}
+ slog.InfoContext(appCtx, "Database initialized successfully")
defer func() {
cancelApp()
if closeErr := db.Close(); closeErr != nil {
@@ -84,6 +83,31 @@ func Bootstrap(ctx context.Context) error {
}
}()
+ // Telemetry identifies this installation by its persisted instance ID, so it starts once the
+ // database is open but before Docker clients and services. Exporter errors go to the console-only logger.
+ instanceID, err := settings.EnsureInstanceID(appCtx, db)
+ if err != nil {
+ return err
+ }
+ providers, err := setupTelemetry(ctx, cfg, slog.Default(), instanceID)
+ if err != nil {
+ return fmt.Errorf("setup telemetry: %w", err)
+ }
+ defer func() {
+ shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+ defer cancel()
+ if shutdownErr := providers.Shutdown(shutdownCtx); shutdownErr != nil {
+ slog.ErrorContext(shutdownCtx, "Error shutting down telemetry", "error", shutdownErr)
+ }
+ }()
+ if providers.loggerProvider != nil {
+ SetupSlogLogger(cfg, otelslog.NewHandler(tracing.InstrumentationName, otelslog.WithLoggerProvider(providers.loggerProvider)))
+ db.Logger = BuildGormLogger(cfg)
+ }
+ if providers.tracerProvider != nil || providers.meterProvider != nil || providers.loggerProvider != nil {
+ slog.InfoContext(ctx, "OpenTelemetry export enabled", "traces", providers.tracerProvider != nil, "metrics", providers.meterProvider != nil, "logs", providers.loggerProvider != nil)
+ }
+
app := fx.New(applicationOptions(appCtx, cfg, db, cancelApp))
startCtx, cancelStart := context.WithTimeout(ctx, 5*time.Minute)
@@ -114,7 +138,7 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *database
fx.Supply(cfg, db, cancelApp),
fx.Provide(
func() context.Context { return appCtx },
- newConfiguredHTTPClientInternal,
+ newConfiguredHTTPClient,
),
di.ActorOptions,
di.ServiceOptions,
@@ -136,11 +160,9 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *database
)
}
-// isWeakProductionEncryptionKeyInternal reports whether an explicit
-// ENCRYPTION_KEY is an unprefixed passphrase shorter than 32 characters in
-// production. crypto derives a key from any non-empty passphrase, so this
-// preserves the historical fail-fast rejection of low-entropy production keys.
-func isWeakProductionEncryptionKeyInternal(encryptionKey, localEnvironment string, agentMode bool) bool {
+// isWeakProductionEncryptionKey reports whether a production ENCRYPTION_KEY is an
+// unprefixed passphrase shorter than 32 characters.
+func isWeakProductionEncryptionKey(encryptionKey, localEnvironment string, agentMode bool) bool {
if localEnvironment != "production" || agentMode {
return false
}
@@ -151,13 +173,7 @@ func isWeakProductionEncryptionKeyInternal(encryptionKey, localEnvironment strin
return len(strings.TrimPrefix(key, "raw:")) < 32
}
-func warnDeprecatedEnvVarsInternal(ctx context.Context, cfg *config.Config) {
- for _, envName := range cfg.DeprecatedEnvVarsSet() {
- slog.WarnContext(ctx, "Deprecated environment variable is set and no longer used; remove it from your environment", "env", envName)
- }
-}
-
-func newConfiguredHTTPClientInternal(cfg *config.Config) *http.Client {
+func newConfiguredHTTPClient(cfg *config.Config) *http.Client {
options := httpxtypes.ClientOptions{
Timeout: 10 * time.Second,
TLSHandshakeTimeout: 5 * time.Second,
@@ -172,9 +188,8 @@ func newConfiguredHTTPClientInternal(cfg *config.Config) *http.Client {
type initializeStartupStateParams struct {
fx.In
- AppCtx context.Context
- Config *config.Config
- HTTPClient *http.Client
+ AppCtx context.Context
+ Config *config.Config
Volume *volume.VolumeService
Settings *settings.SettingsService
@@ -193,7 +208,6 @@ type initializeStartupStateParams struct {
func initializeStartupState(p initializeStartupStateParams) {
appCtx := p.AppCtx
cfg := p.Config
- httpClient := p.HTTPClient
if p.Volume != nil {
startup.CleanupOrphanedVolumeHelpers(appCtx, p.Volume.CleanupOrphanedVolumeHelpers)
@@ -208,14 +222,16 @@ func initializeStartupState(p initializeStartupStateParams) {
AdminStaticAPIKey: cfg.AdminStaticAPIKey,
}
- warnDeprecatedEnvVarsInternal(appCtx, cfg)
+ for _, envName := range cfg.DeprecatedEnvVarsSet() {
+ slog.WarnContext(appCtx, "Deprecated environment variable is set and no longer used; remove it from your environment", "env", envName)
+ }
startup.LoadAgentToken(appCtx, runtimeCfg, p.Settings.GetStringSetting)
startup.EnsureEncryptionKey(appCtx, runtimeCfg, p.Settings.EnsureEncryptionKey)
cfg.AgentToken = runtimeCfg.AgentToken
cfg.EncryptionKey = runtimeCfg.EncryptionKey
- if isWeakProductionEncryptionKeyInternal(cfg.EncryptionKey, string(cfg.Environment), cfg.AgentMode) {
+ if isWeakProductionEncryptionKey(cfg.EncryptionKey, string(cfg.Environment), cfg.AgentMode) {
panic("ENCRYPTION_KEY passphrase must be at least 32 characters in production (or use a hex:/base64: encoded 32-byte key)")
}
@@ -237,7 +253,20 @@ func initializeStartupState(p initializeStartupStateParams) {
if err := p.Environment.EnsureLocalEnvironment(appCtx, cfg.AppUrl); err != nil {
slog.WarnContext(appCtx, "Failed to ensure local environment", "error", err)
}
- initializeGitOpsStartupStateInternal(appCtx, p.GitOpsSync)
+ if p.GitOpsSync != nil {
+ if err := p.GitOpsSync.CleanupOrphanedSyncsOnStartup(appCtx); err != nil {
+ slog.WarnContext(appCtx, "Failed to clean up orphaned GitOps syncs on startup", "error", err)
+ }
+ if err := p.GitOpsSync.CleanupLeakedScratchDirsOnStartup(appCtx); err != nil {
+ slog.WarnContext(appCtx, "Failed to clean up leaked GitOps scratch directories on startup", "error", err)
+ }
+ if err := p.GitOpsSync.CleanupLeakedCloneDirsOnStartup(appCtx); err != nil {
+ slog.WarnContext(appCtx, "Failed to clean up leaked git clone directories on startup", "error", err)
+ }
+ if err := p.GitOpsSync.ReconcileDirectorySyncProjectsOnStartup(appCtx); err != nil {
+ slog.WarnContext(appCtx, "Failed to reconcile directory GitOps projects on startup", "error", err)
+ }
+ }
p.Vulnerability.ImportLegacyReportFiles(appCtx)
p.Vulnerability.BackfillScanItems(appCtx)
if p.Project != nil {
@@ -251,9 +280,7 @@ func initializeStartupState(p initializeStartupStateParams) {
slog.WarnContext(appCtx, "Failed to reconcile edge environment statuses on startup", "error", err)
}
- // Global variables are a manager resource: import any pre-existing local
- // .env.global once, then materialize the effective set everywhere. Agents
- // only serve the per-environment variables endpoint the manager pushes to.
+ // Global variables are manager-owned: import a legacy local .env.global once, then sync everywhere.
p.Environment.SetVariableSyncer(p.Variable)
if err := p.Variable.ImportLegacyLocalEnvFile(appCtx); err != nil {
slog.WarnContext(appCtx, "Failed to import legacy global variables", "error", err)
@@ -285,10 +312,8 @@ func initializeStartupState(p initializeStartupStateParams) {
startup.InitializeNonAgentFeatures(appCtx, runtimeCfg,
p.Role.EnsureBuiltInRoles,
func(ctx context.Context) error {
- // Backfill legacy users.roles first so CreateDefaultAdmin's
- // zero-global-admin recovery gate sees upgraded assignments.
- // Runs once (kv completion marker); runRoleStartupTasks repeats
- // the call for agent-mode and error-retry coverage.
+ // Backfill legacy users.roles first so CreateDefaultAdmin's zero-global-admin gate sees them;
+ // runRoleStartupTasks repeats it for agent mode and retries.
if err := p.Role.BackfillLegacyRoleAssignments(ctx); err != nil {
slog.WarnContext(ctx, "Failed to backfill legacy role assignments before admin bootstrap", "error", err)
}
@@ -304,14 +329,11 @@ func initializeStartupState(p initializeStartupStateParams) {
)
startup.CleanupUnknownSettings(appCtx, p.Settings)
- runRoleStartupTasks(appCtx, p.Role, p.ApiKey, cfg, cfg.AgentMode)
+ runRoleStartupTasks(appCtx, p.Role, p.ApiKey, cfg)
- // Auto-pair only applies in Edge mode (where the agent's outbound tunnel is the
- // only path to the manager). Direct mode is passive — the manager dials the agent's
- // HTTP server on TCP 3553, and the manager-side health-check promotes the env to
- // Online once reachability is confirmed.
+ // Only edge agents auto-pair; direct agents are passive and the manager dials them.
if cfg.AgentMode && cfg.EdgeAgent && cfg.AgentToken != "" && cfg.ManagerApiUrl != "" {
- if err := handleAgentBootstrapPairing(appCtx, cfg, httpClient); err != nil {
+ if err := handleAgentBootstrapPairing(appCtx, cfg); err != nil {
slog.WarnContext(appCtx, "Failed to auto-pair agent with manager", "error", err)
}
} else if cfg.AgentMode && !cfg.EdgeAgent {
@@ -319,36 +341,14 @@ func initializeStartupState(p initializeStartupStateParams) {
}
}
-func initializeGitOpsStartupStateInternal(appCtx context.Context, gitOpsSync *gitops.GitOpsSyncService) {
- if gitOpsSync == nil {
- return
- }
- if err := gitOpsSync.CleanupOrphanedSyncsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to clean up orphaned GitOps syncs on startup", "error", err)
- }
-
- if err := gitOpsSync.CleanupLeakedScratchDirsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to clean up leaked GitOps scratch directories on startup", "error", err)
- }
- if err := gitOpsSync.CleanupLeakedCloneDirsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to clean up leaked git clone directories on startup", "error", err)
- }
- if err := gitOpsSync.ReconcileDirectorySyncProjectsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to reconcile directory GitOps projects on startup", "error", err)
- }
-}
-
-func runRoleStartupTasks(ctx context.Context, roleService *role.RoleService, apiKeyService *apikey.ApiKeyService, cfg *config.Config, agentMode bool) {
+func runRoleStartupTasks(ctx context.Context, roleService *role.RoleService, apiKeyService *apikey.ApiKeyService, cfg *config.Config) {
if roleService == nil {
return
}
if err := roleService.EnsureBuiltInRoles(ctx); err != nil {
slog.ErrorContext(ctx, "Failed to reconcile built-in roles", "error", err)
}
- // Backfill must run AFTER EnsureBuiltInRoles (it references the role IDs
- // seeded there) and BEFORE BackfillApiKeyPermissions / AssertGlobalAdminExists
- // (both consult the assignments table this populates). It is a no-op once
- // its kv completion marker exists.
+ // Runs after EnsureBuiltInRoles seeds role IDs and before the API key backfill and admin guard read assignments.
if err := roleService.BackfillLegacyRoleAssignments(ctx); err != nil {
slog.ErrorContext(ctx, "Failed to backfill legacy users.roles into user_role_assignments", "error", err)
}
@@ -357,12 +357,10 @@ func runRoleStartupTasks(ctx context.Context, roleService *role.RoleService, api
slog.WarnContext(ctx, "Failed to backfill API key permissions", "error", err)
}
}
- if cfg != nil {
- if err := roleService.ReconcileEnvOidcMappings(ctx, cfg.OidcRoleMappings); err != nil {
- slog.ErrorContext(ctx, "Failed to reconcile OIDC_ROLE_MAPPINGS", "error", err)
- }
+ if err := roleService.ReconcileEnvOidcMappings(ctx, cfg.OidcRoleMappings); err != nil {
+ slog.ErrorContext(ctx, "Failed to reconcile OIDC_ROLE_MAPPINGS", "error", err)
}
- if agentMode {
+ if cfg.AgentMode {
return
}
if err := roleService.AssertGlobalAdminExists(ctx); err != nil {
@@ -374,11 +372,18 @@ func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, cfg *config.
var stop func(context.Context) error
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
- var startErr error
- stop, startErr = startEdgeTunnelClientIfConfigured(appCtx, cfg, router)
- if startErr != nil {
- slog.ErrorContext(appCtx, "Failed to start edge tunnel client", "error", startErr)
+ if !cfg.EdgeAgent || cfg.ManagerApiUrl == "" || cfg.AgentToken == "" {
+ return nil
}
+ edgeCfg := buildEdgeRuntimeConfigInternal(cfg)
+ slog.InfoContext(appCtx, "Starting edge agent session client", edge.StartupLogAttrs(edgeCfg)...)
+ tunnelStop, err := edge.StartTunnelClient(appCtx, edgeCfg, router)
+ if err != nil {
+ slog.ErrorContext(appCtx, "Failed to start edge tunnel client", "error", err)
+ return nil
+ }
+ stop = tunnelStop
+ slog.InfoContext(appCtx, "Edge tunnel client started", "managerUrl", cfg.ManagerApiUrl)
return nil
},
OnStop: func(ctx context.Context) error {
@@ -399,40 +404,7 @@ func registerAppCancelHook(lc fx.Lifecycle, cancelApp context.CancelFunc) {
})
}
-func startEdgeTunnelClientIfConfigured(appCtx context.Context, cfg *config.Config, router http.Handler) (func(context.Context) error, error) {
- managerEndpointConfigured := cfg.ManagerApiUrl != ""
- if !cfg.EdgeAgent || !managerEndpointConfigured || cfg.AgentToken == "" {
- return nil, nil
- }
-
- edgeCfg := &edge.Config{
- EdgeAgent: cfg.EdgeAgent,
- EdgeTransport: cfg.EdgeTransport,
- EdgeReconnectInterval: cfg.EdgeReconnectInterval,
- EdgeMTLSMode: cfg.EdgeMTLSMode,
- EdgeMTLSCAFile: cfg.EdgeMTLSCAFile,
- EdgeMTLSCertFile: cfg.EdgeMTLSCertFile,
- EdgeMTLSKeyFile: cfg.EdgeMTLSKeyFile,
- EdgeMTLSServerName: cfg.EdgeMTLSServerName,
- EdgeMTLSAssetsDir: cfg.EdgeMTLSAssetsDir,
- AppURL: cfg.GetAppURL(),
- ManagerApiUrl: cfg.ManagerApiUrl,
- AgentToken: cfg.AgentToken,
- Port: cfg.Port,
- Listen: cfg.Listen,
- }
-
- slog.InfoContext(appCtx, "Starting edge agent session client", edge.StartupLogAttrs(edgeCfg)...)
- stop, err := edge.StartTunnelClient(appCtx, edgeCfg, router)
- if err != nil {
- return nil, fmt.Errorf("failed to start edge tunnel client: %w", err)
- }
-
- slog.InfoContext(appCtx, "Edge tunnel client started", "managerUrl", cfg.ManagerApiUrl)
- return stop, nil
-}
-
-func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config, httpClient *http.Client) error {
+func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config) error {
slog.InfoContext(ctx, "Agent mode detected with token, attempting auto-pairing", "managerUrl", cfg.ManagerApiUrl)
pairURL := strings.TrimRight(httpx.ManagerBaseURL(cfg.ManagerApiUrl), "/") + "/api/environments/pair"
@@ -447,20 +419,9 @@ func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config, httpCl
req.Header.Set("X-Api-Key", cfg.AgentToken)
- if cfg.EdgeAgent && strings.TrimSpace(cfg.ManagerApiUrl) != "" {
- edgeClient, edgeErr := edge.NewManagerHTTPClient(&edge.Config{
- ManagerApiUrl: cfg.ManagerApiUrl,
- EdgeMTLSMode: cfg.EdgeMTLSMode,
- EdgeMTLSCAFile: cfg.EdgeMTLSCAFile,
- EdgeMTLSCertFile: cfg.EdgeMTLSCertFile,
- EdgeMTLSKeyFile: cfg.EdgeMTLSKeyFile,
- EdgeMTLSServerName: cfg.EdgeMTLSServerName,
- EdgeMTLSAssetsDir: cfg.EdgeMTLSAssetsDir,
- }, 10*time.Second)
- if edgeErr != nil {
- return fmt.Errorf("failed to configure edge pairing client: %w", edgeErr)
- }
- httpClient = edgeClient
+ httpClient, err := edge.NewManagerHTTPClient(buildEdgeRuntimeConfigInternal(cfg), 10*time.Second)
+ if err != nil {
+ return fmt.Errorf("failed to configure edge pairing client: %w", err)
}
resp, err := httpClient.Do(req)
@@ -471,35 +432,17 @@ func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config, httpCl
body, _ := io.ReadAll(resp.Body)
- switch resp.StatusCode {
- case http.StatusOK:
+ switch {
+ case resp.StatusCode == http.StatusOK:
slog.InfoContext(ctx, "Successfully paired agent with manager", "managerUrl", cfg.ManagerApiUrl)
return nil
- case http.StatusBadRequest:
- // Environment is not in pending status - already paired, this is fine
- if strings.Contains(string(body), "not in pending status") {
- slog.InfoContext(ctx, "Agent already paired with manager", "managerUrl", cfg.ManagerApiUrl)
- return nil
- }
- return fmt.Errorf("pairing failed with status %d: %s", resp.StatusCode, string(body))
- case http.StatusUnauthorized:
- // Invalid API key - could be already paired with a different key, or key was deleted
- // This is not fatal; the agent can still function if it has a valid token configured
+ case resp.StatusCode == http.StatusBadRequest && strings.Contains(string(body), "not in pending status"):
+ slog.InfoContext(ctx, "Agent already paired with manager", "managerUrl", cfg.ManagerApiUrl)
+ return nil
+ case resp.StatusCode == http.StatusUnauthorized:
+ // Not fatal: the key may belong to an earlier pairing, and a configured token still works.
slog.DebugContext(ctx, "Pairing skipped - API key not recognized (agent may already be paired)", "managerUrl", cfg.ManagerApiUrl)
return nil
- default:
- return fmt.Errorf("pairing failed with status %d: %s", resp.StatusCode, string(body))
}
-}
-
-func initializeDBAndMigrateInternal(ctx context.Context, cfg *config.Config) (*database.DB, error) {
- db, err := database.Initialize(ctx, cfg.DatabaseURL, database.MigrationOptions{
- AllowDowngrade: cfg.AllowDowngrade,
- })
- if err != nil {
- return nil, fmt.Errorf("failed to initialize database: %w", err)
- }
-
- slog.InfoContext(ctx, "Database initialized successfully")
- return db, nil
+ return fmt.Errorf("pairing failed with status %d: %s", resp.StatusCode, string(body))
}
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index 25e245226..257cea7f0 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -40,21 +40,21 @@ import (
tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
)
-type blockingBusWatcherInternal struct {
+type blockingBusWatcher struct {
started chan struct{}
stopped chan struct{}
}
-func (w *blockingBusWatcherInternal) Name() string { return "blocking" }
+func (w *blockingBusWatcher) Name() string { return "blocking" }
-func (w *blockingBusWatcherInternal) Start(ctx context.Context) error {
+func (w *blockingBusWatcher) Start(ctx context.Context) error {
close(w.started)
<-ctx.Done()
close(w.stopped)
return nil
}
-func (w *blockingBusWatcherInternal) RunNow(context.Context) error { return nil }
+func (w *blockingBusWatcher) RunNow(context.Context) error { return nil }
func TestNormalizeTunnelGRPCRequestPath(t *testing.T) {
fullMethodPath := tunnelpb.TunnelService_Connect_FullMethodName
@@ -90,10 +90,8 @@ func TestNormalizeTunnelGRPCRequestPath(t *testing.T) {
})
t.Run("legacy /api/tunnel/connect is rewritten to gRPC method", func(t *testing.T) {
- // Regression: PR #2722 removed this branch, breaking the edge agent's
- // gRPC transport. The agent client uses /api/tunnel/connect as its
- // gRPC method path so reverse proxies can route tunnel traffic with
- // a stable URL instead of the proto-generated gRPC service name.
+ // Edge agents use /api/tunnel/connect as their gRPC method path so reverse proxies
+ // can route tunnel traffic on a stable URL.
req := httptest.NewRequest("POST", "/api/tunnel/connect", http.NoBody)
normalized := normalizeTunnelGRPCRequestPathInternal(req)
@@ -256,11 +254,8 @@ func TestHTTP2APIResponsesDoNotUseAPIGzip(t *testing.T) {
}
func TestH2CStreamSurvivesPastReadHeaderTimeout(t *testing.T) {
- // Regression guard for the go1.26.6 CVE-2026-56853 backport, which arms
- // ReadHeaderTimeout on the raw conn before the h2c preface sniff and never
- // clears it on the HTTP/2 handoff. Without ClearReadDeadline in the h2c
- // handler wrapper, the connection — and every gRPC tunnel stream on it —
- // dies with an i/o timeout exactly ReadHeaderTimeout after accept.
+ // go1.26.6 arms ReadHeaderTimeout before the h2c preface sniff and never clears it, so
+ // without ClearReadDeadline every h2c stream dies ReadHeaderTimeout after accept.
const readHeaderTimeout = 250 * time.Millisecond
release := make(chan struct{})
@@ -300,9 +295,7 @@ func TestH2CStreamSurvivesPastReadHeaderTimeout(t *testing.T) {
defer func() { _ = resp.Body.Close() }()
require.Equal(t, "HTTP/2.0", resp.Proto)
- // Keep the stream idle well past ReadHeaderTimeout before the handler
- // writes again; on a connection with the stale deadline still armed the
- // read below fails instead of returning the body.
+ // Idle past ReadHeaderTimeout; a stale deadline would fail the read below.
time.Sleep(4 * readHeaderTimeout)
close(release)
@@ -423,7 +416,7 @@ func TestJobSchedulerStopCancelsItsPrivateContext(t *testing.T) {
executed <- run
return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}, nil)
- watcher := &blockingBusWatcherInternal{
+ watcher := &blockingBusWatcher{
started: make(chan struct{}),
stopped: make(chan struct{}),
}
@@ -468,7 +461,7 @@ func TestApplicationOptionsValidate(t *testing.T) {
require.NoError(t, err)
}
-func TestPrepareServerTLSInternal_AgentModeSkipsManagerMTLSValidation(t *testing.T) {
+func TestPrepareServerTLS_AgentModeSkipsManagerMTLSValidation(t *testing.T) {
cfg := &config.Config{
AgentMode: true,
EdgeMTLSMode: "required",
@@ -484,7 +477,7 @@ func TestPrepareServerTLSInternal_AgentModeSkipsManagerMTLSValidation(t *testing
assert.Equal(t, "required", edgeCfg.EdgeMTLSMode)
}
-func TestPrepareServerTLSInternal_AllowsExternalMTLSTermination(t *testing.T) {
+func TestPrepareServerTLS_AllowsExternalMTLSTermination(t *testing.T) {
libcrypto.InitEncryption(&libcrypto.Config{
EncryptionKey: "test-encryption-key-for-edge-mtls-32bytes-min",
Environment: "test",
@@ -511,8 +504,8 @@ func TestPrepareServerTLSInternal_AllowsExternalMTLSTermination(t *testing.T) {
}
func TestIsWeakProductionEncryptionKey(t *testing.T) {
- assert.True(t, isWeakProductionEncryptionKeyInternal("short", "production", false))
- assert.False(t, isWeakProductionEncryptionKeyInternal("test-encryption-key-for-edge-mtls-32bytes-min", "production", false))
- assert.False(t, isWeakProductionEncryptionKeyInternal("hex:abc", "production", false))
- assert.False(t, isWeakProductionEncryptionKeyInternal("short", "development", false))
+ assert.True(t, isWeakProductionEncryptionKey("short", "production", false))
+ assert.False(t, isWeakProductionEncryptionKey("test-encryption-key-for-edge-mtls-32bytes-min", "production", false))
+ assert.False(t, isWeakProductionEncryptionKey("hex:abc", "production", false))
+ assert.False(t, isWeakProductionEncryptionKey("short", "development", false))
}
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 19143e001..b1fe7e37b 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -393,7 +393,7 @@ func registerDynamicJobs(params dynamicJobsParams) error {
params.JobSchedule.RunEnvironmentHealthNow = func(ctx context.Context) error {
return params.Environment.RunHealthChecksNow(ctx)
}
- params.Environment.RegisterHealthJobsOnStartup(params.AppCtx)
+ params.Environment.RescheduleHealthJobs(params.AppCtx)
}
return nil
}
diff --git a/backend/internal/bootstrap/logger_bootstrap.go b/backend/internal/bootstrap/logger_bootstrap.go
index aff9cf68d..b6f6bf4a7 100644
--- a/backend/internal/bootstrap/logger_bootstrap.go
+++ b/backend/internal/bootstrap/logger_bootstrap.go
@@ -11,12 +11,22 @@ import (
"github.com/lmittmann/tint"
slogGorm "github.com/orandin/slog-gorm"
+ "go.getarcane.app/streams/logs"
"gorm.io/gorm"
"gorm.io/gorm/logger"
+ "github.com/getarcaneapp/arcane/backend/v2/api/ws"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
)
+// logLevels maps LOG_LEVEL values to slog levels; unknown values map to the zero value, slog.LevelInfo.
+var logLevels = map[string]slog.Level{
+ "debug": slog.LevelDebug,
+ "warn": slog.LevelWarn,
+ "warning": slog.LevelWarn,
+ "error": slog.LevelError,
+}
+
type requestLogHandler struct {
handler slog.Handler
}
@@ -123,7 +133,10 @@ func (h *requestLogHandler) Handle(ctx context.Context, r slog.Record) error {
condensedAttrs = append(condensedAttrs, debugAttrs...)
for _, a := range attrs {
- condensedAttrs = append(condensedAttrs, normalizeRequestLogErrorInternal(a))
+ if value, ok := a.Value.Any().(map[string]any); ok && a.Key == "error" {
+ a = slog.String("error", fmt.Sprint(value["message"]))
+ }
+ condensedAttrs = append(condensedAttrs, a)
}
newRecord := slog.NewRecord(r.Time, r.Level, r.Message, r.PC)
@@ -132,25 +145,6 @@ func (h *requestLogHandler) Handle(ctx context.Context, r slog.Record) error {
return h.handler.Handle(ctx, newRecord)
}
-func normalizeRequestLogErrorInternal(a slog.Attr) slog.Attr {
- if a.Key != "error" {
- return a
- }
- if value, ok := a.Value.Any().(map[string]any); ok {
- return slog.String("error", fmt.Sprint(value["message"]))
- }
- return a
-}
-
-func replaceErrorAttrInternal(_ []string, a slog.Attr) slog.Attr {
- if a.Value.Kind() == slog.KindAny {
- if err, ok := a.Value.Any().(error); ok {
- return slog.String(a.Key, err.Error())
- }
- }
- return a
-}
-
func (h *requestLogHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
return &requestLogHandler{handler: h.handler.WithAttrs(attrs)}
}
@@ -187,43 +181,92 @@ func (h *attrFilterHandler) WithGroup(name string) slog.Handler {
return &attrFilterHandler{handler: h.handler.WithGroup(name), dropKeys: h.dropKeys}
}
-func SetupSlogLogger(cfg *config.Config) {
- var lvl slog.Level
- switch strings.ToLower(cfg.LogLevel) {
- case "debug":
- lvl = slog.LevelDebug
- case "warn", "warning":
- lvl = slog.LevelWarn
- case "error":
- lvl = slog.LevelError
- default:
- lvl = slog.LevelInfo
+// telemetryLogHandler applies LOG_LEVEL to the OpenTelemetry bridge and strips raw query strings,
+// URL queries, and webhook trigger tokens before records are exported.
+type telemetryLogHandler struct {
+ handler slog.Handler
+ level slog.Leveler
+}
+
+func (h *telemetryLogHandler) Enabled(ctx context.Context, level slog.Level) bool {
+ return level >= h.level.Level() && h.handler.Enabled(ctx, level)
+}
+
+func (h *telemetryLogHandler) Handle(ctx context.Context, r slog.Record) error {
+ redacted := slog.NewRecord(r.Time, r.Level, redactSecrets(r.Message), r.PC)
+ r.Attrs(func(a slog.Attr) bool {
+ redacted.AddAttrs(redactLogAttrs([]slog.Attr{a})...)
+ return true
+ })
+ return h.handler.Handle(ctx, redacted)
+}
+
+func (h *telemetryLogHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
+ return &telemetryLogHandler{handler: h.handler.WithAttrs(redactLogAttrs(attrs)), level: h.level}
+}
+
+func (h *telemetryLogHandler) WithGroup(name string) slog.Handler {
+ return &telemetryLogHandler{handler: h.handler.WithGroup(name), level: h.level}
+}
+
+func redactLogAttrs(attrs []slog.Attr) []slog.Attr {
+ redacted := make([]slog.Attr, 0, len(attrs))
+ for _, a := range attrs {
+ a.Value = a.Value.Resolve()
+ if err, ok := a.Value.Any().(error); ok && a.Value.Kind() == slog.KindAny {
+ a.Value = slog.StringValue(err.Error())
+ }
+ switch {
+ case a.Key == "query":
+ continue
+ case a.Value.Kind() == slog.KindString:
+ a.Value = slog.StringValue(redactSecrets(a.Value.String()))
+ case a.Value.Kind() == slog.KindGroup:
+ a.Value = slog.GroupValue(redactLogAttrs(a.Value.Group())...)
+ }
+ redacted = append(redacted, a)
}
+ return redacted
+}
+// SetupSlogLogger installs the console logger and the diagnostics log tee. When
+// telemetryHandler is set, records are also sent to it at the same level.
+func SetupSlogLogger(cfg *config.Config, telemetryHandler slog.Handler) {
lv := new(slog.LevelVar)
- lv.Set(lvl)
+ lv.Set(logLevels[strings.ToLower(cfg.LogLevel)])
+
+ replaceErrorAttr := func(_ []string, a slog.Attr) slog.Attr {
+ if err, ok := a.Value.Any().(error); ok && a.Value.Kind() == slog.KindAny {
+ return slog.String(a.Key, err.Error())
+ }
+ return a
+ }
var h slog.Handler
if cfg.LogJson {
h = slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
Level: lv,
- ReplaceAttr: replaceErrorAttrInternal,
+ ReplaceAttr: replaceErrorAttr,
})
} else {
h = tint.NewTextHandler(os.Stdout, &tint.Options{
Level: lv,
TimeFormat: "Jan 02 15:04:05.000",
- ReplaceAttr: replaceErrorAttrInternal,
+ ReplaceAttr: replaceErrorAttr,
})
}
+ if telemetryHandler != nil {
+ h = slog.NewMultiHandler(h, &telemetryLogHandler{handler: telemetryHandler, level: lv})
+ }
+
h = &requestLogHandler{handler: h}
- slog.SetDefault(slog.New(h))
+ slog.SetDefault(slog.New(logs.NewSlogHandler(h, ws.LogBroadcaster())))
}
func BuildGormLogger(cfg *config.Config) logger.Interface {
- lvl := strings.ToLower(cfg.LogLevel)
+ lvl := logLevels[strings.ToLower(cfg.LogLevel)]
filteredHandler := &attrFilterHandler{
handler: slog.Default().Handler(),
@@ -233,27 +276,14 @@ func BuildGormLogger(cfg *config.Config) logger.Interface {
opts := []slogGorm.Option{
slogGorm.WithHandler(filteredHandler),
slogGorm.WithSlowThreshold(200 * time.Millisecond),
+ slogGorm.SetLogLevel(slogGorm.DefaultLogType, lvl),
+ slogGorm.SetLogLevel(slogGorm.ErrorLogType, slog.LevelError),
+ slogGorm.SetLogLevel(slogGorm.SlowQueryLogType, slog.LevelWarn),
}
-
- var defaultTypeLevel slog.Level
- switch lvl {
- case "debug":
- defaultTypeLevel = slog.LevelDebug
- // Trace all SQL messages only in debug
+ // Trace all SQL messages only in debug.
+ if lvl == slog.LevelDebug {
opts = append(opts, slogGorm.WithTraceAll())
- case "warn", "warning":
- defaultTypeLevel = slog.LevelWarn
- case "error":
- defaultTypeLevel = slog.LevelError
- default:
- defaultTypeLevel = slog.LevelInfo
}
- opts = append(opts,
- slogGorm.SetLogLevel(slogGorm.DefaultLogType, defaultTypeLevel),
- slogGorm.SetLogLevel(slogGorm.ErrorLogType, slog.LevelError),
- slogGorm.SetLogLevel(slogGorm.SlowQueryLogType, slog.LevelWarn),
- )
-
return gormLogger{Interface: slogGorm.New(opts...)}
}
diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.go
index c8d08e39a..cbbb3dffe 100644
--- a/backend/internal/bootstrap/router_bootstrap.go
+++ b/backend/internal/bootstrap/router_bootstrap.go
@@ -7,10 +7,12 @@ import (
"net"
"net/http"
"path"
+ "slices"
"strings"
"github.com/getarcaneapp/arcane/types/v2"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
+ "github.com/labstack/echo-otel/v5"
"github.com/labstack/echo/v5"
echomiddleware "github.com/labstack/echo/v5/middleware"
"github.com/samber/slog-echo/v2"
@@ -24,6 +26,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/federated"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/telemetry"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/cookie"
@@ -44,6 +47,7 @@ var loggerSkipPatterns = []string{
"GET /api/environments/*/ws/projects/*/logs",
"GET /api/environments/*/ws/system/stats",
"GET /api/stream",
+ "POST /api/telemetry/*",
"GET /_app/*",
"GET /img",
"GET /api/health",
@@ -52,52 +56,7 @@ var loggerSkipPatterns = []string{
"PUT /api/environments/*/uploads/*/*/chunks/*",
}
-func shouldLogRequestInternal(c *echo.Context, _ error) bool {
- mp := c.Request().Method + " " + c.Request().URL.Path
- for _, pat := range loggerSkipPatterns {
- if pat == mp {
- return false
- }
- if before, ok := strings.CutSuffix(pat, "/*"); ok {
- if strings.HasPrefix(mp, before) {
- return false
- }
- }
- if ok, _ := path.Match(pat, mp); ok {
- return false
- }
- if strings.HasSuffix(pat, "/") && strings.HasPrefix(mp, pat) {
- return false
- }
- }
- return true
-}
-
-// requestLoggerMiddlewareInternal wraps slog-echo and filters out internal
-// edge tunnel requests plus high-volume endpoints (health, WS, static).
-func requestLoggerMiddlewareInternal() echo.MiddlewareFunc {
- loggerMiddleware := slogecho.NewWithConfig(slog.Default(), slogecho.Config{
- DefaultLevel: slog.LevelInfo,
- ClientErrorLevel: slog.LevelWarn,
- ServerErrorLevel: slog.LevelError,
- Filters: []slogecho.Filter{shouldLogRequestInternal},
- WithRequestID: true,
- })
-
- return func(next echo.HandlerFunc) echo.HandlerFunc {
- return func(c *echo.Context) error {
- if edge.IsInternalTunnelRequest(c.Request().Context()) {
- return next(c)
- }
- if c.Request().Body == nil {
- c.Request().Body = http.NoBody
- }
- return loggerMiddleware(next)(c)
- }
- }
-}
-
-func createAuthValidatorInternal(deps api.HandlerDeps) middleware.AuthValidator {
+func createAuthValidator(deps api.HandlerDeps) middleware.AuthValidator {
resolveUser := func(ctx context.Context, user *usertypes.Actor) *authz.PermissionSet {
ps, err := deps.Role.Service().ResolvePermissions(ctx, user.ID)
if err != nil || ps == nil {
@@ -134,26 +93,21 @@ func createAuthValidatorInternal(deps api.HandlerDeps) middleware.AuthValidator
return nil, nil, false
}
- user, ok := verifyRouterRequestUserInternal(ctx, deps.Auth.Service(), req)
- if !ok {
+ // Bearer tokens take precedence over the browser token cookie.
+ verify := deps.Auth.Service().VerifyBrowserToken
+ token, err := cookie.GetTokenCookie(req)
+ if bearer, ok := strings.CutPrefix(req.Header.Get("Authorization"), "Bearer "); ok {
+ verify, token, err = deps.Auth.Service().VerifyToken, bearer, nil
+ }
+ if err != nil || token == "" {
return nil, nil, false
}
- return resolveUser(ctx, user), user, true
- }
-}
-
-func verifyRouterRequestUserInternal(ctx context.Context, authService *auth.AuthService, req *http.Request) (*usertypes.Actor, bool) {
- if authHeader := req.Header.Get("Authorization"); strings.HasPrefix(authHeader, "Bearer ") {
- user, _, err := authService.VerifyToken(ctx, strings.TrimPrefix(authHeader, "Bearer "))
- return user.Actor(), err == nil && user != nil
- }
-
- browserToken, err := cookie.GetTokenCookie(req)
- if err != nil || browserToken == "" {
- return nil, false
+ user, _, err := verify(ctx, token)
+ if err != nil || user == nil {
+ return nil, nil, false
+ }
+ return resolveUser(ctx, user.Actor()), user.Actor(), true
}
- user, _, err := authService.VerifyBrowserToken(ctx, browserToken)
- return user.Actor(), err == nil && user != nil
}
type RouterParams struct {
@@ -167,45 +121,84 @@ type RouterParams struct {
TunnelRegistry *edge.TunnelRegistry
}
-// newEchoInternal builds the Echo instance with path parameter values
-// percent-decoded before they reach handlers (RFC 3986 §6.2.2.2); route
-// matching still happens on the escaped path, so an encoded "/" cannot
-// change which route matches.
-func newEchoInternal() *echo.Echo {
- return echo.NewWithConfig(echo.Config{
- Router: echo.NewRouter(echo.RouterConfig{
- AllowOverwritingRoute: true,
- UnescapePathParamValues: true,
- }),
- })
-}
-
func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
ctx := p.Context
cfg := p.Config
deps := p.HandlerDeps
- e := newEchoInternal()
+ // Path params are decoded after matching on the escaped path, so an encoded "/" cannot change the route.
+ e := echo.NewWithConfig(echo.Config{
+ Router: echo.NewRouter(echo.RouterConfig{AllowOverwritingRoute: true, UnescapePathParamValues: true}),
+ })
- trustedProxyNets := parseTrustedProxyCIDRsInternal(cfg.TrustedProxies)
+ var trustedProxyNets []*net.IPNet
+ // Trust only the configured ranges, not Echo's built-in loopback/link-local/private defaults.
+ trustOpts := []echo.TrustOption{echo.TrustLoopback(false), echo.TrustLinkLocal(false), echo.TrustPrivateNet(false)}
+ for cidr := range strings.SplitSeq(cfg.TrustedProxies, ",") {
+ cidr = strings.TrimSpace(cidr)
+ if cidr == "" {
+ continue
+ }
+ _, ipnet, err := net.ParseCIDR(cidr)
+ if err != nil {
+ slog.WarnContext(ctx, "invalid TRUSTED_PROXIES CIDR, ignoring", "cidr", cidr, "error", err)
+ continue
+ }
+ trustedProxyNets = append(trustedProxyNets, ipnet)
+ trustOpts = append(trustOpts, echo.TrustIPRange(ipnet))
+ }
if cfg.TrustedProxies != "" && len(trustedProxyNets) == 0 {
slog.WarnContext(ctx, "TRUSTED_PROXIES set but no valid CIDRs found; falling back to direct IP extraction")
}
if len(trustedProxyNets) == 0 {
e.IPExtractor = echo.ExtractIPDirect()
} else {
- // Trust only the configured ranges, not Echo's built-in loopback/link-local/private defaults.
- opts := []echo.TrustOption{echo.TrustLoopback(false), echo.TrustLinkLocal(false), echo.TrustPrivateNet(false)}
- for _, ipnet := range trustedProxyNets {
- opts = append(opts, echo.TrustIPRange(ipnet))
- }
- e.IPExtractor = echo.ExtractIPFromXFFHeader(opts...)
+ e.IPExtractor = echo.ExtractIPFromXFFHeader(trustOpts...)
}
e.Use(echomiddleware.Recover())
e.Use(echomiddleware.RequestID())
- e.Use(requestLoggerMiddlewareInternal())
- e.Use(secureCookieContextMiddlewareInternal(trustedProxyNets))
+ // Trace REST API calls only, skipping long-lived streams, tunnels, health probes, and the browser trace relay.
+ e.Use(echootel.NewMiddlewareWithConfig(echootel.Config{Skipper: func(c *echo.Context) bool {
+ req := c.Request()
+ p := req.URL.Path
+ return !strings.HasPrefix(p, "/api/") || p == "/api/health" ||
+ strings.HasPrefix(p, "/api/telemetry/") || strings.HasPrefix(p, "/api/tunnel/") ||
+ httpx.IsWebSocketUpgradeRequest(req) || strings.Contains(req.Header.Get("Accept"), "text/event-stream")
+ }}))
+ e.Use(middleware.SpanEnvironment(deps.Environment.Service().ResolveEnvironmentName))
+ requestLogger := slogecho.NewWithConfig(slog.Default(), slogecho.Config{
+ DefaultLevel: slog.LevelInfo,
+ ClientErrorLevel: slog.LevelWarn,
+ ServerErrorLevel: slog.LevelError,
+ Filters: []slogecho.Filter{func(c *echo.Context, _ error) bool {
+ mp := c.Request().Method + " " + c.Request().URL.Path
+ return !slices.ContainsFunc(loggerSkipPatterns, func(pat string) bool {
+ if prefix, ok := strings.CutSuffix(pat, "/*"); ok && strings.HasPrefix(mp, prefix) {
+ return true
+ }
+ matched, _ := path.Match(pat, mp)
+ return matched
+ })
+ }},
+ WithRequestID: true,
+ WithTraceID: true,
+ WithSpanID: true,
+ })
+ // Internal edge tunnel requests are never logged.
+ e.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
+ logged := requestLogger(next)
+ return func(c *echo.Context) error {
+ if edge.IsInternalTunnelRequest(c.Request().Context()) {
+ return next(c)
+ }
+ if c.Request().Body == nil {
+ c.Request().Body = http.NoBody
+ }
+ return logged(c)
+ }
+ })
+ e.Use(secureCookieContextMiddleware(trustedProxyNets))
authMiddleware := p.AuthMiddleware
e.Use(middleware.NewCORSMiddleware(cfg).Add())
@@ -234,9 +227,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
apiGroup.Use(middleware.PerTokenRateLimitForPaths(
[]string{"/api/webhooks/trigger/:token"}, 60, 10, deps.Webhook.Service().IsKnownToken,
))
- // Agent event ingestion authenticates on the agent token alone and sits
- // outside the auth middleware, so it needs its own brute-force ceiling.
- // The allowance is generous because busy agents legitimately batch events.
+ // Agent event ingestion sits outside the auth middleware, so it needs its own ceiling; agents batch events.
apiGroup.Use(middleware.PerIPRateLimitForPaths(
[]string{"/api/events"}, 60, 30,
))
@@ -253,6 +244,9 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
// Register public webhook trigger endpoint before auth middleware (token in URL is the sole auth)
api.RegisterWebhookTrigger(apiGroup, deps.Webhook.Service(), handlerAppCtx)
federated.RegisterFederatedTokenExchange(apiGroup, deps.Federated)
+ if !cfg.AgentMode {
+ telemetry.RegisterRoutes(apiGroup, telemetry.NewService())
+ }
deps.Event.RegisterAgentRoutes(apiGroup, func(ctx context.Context, token string) (string, error) {
env, err := deps.Environment.Service().ResolveEnvironmentByAccessToken(ctx, token)
if err != nil {
@@ -270,7 +264,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
types.LocalDockerEnvironmentID,
"id",
envResolver,
- createAuthValidatorInternal(deps),
+ createAuthValidator(deps),
permissionMatcher,
p.TunnelRegistry,
httpx.ValidateWebSocketOrigin(cfg.GetAppURL()),
@@ -279,10 +273,8 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
humaAPI := api.SetupAPI(e, apiGroup, handlerAppCtx, cfg, deps)
- // Populate the proxy permission matcher from the registered API surface so
- // remote-environment requests are authorized with the same permissions
- // enforced locally. The matcher pointer is shared with the proxy middleware
- // constructed above; population completes here, before the server serves traffic.
+ // Fill the proxy's shared matcher from the local API surface before serving so remote
+ // requests are authorized with the same permissions.
permissionMatcher.CollectFromHumaAPI(humaAPI)
ws.AddProxiedPermissions(permissionMatcher)
@@ -293,8 +285,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
// Remaining echo handlers (WebSocket/streaming)
ws.NewWebSocketHandler(apiGroup, deps.Project.Service(), deps.Container.Service(), deps.Swarm.Service(), deps.System.Service(), deps.Diagnostics, authMiddleware, cfg)
- // Register edge tunnel endpoint for manager to accept agent connections
- // This is only registered when NOT in agent mode (i.e., running as manager)
+ // Managers accept agent connections on the edge tunnel endpoint.
var tunnelServer *edge.TunnelServer
if !cfg.AgentMode {
tunnelServer = registerEdgeTunnelRoutes(ctx, p.Lifecycle, cfg, apiGroup, deps.Environment.Service(), deps.Event.Service(), deps.Notification.Service(), p.TunnelRegistry)
@@ -327,42 +318,22 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
return e, tunnelServer
}
-// parseTrustedProxyCIDRsInternal parses TRUSTED_PROXIES into a list of
-// validated networks. Invalid entries are logged and skipped.
-func parseTrustedProxyCIDRsInternal(raw string) []*net.IPNet {
- if raw == "" {
- return nil
- }
- var nets []*net.IPNet
- for cidr := range strings.SplitSeq(raw, ",") {
- cidr = strings.TrimSpace(cidr)
- if cidr == "" {
- continue
- }
- _, ipnet, err := net.ParseCIDR(cidr)
- if err != nil {
- slog.WarnContext(context.Background(), "invalid TRUSTED_PROXIES CIDR, ignoring", "cidr", cidr, "error", err) //nolint:forbidigo // Startup configuration parsing has no request context.
- continue
- }
- nets = append(nets, ipnet)
- }
- return nets
-}
-
-// secureCookieContextMiddlewareInternal records whether the request should be
-// treated as HTTPS for cookie-emitting handlers. X-Forwarded-Proto is honored
-// ONLY when the direct TCP peer is in TRUSTED_PROXIES — an untrusted client
-// setting the header directly cannot trick the server into issuing Secure /
-// __Host- cookies over plain HTTP.
-func secureCookieContextMiddlewareInternal(trustedProxyNets []*net.IPNet) echo.MiddlewareFunc {
+// secureCookieContextMiddleware marks requests as HTTPS for cookies. X-Forwarded-Proto is
+// honored only when the direct TCP peer is in TRUSTED_PROXIES.
+func secureCookieContextMiddleware(trustedProxyNets []*net.IPNet) echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c *echo.Context) error {
req := c.Request()
secure := req.TLS != nil
- if !secure && len(trustedProxyNets) > 0 &&
- strings.EqualFold(req.Header.Get("X-Forwarded-Proto"), "https") &&
- remoteAddrInTrustedProxiesInternal(req.RemoteAddr, trustedProxyNets) {
- secure = true
+ if !secure && len(trustedProxyNets) > 0 && strings.EqualFold(req.Header.Get("X-Forwarded-Proto"), "https") {
+ host, _, err := net.SplitHostPort(req.RemoteAddr)
+ if err != nil {
+ host = req.RemoteAddr
+ }
+ // Unparseable remote addresses are untrusted.
+ if ip := net.ParseIP(host); ip != nil {
+ secure = slices.ContainsFunc(trustedProxyNets, func(n *net.IPNet) bool { return n.Contains(ip) })
+ }
}
if secure {
c.SetRequest(req.WithContext(context.WithValue(req.Context(), cookie.SecureCookieContextKey{}, true)))
@@ -371,23 +342,3 @@ func secureCookieContextMiddlewareInternal(trustedProxyNets []*net.IPNet) echo.M
}
}
}
-
-// remoteAddrInTrustedProxiesInternal reports whether the direct TCP peer
-// address of the request falls within any of the configured trusted-proxy
-// networks. Unparseable remote addresses are treated as untrusted.
-func remoteAddrInTrustedProxiesInternal(remoteAddr string, nets []*net.IPNet) bool {
- host, _, err := net.SplitHostPort(remoteAddr)
- if err != nil {
- host = remoteAddr
- }
- ip := net.ParseIP(host)
- if ip == nil {
- return false
- }
- for _, n := range nets {
- if n.Contains(ip) {
- return true
- }
- }
- return false
-}
diff --git a/backend/internal/bootstrap/router_bootstrap_test.go b/backend/internal/bootstrap/router_bootstrap_test.go
index 092882afa..657499458 100644
--- a/backend/internal/bootstrap/router_bootstrap_test.go
+++ b/backend/internal/bootstrap/router_bootstrap_test.go
@@ -13,15 +13,39 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "github.com/getarcaneapp/arcane/backend/v2/api"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/auth"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/container"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/project"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/system"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/cookie"
)
-func TestNewEchoInternal_DecodesPathParams(t *testing.T) {
- router := newEchoInternal()
- api := humaecho.NewWithGroup(router, router.Group("/api"), huma.DefaultConfig("test", "1.0.0"))
+func TestNewRouter_DecodesPathParams(t *testing.T) {
+ cfg := &config.Config{
+ AgentMode: true,
+ AppUrl: "http://localhost:3552",
+ Environment: config.AppEnvironmentTest,
+ }
+ router, _ := newRouter(RouterParams{
+ Context: t.Context(),
+ Config: cfg,
+ HandlerDeps: api.HandlerDeps{
+ Project: project.New(&project.ProjectService{}, nil),
+ Container: container.New(&container.ContainerService{}, nil, nil, nil),
+ Swarm: swarm.New(&swarm.SwarmService{}, nil, nil, nil),
+ System: system.New(&system.SystemService{}, nil, nil, nil),
+ },
+ AuthMiddleware: auth.NewAuthMiddleware(nil, cfg),
+ TunnelRegistry: edge.NewTunnelRegistry(),
+ })
+ humaAPI := humaecho.NewWithGroup(router, router.Group("/api"), huma.DefaultConfig("test", "1.0.0"))
var got string
- huma.Register(api, huma.Operation{
+ huma.Register(humaAPI, huma.Operation{
OperationID: "get-image",
Method: http.MethodGet,
Path: "/environments/{id}/images/{imageId}",
@@ -66,7 +90,7 @@ func TestSecureCookieContextMiddleware_TrustGating(t *testing.T) {
c := e.NewContext(req, rec)
var observed bool
- handler := secureCookieContextMiddlewareInternal(nets)(func(c *echo.Context) error {
+ handler := secureCookieContextMiddleware(nets)(func(c *echo.Context) error {
observed = cookie.SecureCookieFromContext(c.Request().Context())
return nil
})
diff --git a/backend/internal/bootstrap/telemetry_bootstrap.go b/backend/internal/bootstrap/telemetry_bootstrap.go
new file mode 100644
index 000000000..a5626a6b4
--- /dev/null
+++ b/backend/internal/bootstrap/telemetry_bootstrap.go
@@ -0,0 +1,195 @@
+package bootstrap
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "regexp"
+ "slices"
+ "sync"
+
+ "go.getarcane.app/kit/pkg"
+ "go.opentelemetry.io/contrib/exporters/autoexport"
+ "go.opentelemetry.io/contrib/instrumentation/runtime"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/log/global"
+ "go.opentelemetry.io/otel/propagation"
+ "go.opentelemetry.io/otel/sdk/log"
+ "go.opentelemetry.io/otel/sdk/metric"
+ "go.opentelemetry.io/otel/sdk/resource"
+ "go.opentelemetry.io/otel/sdk/trace"
+ "go.opentelemetry.io/otel/semconv/v1.40.0"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/telemetry"
+)
+
+// telemetryProviders owns the providers enabled through the standard OTEL_*
+// exporter variables. A nil provider means that signal is off.
+type telemetryProviders struct {
+ tracerProvider *trace.TracerProvider
+ meterProvider *metric.MeterProvider
+ loggerProvider *log.LoggerProvider
+
+ shutdownOnce sync.Once
+ shutdownErr error
+}
+
+// setupTelemetry installs global providers for each OTEL_*_EXPORTER that is set
+// to something other than none. Exporter failures are reported to errorLogger.
+func setupTelemetry(ctx context.Context, cfg *config.Config, errorLogger *slog.Logger, instanceID string) (*telemetryProviders, error) {
+ tel := &telemetryProviders{}
+ if telemetry.SDKDisabled() {
+ return tel, nil
+ }
+ fail := func(err error) (*telemetryProviders, error) {
+ return nil, errors.Join(err, tel.Shutdown(context.WithoutCancel(ctx)))
+ }
+
+ mode := "manager"
+ if cfg.AgentMode {
+ mode = kit.Ternary(cfg.EdgeAgent, "edge", "agent")
+ }
+ // OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES may override the defaults, but never the
+ // persisted instance ID or mode that identify this installation.
+ res, err := resource.New(ctx,
+ resource.WithTelemetrySDK(),
+ resource.WithAttributes(semconv.ServiceName("arcane"), semconv.ServiceVersion(config.Version)),
+ resource.WithFromEnv(),
+ resource.WithAttributes(semconv.ServiceInstanceID(instanceID), attribute.String("arcane.mode", mode)),
+ )
+ if err != nil {
+ return fail(fmt.Errorf("create telemetry resource: %w", err))
+ }
+
+ // The fallbacks return nil so an unset OTEL_*_EXPORTER means off, not otlp.
+ spanExporter, err := autoexport.NewSpanExporter(ctx, autoexport.WithFallbackSpanExporter(func(context.Context) (trace.SpanExporter, error) {
+ return nil, nil //nolint:nilnil // nil means tracing is off.
+ }))
+ if err != nil {
+ return fail(fmt.Errorf("create trace exporter: %w", err))
+ }
+ if spanExporter != nil && !autoexport.IsNoneSpanExporter(spanExporter) {
+ tel.tracerProvider = trace.NewTracerProvider(trace.WithResource(res), trace.WithBatcher(redactingExporter{spanExporter}))
+ }
+
+ // Adds go.schedule.duration, which runtime.Start does not report; OTEL_METRICS_PRODUCERS overrides it.
+ autoexport.WithFallbackMetricProducer(func(context.Context) (metric.Producer, error) { return runtime.NewProducer(), nil })
+ metricReader, err := autoexport.NewMetricReader(ctx, autoexport.WithFallbackMetricReader(func(context.Context) (metric.Reader, error) {
+ return nil, nil //nolint:nilnil // nil means metrics are off.
+ }))
+ if err != nil {
+ return fail(fmt.Errorf("create metric reader: %w", err))
+ }
+ if metricReader != nil && !autoexport.IsNoneMetricReader(metricReader) {
+ tel.meterProvider = metric.NewMeterProvider(metric.WithResource(res), metric.WithReader(metricReader))
+ if err = runtime.Start(runtime.WithMeterProvider(tel.meterProvider)); err != nil {
+ return fail(fmt.Errorf("start runtime metrics: %w", err))
+ }
+ }
+
+ logExporter, err := autoexport.NewLogExporter(ctx, autoexport.WithFallbackLogExporter(func(context.Context) (log.Exporter, error) {
+ return nil, nil //nolint:nilnil // nil means log export is off.
+ }))
+ if err != nil {
+ return fail(fmt.Errorf("create log exporter: %w", err))
+ }
+ if logExporter != nil && !autoexport.IsNoneLogExporter(logExporter) {
+ tel.loggerProvider = log.NewLoggerProvider(log.WithResource(res), log.WithProcessor(log.NewBatchProcessor(logExporter)))
+ }
+
+ if tel.tracerProvider == nil && tel.meterProvider == nil && tel.loggerProvider == nil {
+ return tel, nil
+ }
+ otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
+ errorLogger.WarnContext(ctx, "OpenTelemetry export failed", "error", err)
+ }))
+ if tel.tracerProvider != nil {
+ otel.SetTracerProvider(tel.tracerProvider)
+ otel.SetTextMapPropagator(propagation.TraceContext{})
+ }
+ if tel.meterProvider != nil {
+ otel.SetMeterProvider(tel.meterProvider)
+ }
+ if tel.loggerProvider != nil {
+ global.SetLoggerProvider(tel.loggerProvider)
+ }
+ return tel, nil
+}
+
+// Shutdown flushes and stops every provider. It is safe to call more than once.
+func (t *telemetryProviders) Shutdown(ctx context.Context) error {
+ t.shutdownOnce.Do(func() {
+ var errs []error
+ if t.tracerProvider != nil {
+ errs = append(errs, t.tracerProvider.Shutdown(ctx))
+ }
+ if t.meterProvider != nil {
+ errs = append(errs, t.meterProvider.Shutdown(ctx))
+ }
+ if t.loggerProvider != nil {
+ errs = append(errs, t.loggerProvider.Shutdown(ctx))
+ }
+ t.shutdownErr = errors.Join(errs...)
+ })
+ return t.shutdownErr
+}
+
+var (
+ urlUserinfo = regexp.MustCompile(`(https?://)[^@/\s"']*@`)
+ urlQuery = regexp.MustCompile(`(https?://[^\s?#"']*)[?#][^\s"']*`)
+ webhookToken = regexp.MustCompile(`(/webhooks/trigger/)[^/?#\s"']+`)
+)
+
+// redactSecrets strips URL credentials, queries, and fragments, and masks webhook trigger tokens, which travel in the path.
+func redactSecrets(s string) string {
+ s = urlUserinfo.ReplaceAllString(s, "$1")
+ return webhookToken.ReplaceAllString(urlQuery.ReplaceAllString(s, "$1"), "${1}:token")
+}
+
+// redactingExporter strips URL queries and fragments from span attributes, events, and statuses before export.
+type redactingExporter struct{ trace.SpanExporter }
+
+func (e redactingExporter) ExportSpans(ctx context.Context, spans []trace.ReadOnlySpan) error {
+ redacted := make([]trace.ReadOnlySpan, len(spans))
+ for i, span := range spans {
+ redacted[i] = redactedSpan{span}
+ }
+ return e.SpanExporter.ExportSpans(ctx, redacted)
+}
+
+type redactedSpan struct{ trace.ReadOnlySpan }
+
+func (s redactedSpan) Attributes() []attribute.KeyValue {
+ return redactAttributes(s.ReadOnlySpan.Attributes())
+}
+
+func (s redactedSpan) Events() []trace.Event {
+ events := slices.Clone(s.ReadOnlySpan.Events())
+ for i := range events {
+ events[i].Attributes = redactAttributes(events[i].Attributes)
+ }
+ return events
+}
+
+func (s redactedSpan) Status() trace.Status {
+ status := s.ReadOnlySpan.Status()
+ status.Description = redactSecrets(status.Description)
+ return status
+}
+
+func redactAttributes(attrs []attribute.KeyValue) []attribute.KeyValue {
+ redacted := make([]attribute.KeyValue, 0, len(attrs))
+ for _, kv := range attrs {
+ switch {
+ case kv.Key == "url.query":
+ continue
+ case kv.Value.Type() == attribute.STRING:
+ kv.Value = attribute.StringValue(redactSecrets(kv.Value.AsString()))
+ }
+ redacted = append(redacted, kv)
+ }
+ return redacted
+}
diff --git a/backend/internal/container/children/stats/service.go b/backend/internal/container/children/stats/service.go
index 6948eca13..30f4e0617 100644
--- a/backend/internal/container/children/stats/service.go
+++ b/backend/internal/container/children/stats/service.go
@@ -1,4 +1,4 @@
-// Package stats owns container resource sampling, resource sorting and the
+// Package stats owns container resource sampling, resource metrics, resource sorting and the
// live stats stream.
package stats
@@ -10,6 +10,8 @@ import (
"fmt"
"io"
"log/slog"
+ "maps"
+ "slices"
"strings"
"sync"
"time"
@@ -18,8 +20,11 @@ import (
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
"github.com/samber/hot"
+ "go.getarcane.app/docker"
"go.getarcane.app/kit/pkg"
"go.getarcane.app/streams/stats"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/singleflight"
@@ -33,6 +38,7 @@ const (
ContainerResourceSampleCacheSize = 4096
ContainerResourceCollectConcurrency = 16
ContainerResourceCollectTimeout = 3 * time.Second
+ ContainerMetricsCollectTimeout = 20 * time.Second
)
// Service collects container stats through the parent's Docker client.
@@ -40,7 +46,8 @@ type Service struct {
getClient func(context.Context) (*client.Client, error)
dockerHost func() string
history stats.Store
- cache *hot.HotCache[string, containertypes.ResourceSample]
+ cache *hot.HotCache[string, container.StatsResponse]
+ inspectCache *hot.HotCache[string, container.InspectResponse]
flight singleflight.Group
sampleTimeout time.Duration
batchTimeout time.Duration
@@ -57,10 +64,15 @@ func New(getClient func(context.Context) (*client.Client, error), dockerHost fun
return &Service{
getClient: getClient,
dockerHost: dockerHost,
- cache: hot.NewHotCache[string, containertypes.ResourceSample](hot.LRU, ContainerResourceSampleCacheSize).
+ cache: hot.NewHotCache[string, container.StatsResponse](hot.LRU, ContainerResourceSampleCacheSize).
WithTTL(cacheTTL).
WithJanitor().
Build(),
+ // Restart count and start time only change on restart, so metrics reuse inspects for a minute.
+ inspectCache: hot.NewHotCache[string, container.InspectResponse](hot.LRU, ContainerResourceSampleCacheSize).
+ WithTTL(time.Minute).
+ WithJanitor().
+ Build(),
sampleTimeout: sampleTimeout,
batchTimeout: batchTimeout,
}
@@ -107,12 +119,7 @@ func (s *Service) Stream(ctx context.Context, containerID string, statsChan chan
StatsResponse: statsData,
CurrentHistorySample: stats.BuildSample(statsData),
}
- payload.StatsHistory = s.history.Record(
- containerID,
- payload.CurrentHistorySample,
- !historySent,
- recordedAt,
- )
+ payload.StatsHistory = s.history.Record(containerID, payload.CurrentHistorySample, !historySent, recordedAt)
historySent = true
select {
@@ -123,26 +130,45 @@ func (s *Service) Stream(ctx context.Context, containerID string, statsChan chan
}
}
-// Collect fetches one sample per running container. Individual failures leave
-// the container unsampled; a batch timeout or cancellation fails the request
-// so partial lists never sort as global.
+// Collect fetches one sample per running container. Individual failures leave the container
+// unsampled; a batch timeout or cancellation fails so partial lists never sort as global.
func (s *Service) Collect(ctx context.Context, items []containertypes.Summary) (map[string]*containertypes.ResourceSample, error) {
- batchCtx, cancel := context.WithTimeout(ctx, s.batchTimeout)
+ ids := make([]string, 0, len(items))
+ for i := range items {
+ if items[i].State == string(container.StateRunning) {
+ ids = append(ids, items[i].ID)
+ }
+ }
+ raw, err := collect(ctx, s.batchTimeout, ids, s.fetch)
+ if err != nil {
+ return nil, fmt.Errorf("failed to collect container resource samples: %w", err)
+ }
+ samples := make(map[string]*containertypes.ResourceSample, len(raw))
+ for id, statsData := range raw {
+ built := stats.BuildSample(statsData)
+ samples[id] = &containertypes.ResourceSample{
+ CPUPercent: float64(built.CPUTenths) / 10,
+ MemoryUsageBytes: built.MemoryUsageBytes,
+ MemoryLimitBytes: statsData.MemoryStats.Limit,
+ SampleTime: statsData.Read,
+ }
+ }
+ return samples, nil
+}
+
+// collect runs fetch for each ID with bounded concurrency under timeout. Individual failures are
+// skipped; on batch timeout or cancellation it returns the partial result with the error.
+func collect[T any](ctx context.Context, timeout time.Duration, ids []string, fetch func(context.Context, string) (T, error)) (map[string]T, error) {
+ batchCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
g, groupCtx := errgroup.WithContext(batchCtx)
g.SetLimit(ContainerResourceCollectConcurrency)
-
- samples := make(map[string]*containertypes.ResourceSample, len(items))
+ results := make(map[string]T, len(ids))
var mu sync.Mutex
-
- for i := range items {
- if items[i].State != string(container.StateRunning) {
- continue
- }
- id := items[i].ID
+ for _, id := range ids {
g.Go(func() error {
- sample, err := s.fetchInternal(groupCtx, id)
+ result, err := fetch(groupCtx, id)
if err != nil {
if groupCtx.Err() != nil {
return groupCtx.Err()
@@ -151,93 +177,200 @@ func (s *Service) Collect(ctx context.Context, items []containertypes.Summary) (
return nil
}
mu.Lock()
- samples[id] = sample
+ results[id] = result
mu.Unlock()
return nil
})
}
-
- if err := g.Wait(); err != nil {
- return nil, fmt.Errorf("failed to collect container resource samples: %w", err)
- }
- return samples, nil
-}
-
-// cacheKeyInternal isolates cache and coalescing entries by Docker daemon and
-// container ID.
-func (s *Service) cacheKeyInternal(containerID string) string {
- return s.dockerHost() + "\x00" + containerID
+ return results, g.Wait()
}
-// fetchInternal returns a cached sample when fresh and coalesces concurrent
-// fetches for the same container.
-func (s *Service) fetchInternal(ctx context.Context, containerID string) (*containertypes.ResourceSample, error) {
- key := s.cacheKeyInternal(containerID)
+// fetch returns a cached stats sample when fresh and coalesces concurrent fetches per daemon and
+// container. IncludePreviousSample gives the CPU delta a valid previous sample.
+func (s *Service) fetch(ctx context.Context, containerID string) (container.StatsResponse, error) {
+ key := s.dockerHost() + "\x00" + containerID
if sample, ok, _ := s.cache.Get(key); ok {
- return &sample, nil
+ return sample, nil
}
ch := s.flight.DoChan(key, func() (any, error) {
flightCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.sampleTimeout)
defer cancel()
- return s.sampleInternal(flightCtx, containerID)
+
+ dockerClient, err := s.getClient(flightCtx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to connect to Docker: %w", err)
+ }
+ statsResponse, err := dockerClient.ContainerStats(flightCtx, containerID, client.ContainerStatsOptions{Stream: false, IncludePreviousSample: true})
+ if err != nil {
+ return nil, fmt.Errorf("failed to fetch container stats: %w", err)
+ }
+ defer func() { _ = statsResponse.Body.Close() }()
+
+ var statsData container.StatsResponse
+ if decodeErr := json.UnmarshalDecode(jsontext.NewDecoder(statsResponse.Body), &statsData); decodeErr != nil {
+ return nil, fmt.Errorf("failed to decode container stats: %w", decodeErr)
+ }
+ if statsData.Read.IsZero() {
+ statsData.Read = time.Now()
+ }
+ s.cache.Set(key, statsData)
+ return statsData, nil
})
fetchCtx, cancel := context.WithTimeout(ctx, s.sampleTimeout)
defer cancel()
-
select {
case <-fetchCtx.Done():
- return nil, fetchCtx.Err()
+ return container.StatsResponse{}, fetchCtx.Err()
case res := <-ch:
if res.Err != nil {
- return nil, res.Err
+ return container.StatsResponse{}, res.Err
}
- sample, ok := res.Val.(*containertypes.ResourceSample)
+ statsData, ok := res.Val.(container.StatsResponse)
if !ok {
- return nil, errors.New("resource sample flight returned unexpected type")
+ return container.StatsResponse{}, errors.New("resource sample flight returned unexpected type")
}
- return sample, nil
+ return statsData, nil
}
}
-// sampleInternal performs the non-streaming Docker stats call.
-// IncludePreviousSample gives the CPU delta a valid previous sample.
-func (s *Service) sampleInternal(ctx context.Context, containerID string) (*containertypes.ResourceSample, error) {
- dockerClient, err := s.getClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
+type resourceSample struct {
+ stats container.StatsResponse
+ inspect *container.InspectResponse
+}
- statsResponse, err := dockerClient.ContainerStats(ctx, containerID, client.ContainerStatsOptions{Stream: false, IncludePreviousSample: true})
- if err != nil {
- return nil, fmt.Errorf("failed to fetch container stats: %w", err)
+type resourceInstruments struct {
+ cpuTime metric.Float64ObservableCounter
+ cpuUtilization, uptime metric.Float64ObservableGauge
+ memoryUsage, memoryLimit metric.Int64ObservableGauge
+ pids metric.Int64ObservableGauge
+ networkIO, diskIO metric.Int64ObservableCounter
+ restarts metric.Int64ObservableCounter
+}
+
+// ObserveResources reports arcane.container.* metrics for running containers from list, sharing the
+// sample cache and flights with Collect so overlapping collections reuse Docker calls.
+func (s *Service) ObserveResources(meter metric.Meter, list func(context.Context) ([]container.Summary, error)) error {
+ var r resourceInstruments
+ var errs [9]error
+ r.cpuTime, errs[0] = meter.Float64ObservableCounter("arcane.container.cpu.time",
+ metric.WithDescription("Cumulative CPU time consumed by a running container"), metric.WithUnit("s"))
+ r.cpuUtilization, errs[1] = meter.Float64ObservableGauge("arcane.container.cpu.utilization",
+ metric.WithDescription("Container CPU usage as a fraction of total host CPU capacity"), metric.WithUnit("1"))
+ r.memoryUsage, errs[2] = meter.Int64ObservableGauge("arcane.container.memory.usage",
+ metric.WithDescription("Container memory usage excluding inactive page cache"), metric.WithUnit("By"))
+ r.memoryLimit, errs[3] = meter.Int64ObservableGauge("arcane.container.memory.limit",
+ metric.WithDescription("Container memory limit, or host memory when unlimited"), metric.WithUnit("By"))
+ r.pids, errs[4] = meter.Int64ObservableGauge("arcane.container.pids",
+ metric.WithDescription("Processes and threads running in a container"), metric.WithUnit("{process}"))
+ r.networkIO, errs[5] = meter.Int64ObservableCounter("arcane.container.network.io",
+ metric.WithDescription("Bytes received and transmitted across all container network interfaces"), metric.WithUnit("By"))
+ r.diskIO, errs[6] = meter.Int64ObservableCounter("arcane.container.disk.io",
+ metric.WithDescription("Bytes read from and written to block devices by a container"), metric.WithUnit("By"))
+ r.restarts, errs[7] = meter.Int64ObservableCounter("arcane.container.restarts",
+ metric.WithDescription("Times Docker restarted the container under its restart policy"), metric.WithUnit("{restart}"))
+ r.uptime, errs[8] = meter.Float64ObservableGauge("arcane.container.uptime",
+ metric.WithDescription("Time since the container last started"), metric.WithUnit("s"))
+ if err := errors.Join(errs[:]...); err != nil {
+ return err
}
- defer func() { _ = statsResponse.Body.Close() }()
- var statsData container.StatsResponse
- if unmarshalDecodeErr := json.UnmarshalDecode(jsontext.NewDecoder(statsResponse.Body), &statsData); unmarshalDecodeErr != nil {
- return nil, fmt.Errorf("failed to decode container stats: %w", unmarshalDecodeErr)
+ _, err := meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
+ containers, err := list(ctx)
+ if err != nil {
+ slog.DebugContext(ctx, "Skipping container resource metrics", "error", err)
+ return nil
+ }
+ running := make(map[string]container.Summary, len(containers))
+ for _, c := range containers {
+ if c.State == container.StateRunning {
+ running[c.ID] = c
+ }
+ }
+ samples, err := collect(ctx, ContainerMetricsCollectTimeout, slices.Collect(maps.Keys(running)), func(ctx context.Context, id string) (resourceSample, error) {
+ statsData, fetchErr := s.fetch(ctx, id)
+ if fetchErr != nil {
+ return resourceSample{}, fetchErr
+ }
+ sample := resourceSample{stats: statsData}
+ if cached, ok, _ := s.inspectCache.Get(id); ok {
+ sample.inspect = &cached
+ } else if dockerClient, clientErr := s.getClient(ctx); clientErr == nil {
+ if inspect, inspectErr := dockerClient.ContainerInspect(ctx, id, client.ContainerInspectOptions{}); inspectErr == nil {
+ s.inspectCache.Set(id, inspect.Container)
+ sample.inspect = &inspect.Container
+ }
+ }
+ return sample, nil
+ })
+ if err != nil {
+ slog.DebugContext(ctx, "Container resource metrics are partial", "error", err)
+ }
+ for id, sample := range samples {
+ r.observe(o, running[id], sample)
+ }
+ return nil
+ }, r.cpuTime, r.cpuUtilization, r.memoryUsage, r.memoryLimit, r.pids, r.networkIO, r.diskIO, r.restarts, r.uptime)
+ return err
+}
+
+// observe records one container's sample, labelled by name, image and Compose project/service.
+func (r *resourceInstruments) observe(o metric.Observer, c container.Summary, sample resourceSample) {
+ attrs := []attribute.KeyValue{
+ attribute.String("container.name", docker.ContainerNameFromNames(c.Names)),
+ attribute.String("container.image.name", c.Image),
+ }
+ if project := docker.ComposeProjectLabel(c.Labels); project != "" {
+ attrs = append(attrs, attribute.String("compose.project", project))
+ }
+ if service := docker.ComposeServiceLabel(c.Labels); service != "" {
+ attrs = append(attrs, attribute.String("compose.service", service))
+ }
+ labels := metric.WithAttributes(attrs...)
+ with := func(kv attribute.KeyValue) metric.MeasurementOption {
+ return metric.WithAttributes(append(attrs[:len(attrs):len(attrs)], kv)...)
}
+ statsData := sample.stats
built := stats.BuildSample(statsData)
- sample := &containertypes.ResourceSample{
- CPUPercent: float64(built.CPUTenths) / 10,
- MemoryUsageBytes: built.MemoryUsageBytes,
- MemoryLimitBytes: statsData.MemoryStats.Limit,
- SampleTime: statsData.Read,
+ o.ObserveFloat64(r.cpuTime, float64(statsData.CPUStats.CPUUsage.TotalUsage)/float64(time.Second), labels)
+ o.ObserveFloat64(r.cpuUtilization, float64(built.CPUTenths)/1000, labels)
+ o.ObserveInt64(r.memoryUsage, int64(built.MemoryUsageBytes), labels)
+ o.ObserveInt64(r.memoryLimit, int64(statsData.MemoryStats.Limit), labels)
+ o.ObserveInt64(r.pids, int64(statsData.PidsStats.Current), labels)
+
+ var rx, tx, read, write uint64
+ for _, network := range statsData.Networks {
+ rx += network.RxBytes
+ tx += network.TxBytes
}
- if sample.SampleTime.IsZero() {
- sample.SampleTime = time.Now()
+ for _, entry := range statsData.BlkioStats.IoServiceBytesRecursive {
+ switch strings.ToLower(entry.Op) {
+ case "read":
+ read += entry.Value
+ case "write":
+ write += entry.Value
+ }
}
+ o.ObserveInt64(r.networkIO, int64(rx), with(attribute.String("network.io.direction", "receive")))
+ o.ObserveInt64(r.networkIO, int64(tx), with(attribute.String("network.io.direction", "transmit")))
+ o.ObserveInt64(r.diskIO, int64(read), with(attribute.String("disk.io.direction", "read")))
+ o.ObserveInt64(r.diskIO, int64(write), with(attribute.String("disk.io.direction", "write")))
- s.cache.Set(s.cacheKeyInternal(containerID), *sample)
- return sample, nil
+ if sample.inspect == nil {
+ return
+ }
+ o.ObserveInt64(r.restarts, int64(sample.inspect.RestartCount), labels)
+ if sample.inspect.State != nil {
+ if startedAt, err := time.Parse(time.RFC3339Nano, sample.inspect.State.StartedAt); err == nil {
+ o.ObserveFloat64(r.uptime, time.Since(startedAt).Seconds(), labels)
+ }
+ }
}
-// ContainerResourceSampleSort orders summaries by their collected sample
-// value, keeping unsampled containers last in both directions and breaking
-// ties by name, then ID. Valid zero values sort as zero, not as unavailable.
+// ContainerResourceSampleSort orders summaries by sample value with unsampled containers last in both
+// directions, breaking ties by name then ID. Valid zero values sort as zero, not as unavailable.
func ContainerResourceSampleSort(sort string, descending bool) pagination.SortOption[containertypes.Summary] {
value := kit.Ternary(
sort == containertypes.SortCPUUsage,
diff --git a/backend/internal/container/module.go b/backend/internal/container/module.go
index 9e46bc57e..45238a572 100644
--- a/backend/internal/container/module.go
+++ b/backend/internal/container/module.go
@@ -6,6 +6,7 @@ import (
"net/http"
"github.com/danielgtaylor/huma/v2"
+ "go.opentelemetry.io/otel"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
@@ -13,6 +14,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
// Module wires the container domain and mounts its routes.
@@ -23,8 +25,13 @@ type Module struct {
activityService *activity.ActivityService
}
-// New wires container routes around an existing service.
+// New wires container routes around an existing service and registers per-container resource metrics once.
func New(service *ContainerService, dockerService *docker.DockerClientService, settingsService *settings.SettingsService, activityService *activity.ActivityService) *Module {
+ if service != nil && service.stats != nil && service.dockerService != nil {
+ if err := service.stats.ObserveResources(otel.Meter(tracing.InstrumentationName), service.dockerService.ListContainers); err != nil {
+ otel.Handle(err)
+ }
+ }
return &Module{service: service, dockerService: dockerService, settingsService: settingsService, activityService: activityService}
}
diff --git a/backend/internal/dashboard/module.go b/backend/internal/dashboard/module.go
index f27cc6d80..767fe7232 100644
--- a/backend/internal/dashboard/module.go
+++ b/backend/internal/dashboard/module.go
@@ -1,17 +1,18 @@
-// Package dashboard owns the dashboard snapshot: the service that assembles it
-// from the container, project, image, volume and vulnerability domains, and the
-// HTTP surface that serves it.
+// Package dashboard owns the dashboard snapshot assembled from the container, project, image,
+// volume and vulnerability domains, the HTTP surface that serves it, and local inventory metrics.
package dashboard
import (
"net/http"
"github.com/danielgtaylor/huma/v2"
+ "go.opentelemetry.io/otel"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
// Module is the dashboard domain's wiring seam: it owns the service and the
@@ -21,8 +22,13 @@ type Module struct {
handler *DashboardHandler
}
-// New wires dashboard routes around an existing service.
+// New wires dashboard routes around an existing service and registers its inventory metrics once.
func New(service *DashboardService, environmentService *environment.EnvironmentService) *Module {
+ if service != nil {
+ if err := service.ObserveInventory(otel.Meter(tracing.InstrumentationName)); err != nil {
+ otel.Handle(err)
+ }
+ }
return &Module{service: service, handler: NewHandler(service, environmentService)}
}
diff --git a/backend/internal/dashboard/service.go b/backend/internal/dashboard/service.go
index 54e427f1b..676c72e5a 100644
--- a/backend/internal/dashboard/service.go
+++ b/backend/internal/dashboard/service.go
@@ -1,25 +1,30 @@
package dashboard
import (
+ "cmp"
"context"
"errors"
"fmt"
+ "log/slog"
"sort"
"strconv"
"sync/atomic"
"time"
- "github.com/getarcaneapp/arcane/types/v2/base"
containertypes "github.com/getarcaneapp/arcane/types/v2/container"
"github.com/getarcaneapp/arcane/types/v2/dashboard"
imagetypes "github.com/getarcaneapp/arcane/types/v2/image"
versiontypes "github.com/getarcaneapp/arcane/types/v2/version"
volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"
dockercontainer "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/api/types/swarm"
+ "github.com/moby/moby/client"
"go.getarcane.app/docker"
"go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/cgroup"
"go.getarcane.app/updater/labels"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/singleflight"
@@ -34,7 +39,9 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/version"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/iconcatalog"
)
@@ -43,6 +50,7 @@ const (
dashboardSnapshotPreloadLimit = 50
dashboardSnapshotCacheTTL = 5 * time.Second
dashboardSnapshotBuildTimeout = 30 * time.Second
+ dashboardInventoryCollectTimeout = 15 * time.Second
)
type DashboardService struct {
@@ -57,18 +65,13 @@ type DashboardService struct {
versionService *version.VersionService
volumeService *volume.VolumeService
- // snapshotFlight + snapshotCache share one snapshot build across all
- // concurrent dashboard consumers (HTTP first paint + every stream
- // subscriber). Cached snapshots are shared pointers and must be treated
- // as immutable by callers. Indexed by
- // [includeTables][debugAllGood][iconCatalog]: full snapshots resolve
- // container icons against the requesting user's icon catalog, so each
- // catalog gets its own entry.
+ // Shared, immutable snapshot builds indexed by [includeTables][debugAllGood][iconCatalog];
+ // full snapshots resolve icons per catalog, so each catalog gets its own entry.
snapshotFlight singleflight.Group
- snapshotCache [2][2][2]atomic.Pointer[dashboardSnapshotCacheEntryInternal]
+ snapshotCache [2][2][2]atomic.Pointer[dashboardSnapshotCacheEntry]
}
-type dashboardSnapshotCacheEntryInternal struct {
+type dashboardSnapshotCacheEntry struct {
snapshot *dashboard.Snapshot
builtAt time.Time
}
@@ -103,16 +106,10 @@ func NewDashboardService(
}
}
-// GetSnapshot returns the dashboard snapshot, shared across all concurrent
-// consumers (HTTP first paint + every stream subscriber) through a short-TTL
-// cache; the result is a shared pointer and must not be mutated.
-// includeTables controls whether the first-page container/image tables are
-// built: stream subscribers pass false since the all-environments dashboard
-// only reads the aggregate counters, skipping the per-container DTO builds,
-// sorting, and icon resolution entirely.
+// GetSnapshot returns a short-TTL cached snapshot shared by all consumers; it must not be mutated.
+// includeTables builds the first-page tables; stream subscribers pass false and read only counters.
func (s *DashboardService) GetSnapshot(ctx context.Context, options DashboardActionItemsOptions, includeTables bool) (*dashboard.Snapshot, error) {
- // Trimmed snapshots skip icon resolution entirely, so every consumer
- // shares one entry regardless of the requesting user's catalog.
+ // Trimmed snapshots skip icon resolution, so every consumer shares one entry.
catalog := iconcatalog.DefaultCatalog
if includeTables {
catalog = iconcatalog.Normalize(project.IconCatalogForContext(ctx))
@@ -128,8 +125,7 @@ func (s *DashboardService) GetSnapshot(ctx context.Context, options DashboardAct
key := strconv.FormatBool(includeTables) + ":" + strconv.FormatBool(options.DebugAllGood) + ":" + catalog
result, err, _ := s.snapshotFlight.Do(key, func() (any, error) {
- // Re-check under the flight: a caller that queued behind the winner
- // finds the fresh entry here instead of rebuilding.
+ // Re-check so a caller queued behind the winner reuses its fresh entry.
if entry := slot.Load(); entry != nil && time.Since(entry.builtAt) < dashboardSnapshotCacheTTL {
return entry.snapshot, nil
}
@@ -137,11 +133,11 @@ func (s *DashboardService) GetSnapshot(ctx context.Context, options DashboardAct
buildCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dashboardSnapshotBuildTimeout)
defer cancel()
- snapshot, err := s.buildSnapshotInternal(buildCtx, options, includeTables)
+ snapshot, err := s.buildSnapshot(buildCtx, options, includeTables)
if err != nil {
return nil, err
}
- slot.Store(&dashboardSnapshotCacheEntryInternal{snapshot: snapshot, builtAt: time.Now()})
+ slot.Store(&dashboardSnapshotCacheEntry{snapshot: snapshot, builtAt: time.Now()})
return snapshot, nil
})
if err != nil {
@@ -154,7 +150,7 @@ func (s *DashboardService) GetSnapshot(ctx context.Context, options DashboardAct
return snapshot, nil
}
-func (s *DashboardService) buildSnapshotInternal(ctx context.Context, options DashboardActionItemsOptions, includeTables bool) (*dashboard.Snapshot, error) {
+func (s *DashboardService) buildSnapshot(ctx context.Context, options DashboardActionItemsOptions, includeTables bool) (*dashboard.Snapshot, error) {
if s.dockerService == nil {
return nil, errors.New("docker service not available")
}
@@ -228,15 +224,14 @@ func (s *DashboardService) buildSnapshotInternal(ctx context.Context, options Da
imageUsageCounts := dockerInternal.CountImageUsage(dockerImages, imageConsumers)
- // Uses the unfiltered container list so a volume mounted only by an internal
- // container still counts as in use, matching the volumes page.
+ // Unfiltered containers keep volumes mounted only by internal containers in use, matching the volumes page.
var volumeUsageCounts *volumetypes.UsageCounts
if s.volumeService != nil && dockerSnapshot.Volumes != nil {
counts := s.volumeService.CountUsageFromSnapshot(dockerSnapshot.Volumes.Items, dockerContainers)
volumeUsageCounts = &counts
}
- actionItems, err := s.buildActionItemsForSnapshotInternal(ctx, options, filteredContainers, dockerContainers)
+ actionItems, err := s.buildActionItems(ctx, options, containerCounts.StoppedContainers, dockerContainers)
if err != nil {
return nil, err
}
@@ -246,15 +241,17 @@ func (s *DashboardService) buildSnapshotInternal(ctx context.Context, options Da
versionInfo = s.versionService.GetAppVersionInfo(ctx)
}
+ preloadParams := pagination.QueryParams{Params: pagination.Params{Limit: dashboardSnapshotPreloadLimit}}
+
return &dashboard.Snapshot{
Containers: dashboard.SnapshotContainers{
Data: containerPage,
Counts: containerCounts,
- Pagination: buildDashboardPaginationResponseInternal(len(filteredContainers), dashboardSnapshotPreloadLimit),
+ Pagination: handlerutil.PaginationResponse(pagination.BuildResponse(int64(len(filteredContainers)), int64(len(filteredContainers)), preloadParams)),
},
Images: dashboard.SnapshotImages{
Data: imagePage,
- Pagination: buildDashboardPaginationResponseInternal(len(dockerImages), dashboardSnapshotPreloadLimit),
+ Pagination: handlerutil.PaginationResponse(pagination.BuildResponse(int64(len(dockerImages)), int64(len(dockerImages)), preloadParams)),
},
ImageUsageCounts: imageUsageCounts,
VolumeUsageCounts: volumeUsageCounts,
@@ -264,15 +261,9 @@ func (s *DashboardService) buildSnapshotInternal(ctx context.Context, options Da
}, nil
}
-// buildActionItemsForSnapshotInternal derives the dashboard's action badges.
-// filteredContainers drives the stopped-container badge; allContainers is the raw
-// snapshot list, passed to the update count so it need not re-list containers.
-func (s *DashboardService) buildActionItemsForSnapshotInternal(
- ctx context.Context,
- options DashboardActionItemsOptions,
- filteredContainers []dockercontainer.Summary,
- allContainers []dockercontainer.Summary,
-) (*dashboard.ActionItems, error) {
+// buildActionItems derives the dashboard's action badges; allContainers is the raw snapshot
+// list, reused so the update count need not re-list containers.
+func (s *DashboardService) buildActionItems(ctx context.Context, options DashboardActionItemsOptions, stoppedContainers int, allContainers []dockercontainer.Summary) (*dashboard.ActionItems, error) {
if options.DebugAllGood {
return &dashboard.ActionItems{Items: []dashboard.ActionItem{}}, nil
}
@@ -288,33 +279,56 @@ func (s *DashboardService) buildActionItemsForSnapshotInternal(
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "dashboard action item worker")
- count, err := s.getPendingResourceUpdatesCountInternal(groupCtx, allContainers)
+ if s.db == nil || s.dockerService == nil {
+ return nil
+ }
+ standaloneContainers := make([]dockercontainer.Summary, 0, len(allContainers))
+ for _, c := range container.FilterExcludedContainers(allContainers, false, false) {
+ if docker.ComposeProjectLabel(c.Labels) == "" {
+ standaloneContainers = append(standaloneContainers, c)
+ }
+ }
+ containerCount, err := s.getPendingContainerUpdatesCount(groupCtx, standaloneContainers)
if err != nil {
return err
}
- pendingResourceUpdates = count
+ // Hidden service identities stay so project counting can exclude their Compose and cached image records.
+ projectCount := 0
+ if s.projectService != nil {
+ if projectCount, err = s.projectService.CountProjectsWithPendingUpdates(groupCtx, allContainers); err != nil {
+ return fmt.Errorf("failed to count projects with updates: %w", err)
+ }
+ }
+ pendingResourceUpdates = containerCount + projectCount
return nil
})
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "dashboard action item worker")
- count, err := s.getActionableVulnerabilitiesCountInternal(groupCtx)
- if err != nil {
- return err
+ if s.vulnerabilityService == nil {
+ return nil
}
- actionableVulnerabilities = count
- return nil
+ actionableVulnerabilities, workerErr = s.vulnerabilityService.ActionableCountExcludingIgnored(groupCtx)
+ return workerErr
})
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "dashboard action item worker")
- count, err := s.getExpiringAPIKeysCountInternal(groupCtx)
+ if s.db == nil {
+ return nil
+ }
+ var count int64
+ err := s.db.WithContext(groupCtx).
+ Model(&apikey.ApiKey{}).
+ Where("expires_at IS NOT NULL").
+ Where("expires_at <= ?", time.Now().Add(defaultDashboardAPIKeyExpiryWindow)).
+ Count(&count).Error
if err != nil {
- return err
+ return fmt.Errorf("failed to count expiring API keys: %w", err)
}
- expiringAPIKeys = count
+ expiringAPIKeys = int(count)
return nil
})
@@ -322,99 +336,24 @@ func (s *DashboardService) buildActionItemsForSnapshotInternal(
return nil, err
}
- stoppedContainers := 0
- for _, container := range filteredContainers {
- if container.State != "running" {
- stoppedContainers++
- }
- }
-
- return buildDashboardActionItemsInternal(stoppedContainers, pendingResourceUpdates, actionableVulnerabilities, expiringAPIKeys), nil
-}
-
-func buildDashboardActionItemsInternal(
- stoppedContainers int,
- pendingResourceUpdates int,
- actionableVulnerabilities int,
- expiringAPIKeys int,
-) *dashboard.ActionItems {
- actionItems := make([]dashboard.ActionItem, 0, 4)
-
- if stoppedContainers > 0 {
- actionItems = append(actionItems, dashboard.ActionItem{
- Kind: dashboard.ActionItemKindStoppedContainers,
- Count: stoppedContainers,
- Severity: dashboard.ActionItemSeverityWarning,
- })
- }
-
- if pendingResourceUpdates > 0 {
- actionItems = append(actionItems, dashboard.ActionItem{
- Kind: dashboard.ActionItemKindImageUpdates,
- Count: pendingResourceUpdates,
- Severity: dashboard.ActionItemSeverityWarning,
- })
- }
-
- if actionableVulnerabilities > 0 {
- actionItems = append(actionItems, dashboard.ActionItem{
- Kind: dashboard.ActionItemKindActionableVulnerabilities,
- Count: actionableVulnerabilities,
- Severity: dashboard.ActionItemSeverityCritical,
- })
- }
-
- if expiringAPIKeys > 0 {
- actionItems = append(actionItems, dashboard.ActionItem{
- Kind: dashboard.ActionItemKindExpiringKeys,
- Count: expiringAPIKeys,
- Severity: dashboard.ActionItemSeverityWarning,
- })
- }
-
- return &dashboard.ActionItems{Items: actionItems}
-}
-
-// getPendingResourceUpdatesCountInternal counts standalone containers and projects
-// with a pending image update. allContainers is the snapshot's container list,
-// reused rather than re-listed: this used to issue its own GetAllContainers on
-// top of the two the project count triggered, for a single badge number.
-func (s *DashboardService) getPendingResourceUpdatesCountInternal(ctx context.Context, allContainers []dockercontainer.Summary) (int, error) {
- if s.db == nil || s.dockerService == nil {
- return 0, nil
- }
-
- filteredContainers := container.FilterExcludedContainers(allContainers, false, false)
- standaloneContainers := filterStandaloneDockerContainersInternal(filteredContainers)
- containerCount, err := s.getPendingContainerUpdatesCountInternal(ctx, standaloneContainers)
- if err != nil {
- return 0, err
- }
-
- // Keep hidden service identities so project counting can exclude their Compose and cached image records.
- projectCount, err := s.getPendingProjectUpdatesCountInternal(ctx, allContainers)
- if err != nil {
- return 0, err
+ candidates := []dashboard.ActionItem{
+ {Kind: dashboard.ActionItemKindStoppedContainers, Count: stoppedContainers, Severity: dashboard.ActionItemSeverityWarning},
+ {Kind: dashboard.ActionItemKindImageUpdates, Count: pendingResourceUpdates, Severity: dashboard.ActionItemSeverityWarning},
+ {Kind: dashboard.ActionItemKindActionableVulnerabilities, Count: actionableVulnerabilities, Severity: dashboard.ActionItemSeverityCritical},
+ {Kind: dashboard.ActionItemKindExpiringKeys, Count: expiringAPIKeys, Severity: dashboard.ActionItemSeverityWarning},
}
-
- return containerCount + projectCount, nil
-}
-
-func filterStandaloneDockerContainersInternal(containers []dockercontainer.Summary) []dockercontainer.Summary {
- filtered := make([]dockercontainer.Summary, 0, len(containers))
- for _, c := range containers {
- if docker.ComposeProjectLabel(c.Labels) != "" {
- continue
+ items := make([]dashboard.ActionItem, 0, len(candidates))
+ for _, item := range candidates {
+ if item.Count > 0 {
+ items = append(items, item)
}
- filtered = append(filtered, c)
}
- return filtered
+ return &dashboard.ActionItems{Items: items}, nil
}
-// getPendingContainerUpdatesCountInternal counts standalone containers whose
-// stored check reports an update, through the same lookup the container list
-// uses. Containers sharing one image each count once.
-func (s *DashboardService) getPendingContainerUpdatesCountInternal(ctx context.Context, containers []dockercontainer.Summary) (int, error) {
+// getPendingContainerUpdatesCount counts standalone containers whose stored check reports an
+// update, using the container list's lookup. Containers sharing one image each count once.
+func (s *DashboardService) getPendingContainerUpdatesCount(ctx context.Context, containers []dockercontainer.Summary) (int, error) {
if s.imageService == nil || len(containers) == 0 {
return 0, nil
}
@@ -431,60 +370,134 @@ func (s *DashboardService) getPendingContainerUpdatesCountInternal(ctx context.C
return count, nil
}
-func (s *DashboardService) getPendingProjectUpdatesCountInternal(ctx context.Context, allContainers []dockercontainer.Summary) (int, error) {
- if s.projectService == nil {
- return 0, nil
- }
+type inventoryGauges struct {
+ containers, health, images, imageSize, volumes, networks, projects metric.Int64ObservableGauge
+ swarmNodes, swarmServices, swarmTasks metric.Int64ObservableGauge
+}
- count, err := s.projectService.CountProjectsWithPendingUpdates(ctx, allContainers)
- if err != nil {
- return 0, fmt.Errorf("failed to count projects with updates: %w", err)
- }
+// ObserveInventory reports the local Docker engine's inventory through arcane.docker.* and arcane.projects gauges.
+func (s *DashboardService) ObserveInventory(meter metric.Meter) error {
+ var g inventoryGauges
+ var errs [10]error
+ g.containers, errs[0] = meter.Int64ObservableGauge("arcane.docker.containers",
+ metric.WithDescription("Containers on the local Docker engine by state"), metric.WithUnit("{container}"))
+ g.health, errs[1] = meter.Int64ObservableGauge("arcane.docker.containers.health",
+ metric.WithDescription("Containers with a healthcheck by health status"), metric.WithUnit("{container}"))
+ g.images, errs[2] = meter.Int64ObservableGauge("arcane.docker.images",
+ metric.WithDescription("Images on the local Docker engine by usage"), metric.WithUnit("{image}"))
+ g.imageSize, errs[3] = meter.Int64ObservableGauge("arcane.docker.images.size",
+ metric.WithDescription("Total size of images on the local Docker engine"), metric.WithUnit("By"))
+ g.volumes, errs[4] = meter.Int64ObservableGauge("arcane.docker.volumes",
+ metric.WithDescription("Volumes on the local Docker engine by usage, excluding Arcane-internal volumes"), metric.WithUnit("{volume}"))
+ g.networks, errs[5] = meter.Int64ObservableGauge("arcane.docker.networks",
+ metric.WithDescription("Networks on the local Docker engine by usage; built-in networks report as default"), metric.WithUnit("{network}"))
+ g.projects, errs[6] = meter.Int64ObservableGauge("arcane.projects",
+ metric.WithDescription("Arcane Compose projects by status"), metric.WithUnit("{project}"))
+ g.swarmNodes, errs[7] = meter.Int64ObservableGauge("arcane.docker.swarm.nodes",
+ metric.WithDescription("Swarm nodes by role and state, reported by swarm managers"), metric.WithUnit("{node}"))
+ g.swarmServices, errs[8] = meter.Int64ObservableGauge("arcane.docker.swarm.services",
+ metric.WithDescription("Swarm services, reported by swarm managers"), metric.WithUnit("{service}"))
+ g.swarmTasks, errs[9] = meter.Int64ObservableGauge("arcane.docker.swarm.service.tasks",
+ metric.WithDescription("Running and desired tasks per swarm service, reported by swarm managers"), metric.WithUnit("{task}"))
+ if err := errors.Join(errs[:]...); err != nil {
+ return err
+ }
+
+ _, err := meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
+ ctx, cancel := context.WithTimeout(ctx, dashboardInventoryCollectTimeout)
+ defer cancel()
+ snapshot, err := s.dockerService.GetSnapshot(ctx, environment.LocalEnvironmentID)
+ if err != nil {
+ slog.DebugContext(ctx, "Skipping Docker inventory metrics", "error", err)
+ return nil
+ }
- return count, nil
-}
+ states := map[dockercontainer.ContainerState]int64{
+ dockercontainer.StateCreated: 0, dockercontainer.StateRunning: 0, dockercontainer.StatePaused: 0, dockercontainer.StateRestarting: 0,
+ dockercontainer.StateRemoving: 0, dockercontainer.StateExited: 0, dockercontainer.StateDead: 0,
+ }
+ health := map[dockercontainer.HealthStatus]int64{dockercontainer.Starting: 0, dockercontainer.Healthy: 0, dockercontainer.Unhealthy: 0}
+ for _, c := range snapshot.Containers {
+ states[c.State]++
+ if c.Health != nil && c.Health.Status != dockercontainer.NoHealthcheck {
+ health[c.Health.Status]++
+ }
+ }
+ for state, n := range states {
+ o.ObserveInt64(g.containers, n, metric.WithAttributes(attribute.String("container.state", string(state))))
+ }
+ for status, n := range health {
+ o.ObserveInt64(g.health, n, metric.WithAttributes(attribute.String("container.health.status", string(status))))
+ }
-func (s *DashboardService) getActionableVulnerabilitiesCountInternal(ctx context.Context) (int, error) {
- if s.vulnerabilityService == nil {
- return 0, nil
- }
+ images := dockerInternal.CountImageUsage(snapshot.Images, container.FilterExcludedContainers(snapshot.Containers, false, true))
+ o.ObserveInt64(g.images, int64(images.Inuse), metric.WithAttributes(attribute.String("usage", "in_use")))
+ o.ObserveInt64(g.images, int64(images.Unused), metric.WithAttributes(attribute.String("usage", "unused")))
+ o.ObserveInt64(g.imageSize, images.TotalSize)
- return s.vulnerabilityService.ActionableCountExcludingIgnored(ctx)
-}
+ if s.volumeService != nil && snapshot.Volumes != nil {
+ volumes := s.volumeService.CountUsageFromSnapshot(snapshot.Volumes.Items, snapshot.Containers)
+ o.ObserveInt64(g.volumes, int64(volumes.Inuse), metric.WithAttributes(attribute.String("usage", "in_use")))
+ o.ObserveInt64(g.volumes, int64(volumes.Unused), metric.WithAttributes(attribute.String("usage", "unused")))
+ }
-func (s *DashboardService) getExpiringAPIKeysCountInternal(ctx context.Context) (int, error) {
- if s.db == nil {
- return 0, nil
- }
+ if len(snapshot.Networks) > 0 {
+ inuse, unused, total := dockerInternal.CountNetworkUsage(snapshot.Networks, snapshot.Containers)
+ o.ObserveInt64(g.networks, int64(inuse), metric.WithAttributes(attribute.String("usage", "in_use")))
+ o.ObserveInt64(g.networks, int64(unused), metric.WithAttributes(attribute.String("usage", "unused")))
+ o.ObserveInt64(g.networks, int64(total-inuse-unused), metric.WithAttributes(attribute.String("usage", "default")))
+ }
- var count int64
- err := s.db.WithContext(ctx).
- Model(&apikey.ApiKey{}).
- Where("expires_at IS NOT NULL").
- Where("expires_at <= ?", time.Now().Add(defaultDashboardAPIKeyExpiryWindow)).
- Count(&count).Error
- if err != nil {
- return 0, fmt.Errorf("failed to count expiring API keys: %w", err)
- }
+ if s.projectService != nil {
+ if counts, countErr := s.projectService.GetProjectStatusCounts(ctx); countErr != nil {
+ slog.DebugContext(ctx, "Skipping project inventory metrics", "error", countErr)
+ } else {
+ other := counts.TotalProjects - counts.RunningProjects - counts.StoppedProjects - counts.ArchivedProjects
+ for status, n := range map[string]int{"running": counts.RunningProjects, "stopped": counts.StoppedProjects, "archived": counts.ArchivedProjects, "unknown": other} {
+ o.ObserveInt64(g.projects, int64(n), metric.WithAttributes(attribute.String("project.status", status)))
+ }
+ }
+ }
- return int(count), nil
+ s.observeSwarm(ctx, o, &g)
+ return nil
+ }, g.containers, g.health, g.images, g.imageSize, g.volumes, g.networks, g.projects, g.swarmNodes, g.swarmServices, g.swarmTasks)
+ return err
}
-func buildDashboardPaginationResponseInternal(totalItems, limit int) base.PaginationResponse {
- if limit <= 0 {
- limit = dashboardSnapshotPreloadLimit
+// observeSwarm reports nodes and services only when the local engine is a swarm manager; the ping keeps non-swarm hosts cheap.
+func (s *DashboardService) observeSwarm(ctx context.Context, o metric.Observer, g *inventoryGauges) {
+ cli, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return
+ }
+ ping, err := cli.Ping(ctx, client.PingOptions{})
+ if err != nil || ping.SwarmStatus == nil || !ping.SwarmStatus.ControlAvailable {
+ return
}
- totalPages := 1
- if totalItems > 0 {
- totalPages = (totalItems + limit - 1) / limit
+ if nodes, nodeErr := cli.NodeList(ctx, client.NodeListOptions{}); nodeErr != nil {
+ slog.DebugContext(ctx, "Skipping swarm node metrics", "error", nodeErr)
+ } else {
+ counts := map[[2]string]int64{}
+ for _, node := range nodes.Items {
+ counts[[2]string{string(node.Spec.Role), string(node.Status.State)}]++
+ }
+ for key, n := range counts {
+ o.ObserveInt64(g.swarmNodes, n, metric.WithAttributes(attribute.String("swarm.node.role", key[0]), attribute.String("swarm.node.state", key[1])))
+ }
}
- return base.PaginationResponse{
- TotalPages: int64(totalPages),
- TotalItems: int64(totalItems),
- CurrentPage: 1,
- ItemsPerPage: limit,
- GrandTotalItems: int64(totalItems),
+ services, err := cli.ServiceList(ctx, client.ServiceListOptions{Status: true})
+ if err != nil {
+ slog.DebugContext(ctx, "Skipping swarm service metrics", "error", err)
+ return
+ }
+ o.ObserveInt64(g.swarmServices, int64(len(services.Items)))
+ for _, service := range services.Items {
+ status := cmp.Or(service.ServiceStatus, &swarm.ServiceStatus{})
+ name := attribute.String("swarm.service.name", service.Spec.Name)
+ o.ObserveInt64(g.swarmTasks, int64(status.RunningTasks), metric.WithAttributes(name, attribute.String("swarm.task.state", "running")))
+ o.ObserveInt64(g.swarmTasks, int64(status.DesiredTasks), metric.WithAttributes(name, attribute.String("swarm.task.state", "desired")))
}
}
diff --git a/backend/internal/dashboard/service_test.go b/backend/internal/dashboard/service_test.go
index 7a676e062..336d5be7a 100644
--- a/backend/internal/dashboard/service_test.go
+++ b/backend/internal/dashboard/service_test.go
@@ -47,8 +47,9 @@ func setupDashboardServiceTestDB(t *testing.T) (*database.DB, *settings.Settings
require.NoError(t, db.AutoMigrate(&apikey.ApiKey{}, &environment.Environment{}, &imageupdate.ImageUpdateRecord{}, &project.Project{}, &settings.SettingVariable{}))
databaseDB := &database.DB{DB: db}
- settingsSvc, err := newSettingsServiceForTestInternal(t.Context(), t, databaseDB)
+ settingsSvc, err := settings.NewSettingsService(t.Context(), databaseDB)
require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, settingsSvc.Stop(context.WithoutCancel(t.Context()))) })
return databaseDB, settingsSvc
}
@@ -185,7 +186,8 @@ func TestDashboardService_GetSnapshot_ReturnsDashboardSnapshot(t *testing.T) {
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
require.NoError(t, settingsSvc.SetStringSetting(t.Context(), "projectsDirectory", projectsDir))
require.NoError(t, settingsSvc.SetStringSetting(t.Context(), "autoUpdateExcludedContainers", "stopped-app"))
- projectPath := createComposeProjectDirInternal(t, projectsDir, "project-with-update")
+ projectPath := filepath.Join(projectsDir, "project-with-update")
+ require.NoError(t, os.MkdirAll(projectPath, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(projectPath, "compose.yaml"), []byte("services:\n app:\n image: repo/worker:latest\n"), 0o644))
dirName := "project-with-update"
require.NoError(t, db.WithContext(t.Context()).Create(&project.Project{
@@ -266,7 +268,7 @@ func TestDashboardService_GetSnapshot_DebugAllGoodOnlyClearsActionItems(t *testi
require.Empty(t, snapshot.ActionItems.Items)
}
-func TestDashboardService_GetSnapshot_EnrichesPinnedReferencesInternal(t *testing.T) {
+func TestDashboardService_GetSnapshot_EnrichesPinnedReferences(t *testing.T) {
db, settingsSvc := setupDashboardServiceTestDB(t)
pinnedRef := "ghcr.io/syncthing/syncthing:2.1.3@sha256:8c8ff37ab6aa8be23b700648a90fa9412e214852e9fd6ea8477c8334792daec0"
@@ -306,29 +308,6 @@ func TestDashboardService_GetSnapshot_EnrichesPinnedReferencesInternal(t *testin
assert.Equal(t, []string{pinnedRef}, snapshot.Images.Data[0].PinnedReferences)
}
-// createComposeProjectDirInternal writes a minimal single-service compose project under
-// root and returns its path.
-func createComposeProjectDirInternal(t *testing.T, root, name string) string {
- t.Helper()
-
- projectPath := filepath.Join(root, name)
- require.NoError(t, os.MkdirAll(projectPath, 0o755))
- require.NoError(t, os.WriteFile(filepath.Join(projectPath, "compose.yaml"), []byte("services:\n app:\n image: nginx:alpine\n"), 0o600))
-
- return projectPath
-}
-
-// newSettingsServiceForTestInternal builds a SettingsService backed by its own actor runtime,
-// stopped when the test ends.
-func newSettingsServiceForTestInternal(ctx context.Context, t testing.TB, db *database.DB) (*settings.SettingsService, error) {
- t.Helper()
- svc, err := settings.NewSettingsService(ctx, db)
- if err == nil {
- t.Cleanup(func() { require.NoError(t, svc.Stop(context.WithoutCancel(t.Context()))) })
- }
- return svc, err
-}
-
func TestDashboardService_GetSnapshot_TrimmedOmitsTablesAndSharesBuilds(t *testing.T) {
db, settingsSvc := setupDashboardServiceTestDB(t)
@@ -463,27 +442,27 @@ func TestPendingContainerCountUsesScopedTagRecords(t *testing.T) {
{ID: "first", Image: "app:1.2.3", ImageID: "shared", Labels: map[string]string{labels.LabelUpdateStrategy: "auto"}},
{ID: "unchecked", Image: "app:1.2.3", ImageID: "shared", Labels: map[string]string{labels.LabelUpdateStrategy: "auto"}},
}
- count, err := service.getPendingContainerUpdatesCountInternal(t.Context(), containers)
+ count, err := service.getPendingContainerUpdatesCount(t.Context(), containers)
require.NoError(t, err)
require.Equal(t, 1, count)
containers = append(containers, dockercontainer.Summary{ID: "digest", Image: "app:latest", ImageID: "shared"})
- count, err = service.getPendingContainerUpdatesCountInternal(t.Context(), containers)
+ count, err = service.getPendingContainerUpdatesCount(t.Context(), containers)
require.NoError(t, err)
require.Equal(t, 2, count, "a digest record under another tag of the running image counts")
containers = append(containers, dockercontainer.Summary{ID: "digest-twin", Image: "app:latest", ImageID: "shared"})
- count, err = service.getPendingContainerUpdatesCountInternal(t.Context(), containers)
+ count, err = service.getPendingContainerUpdatesCount(t.Context(), containers)
require.NoError(t, err)
require.Equal(t, 3, count, "separate containers sharing one image each count once")
containers[0].Labels[labels.LabelUpdater] = "off"
- count, err = service.getPendingContainerUpdatesCountInternal(t.Context(), containers)
+ count, err = service.getPendingContainerUpdatesCount(t.Context(), containers)
require.NoError(t, err)
require.Equal(t, 3, count, "disabling automatic installation keeps the pending update visible")
containers[0].Labels[imageref.UpdateCheckLabel] = "false"
- count, err = service.getPendingContainerUpdatesCountInternal(t.Context(), containers)
+ count, err = service.getPendingContainerUpdatesCount(t.Context(), containers)
require.NoError(t, err)
require.Equal(t, 2, count, "disabling update checks suppresses the result")
containers[2].ImageID = ""
- count, err = service.getPendingContainerUpdatesCountInternal(t.Context(), containers)
+ count, err = service.getPendingContainerUpdatesCount(t.Context(), containers)
require.NoError(t, err)
require.Equal(t, 1, count, "a missing runtime image ID inherits no digest check")
}
diff --git a/backend/internal/database/database.go b/backend/internal/database/database.go
index 9c57638ce..7aa4135fd 100644
--- a/backend/internal/database/database.go
+++ b/backend/internal/database/database.go
@@ -23,6 +23,7 @@ import (
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/gorm/logger"
+ "gorm.io/plugin/opentelemetry/metrics"
sqliteutil "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/sqlite"
"github.com/getarcaneapp/arcane/backend/v2/resources"
@@ -137,11 +138,30 @@ func Initialize(ctx context.Context, databaseURL string, options MigrationOption
if cancellationErr := ctx.Err(); cancellationErr != nil {
return nil, cancellationErr
}
-
sqlDB, err := db.SQLDB()
if err != nil {
return nil, fmt.Errorf("failed to get sql.DB: %w", err)
}
+ // Statement spans and pool metrics are no-ops until telemetry is enabled.
+ cb := gormDB.Callback()
+ err = errors.Join(
+ cb.Create().Before("gorm:create").Register("arcane:otel_before_create", startStatementSpan),
+ cb.Create().After("gorm:create").Register("arcane:otel_after_create", endStatementSpan),
+ cb.Query().Before("gorm:query").Register("arcane:otel_before_query", startStatementSpan),
+ cb.Query().After("gorm:query").Register("arcane:otel_after_query", endStatementSpan),
+ cb.Update().Before("gorm:update").Register("arcane:otel_before_update", startStatementSpan),
+ cb.Update().After("gorm:update").Register("arcane:otel_after_update", endStatementSpan),
+ cb.Delete().Before("gorm:delete").Register("arcane:otel_before_delete", startStatementSpan),
+ cb.Delete().After("gorm:delete").Register("arcane:otel_after_delete", endStatementSpan),
+ cb.Row().Before("gorm:row").Register("arcane:otel_before_row", startStatementSpan),
+ cb.Row().After("gorm:row").Register("arcane:otel_after_row", endStatementSpan),
+ cb.Raw().Before("gorm:raw").Register("arcane:otel_before_raw", startStatementSpan),
+ cb.Raw().After("gorm:raw").Register("arcane:otel_after_raw", endStatementSpan),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("failed to register database instrumentation: %w", err)
+ }
+ metrics.ReportDBStatsMetrics(sqlDB)
if migrationErr := migrateDatabase(ctx, sqlDB, dbProvider, options, latestMigrationVersion); migrationErr != nil {
slog.ErrorContext(ctx, "Failed to run migrations", "error", migrationErr)
return nil, fmt.Errorf("failed to run migrations: %w", migrationErr)
diff --git a/backend/internal/database/tracing.go b/backend/internal/database/tracing.go
new file mode 100644
index 000000000..326da6a42
--- /dev/null
+++ b/backend/internal/database/tracing.go
@@ -0,0 +1,57 @@
+package database
+
+import (
+ "database/sql"
+ "errors"
+ "strings"
+
+ "go.getarcane.app/kit/pkg"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
+ "gorm.io/gorm"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
+)
+
+const statementSpanKey = "arcane:otel_span"
+
+var statementTracer = otel.Tracer(tracing.InstrumentationName)
+
+// startStatementSpan opens a client span only inside an existing trace, so
+// background queries do not create root traces.
+func startStatementSpan(tx *gorm.DB) {
+ ctx := tx.Statement.Context
+ if !trace.SpanContextFromContext(ctx).IsValid() {
+ return
+ }
+ _, span := statementTracer.Start(ctx, "db", trace.WithSpanKind(trace.SpanKindClient))
+ tx.InstanceSet(statementSpanKey, span)
+}
+
+// endStatementSpan names and ends the span with the placeholder SQL; parameter values are never recorded.
+func endStatementSpan(tx *gorm.DB) {
+ value, _ := tx.InstanceGet(statementSpanKey)
+ span, ok := value.(trace.Span)
+ if !ok {
+ return
+ }
+
+ query := tx.Statement.SQL.String()
+ operation, _, _ := strings.Cut(strings.TrimSpace(query), " ")
+ operation = strings.ToLower(operation)
+ if name := strings.TrimSpace(operation + " " + tx.Statement.Table); name != "" {
+ span.SetName(name)
+ }
+ span.SetAttributes(
+ attribute.String("db.system.name", kit.Ternary(tx.Name() == dbProviderPostgres, "postgresql", "sqlite")),
+ attribute.String("db.query.text", query),
+ attribute.String("db.operation.name", operation),
+ attribute.String("db.collection.name", tx.Statement.Table),
+ )
+ err := tx.Error
+ if errors.Is(err, gorm.ErrRecordNotFound) || errors.Is(err, sql.ErrNoRows) {
+ err = nil
+ }
+ tracing.End(span, err)
+}
diff --git a/backend/internal/docker/service.go b/backend/internal/docker/service.go
index 26f10d5ee..5c351238f 100644
--- a/backend/internal/docker/service.go
+++ b/backend/internal/docker/service.go
@@ -14,11 +14,8 @@ import (
"github.com/getarcaneapp/arcane/types/v2/dashboard"
imagetypes "github.com/getarcaneapp/arcane/types/v2/image"
"github.com/moby/moby/api/types/container"
- "github.com/moby/moby/api/types/events"
"github.com/moby/moby/api/types/image"
- "github.com/moby/moby/api/types/mount"
"github.com/moby/moby/api/types/network"
- "github.com/moby/moby/api/types/volume"
"github.com/moby/moby/client"
"go.getarcane.app/docker"
"go.getarcane.app/docker/compat"
@@ -53,10 +50,8 @@ type DockerClientService struct {
mu sync.Mutex
eventBus *bus.DockerEventBus
- // Coalesce concurrent full-inventory list calls so overlapping requests
- // (e.g. the Homepage widget's simultaneous counts endpoints) decode one
- // Docker response instead of one per caller. Per-instance on purpose:
- // a WithClient copy talks to a different daemon and must not share flights.
+ // Coalesce concurrent full-inventory lists into one Docker response; per-instance
+ // because a WithClient copy talks to a different daemon.
imageListFlight singleflight.Group
containerListFlight singleflight.Group
}
@@ -82,28 +77,12 @@ func (s *DockerClientService) WithClient(dockerClient *client.Client) *DockerCli
}
}
-func newDockerClientInternal(ctx context.Context, host string) (*client.Client, error) {
- apiVersion, err := detectDockerAPIVersionInternal(ctx, host)
- if err != nil {
- return nil, err
- }
-
- configuredClient, err := newDockerClientWithAPIVersionInternal(host, apiVersion)
- if err != nil {
- return nil, err
- }
-
- return configuredClient, nil
-}
-
-func detectDockerAPIVersionInternal(ctx context.Context, host string) (string, error) {
- probeClient, err := client.New(
- client.WithHost(host),
- )
+func detectDockerAPIVersion(ctx context.Context, host string) (string, error) {
+ probeClient, err := client.New(client.WithHost(host))
if err != nil {
return "", fmt.Errorf("failed to create Docker probe client: %w", err)
}
- defer closeDockerClientInternal(ctx, probeClient, "probe")
+ defer closeDockerClient(ctx, probeClient, "probe")
ctx, cancel := context.WithTimeout(ctx, dockerClientNegotiationTimeout)
defer cancel()
@@ -122,19 +101,7 @@ func detectDockerAPIVersionInternal(ctx context.Context, host string) (string, e
return apiVersion, nil
}
-func newDockerClientWithAPIVersionInternal(host, apiVersion string) (*client.Client, error) {
- configuredClient, err := client.New(
- client.WithHost(host),
- client.WithAPIVersion(apiVersion),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to configure Docker client API version %s: %w", apiVersion, err)
- }
-
- return configuredClient, nil
-}
-
-func closeDockerClientInternal(ctx context.Context, cli *client.Client, purpose string) {
+func closeDockerClient(ctx context.Context, cli *client.Client, purpose string) {
if cli == nil {
return
}
@@ -155,16 +122,20 @@ func (s *DockerClientService) GetClient(ctx context.Context) (*client.Client, er
}
s.mu.Unlock()
- cli, err := newDockerClientInternal(ctx, s.config.DockerHost)
+ apiVersion, err := detectDockerAPIVersion(ctx, s.config.DockerHost)
if err != nil {
return nil, fmt.Errorf("failed to create Docker client: %w", err)
}
+ cli, err := client.New(client.WithHost(s.config.DockerHost), client.WithAPIVersion(apiVersion))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create Docker client: failed to configure Docker client API version %s: %w", apiVersion, err)
+ }
s.mu.Lock()
if s.Client != nil {
existingClient := s.Client
s.mu.Unlock()
- closeDockerClientInternal(ctx, cli, "unused after concurrent initialization")
+ closeDockerClient(ctx, cli, "unused after concurrent initialization")
return existingClient, nil
}
@@ -179,7 +150,7 @@ func (s *DockerClientService) GetClient(ctx context.Context) (*client.Client, er
// RefreshClient probes the Docker daemon and recreates the cached client when
// the daemon's effective API version changed.
func (s *DockerClientService) RefreshClient(ctx context.Context) error {
- apiVersion, err := detectDockerAPIVersionInternal(ctx, s.config.DockerHost)
+ apiVersion, err := detectDockerAPIVersion(ctx, s.config.DockerHost)
if err != nil {
return fmt.Errorf("failed to refresh Docker client: %w", err)
}
@@ -192,16 +163,16 @@ func (s *DockerClientService) RefreshClient(ctx context.Context) error {
}
s.mu.Unlock()
- cli, err := newDockerClientWithAPIVersionInternal(s.config.DockerHost, apiVersion)
+ cli, err := client.New(client.WithHost(s.config.DockerHost), client.WithAPIVersion(apiVersion))
if err != nil {
- return fmt.Errorf("failed to refresh Docker client: %w", err)
+ return fmt.Errorf("failed to refresh Docker client: failed to configure Docker client API version %s: %w", apiVersion, err)
}
s.mu.Lock()
if s.Client != nil && apiVersion == s.clientVersion {
s.clientLastProbe = time.Now()
s.mu.Unlock()
- closeDockerClientInternal(ctx, cli, "unused after concurrent refresh")
+ closeDockerClient(ctx, cli, "unused after concurrent refresh")
return nil
}
@@ -211,7 +182,7 @@ func (s *DockerClientService) RefreshClient(ctx context.Context) error {
s.clientLastProbe = time.Now()
s.mu.Unlock()
- closeDockerClientInternal(ctx, oldClient, "replaced")
+ closeDockerClient(ctx, oldClient, "replaced")
return nil
}
@@ -231,7 +202,7 @@ func (s *DockerClientService) Close() {
s.Client = nil
s.mu.Unlock()
- closeDockerClientInternal(context.Background(), oldClient, "cached") //nolint:forbidigo // Shutdown close runs without a request context.
+ closeDockerClient(context.Background(), oldClient, "cached") //nolint:forbidigo // Shutdown close runs without a request context.
}
func (s *DockerClientService) EventBus() *bus.DockerEventBus {
@@ -249,77 +220,53 @@ func (s *DockerClientService) WatchEvents(ctx context.Context) {
eventBackoff.MaxInterval = 30 * time.Second
for ctx.Err() == nil {
- dockerClient, err := s.GetClient(ctx)
- if err != nil {
+ if dockerClient, err := s.GetClient(ctx); err != nil {
slog.WarnContext(ctx, "failed to connect to Docker event stream", "error", err)
- if !sleepDockerEventBackoffInternal(ctx, eventBackoff) {
- return
+ } else {
+ result := dockerClient.Events(ctx, client.EventsListOptions{})
+ streamStart := time.Now()
+ var streamErr error
+ stream:
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case msg, ok := <-result.Messages:
+ if !ok {
+ break stream
+ }
+ // Stamp missing daemon timestamps once, before normal and overflow delivery diverge.
+ if msg.TimeNano == 0 {
+ msg.TimeNano = time.Now().UnixNano()
+ }
+ s.EventBus().Publish(msg)
+ case recvErr, ok := <-result.Err:
+ streamErr = kit.Ternary(ok, recvErr, nil)
+ break stream
+ }
+ }
+ if streamErr != nil && !errors.Is(streamErr, context.Canceled) && !errors.Is(streamErr, io.EOF) {
+ slog.WarnContext(ctx, "Docker event stream stopped", "error", streamErr)
+ }
+ // Only a stream that stayed up counts as recovery, so a daemon that drops connections immediately keeps backing off.
+ if time.Since(streamStart) >= dockerEventStreamHealthyAfter {
+ eventBackoff.Reset()
}
- continue
- }
-
- result := dockerClient.Events(ctx, client.EventsListOptions{})
- streamStart := time.Now()
- err = s.consumeEventsInternal(ctx, result.Messages, result.Err)
- if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, io.EOF) {
- slog.WarnContext(ctx, "Docker event stream stopped", "error", err)
- }
-
- // Only a stream that actually stayed up counts as recovery. Resetting
- // unconditionally before Events() meant a daemon that accepts the
- // connection and drops it immediately was reconnected at the floor
- // interval forever.
- if time.Since(streamStart) >= dockerEventStreamHealthyAfter {
- eventBackoff.Reset()
- }
-
- if !sleepDockerEventBackoffInternal(ctx, eventBackoff) {
- return
}
- }
-}
-func (s *DockerClientService) consumeEventsInternal(ctx context.Context, messages <-chan events.Message, errs <-chan error) error {
- for {
+ timer := time.NewTimer(eventBackoff.NextBackOff())
select {
case <-ctx.Done():
- return ctx.Err()
- case msg, ok := <-messages:
- if !ok {
- return nil
- }
- // Stamp missing daemon timestamps once, before normal and overflow delivery diverge.
- if msg.TimeNano == 0 {
- msg.TimeNano = time.Now().UnixNano()
- }
- s.EventBus().Publish(msg)
- case err, ok := <-errs:
- return kit.Ternary(!ok, nil, err)
+ timer.Stop()
+ return
+ case <-timer.C:
}
}
}
-func sleepDockerEventBackoffInternal(ctx context.Context, eventBackoff *backoff.ExponentialBackOff) bool {
- delay := eventBackoff.NextBackOff()
- timer := time.NewTimer(delay)
- defer timer.Stop()
- select {
- case <-ctx.Done():
- return false
- case <-timer.C:
- return true
- }
-}
-
-// listCoalescedInternal runs op through the given singleflight group so
-// concurrent callers share one Docker request and one decoded result. The
-// flight runs on a context detached from the initiating caller, bounded only
-// by op's own Docker API timeout, so one canceled waiter cannot cancel work
-// other waiters still need; each waiter stops waiting when its own context
-// ends. Results and errors are shared only among concurrent callers —
-// singleflight drops the key once the flight completes, so nothing is cached.
-// Callers must treat the shared result as immutable.
-func listCoalescedInternal[T any](ctx context.Context, group *singleflight.Group, op func(context.Context) (T, error)) (T, error) {
+// listCoalesced shares one detached op among concurrent callers; each waiter stops on its own
+// context, nothing is cached, and the shared result must be treated as immutable.
+func listCoalesced[T any](ctx context.Context, group *singleflight.Group, op func(context.Context) (T, error)) (T, error) {
var zero T
ch := group.DoChan("list", func() (any, error) {
@@ -341,81 +288,56 @@ func listCoalescedInternal[T any](ctx context.Context, group *singleflight.Group
}
}
-func (s *DockerClientService) apiTimeoutInternal() time.Duration {
- if s.settingsService == nil {
- return timeouts.DefaultDockerAPI
+// callAPI runs op against the cached client under the configured Docker API timeout.
+func callAPI[T any](ctx context.Context, s *DockerClientService, op func(context.Context, *client.Client) (T, error)) (T, error) {
+ dockerClient, err := s.GetClient(ctx)
+ if err != nil {
+ var zero T
+ return zero, fmt.Errorf("failed to connect to Docker: %w", err)
}
- return timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI)
+ timeout := timeouts.DefaultDockerAPI
+ if s.settingsService != nil {
+ timeout = timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI)
+ }
+ apiCtx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+ return op(apiCtx, dockerClient)
}
func (s *DockerClientService) ListContainers(ctx context.Context) ([]container.Summary, error) {
- return listCoalescedInternal(ctx, &s.containerListFlight, func(ctx context.Context) ([]container.Summary, error) {
- dockerClient, err := s.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- apiCtx, cancel := context.WithTimeout(ctx, s.apiTimeoutInternal())
- defer cancel()
-
- containerList, err := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})
- if err != nil {
- return nil, fmt.Errorf("failed to list Docker containers: %w", err)
- }
- return containerList.Items, nil
+ return listCoalesced(ctx, &s.containerListFlight, func(ctx context.Context) ([]container.Summary, error) {
+ return callAPI(ctx, s, func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {
+ containerList, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ if err != nil {
+ return nil, fmt.Errorf("failed to list Docker containers: %w", err)
+ }
+ return containerList.Items, nil
+ })
})
}
func (s *DockerClientService) ListImages(ctx context.Context) ([]image.Summary, error) {
- return listCoalescedInternal(ctx, &s.imageListFlight, func(ctx context.Context) ([]image.Summary, error) {
- dockerClient, err := s.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- apiCtx, cancel := context.WithTimeout(ctx, s.apiTimeoutInternal())
- defer cancel()
+ return listCoalesced(ctx, &s.imageListFlight, func(ctx context.Context) ([]image.Summary, error) {
+ return callAPI(ctx, s, func(ctx context.Context, dockerClient *client.Client) ([]image.Summary, error) {
+ imageList, err := dockerClient.ImageList(ctx, client.ImageListOptions{All: true})
+ if err != nil {
+ return nil, fmt.Errorf("failed to list Docker images: %w", err)
+ }
+ return imageList.Items, nil
+ })
+ })
+}
- imageList, err := dockerClient.ImageList(apiCtx, client.ImageListOptions{All: true})
+func (s *DockerClientService) listNetworks(ctx context.Context) ([]network.Summary, error) {
+ return callAPI(ctx, s, func(ctx context.Context, dockerClient *client.Client) ([]network.Summary, error) {
+ networkList, err := compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})
if err != nil {
- return nil, fmt.Errorf("failed to list Docker images: %w", err)
+ return nil, fmt.Errorf("failed to list Docker networks: %w", err)
}
- return imageList.Items, nil
+ return networkList.Items, nil
})
}
-func (s *DockerClientService) listNetworksInternal(ctx context.Context) ([]network.Summary, error) {
- dockerClient, err := s.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- apiCtx, cancel := context.WithTimeout(ctx, s.apiTimeoutInternal())
- defer cancel()
-
- networkList, err := compat.NetworkListWithCompatibility(apiCtx, dockerClient, client.NetworkListOptions{})
- if err != nil {
- return nil, fmt.Errorf("failed to list Docker networks: %w", err)
- }
- return networkList.Items, nil
-}
-
-func (s *DockerClientService) listVolumesInternal(ctx context.Context) (*client.VolumeListResult, error) {
- dockerClient, err := s.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- apiCtx, cancel := context.WithTimeout(ctx, s.apiTimeoutInternal())
- defer cancel()
-
- volResp, err := dockerClient.VolumeList(apiCtx, client.VolumeListOptions{})
- if err != nil {
- return nil, fmt.Errorf("failed to list Docker volumes: %w", err)
- }
- return &volResp, nil
-}
-
func (s *DockerClientService) GetSnapshot(ctx context.Context, envID string) (*dashboard.DockerSnapshot, error) {
g, groupCtx := errgroup.WithContext(ctx)
@@ -442,7 +364,7 @@ func (s *DockerClientService) GetSnapshot(ctx context.Context, envID string) (*d
defer utils.RecoverToError(&workerErr, "docker snapshot worker")
var err error
- networks, err = s.listNetworksInternal(groupCtx)
+ networks, err = s.listNetworks(groupCtx)
if err != nil {
slog.WarnContext(groupCtx, "failed to list Docker networks for snapshot", "error", err)
}
@@ -452,7 +374,13 @@ func (s *DockerClientService) GetSnapshot(ctx context.Context, envID string) (*d
defer utils.RecoverToError(&workerErr, "docker snapshot worker")
var err error
- volumes, err = s.listVolumesInternal(groupCtx)
+ volumes, err = callAPI(groupCtx, s, func(ctx context.Context, dockerClient *client.Client) (*client.VolumeListResult, error) {
+ volResp, listErr := dockerClient.VolumeList(ctx, client.VolumeListOptions{})
+ if listErr != nil {
+ return nil, fmt.Errorf("failed to list Docker volumes: %w", listErr)
+ }
+ return &volResp, nil
+ })
if err != nil {
slog.WarnContext(groupCtx, "failed to list Docker volumes for snapshot", "error", err)
}
@@ -518,9 +446,8 @@ func (s *DockerClientService) GetAllImages(ctx context.Context) ([]image.Summary
return images, CountImageUsage(images, containers), nil
}
-// CountImageUsage tallies in-use, unused, and total image counts, plus the
-// summed size of all images. An image is in use when a container references
-// its ID.
+// CountImageUsage tallies in-use, unused, and total image counts plus the summed
+// image size. An image is in use when a container references its ID.
func CountImageUsage(images []image.Summary, containers []container.Summary) imagetypes.UsageCounts {
inUseImageIDs := make(map[string]struct{}, len(containers))
for _, c := range containers {
@@ -549,65 +476,27 @@ func (s *DockerClientService) GetAllNetworks(ctx context.Context) ([]network.Sum
if err != nil {
return nil, 0, 0, 0, err
}
- inUseByID, inUseByName := docker.BuildNetworkUsageMaps(containers)
-
- networks, err := s.listNetworksInternal(ctx)
+ networks, err := s.listNetworks(ctx)
if err != nil {
return nil, 0, 0, 0, err
}
-
- var inuse, unused, total int
- for _, n := range networks {
- total++ // total includes all networks (including defaults)
-
- // Only count non-default networks towards in-use/unused breakdown
- if !docker.IsDefaultNetwork(n.Name) {
- used := inUseByID[n.ID] || inUseByName[n.Name]
- if used {
- inuse++
- } else {
- unused++
- }
- }
- }
-
- // Return order: inuse, unused, total (matches handler expectations)
+ inuse, unused, total := CountNetworkUsage(networks, containers)
return networks, inuse, unused, total, nil
}
-func (s *DockerClientService) GetAllVolumes(ctx context.Context) ([]*volume.Volume, int, int, int, error) {
- containers, err := s.ListContainers(ctx)
- if err != nil {
- return nil, 0, 0, 0, err
- }
- ref := make(map[string]int64, len(containers))
- for _, c := range containers {
- for _, m := range c.Mounts {
- if m.Type == mount.TypeVolume && m.Name != "" {
- ref[m.Name]++
- }
- }
- }
-
- volResp, err := s.listVolumesInternal(ctx)
- if err != nil {
- return nil, 0, 0, 0, err
- }
- volumeItems := volResp.Items
- volumes := make([]*volume.Volume, 0, len(volumeItems))
- for i := range volumeItems {
- volumes = append(volumes, &volumeItems[i])
- }
-
- var inuse, unused, total int
- for _, v := range volumes {
+// CountNetworkUsage returns in-use and unused counts for non-default networks, and the total of all networks.
+func CountNetworkUsage(networks []network.Summary, containers []container.Summary) (inuse, unused, total int) {
+ inUseByID, inUseByName := docker.BuildNetworkUsageMaps(containers)
+ for _, n := range networks {
total++
- if ref[v.Name] > 0 {
+ if docker.IsDefaultNetwork(n.Name) {
+ continue
+ }
+ if inUseByID[n.ID] || inUseByName[n.Name] {
inuse++
} else {
unused++
}
}
-
- return volumes, inuse, unused, total, nil
+ return inuse, unused, total
}
diff --git a/backend/internal/docker/service_test.go b/backend/internal/docker/service_test.go
index 894b9e5a4..e1e537ceb 100644
--- a/backend/internal/docker/service_test.go
+++ b/backend/internal/docker/service_test.go
@@ -2,10 +2,11 @@ package docker
import (
"context"
+ "encoding/json"
"net"
"net/http"
"net/http/httptest"
- "regexp"
+ "path"
"sync/atomic"
"testing"
"time"
@@ -22,7 +23,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
)
-func TestNewDockerClient_PinsEffectiveAPIVersion(t *testing.T) {
+func TestDockerClientService_GetClientPinsEffectiveAPIVersion(t *testing.T) {
t.Setenv("DOCKER_API_VERSION", "1.54")
t.Setenv("DOCKER_HOST", "tcp://docker-from-env:2375")
@@ -45,13 +46,12 @@ func TestNewDockerClient_PinsEffectiveAPIVersion(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- server := newDockerPingTestServerInternal(t, tt.pingAPIVersion)
+ server := newDockerPingTestServer(t, func() string { return tt.pingAPIVersion })
+ svc := newDockerClientServiceForTest(t, server.URL)
+ t.Cleanup(svc.Close)
- cli, err := newDockerClientInternal(t.Context(), server.URL)
+ cli, err := svc.GetClient(t.Context())
require.NoError(t, err)
- t.Cleanup(func() {
- _ = cli.Close()
- })
assert.Equal(t, server.URL, cli.DaemonHost())
assert.Equal(t, tt.expectedVersion, cli.ClientVersion())
@@ -61,8 +61,8 @@ func TestNewDockerClient_PinsEffectiveAPIVersion(t *testing.T) {
}
func TestDockerClientService_GetClientReturnsCachedClientUntilRefresh(t *testing.T) {
- server := newDockerPingTestServerInternal(t, "1.41")
- svc := newDockerClientServiceForTestInternal(t, server.URL)
+ server := newDockerPingTestServer(t, func() string { return "1.41" })
+ svc := newDockerClientServiceForTest(t, server.URL)
firstClient, err := svc.GetClient(t.Context())
require.NoError(t, err)
@@ -80,10 +80,10 @@ func TestDockerClientService_GetClientReturnsCachedClientUntilRefresh(t *testing
func TestDockerClientService_RefreshClientRecreatesCachedClientAfterAPIVersionChange(t *testing.T) {
apiVersion := atomic.Value{}
apiVersion.Store("1.41")
- server := newDockerPingTestServerWithVersionInternal(t, func() string {
+ server := newDockerPingTestServer(t, func() string {
return apiVersion.Load().(string)
})
- svc := newDockerClientServiceForTestInternal(t, server.URL)
+ svc := newDockerClientServiceForTest(t, server.URL)
firstClient, err := svc.GetClient(t.Context())
require.NoError(t, err)
@@ -107,13 +107,23 @@ func TestDockerClientService_RefreshClientRecreatesCachedClientAfterAPIVersionCh
func TestDockerClientService_RefreshClientClosesOldCachedClientWhenReplaced(t *testing.T) {
var oldServerClosedConnections atomic.Int32
- oldServer := newDockerPingTestServerWithConnStateInternal(t, "1.41", func(_ net.Conn, state http.ConnState) {
+ oldServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/_ping" {
+ http.NotFound(w, r)
+ return
+ }
+ w.Header().Set("Api-Version", "1.41")
+ w.WriteHeader(http.StatusOK)
+ }))
+ oldServer.Config.ConnState = func(_ net.Conn, state http.ConnState) {
if state == http.StateClosed {
oldServerClosedConnections.Add(1)
}
- })
- newServer := newDockerPingTestServerInternal(t, "1.42")
- svc := newDockerClientServiceForTestInternal(t, oldServer.URL)
+ }
+ oldServer.Start()
+ t.Cleanup(oldServer.Close)
+ newServer := newDockerPingTestServer(t, func() string { return "1.42" })
+ svc := newDockerClientServiceForTest(t, oldServer.URL)
firstClient, err := svc.GetClient(t.Context())
require.NoError(t, err)
@@ -143,7 +153,7 @@ func TestDockerClientService_RefreshClientClosesOldCachedClientWhenReplaced(t *t
func TestDockerClientService_RefreshClientProbeFailureKeepsCachedClient(t *testing.T) {
var failProbe atomic.Bool
- server := newDockerPingTestServerWithHandlerInternal(t, func(w http.ResponseWriter, r *http.Request) {
+ server := newDockerPingTestServerWithHandler(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/_ping" {
http.NotFound(w, r)
return
@@ -155,7 +165,7 @@ func TestDockerClientService_RefreshClientProbeFailureKeepsCachedClient(t *testi
w.Header().Set("Api-Version", "1.41")
w.WriteHeader(http.StatusOK)
})
- svc := newDockerClientServiceForTestInternal(t, server.URL)
+ svc := newDockerClientServiceForTest(t, server.URL)
firstClient, err := svc.GetClient(t.Context())
require.NoError(t, err)
@@ -172,33 +182,50 @@ func TestDockerClientService_RefreshClientProbeFailureKeepsCachedClient(t *testi
assert.False(t, svc.clientLastProbe.IsZero())
}
-func TestDockerClientService_PublishesDaemonImageEventsInternal(t *testing.T) {
- svc := NewDockerClientService(t.Context(), nil, &config.Config{}, nil)
+func TestDockerClientService_PublishesDaemonImageEvents(t *testing.T) {
+ expected := events.Message{Type: events.ImageEventType, Action: events.ActionPull, TimeNano: 123, Actor: events.Actor{ID: "nginx:latest"}}
+ payload, err := json.Marshal(expected)
+ require.NoError(t, err)
+ server := newDockerPingTestServerWithHandler(t, func(w http.ResponseWriter, r *http.Request) {
+ switch path.Base(r.URL.Path) {
+ case "_ping":
+ w.Header().Set("Api-Version", "1.41")
+ w.WriteHeader(http.StatusOK)
+ case "events":
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write(payload)
+ default:
+ http.NotFound(w, r)
+ }
+ })
+ svc := newDockerClientServiceForTest(t, server.URL)
t.Cleanup(svc.Close)
eventCh, unsubscribe := svc.EventBus().Subscribe(events.ImageEventType)
t.Cleanup(unsubscribe)
- messages := make(chan events.Message, 1)
- expected := events.Message{Type: events.ImageEventType, Action: events.ActionPull, TimeNano: 123, Actor: events.Actor{ID: "nginx:latest"}}
- messages <- expected
- close(messages)
- require.NoError(t, svc.consumeEventsInternal(t.Context(), messages, nil))
+ stop, err := concurrency.StartSupervised(t.Context(), "Docker event watcher", func(ctx context.Context) error {
+ svc.WatchEvents(ctx)
+ return nil
+ })
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = stop(context.WithoutCancel(t.Context())) })
+
select {
case message := <-eventCh:
require.Equal(t, expected, message)
- default:
+ case <-time.After(time.Second):
t.Fatal("daemon image event was not published")
}
}
-func TestDockerClientService_EventActorStopCancelsAndJoinsStreamInternal(t *testing.T) {
+func TestDockerClientService_EventActorStopCancelsAndJoinsStream(t *testing.T) {
streamStarted := make(chan struct{})
streamStopped := make(chan struct{})
- server := newDockerPingTestServerWithHandlerInternal(t, func(w http.ResponseWriter, r *http.Request) {
- switch dockerTestPathInternal(r.URL.Path) {
- case "/_ping":
+ server := newDockerPingTestServerWithHandler(t, func(w http.ResponseWriter, r *http.Request) {
+ switch path.Base(r.URL.Path) {
+ case "_ping":
w.Header().Set("Api-Version", "1.41")
w.WriteHeader(http.StatusOK)
- case "/events":
+ case "events":
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if flusher, ok := w.(http.Flusher); ok {
@@ -212,7 +239,7 @@ func TestDockerClientService_EventActorStopCancelsAndJoinsStreamInternal(t *test
}
})
- service := newDockerClientServiceForTestInternal(t, server.URL)
+ service := newDockerClientServiceForTest(t, server.URL)
t.Cleanup(service.Close)
eventsCh, unsubscribe := service.EventBus().Subscribe(events.ImageEventType)
@@ -275,26 +302,14 @@ func TestCountImageUsage_NoImages(t *testing.T) {
assert.Equal(t, imagetypes.UsageCounts{}, counts)
}
-func newDockerClientServiceForTestInternal(t *testing.T, host string) *DockerClientService {
+func newDockerClientServiceForTest(t *testing.T, host string) *DockerClientService {
return NewDockerClientService(t.Context(), nil, &config.Config{DockerHost: host}, nil)
}
-func dockerTestPathInternal(path string) string {
- return regexp.MustCompile(`^/v\d+\.\d+`).ReplaceAllString(path, "")
-}
-
-func newDockerPingTestServerInternal(t *testing.T, apiVersion string) *httptest.Server {
- t.Helper()
-
- return newDockerPingTestServerWithVersionInternal(t, func() string {
- return apiVersion
- })
-}
-
-func newDockerPingTestServerWithVersionInternal(t *testing.T, apiVersion func() string) *httptest.Server {
+func newDockerPingTestServer(t *testing.T, apiVersion func() string) *httptest.Server {
t.Helper()
- return newDockerPingTestServerWithHandlerInternal(t, func(w http.ResponseWriter, r *http.Request) {
+ return newDockerPingTestServerWithHandler(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/_ping" {
http.NotFound(w, r)
return
@@ -307,26 +322,7 @@ func newDockerPingTestServerWithVersionInternal(t *testing.T, apiVersion func()
})
}
-func newDockerPingTestServerWithConnStateInternal(t *testing.T, apiVersion string, connState func(net.Conn, http.ConnState)) *httptest.Server {
- t.Helper()
-
- server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.URL.Path != "/_ping" {
- http.NotFound(w, r)
- return
- }
-
- w.Header().Set("Api-Version", apiVersion)
- w.WriteHeader(http.StatusOK)
- }))
- server.Config.ConnState = connState
- server.Start()
- t.Cleanup(server.Close)
-
- return server
-}
-
-func newDockerPingTestServerWithHandlerInternal(t *testing.T, handler http.HandlerFunc) *httptest.Server {
+func newDockerPingTestServerWithHandler(t *testing.T, handler http.HandlerFunc) *httptest.Server {
t.Helper()
server := httptest.NewServer(handler)
diff --git a/backend/internal/environment/handler.go b/backend/internal/environment/handler.go
index 856440a48..4a174aedd 100644
--- a/backend/internal/environment/handler.go
+++ b/backend/internal/environment/handler.go
@@ -6,7 +6,6 @@ import (
"cmp"
"context"
"crypto/x509"
- "encoding/json/v2"
"encoding/pem"
"errors"
"fmt"
@@ -24,7 +23,6 @@ import (
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/types/v2/base"
"github.com/getarcaneapp/arcane/types/v2/environment"
- usertypes "github.com/getarcaneapp/arcane/types/v2/user"
"github.com/getarcaneapp/arcane/types/v2/version"
kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/kit/pkg/mapping"
@@ -42,20 +40,16 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/userctx"
)
const (
- localDockerEnvironmentID = "0"
-
- // Re-applies in-memory runtime state to cached rows, covering poll-mode TTL
- // expiry; CRUD, tunnel and health-check changes arrive on the runtime-change signal.
+ // Re-applies runtime state to cached rows to cover poll-mode TTL expiry.
environmentStreamRuntimeInterval = 5 * time.Second
// Safety net for writes that bypass the runtime-change signal.
environmentStreamReloadInterval = 60 * time.Second
environmentStreamRefreshFloor = 30 * time.Second
- environmentStreamHubKeyInternal = "environments"
+ environmentStreamHubKey = "environments"
)
// EnvironmentHandler handles environment management endpoints.
@@ -68,6 +62,7 @@ type EnvironmentHandler struct {
streamHub *agg.Hub[[]environment.Environment]
activityService activity.Service
appCtx context.Context
+ localVersion func(context.Context) *version.Info
}
type ListEnvironmentsInput struct {
@@ -171,19 +166,11 @@ func NewHandler(
// ListEnvironments returns a paginated list of environments.
func (h *EnvironmentHandler) ListEnvironments(ctx context.Context, input *ListEnvironmentsInput) (*handlerutil.Page[environment.Environment], error) {
- // The list endpoint backs both the environments management page and the
- // environment switcher, so any authenticated caller may reach it. Global
- // listers (sudo, global admins, or holders of the org-level
- // environments:list permission) see every environment; environment-scoped
- // callers see only the environments they hold at least one permission on.
ps, ok := middleware.PermissionsFromContext(ctx)
if !ok {
return nil, huma.Error403Forbidden("permission denied")
}
- var accessibleEnvIDs []string // nil = no restriction
- if !environmentListerSeesAllInternal(ps) {
- accessibleEnvIDs = accessibleEnvironmentIDsInternal(ps)
- }
+ accessibleEnvIDs := accessibleEnvironmentIDs(ps)
params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)
if input.Type != "" {
@@ -195,7 +182,7 @@ func (h *EnvironmentHandler) ListEnvironments(ctx context.Context, input *ListEn
return nil, huma.Error500InternalServerError("Failed to fetch environments")
}
for i := range envs {
- h.applyEdgeRuntimeStateInternal(&envs[i])
+ ApplyEnvironmentRuntimeState(&envs[i])
}
return &handlerutil.Page[environment.Environment]{
@@ -207,60 +194,36 @@ func (h *EnvironmentHandler) ListEnvironments(ctx context.Context, input *ListEn
}, nil
}
-// environmentListerSeesAllInternal reports whether the caller may list every
-// environment. True for sudo callers, global admins, and holders of the
-// org-level environments:list permission (Allows short-circuits on sudo and
-// treats global admins as holding every permission).
-func environmentListerSeesAllInternal(ps *authz.PermissionSet) bool {
- return ps != nil && ps.Allows(authz.PermEnvironmentsList, "")
-}
-
-// accessibleEnvironmentIDsInternal returns the sorted set of environment IDs the
-// caller holds at least one environment-scoped permission on. A non-nil result
-// (possibly empty) restricts the environment list for non-global callers; an
-// empty result yields no environments.
-func accessibleEnvironmentIDsInternal(ps *authz.PermissionSet) []string {
- if ps == nil {
+// accessibleEnvironmentIDs returns nil when the caller may list every environment,
+// otherwise the sorted IDs it holds at least one permission on.
+func accessibleEnvironmentIDs(ps *authz.PermissionSet) []string {
+ switch {
+ case ps == nil:
return []string{}
+ case ps.Allows(authz.PermEnvironmentsList, ""):
+ return nil
+ default:
+ // Never nil: callers treat nil as unrestricted, so a user without grants must get an empty list.
+ ids := slices.AppendSeq(make([]string, 0, len(ps.PerEnv)), maps.Keys(ps.PerEnv))
+ slices.Sort(ids)
+ return ids
}
- ids := slices.Sorted(maps.Keys(ps.PerEnv))
- return ids
-}
-
-// visibleEnvironmentsForInternal filters the shared environment snapshot down
-// to what the caller may see. It reuses the same access rules as
-// ListEnvironments so the stream and the REST list can never disagree about
-// which environments a caller has. The input slice is shared across stream
-// subscribers and must not be mutated.
-func visibleEnvironmentsForInternal(envs []environment.Environment, ps *authz.PermissionSet) []environment.Environment {
- if environmentListerSeesAllInternal(ps) {
- return envs
- }
-
- allowed := make(map[string]struct{}, len(ps.PerEnv))
- for _, envID := range accessibleEnvironmentIDsInternal(ps) {
- allowed[envID] = struct{}{}
- }
-
- filtered := make([]environment.Environment, 0, len(allowed))
- for _, env := range envs {
- if _, ok := allowed[env.ID]; ok {
- filtered = append(filtered, env)
- }
- }
- return filtered
}
func (h *EnvironmentHandler) RunStreamProducer(ctx context.Context, ps *authz.PermissionSet, events chan<- environment.StreamEvent) {
var lastFingerprint uint64
var haveFingerprint bool
var lastSentAt time.Time
+ accessibleEnvIDs := accessibleEnvironmentIDs(ps)
- h.streamHub.Subscribe(ctx, environmentStreamHubKeyInternal, h.runEnvironmentStreamListerInternal, func(all []environment.Environment) bool {
- envs := visibleEnvironmentsForInternal(all, ps)
+ h.streamHub.Subscribe(ctx, environmentStreamHubKey, h.runEnvironmentStreamLister, func(all []environment.Environment) bool {
+ // all is shared across subscribers, so filter a copy.
+ envs := all
+ if accessibleEnvIDs != nil {
+ envs = slices.DeleteFunc(slices.Clone(all), func(env environment.Environment) bool { return !slices.Contains(accessibleEnvIDs, env.ID) })
+ }
fingerprint := kit.Fingerprint(envs)
- // Re-send unchanged state on a floor so relative timestamps in the UI
- // ("last seen 2 minutes ago") keep advancing.
+ // Re-send unchanged state on a floor so relative UI timestamps keep advancing.
if haveFingerprint && fingerprint == lastFingerprint && time.Since(lastSentAt) < environmentStreamRefreshFloor {
return true
}
@@ -276,7 +239,7 @@ func (h *EnvironmentHandler) RunStreamProducer(ctx context.Context, ps *authz.Pe
})
}
-func (h *EnvironmentHandler) runEnvironmentStreamListerInternal(ctx context.Context, publish func([]environment.Environment)) {
+func (h *EnvironmentHandler) runEnvironmentStreamLister(ctx context.Context, publish func([]environment.Environment)) {
changes, unsubscribe := h.environmentService.SubscribeRuntimeChanges()
defer unsubscribe()
@@ -351,100 +314,57 @@ func (h *EnvironmentHandler) CreateEnvironment(ctx context.Context, input *Creat
if input.Body.IsEdge != nil {
env.IsEdge = *input.Body.IsEdge
}
-
- // Determine pairing method
useApiKey := input.Body.UseApiKey != nil && *input.Body.UseApiKey
-
if useApiKey {
- return h.createEnvironmentWithApiKeyInternal(ctx, env, user)
+ env.Status = string(EnvironmentStatusPending)
+ } else if input.Body.AccessToken != nil && *input.Body.AccessToken != "" {
+ env.AccessToken = input.Body.AccessToken
}
- return h.createEnvironmentLegacyInternal(ctx, env, user, input.Body)
-}
-
-func (h *EnvironmentHandler) createEnvironmentWithApiKeyInternal(ctx context.Context, env *Environment, user *usertypes.Actor) (*handlerutil.Out[EnvironmentWithApiKey], error) {
- // New API key-based pairing flow
- env.Status = string(EnvironmentStatusPending)
-
created, err := h.environmentService.CreateEnvironment(ctx, env, &user.ID, &user.Username)
if err != nil {
return nil, huma.Error500InternalServerError("Failed to create environment: " + err.Error())
}
- // Generate API key for environment
- apiKeyDto, err := h.apiKeyService.CreateEnvironmentApiKey(ctx, created.ID)
- if err != nil {
- slog.ErrorContext(ctx, "Failed to create environment API key", "environmentId", created.ID, "error", err.Error())
- return nil, huma.Error500InternalServerError("Failed to create environment API key")
- }
-
- // Store the full API key in AccessToken for manager-to-agent auth.
- apiKey := apiKeyDto.Key
-
- // Link the API key to the environment for manager use.
- updates := map[string]any{
- "api_key_id": apiKeyDto.ID,
- "access_token": apiKey,
- }
- updated, err := h.environmentService.UpdateEnvironment(ctx, created.ID, updates, &user.ID, &user.Username)
- if err != nil {
- // Remove the key so a failed create does not leave an orphaned row
- // behind. ErrApiKeyProtected means the link actually landed and the
- // update failed afterwards, so the key legitimately belongs to the
- // environment and must survive; deleting the environment cascades it.
- if delErr := h.apiKeyService.DeleteApiKey(ctx, apiKeyDto.ID); delErr != nil &&
- !errors.Is(delErr, apikey.ErrApiKeyNotFound) && !errors.Is(delErr, apikey.ErrApiKeyProtected) {
- slog.ErrorContext(ctx, "Failed to clean up unlinked environment API key", "environmentId", created.ID, "error", delErr.Error())
+ var apiKey *string
+ if useApiKey {
+ apiKeyDto, keyErr := h.apiKeyService.CreateEnvironmentApiKey(ctx, created.ID)
+ if keyErr != nil {
+ slog.ErrorContext(ctx, "Failed to create environment API key", "environmentId", created.ID, "error", keyErr.Error())
+ return nil, huma.Error500InternalServerError("Failed to create environment API key")
+ }
+
+ // The full key doubles as the manager-to-agent access token.
+ updates := map[string]any{
+ "api_key_id": apiKeyDto.ID,
+ "access_token": apiKeyDto.Key,
+ }
+ updated, updateErr := h.environmentService.UpdateEnvironment(ctx, created.ID, updates, &user.ID, &user.Username)
+ if updateErr != nil {
+ // ErrApiKeyProtected means the link landed, so the key belongs to the environment and must survive.
+ if delErr := h.apiKeyService.DeleteApiKey(ctx, apiKeyDto.ID); delErr != nil &&
+ !errors.Is(delErr, apikey.ErrApiKeyNotFound) && !errors.Is(delErr, apikey.ErrApiKeyProtected) {
+ slog.ErrorContext(ctx, "Failed to clean up unlinked environment API key", "environmentId", created.ID, "error", delErr.Error())
+ }
+ slog.ErrorContext(ctx, "Failed to link API key to environment", "environmentId", created.ID, "error", updateErr.Error())
+ return nil, huma.Error500InternalServerError("Failed to link API key")
}
- slog.ErrorContext(ctx, "Failed to link API key to environment", "environmentId", created.ID, "error", err.Error())
- return nil, huma.Error500InternalServerError("Failed to link API key")
+ created = updated
+ apiKey = new(apiKeyDto.Key)
+ } else if created.AccessToken != nil && *created.AccessToken != "" {
+ h.triggerEnvironmentResourceSync(ctx, created.ID, created.Name, "environment creation")
}
- created = updated
out, mapErr := mapping.MapOne[*Environment, environment.Environment](created)
if mapErr != nil {
return nil, huma.Error500InternalServerError("Failed to map environment")
}
- h.applyEdgeRuntimeStateInternal(&out)
+ ApplyEnvironmentRuntimeState(&out)
return &handlerutil.Out[EnvironmentWithApiKey]{
Body: base.ApiResponse[EnvironmentWithApiKey]{
Success: true,
- Data: EnvironmentWithApiKey{
- Environment: out,
- ApiKey: new(apiKeyDto.Key),
- },
- },
- }, nil
-}
-
-func (h *EnvironmentHandler) createEnvironmentLegacyInternal(ctx context.Context, env *Environment, user *usertypes.Actor, body environment.Create) (*handlerutil.Out[EnvironmentWithApiKey], error) {
- if body.AccessToken != nil && *body.AccessToken != "" {
- env.AccessToken = body.AccessToken
- }
-
- created, err := h.environmentService.CreateEnvironment(ctx, env, &user.ID, &user.Username)
- if err != nil {
- return nil, huma.Error500InternalServerError("Failed to create environment: " + err.Error())
- }
-
- // Sync registries and git repositories in background (intentionally detached from request context)
- if created.AccessToken != nil && *created.AccessToken != "" {
- h.triggerEnvironmentResourceSyncInternal(ctx, created.ID, created.Name, "environment creation")
- }
-
- out, mapErr := mapping.MapOne[*Environment, environment.Environment](created)
- if mapErr != nil {
- return nil, huma.Error500InternalServerError("Failed to map environment")
- }
- h.applyEdgeRuntimeStateInternal(&out)
-
- return &handlerutil.Out[EnvironmentWithApiKey]{
- Body: base.ApiResponse[EnvironmentWithApiKey]{
- Success: true,
- Data: EnvironmentWithApiKey{
- Environment: out,
- },
+ Data: EnvironmentWithApiKey{Environment: out, ApiKey: apiKey},
},
}, nil
}
@@ -460,9 +380,9 @@ func (h *EnvironmentHandler) GetEnvironment(ctx context.Context, input *GetEnvir
if mapErr != nil {
return nil, huma.Error500InternalServerError("Failed to map environment")
}
- h.applyEdgeRuntimeStateInternal(&out)
+ ApplyEnvironmentRuntimeState(&out)
if env.IsEdge {
- if certInfo, certErr := readGeneratedEdgeMTLSCertificateInfoInternal(h.cfg, env.ID); certErr == nil {
+ if certInfo, certErr := readGeneratedEdgeMTLSCertificateInfo(h.edgeMTLSConfig(), env.ID); certErr == nil {
out.EdgeMTLSCertificate = certInfo
}
}
@@ -477,17 +397,24 @@ func (h *EnvironmentHandler) GetEnvironment(ctx context.Context, input *GetEnvir
// UpdateEnvironment updates an environment.
func (h *EnvironmentHandler) UpdateEnvironment(ctx context.Context, input *UpdateEnvironmentInput) (*handlerutil.Out[environment.Environment], error) {
- isLocalEnv := input.ID == localDockerEnvironmentID
- updates := h.buildUpdateMapInternal(&input.Body, isLocalEnv)
-
- h.handleEnvironmentPairingInternal(ctx, input.ID, &input.Body, updates, isLocalEnv)
-
- user, _ := userctx.CurrentUserFromContext(ctx)
- var userID, username *string
- if user != nil {
- userID = new(user.ID)
- username = new(user.Username)
+ req := &input.Body
+ updates := map[string]any{}
+ if input.ID != LocalEnvironmentID {
+ if req.ApiUrl != nil {
+ updates["api_url"] = *req.ApiUrl
+ }
+ if req.Enabled != nil {
+ updates["enabled"] = *req.Enabled
+ }
+ if req.AccessToken != nil {
+ updates["access_token"] = *req.AccessToken
+ }
+ }
+ if req.Name != nil {
+ updates["name"] = *req.Name
}
+
+ userID, username := currentUserRefs(ctx)
updated, updateErr := h.environmentService.UpdateEnvironment(ctx, input.ID, updates, userID, username)
if updateErr != nil {
apiErr := common.ToAPIError(updateErr)
@@ -497,17 +424,21 @@ func (h *EnvironmentHandler) UpdateEnvironment(ctx context.Context, input *Updat
return nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)
}
- h.triggerPostUpdateTasksInternal(ctx, input.ID, updated, &input.Body)
-
- out, mapErr := mapping.MapOne[*Environment, environment.Environment](updated)
- if mapErr != nil {
- return nil, huma.Error500InternalServerError("Failed to map environment")
+ if updated.Enabled {
+ go func(syncCtx context.Context, envID, envName string) {
+ status, err := h.environmentService.TestConnection(syncCtx, envID, nil)
+ if err != nil {
+ slog.WarnContext(syncCtx, "Failed to test connection after environment update",
+ "environmentId", envID, "environmentName", envName, "status", status, "error", err)
+ }
+ }(context.WithoutCancel(ctx), input.ID, updated.Name)
+ }
+ if updated.AccessToken != nil && *updated.AccessToken != "" && ((req.AccessToken != nil && *req.AccessToken != "") || req.Name != nil) {
+ h.triggerEnvironmentResourceSync(ctx, input.ID, updated.Name, "environment update")
}
- h.applyEdgeRuntimeStateInternal(&out)
- // If regenerating API key, return the new key
var newApiKey *string
- if input.Body.RegenerateApiKey != nil && *input.Body.RegenerateApiKey {
+ if req.RegenerateApiKey != nil && *req.RegenerateApiKey {
localUser, err := handlerutil.RequireUser(ctx)
if err != nil {
return nil, err
@@ -519,24 +450,19 @@ func (h *EnvironmentHandler) UpdateEnvironment(ctx context.Context, input *Updat
return nil, huma.Error500InternalServerError("Failed to regenerate API key")
}
- // Fetch updated environment
updated, err = h.environmentService.GetEnvironmentByID(ctx, input.ID)
if err != nil {
slog.ErrorContext(ctx, "Failed to fetch updated environment", "environmentId", input.ID, "error", err.Error())
return nil, huma.Error500InternalServerError("Failed to fetch updated environment")
}
-
- // Re-map with updated environment data
- out, mapErr = mapping.MapOne[*Environment, environment.Environment](updated)
- if mapErr != nil {
- return nil, huma.Error500InternalServerError("Failed to map environment")
- }
- h.applyEdgeRuntimeStateInternal(&out)
-
newApiKey = new(apiKey)
}
- // Set the API key on the response if regenerated
+ out, mapErr := mapping.MapOne[*Environment, environment.Environment](updated)
+ if mapErr != nil {
+ return nil, huma.Error500InternalServerError("Failed to map environment")
+ }
+ ApplyEnvironmentRuntimeState(&out)
out.ApiKey = newApiKey
return &handlerutil.Out[environment.Environment]{
@@ -547,22 +473,21 @@ func (h *EnvironmentHandler) UpdateEnvironment(ctx context.Context, input *Updat
}, nil
}
-func (h *EnvironmentHandler) applyEdgeRuntimeStateInternal(env *environment.Environment) {
- ApplyEnvironmentRuntimeState(env)
+// currentUserRefs returns the request user's ID and username, or nils without a user.
+func currentUserRefs(ctx context.Context) (userID, username *string) {
+ if user, _ := userctx.CurrentUserFromContext(ctx); user != nil {
+ return new(user.ID), new(user.Username)
+ }
+ return nil, nil
}
// DeleteEnvironment deletes an environment.
func (h *EnvironmentHandler) DeleteEnvironment(ctx context.Context, input *DeleteEnvironmentInput) (*handlerutil.Out[base.MessageResponse], error) {
- if input.ID == localDockerEnvironmentID {
+ if input.ID == LocalEnvironmentID {
return nil, huma.Error400BadRequest("Cannot delete local environment")
}
- user, _ := userctx.CurrentUserFromContext(ctx)
- var userID, username *string
- if user != nil {
- userID = new(user.ID)
- username = new(user.Username)
- }
+ userID, username := currentUserRefs(ctx)
if err := h.environmentService.DeleteEnvironment(ctx, input.ID, userID, username); err != nil {
return nil, huma.Error500InternalServerError("Failed to delete environment: " + err.Error())
}
@@ -619,7 +544,7 @@ func (h *EnvironmentHandler) UpdateHeartbeat(ctx context.Context, input *UpdateH
// PairAgent generates or rotates the local agent pairing token.
func (h *EnvironmentHandler) PairAgent(ctx context.Context, input *PairAgentInput) (*handlerutil.Out[environment.AgentPairResponse], error) {
- if input.ID != localDockerEnvironmentID {
+ if input.ID != LocalEnvironmentID {
return nil, huma.Error404NotFound("Not found")
}
@@ -658,93 +583,20 @@ func (h *EnvironmentHandler) SyncEnvironment(ctx context.Context, input *SyncEnv
return handlerutil.MessageOutput("Environment synced successfully", activityID), nil
}
-func (h *EnvironmentHandler) buildUpdateMapInternal(req *environment.Update, isLocalEnv bool) map[string]any {
- updates := map[string]any{}
-
- if !isLocalEnv {
- if req.ApiUrl != nil {
- updates["api_url"] = *req.ApiUrl
- }
- if req.Enabled != nil {
- updates["enabled"] = *req.Enabled
- }
- }
-
- if req.Name != nil {
- updates["name"] = *req.Name
- }
-
- return updates
-}
-
-func (h *EnvironmentHandler) handleEnvironmentPairingInternal(ctx context.Context, environmentID string, req *environment.Update, updates map[string]any, isLocalEnv bool) {
- _ = ctx
- _ = environmentID
- if isLocalEnv {
- return
- }
-
- if req.AccessToken != nil {
- updates["access_token"] = *req.AccessToken
- }
-}
-
-func (h *EnvironmentHandler) triggerPostUpdateTasksInternal(ctx context.Context, environmentID string, updated *Environment, req *environment.Update) {
- if updated.Enabled {
- detachedCtx := context.WithoutCancel(ctx)
- go func(syncCtx context.Context, envID, envName string) {
- status, err := h.environmentService.TestConnection(syncCtx, envID, nil)
- if err != nil {
- slog.WarnContext(syncCtx, "Failed to test connection after environment update",
- "environmentId", envID, "environmentName", envName, "status", status, "error", err)
- }
- }(detachedCtx, environmentID, updated.Name)
- }
-
- if updated.AccessToken != nil && *updated.AccessToken != "" && ((req.AccessToken != nil && *req.AccessToken != "") || req.Name != nil) {
- h.triggerEnvironmentResourceSyncInternal(ctx, environmentID, updated.Name, "environment update")
- }
-}
-
-func (h *EnvironmentHandler) triggerEnvironmentResourceSyncInternal(ctx context.Context, environmentID, environmentName, reason string) {
+func (h *EnvironmentHandler) triggerEnvironmentResourceSync(ctx context.Context, environmentID, environmentName, reason string) {
h.environmentService.ForgetSyncState(environmentID)
detachedCtx := context.WithoutCancel(ctx)
- go func(syncCtx context.Context, envID, envName, syncReason string) {
- syncCtx, cancel := context.WithTimeout(syncCtx, edge.DefaultProxyTimeout)
- defer cancel()
- if err := h.environmentService.SyncRegistriesToEnvironment(syncCtx, envID); err != nil {
- slog.WarnContext(syncCtx, "Failed to sync registries to environment",
- "environmentId", envID,
- "environmentName", envName,
- "reason", syncReason,
- "error", err.Error())
- }
- }(detachedCtx, environmentID, environmentName, reason)
-
- go func(syncCtx context.Context, envID, envName, syncReason string) {
- syncCtx, cancel := context.WithTimeout(syncCtx, edge.DefaultProxyTimeout)
- defer cancel()
- if err := h.environmentService.SyncS3DestinationsToEnvironment(syncCtx, envID); err != nil {
- slog.WarnContext(syncCtx, "Failed to sync S3 destinations to environment",
- "environmentId", envID,
- "environmentName", envName,
- "reason", syncReason,
- "error", err.Error())
- }
- }(detachedCtx, environmentID, environmentName, reason)
-
- go func(syncCtx context.Context, envID, envName, syncReason string) {
- syncCtx, cancel := context.WithTimeout(syncCtx, edge.DefaultProxyTimeout)
- defer cancel()
- if err := h.environmentService.SyncRepositoriesToEnvironment(syncCtx, envID); err != nil {
- slog.WarnContext(syncCtx, "Failed to sync git repositories to environment",
- "environmentId", envID,
- "environmentName", envName,
- "reason", syncReason,
- "error", err.Error())
- }
- }(detachedCtx, environmentID, environmentName, reason)
+ for _, resource := range h.environmentService.resourceSyncs() {
+ go func() {
+ syncCtx, cancel := context.WithTimeout(detachedCtx, edge.DefaultProxyTimeout)
+ defer cancel()
+ if err := resource.sync(syncCtx, environmentID); err != nil {
+ slog.WarnContext(syncCtx, "Failed to sync resources to environment",
+ "kind", resource.kind, "environmentId", environmentID, "environmentName", environmentName, "reason", reason, "error", err.Error())
+ }
+ }()
+ }
}
// PairEnvironment handles agent pairing callback with API key.
@@ -783,7 +635,7 @@ func (h *EnvironmentHandler) PairEnvironment(ctx context.Context, input *PairEnv
}
slog.InfoContext(ctx, "Environment pairing completed", "environmentId", *envID, "environmentName", env.Name)
- h.triggerEnvironmentResourceSyncInternal(ctx, *envID, env.Name, "environment pairing")
+ h.triggerEnvironmentResourceSync(ctx, *envID, env.Name, "environment pairing")
return handlerutil.MessageOutput("Environment pairing completed successfully", ""), nil
}
@@ -803,16 +655,9 @@ func (h *EnvironmentHandler) GetDeploymentSnippets(ctx context.Context, input *G
return nil, huma.Error400BadRequest("Environment is missing access token")
}
- // Generate snippets with API key
- // Use edge snippets for edge environments
var snippets *DeploymentSnippets
if env.IsEdge {
- snippets, err = h.environmentService.GenerateEdgeDeploymentSnippets(ctx, env.ID, h.cfg.GetAppURL(), *env.AccessToken, &edge.Config{
- EdgeMTLSMode: h.cfg.EdgeMTLSMode,
- EdgeMTLSCAFile: h.cfg.EdgeMTLSCAFile,
- EdgeMTLSAssetsDir: h.cfg.EdgeMTLSAssetsDir,
- AppURL: h.cfg.GetAppURL(),
- })
+ snippets, err = h.environmentService.GenerateEdgeDeploymentSnippets(ctx, env.ID, h.cfg.GetAppURL(), *env.AccessToken, h.edgeMTLSConfig())
} else {
snippets, err = h.environmentService.GenerateDeploymentSnippets(ctx, env.ID, h.cfg.GetAppURL(), env.ApiUrl, *env.AccessToken)
}
@@ -825,14 +670,14 @@ func (h *EnvironmentHandler) GetDeploymentSnippets(ctx context.Context, input *G
if snippets.MTLS != nil {
files := make([]DeploymentSnippetFile, 0, len(snippets.MTLS.Files))
for _, file := range snippets.MTLS.Files {
- sensitive := isSensitiveMTLSAssetNameInternal(file.Name)
entry := DeploymentSnippetFile{
Name: file.Name,
ContainerPath: file.ContainerPath,
Permissions: file.Permissions,
DownloadURL: fmt.Sprintf("/api/environments/%s/deployment/mtls/%s", env.ID, file.Name),
}
- if sensitive {
+ // Secret material is only served through the download endpoint.
+ if isSensitiveMTLSAssetName(file.Name) {
entry.Sensitive = true
} else {
entry.Content = file.Content
@@ -859,7 +704,7 @@ func (h *EnvironmentHandler) GetDeploymentSnippets(ctx context.Context, input *G
}, nil
}
-// GetEnvironmentVersion returns the version of a remote environment.
+// GetEnvironmentVersion returns the version of the local or a remote environment.
func (h *EnvironmentHandler) GetEnvironmentVersion(ctx context.Context, input *GetEnvironmentVersionInput) (*handlerutil.Out[version.Info], error) {
env, err := h.environmentService.GetEnvironmentByID(ctx, input.ID)
if err != nil {
@@ -870,61 +715,15 @@ func (h *EnvironmentHandler) GetEnvironmentVersion(ctx context.Context, input *G
defer cancel()
var versionInfo version.Info
-
- // For edge environments, route through the tunnel
- if env.IsEdge {
- if !edge.HasActiveTunnel(input.ID) {
- if _, ok := edge.RequestTunnelAndWait(reqCtx, input.ID, edge.DefaultTunnelDemandTTL, edge.DefaultTunnelAcquireTimeout()).Get(); !ok {
- return nil, huma.Error503ServiceUnavailable("Edge agent is not connected")
- }
- }
-
- statusCode, respBody, doRequestErr := edge.DoRequest(reqCtx, input.ID, http.MethodGet, "/api/app-version", nil)
- if doRequestErr != nil {
- return nil, huma.Error500InternalServerError("Request via tunnel failed: " + doRequestErr.Error())
- }
- if statusCode != http.StatusOK {
- return nil, huma.Error500InternalServerError(fmt.Sprintf("Unexpected status code: %d", statusCode))
- }
-
- if unmarshalErr := json.Unmarshal(respBody, &versionInfo); unmarshalErr != nil {
- return nil, huma.Error500InternalServerError("Failed to decode version response")
- }
- } else {
- // Direct HTTP request for non-edge environments
- validatedURL, validateErr := httpx.ValidateOutboundHTTPURL(env.ApiUrl)
- if validateErr != nil {
- return nil, huma.Error400BadRequest("Invalid environment API URL")
- }
- validatedURL.RawQuery = ""
- validatedURL.Fragment = ""
- validatedURL.Path = strings.TrimRight(validatedURL.Path, "/") + "/api/app-version"
-
- req, newRequestWithContextErr := http.NewRequestWithContext(reqCtx, http.MethodGet, validatedURL.String(), http.NoBody)
- if newRequestWithContextErr != nil {
- return nil, huma.Error500InternalServerError("Failed to create request")
- }
-
- client := &http.Client{Timeout: 15 * time.Second}
- resp, newRequestWithContextErr := client.Do(req)
- if newRequestWithContextErr != nil {
- return nil, huma.Error500InternalServerError("Request failed: " + newRequestWithContextErr.Error())
- }
- defer func() { _ = resp.Body.Close() }()
-
- if resp.StatusCode != http.StatusOK {
- return nil, huma.Error500InternalServerError(fmt.Sprintf("Unexpected status code: %d", resp.StatusCode))
- }
-
- if unmarshalReadErr := json.UnmarshalRead(resp.Body, &versionInfo); unmarshalReadErr != nil {
- return nil, huma.Error500InternalServerError("Failed to decode version response")
- }
+ if env.ID == LocalEnvironmentID && h.localVersion != nil {
+ versionInfo = *h.localVersion(reqCtx)
+ } else if proxyErr := h.environmentService.ProxyJSONRequestForEnvironment(reqCtx, *env, http.MethodGet, "/api/app-version", nil, &versionInfo); proxyErr != nil {
+ return nil, handlerutil.TranslateRemoteProxyError(proxyErr)
}
- // Update environment status to online since we successfully contacted it
+ // A successful reply proves the environment is online.
if updateErr := h.environmentService.UpdateEnvironmentHeartbeat(ctx, input.ID); updateErr != nil {
slog.WarnContext(ctx, "Failed to update environment heartbeat", "environmentId", input.ID, "error", updateErr)
- // Don't fail the request if heartbeat update fails
}
return &handlerutil.Out[version.Info]{
@@ -937,55 +736,26 @@ func (h *EnvironmentHandler) GetEnvironmentVersion(ctx context.Context, input *G
// DownloadEdgeMTLSCA downloads the Arcane-managed edge mTLS CA certificate.
func (h *EnvironmentHandler) DownloadEdgeMTLSCA(ctx context.Context, _ *DownloadEdgeMTLSCAInput) (*huma.StreamResponse, error) {
- var edgeCfg *edge.Config
- if h.cfg != nil {
- edgeCfg = &edge.Config{
- EdgeMTLSMode: h.cfg.EdgeMTLSMode,
- EdgeMTLSCAFile: h.cfg.EdgeMTLSCAFile,
- EdgeMTLSAssetsDir: h.cfg.EdgeMTLSAssetsDir,
- }
- }
- caPath, err := edge.AvailableManagerMTLSCAPath(edgeCfg)
+ caPath, err := edge.AvailableManagerMTLSCAPath(h.edgeMTLSConfig())
if err != nil {
return nil, huma.Error404NotFound("Arcane-managed edge mTLS CA is not available")
}
- // os.* rather than acfs: the CA path may resolve to a user-configured
- // location anywhere on the host, so no confinement root exists for it.
+ // os rather than acfs: the CA path may be configured anywhere on the host.
caPEM, err := os.ReadFile(caPath)
if err != nil {
slog.ErrorContext(ctx, "Failed to read generated edge mTLS CA", "path", caPath, "error", err.Error())
return nil, huma.Error500InternalServerError("Failed to read Arcane-generated edge mTLS CA")
}
- fileName := filepath.Base(caPath)
- if strings.TrimSpace(fileName) == "" {
- fileName = "ca.crt"
- }
-
- return &huma.StreamResponse{
- Body: func(humaCtx huma.Context) { //nolint:contextcheck // context is obtained from humaCtx.Context()
- humaCtx.SetHeader("Content-Type", "application/x-pem-file")
- humaCtx.SetHeader("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName))
- humaCtx.SetHeader("Content-Length", strconv.Itoa(len(caPEM)))
-
- if written, writeErr := humaCtx.BodyWriter().Write(bytes.Clone(caPEM)); writeErr != nil || written != len(caPEM) {
- slog.WarnContext(humaCtx.Context(), "Failed to stream edge mTLS CA download", "fileName", fileName, "bytesWritten", written, "bytesExpected", len(caPEM), "error", writeErr)
- return
- }
- h.logMTLSAuditEventInternal(humaCtx.Context(), nil, event.EventTypeEnvironmentMTLSDownload,
- "mTLS CA downloaded",
- fmt.Sprintf("Administrator downloaded edge mTLS CA %q", fileName),
- database.JSON{
- "fileName": fileName,
- "kind": "ca",
- })
- },
- }, nil
+ fileName := cmp.Or(strings.TrimSpace(filepath.Base(caPath)), "ca.crt")
+ return h.mtlsDownloadResponse(nil, "application/x-pem-file", fileName, caPEM,
+ "mTLS CA downloaded", fmt.Sprintf("Administrator downloaded edge mTLS CA %q", fileName),
+ database.JSON{"fileName": fileName, "kind": "ca"}), nil
}
func (h *EnvironmentHandler) DownloadEnvironmentMTLSBundle(ctx context.Context, input *DownloadEnvironmentMTLSBundleInput) (*huma.StreamResponse, error) {
- env, files, err := h.loadEnvironmentMTLSFilesInternal(ctx, input.ID)
+ env, files, err := h.loadEnvironmentMTLSFiles(ctx, input.ID)
if err != nil {
return nil, err
}
@@ -994,12 +764,16 @@ func (h *EnvironmentHandler) DownloadEnvironmentMTLSBundle(ctx context.Context,
zipWriter := zip.NewWriter(&archive)
for _, file := range files {
- downloadName := environmentMTLSAssetDownloadNameInternal(env, file.Name)
+ downloadName := environmentMTLSAssetDownloadName(env, file.Name)
header := &zip.FileHeader{
Name: downloadName,
Method: zip.Deflate,
}
- header.SetMode(environmentMTLSAssetFileModeInternal(file))
+ mode := kit.Ternary[os.FileMode](isSensitiveMTLSAssetName(file.Name), 0o600, 0o644)
+ if parsed, parseErr := strconv.ParseUint(strings.TrimSpace(file.Permissions), 8, 32); parseErr == nil && parsed != 0 {
+ mode = os.FileMode(parsed)
+ }
+ header.SetMode(mode)
entry, createErr := zipWriter.CreateHeader(header)
if createErr != nil {
@@ -1018,122 +792,57 @@ func (h *EnvironmentHandler) DownloadEnvironmentMTLSBundle(ctx context.Context,
return nil, huma.Error500InternalServerError("Failed to build mTLS bundle")
}
- fileName := environmentMTLSDownloadBaseNameInternal(env) + "-mtls.zip"
-
- return &huma.StreamResponse{
- Body: func(humaCtx huma.Context) { //nolint:contextcheck // context is obtained from humaCtx.Context()
- humaCtx.SetHeader("Content-Type", "application/zip")
- humaCtx.SetHeader("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName))
- humaCtx.SetHeader("Content-Length", strconv.Itoa(archive.Len()))
-
- if written, writeErr := humaCtx.BodyWriter().Write(archive.Bytes()); writeErr != nil || written != archive.Len() {
- slog.WarnContext(
- humaCtx.Context(),
- "Failed to stream edge mTLS bundle download",
- "environmentId",
- input.ID,
- "fileName",
- fileName,
- "bytesWritten",
- written,
- "bytesExpected",
- archive.Len(),
- "error",
- writeErr,
- )
- return
- }
- h.logMTLSAuditEventInternal(humaCtx.Context(), env, event.EventTypeEnvironmentMTLSDownload,
- "mTLS bundle downloaded",
- fmt.Sprintf("Administrator downloaded edge mTLS bundle %q (%d files)", fileName, len(files)),
- database.JSON{
- "fileName": fileName,
- "kind": "bundle",
- "fileCount": len(files),
- })
- },
- }, nil
+ fileName := environmentMTLSDownloadBaseName(env) + "-mtls.zip"
+ return h.mtlsDownloadResponse(env, "application/zip", fileName, archive.Bytes(),
+ "mTLS bundle downloaded", fmt.Sprintf("Administrator downloaded edge mTLS bundle %q (%d files)", fileName, len(files)),
+ database.JSON{"fileName": fileName, "kind": "bundle", "fileCount": len(files)}), nil
}
func (h *EnvironmentHandler) DownloadEnvironmentMTLSFile(ctx context.Context, input *DownloadEnvironmentMTLSFileInput) (*huma.StreamResponse, error) {
- env, file, err := h.loadEnvironmentMTLSFileInternal(ctx, input.ID, input.FileName)
+ env, files, err := h.loadEnvironmentMTLSFiles(ctx, input.ID)
if err != nil {
return nil, err
}
+ index := slices.IndexFunc(files, func(file DeploymentSnippetFile) bool { return file.Name == input.FileName })
+ if index < 0 {
+ return nil, huma.Error404NotFound("Requested mTLS asset was not found")
+ }
+ file := files[index]
- fileContent := []byte(file.Content)
- downloadName := environmentMTLSAssetDownloadNameInternal(env, file.Name)
+ return h.mtlsDownloadResponse(env, "application/x-pem-file", environmentMTLSAssetDownloadName(env, file.Name), []byte(file.Content),
+ "mTLS asset downloaded", fmt.Sprintf("Administrator downloaded edge mTLS asset %q", file.Name),
+ database.JSON{"fileName": file.Name, "kind": "file", "sensitive": isSensitiveMTLSAssetName(file.Name)}), nil
+}
- return &huma.StreamResponse{
- Body: func(humaCtx huma.Context) { //nolint:contextcheck // context is obtained from humaCtx.Context()
- humaCtx.SetHeader("Content-Type", "application/x-pem-file")
- humaCtx.SetHeader("Content-Disposition", fmt.Sprintf("attachment; filename=%q", downloadName))
- humaCtx.SetHeader("Content-Length", strconv.Itoa(len(fileContent)))
-
- if written, writeErr := humaCtx.BodyWriter().Write(fileContent); writeErr != nil || written != len(fileContent) {
- slog.WarnContext(
- humaCtx.Context(),
- "Failed to stream edge mTLS asset download",
- "environmentId",
- input.ID,
- "fileName",
- file.Name,
- "bytesWritten",
- written,
- "bytesExpected",
- len(
- fileContent,
- ),
- "error",
- writeErr,
- )
- return
- }
- h.logMTLSAuditEventInternal(humaCtx.Context(), env, event.EventTypeEnvironmentMTLSDownload,
- "mTLS asset downloaded",
- fmt.Sprintf("Administrator downloaded edge mTLS asset %q", file.Name),
- database.JSON{
- "fileName": file.Name,
- "kind": "file",
- "sensitive": isSensitiveMTLSAssetNameInternal(file.Name),
- })
- },
- }, nil
+// edgeMTLSConfig maps the handler config to edge mTLS settings, or nil without config.
+func (h *EnvironmentHandler) edgeMTLSConfig() *edge.Config {
+ if h.cfg == nil {
+ return nil
+ }
+ return &edge.Config{
+ EdgeMTLSMode: h.cfg.EdgeMTLSMode,
+ EdgeMTLSCAFile: h.cfg.EdgeMTLSCAFile,
+ EdgeMTLSAssetsDir: h.cfg.EdgeMTLSAssetsDir,
+ AppURL: h.cfg.GetAppURL(),
+ }
}
-func (h *EnvironmentHandler) loadEnvironmentMTLSEnvironmentInternal(ctx context.Context, environmentID string) (*Environment, error) {
+func (h *EnvironmentHandler) loadEnvironmentMTLSFiles(ctx context.Context, environmentID string) (*Environment, []DeploymentSnippetFile, error) {
env, err := h.environmentService.GetEnvironmentByID(ctx, environmentID)
if err != nil {
- return nil, huma.Error404NotFound("Environment not found")
+ return nil, nil, huma.Error404NotFound("Environment not found")
}
-
if !env.IsEdge {
- return nil, huma.Error400BadRequest("Environment is not an edge agent")
+ return nil, nil, huma.Error400BadRequest("Environment is not an edge agent")
}
-
if env.ApiKeyID == nil {
- return nil, huma.Error400BadRequest("Environment does not have an API key configured")
+ return nil, nil, huma.Error400BadRequest("Environment does not have an API key configured")
}
-
if env.AccessToken == nil || *env.AccessToken == "" {
- return nil, huma.Error400BadRequest("Environment is missing access token")
+ return nil, nil, huma.Error400BadRequest("Environment is missing access token")
}
- return env, nil
-}
-
-func (h *EnvironmentHandler) loadEnvironmentMTLSFilesInternal(ctx context.Context, environmentID string) (*Environment, []DeploymentSnippetFile, error) {
- env, err := h.loadEnvironmentMTLSEnvironmentInternal(ctx, environmentID)
- if err != nil {
- return nil, nil, err
- }
-
- snippets, err := h.environmentService.GenerateEdgeDeploymentSnippets(ctx, env.ID, h.cfg.GetAppURL(), *env.AccessToken, &edge.Config{
- EdgeMTLSMode: h.cfg.EdgeMTLSMode,
- EdgeMTLSCAFile: h.cfg.EdgeMTLSCAFile,
- EdgeMTLSAssetsDir: h.cfg.EdgeMTLSAssetsDir,
- AppURL: h.cfg.GetAppURL(),
- })
+ snippets, err := h.environmentService.GenerateEdgeDeploymentSnippets(ctx, env.ID, h.cfg.GetAppURL(), *env.AccessToken, h.edgeMTLSConfig())
if err != nil {
slog.ErrorContext(ctx, "Failed to generate environment mTLS assets", "environmentId", environmentID, "error", err.Error())
return nil, nil, huma.Error500InternalServerError("Failed to generate environment mTLS assets")
@@ -1146,150 +855,95 @@ func (h *EnvironmentHandler) loadEnvironmentMTLSFilesInternal(ctx context.Contex
return env, snippets.MTLS.Files, nil
}
-func (h *EnvironmentHandler) loadEnvironmentMTLSFileInternal(ctx context.Context, environmentID, fileName string) (*Environment, DeploymentSnippetFile, error) {
- env, files, err := h.loadEnvironmentMTLSFilesInternal(ctx, environmentID)
- if err != nil {
- return nil, DeploymentSnippetFile{}, err
- }
+// mtlsDownloadResponse streams an mTLS asset and audits the download; the audit
+// must never include certificate or key material.
+func (h *EnvironmentHandler) mtlsDownloadResponse(env *Environment, contentType, fileName string, content []byte, title, description string, metadata database.JSON) *huma.StreamResponse {
+ return &huma.StreamResponse{
+ Body: func(humaCtx huma.Context) { //nolint:contextcheck // context is obtained from humaCtx.Context()
+ ctx := humaCtx.Context()
+ humaCtx.SetHeader("Content-Type", contentType)
+ humaCtx.SetHeader("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName))
+ humaCtx.SetHeader("Content-Length", strconv.Itoa(len(content)))
- for _, file := range files {
- if file.Name == fileName {
- return env, file, nil
- }
- }
+ if written, writeErr := humaCtx.BodyWriter().Write(content); writeErr != nil || written != len(content) {
+ slog.WarnContext(ctx, "Failed to stream edge mTLS download", "fileName", fileName, "bytesWritten", written, "bytesExpected", len(content), "error", writeErr)
+ return
+ }
+ if h.eventService == nil {
+ return
+ }
- return nil, DeploymentSnippetFile{}, huma.Error404NotFound("Requested mTLS asset was not found")
+ if remoteAddr := strings.TrimSpace(middleware.GetRemoteAddrFromContext(ctx)); remoteAddr != "" {
+ metadata["remoteAddr"] = remoteAddr
+ }
+ userID, username := currentUserRefs(ctx)
+ req := event.CreateEventRequest{
+ Type: event.EventTypeEnvironmentMTLSDownload,
+ Severity: event.EventSeverityInfo,
+ Title: title,
+ Description: description,
+ UserID: userID,
+ Username: username,
+ Metadata: metadata,
+ }
+ if env != nil {
+ req.ResourceType = new("environment")
+ req.ResourceID = new(env.ID)
+ req.ResourceName = new(env.Name)
+ req.EnvironmentID = new(env.ID)
+ }
+ if _, err := h.eventService.CreateEvent(ctx, req); err != nil {
+ slog.WarnContext(ctx, "Failed to record mTLS audit event", "type", string(req.Type), "error", err)
+ }
+ },
+ }
}
-// isSensitiveMTLSAssetNameInternal reports whether the given generated asset
-// filename contains secret material (currently just the agent private key).
-// Sensitive asset contents must not be returned inline in JSON responses; the
-// client should fetch them via the admin-only download endpoint instead.
-func isSensitiveMTLSAssetNameInternal(fileName string) bool {
+// isSensitiveMTLSAssetName reports whether a generated asset holds secret material
+// (the agent private key), which must never be returned inline.
+func isSensitiveMTLSAssetName(fileName string) bool {
name := strings.ToLower(strings.TrimSpace(fileName))
return strings.HasSuffix(name, ".key") || strings.HasSuffix(name, "-key.pem") || strings.HasSuffix(name, "_key.pem")
}
-func environmentMTLSDownloadBaseNameInternal(env *Environment) string {
- baseName := cmp.Or(strings.TrimSpace(env.Name), "environment")
-
- baseName = strings.Map(func(r rune) rune {
+func environmentMTLSDownloadBaseName(env *Environment) string {
+ baseName := strings.Map(func(r rune) rune {
switch {
- case r >= 'a' && r <= 'z':
- return r
- case r >= 'A' && r <= 'Z':
- return r
- case r >= '0' && r <= '9':
+ case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
return r
default:
return '-'
}
- }, baseName)
-
- baseName = cmp.Or(strings.Trim(baseName, "-"), "environment")
-
- return baseName + "-" + env.ID
+ }, strings.TrimSpace(env.Name))
+ return cmp.Or(strings.Trim(baseName, "-"), "environment") + "-" + env.ID
}
-func environmentMTLSAssetDownloadNameInternal(env *Environment, fileName string) string {
- baseName := environmentMTLSDownloadBaseNameInternal(env)
-
+func environmentMTLSAssetDownloadName(env *Environment, fileName string) string {
switch fileName {
case "agent.crt":
- return baseName + ".pem"
+ return environmentMTLSDownloadBaseName(env) + ".pem"
case "agent.key":
- return baseName + ".key"
+ return environmentMTLSDownloadBaseName(env) + ".key"
default:
return fileName
}
}
-func environmentMTLSAssetFileModeInternal(file DeploymentSnippetFile) os.FileMode {
- if parsed, err := strconv.ParseUint(strings.TrimSpace(file.Permissions), 8, 32); err == nil && parsed != 0 {
- return os.FileMode(parsed)
- }
- return kit.Ternary[os.FileMode](isSensitiveMTLSAssetNameInternal(file.Name), 0o600, 0o644)
-}
-
-// logMTLSAuditEventInternal records an audit event for administrator-triggered
-// edge mTLS actions (downloads, bundle exports). Must never include raw
-// certificate content or private key material.
-func (h *EnvironmentHandler) logMTLSAuditEventInternal(ctx context.Context, env *Environment, eventType event.EventType, title, description string, extra database.JSON) {
- if h == nil || h.eventService == nil {
- return
- }
-
- user, _ := userctx.CurrentUserFromContext(ctx)
- var userID, username *string
- if user != nil {
- userID = new(user.ID)
- username = new(user.Username)
- }
-
- if extra == nil {
- extra = database.JSON{}
- }
- if remoteAddr := strings.TrimSpace(middleware.GetRemoteAddrFromContext(ctx)); remoteAddr != "" {
- extra["remoteAddr"] = remoteAddr
- }
-
- req := event.CreateEventRequest{
- Type: eventType,
- Severity: event.EventSeverityInfo,
- Title: title,
- Description: description,
- UserID: userID,
- Username: username,
- Metadata: extra,
- }
- if env != nil {
- envID := env.ID
- req.ResourceType = new("environment")
- req.ResourceID = &envID
- req.ResourceName = new(env.Name)
- req.EnvironmentID = &envID
- }
-
- if _, err := h.eventService.CreateEvent(ctx, req); err != nil {
- slog.WarnContext(ctx, "Failed to record mTLS audit event", "type", string(eventType), "error", err)
- }
-}
-
const edgeMTLSCertificateExpiryWarningWindow = 30 * 24 * time.Hour
-func generatedEdgeMTLSClientCertPathInternal(cfg *config.Config, envID string) (string, error) {
- if cfg == nil {
- return "", errors.New("config not available")
- }
- if edge.NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == edge.EdgeMTLSModeDisabled {
- return "", errors.New("edge mTLS is disabled")
+func readGeneratedEdgeMTLSCertificateInfo(edgeCfg *edge.Config, envID string) (*environment.EdgeMTLSCertificate, error) {
+ if edgeCfg == nil {
+ return nil, errors.New("config not available")
}
-
- edgeCfg := &edge.Config{
- EdgeMTLSAssetsDir: cfg.EdgeMTLSAssetsDir,
+ if edge.NormalizeEdgeMTLSMode(edgeCfg.EdgeMTLSMode) == edge.EdgeMTLSModeDisabled {
+ return nil, errors.New("edge mTLS is disabled")
}
certPath, err := edge.GeneratedManagerClientMTLSCertPath(edgeCfg, envID)
if err != nil {
- return "", fmt.Errorf("resolve generated edge mTLS client certificate path: %w", err)
- }
- // os.* rather than acfs: the assets dir may be user-configured to anywhere
- // on the host, so no confinement root exists for this path.
- if _, statErr := os.Stat(certPath); statErr != nil {
- return "", fmt.Errorf("stat generated edge mTLS client certificate: %w", statErr)
+ return nil, fmt.Errorf("resolve generated edge mTLS client certificate path: %w", err)
}
-
- return certPath, nil
-}
-
-func readGeneratedEdgeMTLSCertificateInfoInternal(cfg *config.Config, envID string) (*environment.EdgeMTLSCertificate, error) {
- certPath, err := generatedEdgeMTLSClientCertPathInternal(cfg, envID)
- if err != nil {
- return nil, err
- }
-
- // os.* rather than acfs: the assets dir may be user-configured to anywhere
- // on the host, so no confinement root exists for this path.
+ // os rather than acfs: the assets dir may be configured anywhere on the host.
certPEM, err := os.ReadFile(certPath)
if err != nil {
return nil, fmt.Errorf("read generated edge mTLS client certificate: %w", err)
@@ -1310,7 +964,7 @@ func readGeneratedEdgeMTLSCertificateInfoInternal(cfg *config.Config, envID stri
remaining := expiresAt.Sub(now)
info := &environment.EdgeMTLSCertificate{
ExpiresAt: &expiresAt,
- DaysRemaining: new(edgeMTLSCertificateDaysRemainingInternal(now, expiresAt)),
+ DaysRemaining: new(int(math.Ceil(max(remaining, 0).Hours() / 24))),
Expired: now.After(expiresAt),
ExpiringSoon: now.Before(expiresAt) && remaining <= edgeMTLSCertificateExpiryWarningWindow,
}
@@ -1321,11 +975,3 @@ func readGeneratedEdgeMTLSCertificateInfoInternal(cfg *config.Config, envID stri
return info, nil
}
-
-func edgeMTLSCertificateDaysRemainingInternal(now, expiresAt time.Time) int {
- remaining := expiresAt.Sub(now)
- if remaining <= 0 {
- return 0
- }
- return int(math.Ceil(remaining.Hours() / 24))
-}
diff --git a/backend/internal/environment/helpers.go b/backend/internal/environment/helpers.go
index 8b087ae95..11336bde9 100644
--- a/backend/internal/environment/helpers.go
+++ b/backend/internal/environment/helpers.go
@@ -5,34 +5,30 @@ import (
"strings"
"sync"
- "github.com/getarcaneapp/arcane/types/v2/environment"
"github.com/samber/hot"
"github.com/samber/mo"
-
- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
)
-// edgeTokenCacheInternal maps an edge agent token to its environment ID. The
-// reverse index lets an environment's entry be dropped when its token rotates,
-// which the TTL cache alone cannot do.
-type edgeTokenCacheInternal struct {
+// edgeTokenCache maps agent tokens to environment IDs; the reverse index drops
+// an environment's entry when its token rotates.
+type edgeTokenCache struct {
mu sync.RWMutex
byToken *hot.HotCache[string, string]
byEnvID map[string]string
}
-// remoteEnvSnapshotCacheInternal holds the latest in-process copy of every
-// enabled, visible, non-local environment, so hot paths avoid a DB round trip.
-type remoteEnvSnapshotCacheInternal struct {
+// remoteEnvSnapshotCache holds every enabled, visible, non-local environment
+// in memory so hot paths avoid a DB round trip.
+type remoteEnvSnapshotCache struct {
mu sync.RWMutex
envs map[string]Environment
seeded bool
revision uint64
}
-// environmentCacheKeyInternal scopes cached records to a cache generation so
-// loads that started before a write can never be served after it.
-type environmentCacheKeyInternal struct {
+// environmentCacheKey scopes cached records to a generation so loads that
+// started before a write are never served after it.
+type environmentCacheKey struct {
gen uint64
id string
}
@@ -43,43 +39,14 @@ const environmentCacheStripes = 64
var environmentCacheSeed = maphash.MakeSeed()
-func environmentCacheStripeInternal(id string) int {
- return int(maphash.String(environmentCacheSeed, id) % environmentCacheStripes)
-}
-
-// runtimeWatchersInternal fans a coalesced wake-up out to everyone watching for
-// environment liveness changes.
-type runtimeWatchersInternal struct {
+// runtimeWatchers fans a coalesced wake-up out to environment liveness watchers.
+type runtimeWatchers struct {
mu sync.Mutex
chans map[int]chan struct{}
seq int
}
-func newEdgeTokenCacheInternal() *edgeTokenCacheInternal {
- return &edgeTokenCacheInternal{
- byToken: hot.NewHotCache[string, string](hot.LRU, 1024).
- WithTTL(edgeTokenCacheTTL).
- WithJanitor().
- Build(),
- byEnvID: make(map[string]string),
- }
-}
-
-// newEnvironmentCacheInternal caches environment records by ID, including
-// misses, for hot paths that only read CRUD-managed fields.
-func newEnvironmentCacheInternal() *hot.HotCache[environmentCacheKeyInternal, Environment] {
- return hot.NewHotCache[environmentCacheKeyInternal, Environment](hot.LRU, 256).
- WithTTL(environmentCacheTTL).
- WithMissingSharedCache().
- WithJanitor().
- Build()
-}
-
-func newRemoteEnvSnapshotCacheInternal() *remoteEnvSnapshotCacheInternal {
- return &remoteEnvSnapshotCacheInternal{envs: make(map[string]Environment)}
-}
-
-func (c *edgeTokenCacheInternal) environmentID(token string) mo.Option[string] {
+func (c *edgeTokenCache) environmentID(token string) mo.Option[string] {
if c == nil || c.byToken == nil || token == "" {
return mo.None[string]()
}
@@ -99,7 +66,7 @@ func (c *edgeTokenCacheInternal) environmentID(token string) mo.Option[string] {
return mo.None[string]()
}
-func (c *edgeTokenCacheInternal) put(envID, token string) {
+func (c *edgeTokenCache) put(envID, token string) {
if c == nil || c.byToken == nil || envID == "" || token == "" {
return
}
@@ -115,7 +82,7 @@ func (c *edgeTokenCacheInternal) put(envID, token string) {
c.byToken.Set(token, envID)
}
-func (c *edgeTokenCacheInternal) invalidate(envID string) {
+func (c *edgeTokenCache) invalidate(envID string) {
if c == nil || envID == "" {
return
}
@@ -131,9 +98,8 @@ func (c *edgeTokenCacheInternal) invalidate(envID string) {
}
}
-// sync replaces an environment's cached token, dropping the entry entirely when
-// the new token is blank.
-func (c *edgeTokenCacheInternal) sync(envID, token string) {
+// sync replaces an environment's cached token, dropping it when the new token is blank.
+func (c *edgeTokenCache) sync(envID, token string) {
if c == nil || envID == "" {
return
}
@@ -145,11 +111,11 @@ func (c *edgeTokenCacheInternal) sync(envID, token string) {
}
}
-func isActiveRemoteEnvironmentInternal(env Environment) bool {
+func isActiveRemoteEnvironment(env Environment) bool {
return env.ID != "" && env.ID != "0" && env.Enabled && !env.Hidden
}
-func (c *remoteEnvSnapshotCacheInternal) get(environmentID string) mo.Option[Environment] {
+func (c *remoteEnvSnapshotCache) get(environmentID string) mo.Option[Environment] {
if c == nil || environmentID == "" {
return mo.None[Environment]()
}
@@ -157,18 +123,18 @@ func (c *remoteEnvSnapshotCacheInternal) get(environmentID string) mo.Option[Env
c.mu.RLock()
envRecord, ok := c.envs[environmentID]
c.mu.RUnlock()
- if !ok || !isActiveRemoteEnvironmentInternal(envRecord) {
+ if !ok || !isActiveRemoteEnvironment(envRecord) {
return mo.None[Environment]()
}
return mo.Some(envRecord)
}
-func (c *remoteEnvSnapshotCacheInternal) put(env Environment) {
+func (c *remoteEnvSnapshotCache) put(env Environment) {
if c == nil {
return
}
- if !isActiveRemoteEnvironmentInternal(env) {
+ if !isActiveRemoteEnvironment(env) {
c.remove(env.ID)
return
}
@@ -179,7 +145,7 @@ func (c *remoteEnvSnapshotCacheInternal) put(env Environment) {
c.mu.Unlock()
}
-func (c *remoteEnvSnapshotCacheInternal) remove(environmentID string) {
+func (c *remoteEnvSnapshotCache) remove(environmentID string) {
if c == nil || environmentID == "" {
return
}
@@ -190,7 +156,7 @@ func (c *remoteEnvSnapshotCacheInternal) remove(environmentID string) {
c.mu.Unlock()
}
-func (c *remoteEnvSnapshotCacheInternal) update(environmentID string, update func(*Environment)) {
+func (c *remoteEnvSnapshotCache) update(environmentID string, update func(*Environment)) {
if c == nil || environmentID == "" || update == nil {
return
}
@@ -204,60 +170,16 @@ func (c *remoteEnvSnapshotCacheInternal) update(environmentID string, update fun
return
}
update(&envRecord)
- if isActiveRemoteEnvironmentInternal(envRecord) {
+ if isActiveRemoteEnvironment(envRecord) {
c.envs[environmentID] = envRecord
} else {
delete(c.envs, environmentID)
}
}
-// filterEnvironmentsByIDInternal returns only the environments whose ID is in
-// allowedIDs. A non-nil but empty allowedIDs yields an empty slice. Used to
-// restrict the runtime-filtered list path to a caller's accessible environments.
-func filterEnvironmentsByIDInternal(items []environment.Environment, allowedIDs []string) []environment.Environment {
- allowed := make(map[string]struct{}, len(allowedIDs))
- for _, id := range allowedIDs {
- allowed[id] = struct{}{}
- }
- out := make([]environment.Environment, 0, len(items))
- for _, item := range items {
- if _, ok := allowed[item.ID]; ok {
- out = append(out, item)
- }
- }
- return out
-}
-
-func environmentTypeMatchesInternal(env environment.Environment, filterValue string) bool {
- return environmentTypeKeyInternal(env) == strings.ToLower(strings.TrimSpace(filterValue))
-}
-
-func environmentTypeKeyInternal(env environment.Environment) string {
- if !env.IsEdge {
- return "http"
- }
- transport := ""
- if env.Connected != nil && *env.Connected && env.EdgeTransport != nil {
- transport = *env.EdgeTransport
- } else if env.LastEdgeTransport != nil {
- // Disconnected or poll-only agents classify by the transport they
- // last used rather than collapsing into the generic edge bucket.
- transport = *env.LastEdgeTransport
- }
- switch strings.ToLower(strings.TrimSpace(transport)) {
- case edge.EdgeTransportWebSocket:
- return "websocket"
- case edge.EdgeTransportGRPC:
- return "grpc"
- default:
- return "edge"
- }
-}
-
-// subscribe returns a channel that receives a coalesced wake-up whenever
-// environment liveness may have changed, plus a function to release it. The
-// channel is never closed; callers select on it alongside their own context.
-func (w *runtimeWatchersInternal) subscribe() (<-chan struct{}, func()) {
+// subscribe returns a never-closed channel that receives a coalesced wake-up on
+// liveness changes, plus a function to release it.
+func (w *runtimeWatchers) subscribe() (<-chan struct{}, func()) {
w.mu.Lock()
defer w.mu.Unlock()
@@ -267,8 +189,7 @@ func (w *runtimeWatchersInternal) subscribe() (<-chan struct{}, func()) {
w.seq++
id := w.seq
- // Capacity 1: a second signal arriving before the watcher wakes is
- // redundant, because the watcher reads live state rather than a queue.
+ // Capacity 1: watchers read live state, so a second pending signal is redundant.
ch := make(chan struct{}, 1)
w.chans[id] = ch
@@ -279,9 +200,8 @@ func (w *runtimeWatchersInternal) subscribe() (<-chan struct{}, func()) {
}
}
-// notify wakes every watcher. It never blocks, so it is safe to call from
-// connection callbacks on the tunnel hot path.
-func (w *runtimeWatchersInternal) notify() {
+// notify wakes every watcher without blocking, so tunnel callbacks may call it.
+func (w *runtimeWatchers) notify() {
if w == nil {
return
}
diff --git a/backend/internal/environment/module.go b/backend/internal/environment/module.go
index 7cefa1057..50e15fe1d 100644
--- a/backend/internal/environment/module.go
+++ b/backend/internal/environment/module.go
@@ -3,7 +3,10 @@
package environment
import (
+ "context"
+
"github.com/danielgtaylor/huma/v2"
+ "github.com/getarcaneapp/arcane/types/v2/version"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -48,12 +51,14 @@ func (m *Module) Handler() *EnvironmentHandler {
return m.handler
}
-func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContext) {
+// RegisterRoutes registers the environment endpoints; localVersion answers version requests for the local environment.
+func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContext, localVersion func(context.Context) *version.Info) {
if m == nil {
RegisterEnvironments(api, NewHandler(nil, nil, nil, nil, nil, nil))
return
}
m.handler.appCtx = appCtx.Context()
+ m.handler.localVersion = localVersion
RegisterEnvironments(api, m.handler)
}
diff --git a/backend/internal/environment/service.go b/backend/internal/environment/service.go
index 2adeba10b..2dff0d14b 100644
--- a/backend/internal/environment/service.go
+++ b/backend/internal/environment/service.go
@@ -5,7 +5,9 @@ import (
"context"
"errors"
"fmt"
+ "hash/maphash"
"log/slog"
+ "maps"
"net/http"
"net/url"
"slices"
@@ -25,6 +27,9 @@ import (
"go.getarcane.app/kit/pkg"
"go.getarcane.app/kit/pkg/mapping"
"go.getarcane.app/sys/crypto"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
@@ -47,36 +52,28 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation"
)
type EnvironmentService struct {
db *database.DB
- httpClient *http.Client
- dockerService *docker.DockerClientService
eventService *event.EventService
settingsService *settings.SettingsService
- edgeTokens *edgeTokenCacheInternal
- remoteEnvs *remoteEnvSnapshotCacheInternal
- environmentCache *hot.HotCache[environmentCacheKeyInternal, Environment]
+ edgeTokens *edgeTokenCache
+ remoteEnvs *remoteEnvSnapshotCache
+ environmentCache *hot.HotCache[environmentCacheKey, Environment]
environmentGens [environmentCacheStripes]atomic.Uint64
- // jobs carries the scheduler and app lifecycle context, injected
- // post-construction via SetScheduler (manager-only). Each enabled environment
- // gets its own health-check job; this replaces the single global
- // environment-health job.
+ // jobs holds one health-check job per enabled environment; SetScheduler wires it on the manager.
jobs *entityjobs.Registry
- // variableSyncer is injected post-construction via SetVariableSyncer
- // (manager-only) to avoid a wire cycle with variable.VariableService.
+ // variableSyncer is set by SetVariableSyncer on the manager to avoid a wire cycle.
variableSyncer VariableSyncer
- // runtimeWatchers receive a coalesced wake-up whenever an environment's
- // liveness changes. See environment_runtime_notify.go.
- runtimeWatchers runtimeWatchersInternal
+ runtimeWatchers runtimeWatchers
- // syncGate skips the periodic registry/S3/repository pushes to an agent
- // while the payload it last accepted is unchanged.
+ // syncGate skips agent config pushes while the last accepted payload is unchanged.
syncGate utils.SyncGate
proxy *proxy.Service
@@ -87,12 +84,11 @@ type EnvironmentService struct {
const (
// LocalEnvironmentID is the reserved ID of the environment Arcane manages directly.
- LocalEnvironmentID = "0"
- localEnvironmentFallbackNameInternal = "Local"
+ LocalEnvironmentID = "0"
+ localEnvironmentFallbackName = "Local"
- // SyncDeliveryExpiry bounds how long an accepted config push is trusted. An
- // agent rebuilt with a fresh data volume while its status stayed online gets
- // everything again within this window without operator action.
+ // SyncDeliveryExpiry bounds how long an accepted config push is trusted, so a
+ // rebuilt agent that stayed online is resynced without operator action.
SyncDeliveryExpiry = time.Hour
)
@@ -103,14 +99,12 @@ var (
)
// VariableSyncer pushes the effective global-variable set to one environment.
-// Implemented by variable.VariableService.
type VariableSyncer interface {
SyncEnvironment(ctx context.Context, envID string) error
ForgetSyncState(envID string)
}
-// ForgetSyncState makes the next sync of every resource group resend to the
-// environment regardless of whether the payload changed.
+// ForgetSyncState makes the next sync of every resource group resend regardless of payload changes.
func (s *EnvironmentService) ForgetSyncState(environmentID string) {
s.syncGate.Forget(environmentID)
if s.variableSyncer != nil {
@@ -130,26 +124,70 @@ func NewEnvironmentService(
httpClient = http.DefaultClient
}
s := &EnvironmentService{
- syncGate: utils.SyncGate{Expiry: SyncDeliveryExpiry},
- db: db,
- httpClient: httpClient,
- dockerService: dockerService,
- eventService: eventService,
- settingsService: settingsService,
- edgeTokens: newEdgeTokenCacheInternal(),
- remoteEnvs: newRemoteEnvSnapshotCacheInternal(),
- environmentCache: newEnvironmentCacheInternal(),
- jobs: entityjobs.New(environmentHealthJobPrefix, environmentHealthAdmissionScopeInternal),
- health: health.New(httpClient, dockerService),
- snippets: snippets.New(eventService),
- swarm: swarm.New(apiKeyService),
+ syncGate: utils.SyncGate{Expiry: SyncDeliveryExpiry},
+ db: db,
+ eventService: eventService,
+ settingsService: settingsService,
+ edgeTokens: &edgeTokenCache{
+ byToken: hot.NewHotCache[string, string](hot.LRU, 1024).WithTTL(edgeTokenCacheTTL).WithJanitor().Build(),
+ byEnvID: make(map[string]string),
+ },
+ remoteEnvs: &remoteEnvSnapshotCache{envs: make(map[string]Environment)},
+ // Caches records by ID, including misses, for hot paths that read only CRUD-managed fields.
+ environmentCache: hot.NewHotCache[environmentCacheKey, Environment](hot.LRU, 256).
+ WithTTL(environmentCacheTTL).
+ WithMissingSharedCache().
+ WithJanitor().
+ Build(),
+ jobs: entityjobs.New(environmentHealthJobPrefix, environmentHealthAdmissionScope),
+ health: health.New(httpClient, dockerService),
+ snippets: snippets.New(eventService),
+ swarm: swarm.New(apiKeyService),
}
s.proxy = proxy.New(db, httpClient, settingsService, &s.syncGate)
+
+ meter := otel.Meter(tracing.InstrumentationName)
+ statusGauge, err := meter.Int64ObservableGauge("arcane.environments",
+ metric.WithDescription("Visible environments by persisted status"),
+ metric.WithUnit("{environment}"),
+ )
+ if err == nil {
+ _, err = meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
+ var rows []struct {
+ Status string
+ Count int64
+ }
+ if scanErr := s.db.WithContext(ctx).Model(&Environment{}).
+ Select("status, COUNT(*) AS count").
+ Where("hidden = ?", false).
+ Group("status").
+ Scan(&rows).Error; scanErr != nil {
+ return fmt.Errorf("failed to count environments by status: %w", scanErr)
+ }
+ // Known statuses report zero so their series drop instead of going stale.
+ counts := map[string]int64{
+ string(EnvironmentStatusOnline): 0,
+ string(EnvironmentStatusStandby): 0,
+ string(EnvironmentStatusOffline): 0,
+ string(EnvironmentStatusError): 0,
+ string(EnvironmentStatusPending): 0,
+ }
+ for _, row := range rows {
+ counts[row.Status] += row.Count
+ }
+ for status, count := range counts {
+ o.ObserveInt64(statusGauge, count, metric.WithAttributes(attribute.String("status", status)))
+ }
+ return nil
+ }, statusGauge)
+ }
+ if err != nil {
+ otel.Handle(err)
+ }
return s
}
-// SetVariableSyncer injects the global-variable syncer. Called during
-// bootstrap on the manager only; agents leave it nil.
+// SetVariableSyncer injects the global-variable syncer on the manager; agents leave it nil.
func (s *EnvironmentService) SetVariableSyncer(syncer VariableSyncer) {
s.variableSyncer = syncer
}
@@ -171,7 +209,7 @@ func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context,
Where("access_token = ?", token).
First(&env).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
- s.logEdgeTokenResolveMissInternal(ctx, token)
+ s.logEdgeTokenResolveMiss(ctx, token)
return "", errors.New("invalid agent token")
}
return "", fmt.Errorf("failed to resolve edge environment by token: %w", err)
@@ -181,16 +219,9 @@ func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context,
return env.ID, nil
}
-// logEdgeTokenResolveMissInternal emits a debug log diagnosing why an agent
-// token failed to resolve to an edge environment. Counts existing edge
-// environments (by access_token presence) so operators can distinguish
-// "no edge envs configured" from "token does not match any configured env".
-// Token contents are never logged in full — only length and a short
-// fingerprint that cannot be reversed.
-func (s *EnvironmentService) logEdgeTokenResolveMissInternal(ctx context.Context, token string) {
- if s == nil || s.db == nil {
- return
- }
+// logEdgeTokenResolveMiss logs edge environment counts at debug level so a token
+// miss can be diagnosed; only the token's length and irreversible fingerprint are logged.
+func (s *EnvironmentService) logEdgeTokenResolveMiss(ctx context.Context, token string) {
if !slog.Default().Enabled(ctx, slog.LevelDebug) {
return
}
@@ -217,9 +248,8 @@ func (s *EnvironmentService) logEdgeTokenResolveMissInternal(ctx context.Context
slog.DebugContext(ctx, "Edge agent token did not match any environment", args...)
}
-// ResolveEnvironmentName looks up an environment and returns the label to show for it.
-// Use this instead of hardcoding a name for a known ID: names are user-editable, so
-// even the local environment's is not fixed.
+// ResolveEnvironmentName returns an environment's display label; names are
+// user-editable, so never hardcode one for a known ID.
func (s *EnvironmentService) ResolveEnvironmentName(ctx context.Context, environmentID string) string {
if strings.TrimSpace(environmentID) == "" {
environmentID = LocalEnvironmentID
@@ -239,12 +269,11 @@ func (s *EnvironmentService) EnsureLocalEnvironment(ctx context.Context, appUrl
err := s.db.WithContext(ctx).Where("id = ?", LocalEnvironmentID).First(&existingEnv).Error
if err == nil {
- // Local environment already exists, ensure ApiUrl matches current appUrl
if existingEnv.ApiUrl != appUrl {
if updateLocalURLErr := s.db.WithContext(ctx).Model(&existingEnv).Update("api_url", appUrl).Error; updateLocalURLErr != nil {
return fmt.Errorf("failed to update local environment api url: %w", updateLocalURLErr)
}
- s.invalidateEnvironmentCacheInternal(LocalEnvironmentID)
+ s.invalidateEnvironmentCache(LocalEnvironmentID)
slog.InfoContext(ctx, "updated local environment api url", "id", LocalEnvironmentID, "url", appUrl)
}
return nil
@@ -254,7 +283,6 @@ func (s *EnvironmentService) EnsureLocalEnvironment(ctx context.Context, appUrl
return fmt.Errorf("failed to check for local environment: %w", err)
}
- // Create the local environment
now := time.Now()
localEnv := &Environment{
ID: LocalEnvironmentID,
@@ -269,7 +297,7 @@ func (s *EnvironmentService) EnsureLocalEnvironment(ctx context.Context, appUrl
if createLocalEnvironmentErr := s.db.WithContext(ctx).Create(localEnv).Error; createLocalEnvironmentErr != nil {
return fmt.Errorf("failed to create local environment: %w", createLocalEnvironmentErr)
}
- s.invalidateEnvironmentCacheInternal(LocalEnvironmentID)
+ s.invalidateEnvironmentCache(LocalEnvironmentID)
slog.InfoContext(ctx, "created local environment record", "id", LocalEnvironmentID)
return nil
@@ -294,29 +322,14 @@ func (s *EnvironmentService) CreateEnvironment(ctx context.Context, env *Environ
return nil, fmt.Errorf("failed to create environment: %w", err)
}
- // Create event in background
- go s.createEnvironmentEvent(
- context.WithoutCancel(
- ctx,
- ),
- env.ID,
- env.Name,
- event.EventTypeEnvironmentCreate,
- "Environment Created",
- fmt.Sprintf(
- "Environment '%s' was created",
- env.Name,
- ),
- event.EventSeveritySuccess,
- userID,
- username,
- )
+ go s.createEnvironmentEvent(context.WithoutCancel(ctx), env.ID, env.Name, event.EventTypeEnvironmentCreate,
+ "Environment Created", fmt.Sprintf("Environment '%s' was created", env.Name), event.EventSeveritySuccess, userID, username)
if env.Enabled {
- s.registerHealthJobInternal(ctx, env.ID)
+ s.registerHealthJob(ctx, env.ID)
}
s.remoteEnvs.put(*env)
- s.invalidateEnvironmentCacheInternal(env.ID)
+ s.invalidateEnvironmentCache(env.ID)
s.NotifyRuntimeStateChanged()
return env, nil
@@ -371,7 +384,7 @@ func (s *EnvironmentService) UpdateEnvironment(ctx context.Context, id string, u
if updateEnvironmentErr := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", id).Updates(updates).Error; updateEnvironmentErr != nil {
return nil, fmt.Errorf("failed to update environment: %w", updateEnvironmentErr)
}
- s.invalidateEnvironmentCacheInternal(id)
+ s.invalidateEnvironmentCache(id)
s.NotifyRuntimeStateChanged()
updated, err := s.GetEnvironmentByID(ctx, id)
@@ -389,45 +402,29 @@ func (s *EnvironmentService) UpdateEnvironment(ctx context.Context, id string, u
if rawEnabled, ok := updates["enabled"]; ok {
if enabled, isBool := rawEnabled.(bool); isBool {
if enabled {
- s.registerHealthJobInternal(ctx, id)
+ s.registerHealthJob(ctx, id)
} else {
- s.removeHealthJobInternal(ctx, id)
+ s.jobs.Unregister(ctx, id)
}
}
}
- // Create event in background (skip for local environment)
- if id != "0" {
- go s.createEnvironmentEvent(
- context.WithoutCancel(
- ctx,
- ),
- id,
- updated.Name,
- event.EventTypeEnvironmentUpdate,
- "Environment Updated",
- fmt.Sprintf(
- "Environment '%s' was updated",
- updated.Name,
- ),
- event.EventSeverityInfo,
- userID,
- username,
- )
+ if id != LocalEnvironmentID {
+ go s.createEnvironmentEvent(context.WithoutCancel(ctx), id, updated.Name, event.EventTypeEnvironmentUpdate,
+ "Environment Updated", fmt.Sprintf("Environment '%s' was updated", updated.Name), event.EventSeverityInfo, userID, username)
}
return updated, nil
}
func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, userID, username *string) error {
- // Get environment details before deletion
env, err := s.GetEnvironmentByID(ctx, id)
if err != nil {
return err
}
// Stop the per-environment health job before the row is removed.
- s.removeHealthJobInternal(ctx, id)
+ s.jobs.Unregister(ctx, id)
var syncIDs []string
if transactionErr := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
@@ -453,13 +450,12 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, u
return nil
}); transactionErr != nil {
if env.Enabled {
- s.registerHealthJobInternal(ctx, env.ID)
+ s.registerHealthJob(ctx, env.ID)
}
return transactionErr
}
- // Deleting an environment orphans its GitOps syncs, whose jobs belong to
- // gitops.GitOpsSyncService's own registry — remove them by name here.
+ // The deleted GitOps syncs' jobs live in the gitops registry, so remove them by name.
if jobScheduler := s.jobs.Scheduler(); jobScheduler != nil {
schedulerCtx := s.jobs.Context(ctx)
for _, syncID := range syncIDs {
@@ -470,33 +466,16 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, u
s.edgeTokens.invalidate(id)
s.ForgetSyncState(id)
s.remoteEnvs.remove(id)
- s.invalidateEnvironmentCacheInternal(id)
+ s.invalidateEnvironmentCache(id)
s.NotifyRuntimeStateChanged()
- // Create event in background
- go s.createEnvironmentEvent(
- context.WithoutCancel(
- ctx,
- ),
- id,
- env.Name,
- event.EventTypeEnvironmentDelete,
- "Environment Deleted",
- fmt.Sprintf(
- "Environment '%s' was deleted",
- env.Name,
- ),
- event.EventSeverityWarning,
- userID,
- username,
- )
+ go s.createEnvironmentEvent(context.WithoutCancel(ctx), id, env.Name, event.EventTypeEnvironmentDelete,
+ "Environment Deleted", fmt.Sprintf("Environment '%s' was deleted", env.Name), event.EventSeverityWarning, userID, username)
return nil
}
-func (
- s *EnvironmentService,
-) createEnvironmentEvent(
+func (s *EnvironmentService) createEnvironmentEvent(
ctx context.Context,
envID, envName string,
eventType event.EventType,
@@ -504,7 +483,7 @@ func (
severity event.EventSeverity,
userID, username *string,
) {
- if s == nil || s.eventService == nil {
+ if s.eventService == nil {
return
}
@@ -534,17 +513,14 @@ func (s *EnvironmentService) RegenerateEnvironmentApiKey(ctx context.Context, en
}
func (s *EnvironmentService) linkEnvironmentApiKey(ctx context.Context, envID, newApiKeyID, apiKey, userID, username, envName string) error {
- // Trim once at the boundary so the value persisted, the value cached,
- // and the value returned by callers (which already TrimSpace before
- // returning) all stay byte-identical. Any divergence here would surface
- // as a 401 "invalid agent token" because lookup is direct equality.
+ // Persisted, cached, and returned keys must stay byte-identical: token lookup is direct equality.
apiKey = strings.TrimSpace(apiKey)
updates := map[string]any{
"api_key_id": newApiKeyID,
"access_token": apiKey,
"status": string(EnvironmentStatusPending),
- "last_seen": nil, // Clear last seen time
+ "last_seen": nil,
}
result := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", envID).Updates(updates)
@@ -552,11 +528,10 @@ func (s *EnvironmentService) linkEnvironmentApiKey(ctx context.Context, envID, n
return fmt.Errorf("failed to update environment with new API key: %w", result.Error)
}
if result.RowsAffected == 0 {
- // A zero-row update would otherwise report a successful rotation while
- // the new key was never linked to anything.
+ // Otherwise a rotation would succeed without linking the new key.
return ErrEnvironmentNotFound
}
- s.invalidateEnvironmentCacheInternal(envID)
+ s.invalidateEnvironmentCache(envID)
s.NotifyRuntimeStateChanged()
s.edgeTokens.sync(envID, apiKey)
@@ -569,24 +544,8 @@ func (s *EnvironmentService) linkEnvironmentApiKey(ctx context.Context, envID, n
env.UpdatedAt = &now
})
- // Create event log in background
- go s.createEnvironmentEvent(
- context.WithoutCancel(
- ctx,
- ),
- envID,
- envName,
- event.EventTypeEnvironmentApiKeyRegenerated,
- "API Key Regenerated",
- "Environment API key was regenerated and status set to pending",
- event.EventSeverityInfo,
- new(
- userID,
- ),
- new(
- username,
- ),
- )
+ go s.createEnvironmentEvent(context.WithoutCancel(ctx), envID, envName, event.EventTypeEnvironmentApiKeyRegenerated,
+ "API Key Regenerated", "Environment API key was regenerated and status set to pending", event.EventSeverityInfo, new(userID), new(username))
return nil
}
@@ -660,19 +619,11 @@ func (s *EnvironmentService) SyncResourcesToEnvironment(ctx context.Context, env
var failedGroups []string
s.ForgetSyncState(environmentID)
- if err := s.SyncRegistriesToEnvironment(ctx, environmentID); err != nil {
- slog.WarnContext(ctx, "Failed to sync registries", "environmentId", environmentID, "error", err.Error())
- failedGroups = append(failedGroups, "container registries")
- }
-
- if err := s.SyncS3DestinationsToEnvironment(ctx, environmentID); err != nil {
- slog.WarnContext(ctx, "Failed to sync S3 destinations", "environmentId", environmentID, "error", err.Error())
- failedGroups = append(failedGroups, "S3 destinations")
- }
-
- if err := s.SyncRepositoriesToEnvironment(ctx, environmentID); err != nil {
- slog.WarnContext(ctx, "Failed to sync git repositories", "environmentId", environmentID, "error", err.Error())
- failedGroups = append(failedGroups, "git repositories")
+ for _, resource := range s.resourceSyncs() {
+ if err := resource.sync(ctx, environmentID); err != nil {
+ slog.WarnContext(ctx, "Failed to sync resources", "kind", resource.kind, "environmentId", environmentID, "error", err.Error())
+ failedGroups = append(failedGroups, resource.kind)
+ }
}
if len(failedGroups) > 0 {
@@ -683,13 +634,27 @@ func (s *EnvironmentService) SyncResourcesToEnvironment(ctx context.Context, env
})
}
+type resourceSync struct {
+ kind string
+ sync func(ctx context.Context, environmentID string) error
+}
+
+// resourceSyncs lists the manager-owned resource groups pushed to remote environments.
+func (s *EnvironmentService) resourceSyncs() []resourceSync {
+ return []resourceSync{
+ {"container registries", s.SyncRegistriesToEnvironment},
+ {"S3 destinations", s.SyncS3DestinationsToEnvironment},
+ {"git repositories", s.SyncRepositoriesToEnvironment},
+ }
+}
+
// DisplayName returns the stored environment name or its readable fallback.
func DisplayName(environmentID, storedName string) string {
if name := strings.TrimSpace(storedName); name != "" {
return name
}
id := strings.TrimSpace(environmentID)
- return kit.Ternary(id == "" || id == LocalEnvironmentID, localEnvironmentFallbackNameInternal, id)
+ return kit.Ternary(id == "" || id == LocalEnvironmentID, localEnvironmentFallbackName, id)
}
const (
@@ -707,15 +672,11 @@ func (s *EnvironmentService) GetActiveRemoteEnvironmentSnapshot(environmentID st
}
// ListActiveRemoteEnvironments returns every enabled, visible, non-local
-// environment from memory, sorted by ID. The first call loads them from the
-// database.
+// environment from memory, sorted by ID; the first call loads them from the database.
func (s *EnvironmentService) ListActiveRemoteEnvironments(ctx context.Context) ([]Environment, error) {
s.remoteEnvs.mu.RLock()
seeded := s.remoteEnvs.seeded
- environments := make([]Environment, 0, len(s.remoteEnvs.envs))
- for _, envRecord := range s.remoteEnvs.envs {
- environments = append(environments, envRecord)
- }
+ environments := slices.AppendSeq(make([]Environment, 0, len(s.remoteEnvs.envs)), maps.Values(s.remoteEnvs.envs))
s.remoteEnvs.mu.RUnlock()
if !seeded {
@@ -732,11 +693,11 @@ func (s *EnvironmentService) ListActiveRemoteEnvironments(ctx context.Context) (
}
// GetEnvironmentByIDCached is GetEnvironmentByID behind a short TTL cache that
-// also remembers unknown IDs. Status and heartbeat fields may be stale.
+// remembers unknown IDs; status and heartbeat fields may be stale.
func (s *EnvironmentService) GetEnvironmentByIDCached(ctx context.Context, id string) (*Environment, error) {
- key := environmentCacheKeyInternal{gen: s.environmentGens[environmentCacheStripeInternal(id)].Load(), id: id}
- envRecord, found, err := s.environmentCache.GetWithLoaders(key, func(keys []environmentCacheKeyInternal) (map[environmentCacheKeyInternal]Environment, error) {
- found := make(map[environmentCacheKeyInternal]Environment, len(keys))
+ key := environmentCacheKey{gen: s.environmentGens[maphash.String(environmentCacheSeed, id)%environmentCacheStripes].Load(), id: id}
+ envRecord, found, err := s.environmentCache.GetWithLoaders(key, func(keys []environmentCacheKey) (map[environmentCacheKey]Environment, error) {
+ found := make(map[environmentCacheKey]Environment, len(keys))
for _, k := range keys {
loaded, loadErr := s.GetEnvironmentByID(ctx, k.id)
if errors.Is(loadErr, ErrEnvironmentNotFound) {
@@ -758,36 +719,24 @@ func (s *EnvironmentService) GetEnvironmentByIDCached(ctx context.Context, id st
return &envRecord, nil
}
-// invalidateEnvironmentCacheInternal starts a new generation for id's stripe
-// after a committed write, orphaning its cached record and in-flight loads.
-func (s *EnvironmentService) invalidateEnvironmentCacheInternal(id string) {
+// invalidateEnvironmentCache starts a new generation for id's stripe after a
+// committed write, orphaning its cached record and in-flight loads.
+func (s *EnvironmentService) invalidateEnvironmentCache(id string) {
if s == nil || s.environmentCache == nil {
return
}
- previous := s.environmentGens[environmentCacheStripeInternal(id)].Add(1) - 1
- s.environmentCache.Delete(environmentCacheKeyInternal{gen: previous, id: id})
-}
-
-// invalidateAllEnvironmentCacheInternal covers writes whose affected IDs are unknown.
-func (s *EnvironmentService) invalidateAllEnvironmentCacheInternal() {
- if s == nil || s.environmentCache == nil {
- return
- }
- for i := range s.environmentGens {
- s.environmentGens[i].Add(1)
- }
- s.environmentCache.Purge()
+ previous := s.environmentGens[maphash.String(environmentCacheSeed, id)%environmentCacheStripes].Add(1) - 1
+ s.environmentCache.Delete(environmentCacheKey{gen: previous, id: id})
}
func (s *EnvironmentService) ListEnvironmentsPaginated(ctx context.Context, params pagination.QueryParams, accessibleEnvIDs []string) ([]environment.Environment, pagination.Response, error) {
if strings.TrimSpace(params.Filters["type"]) != "" {
- return s.listEnvironmentsPaginatedWithRuntimeFiltersInternal(ctx, params, accessibleEnvIDs)
+ return s.listEnvironmentsPaginatedWithRuntimeFilters(ctx, params, accessibleEnvIDs)
}
var envs []Environment
q := s.db.WithContext(ctx).Model(&Environment{}).Where("hidden = ?", false)
- // accessibleEnvIDs == nil means "no restriction". A non-nil slice limits the
- // result to those environment IDs; an empty slice therefore matches nothing.
+ // nil means no restriction; a non-nil slice limits results, so an empty one matches nothing.
switch {
case accessibleEnvIDs == nil:
// no restriction
@@ -821,17 +770,11 @@ func (s *EnvironmentService) ListEnvironmentsPaginated(ctx context.Context, para
return out, paginationResp, nil
}
-func (
- s *EnvironmentService,
-) listEnvironmentsPaginatedWithRuntimeFiltersInternal(
+func (s *EnvironmentService) listEnvironmentsPaginatedWithRuntimeFilters(
ctx context.Context,
params pagination.QueryParams,
accessibleEnvIDs []string,
-) (
- []environment.Environment,
- pagination.Response,
- error,
-) {
+) ([]environment.Environment, pagination.Response, error) {
var envs []Environment
if err := s.db.WithContext(ctx).
Model(&Environment{}).
@@ -845,9 +788,9 @@ func (
return nil, pagination.Response{}, fmt.Errorf("failed to map environments: %w", mapErr)
}
- // nil = no restriction; non-nil restricts to the caller's accessible envs.
+ // nil means no restriction; non-nil restricts to the caller's accessible environments.
if accessibleEnvIDs != nil {
- items = filterEnvironmentsByIDInternal(items, accessibleEnvIDs)
+ items = slices.DeleteFunc(items, func(item environment.Environment) bool { return !slices.Contains(accessibleEnvIDs, item.ID) })
}
for i := range items {
@@ -910,7 +853,27 @@ func (
},
{
Key: "type",
- Fn: environmentTypeMatchesInternal,
+ Fn: func(item environment.Environment, filterValue string) bool {
+ typeKey := "http"
+ if item.IsEdge {
+ // Disconnected or poll-only agents classify by the transport they last used.
+ transport := ""
+ if item.Connected != nil && *item.Connected && item.EdgeTransport != nil {
+ transport = *item.EdgeTransport
+ } else if item.LastEdgeTransport != nil {
+ transport = *item.LastEdgeTransport
+ }
+ switch strings.ToLower(strings.TrimSpace(transport)) {
+ case edge.EdgeTransportWebSocket:
+ typeKey = "websocket"
+ case edge.EdgeTransportGRPC:
+ typeKey = "grpc"
+ default:
+ typeKey = "edge"
+ }
+ }
+ return typeKey == strings.ToLower(strings.TrimSpace(filterValue))
+ },
},
},
}
@@ -964,7 +927,7 @@ func (s *EnvironmentService) ListRemoteEnvironments(ctx context.Context) ([]Envi
var envs []Environment
err := s.db.WithContext(ctx).
Model(&Environment{}).
- Where("id != ?", "0").
+ Where("id != ?", LocalEnvironmentID).
Where("enabled = ?", true).
Where("hidden = ?", false).
Find(&envs).Error
@@ -1004,39 +967,33 @@ func (s *EnvironmentService) NotifyRuntimeStateChanged() {
}
const (
- defaultEnvironmentHealthInterval = "0 */2 * * * *"
- environmentHealthCheckTimeout = 90 * time.Second
- environmentHealthJobPrefix = "environment-health:"
- environmentHealthAdmissionScopeInternal = "environment-health"
+ defaultEnvironmentHealthInterval = "0 */2 * * * *"
+ environmentHealthCheckTimeout = 90 * time.Second
+ environmentHealthJobPrefix = "environment-health:"
+ environmentHealthAdmissionScope = "environment-health"
)
-// SetScheduler injects the job scheduler and app lifecycle context. Called during
-// bootstrap on the manager only (agent mode leaves scheduler nil, so all health-job
-// registration becomes a no-op).
+// SetScheduler injects the job scheduler and app lifecycle context on the manager;
+// agents leave it nil, making health-job registration a no-op.
func (s *EnvironmentService) SetScheduler(ctx context.Context, jobScheduler scheduler.DynamicScheduler, admissionGate *runs.Admission) error {
return s.jobs.SetScheduler(ctx, jobScheduler, admissionGate)
}
-// registerHealthJobInternal schedules one environment's health check on the
-// global environmentHealthInterval (environment health has no per-entity
-// interval); the run body self-cancels cleanly when the environment is gone.
-func (s *EnvironmentService) registerHealthJobInternal(ctx context.Context, envID string) {
+// registerHealthJob schedules one environment's health check on the global
+// environmentHealthInterval; the run self-cancels once the environment is gone.
+func (s *EnvironmentService) registerHealthJob(ctx context.Context, envID string) {
s.jobs.Register(ctx, envID,
func(ctx context.Context) string {
sched := s.settingsService.GetStringSetting(ctx, "environmentHealthInterval", defaultEnvironmentHealthInterval)
return kit.Ternary(sched == "", defaultEnvironmentHealthInterval, sched)
},
- func(ctx context.Context) (scheduler.Outcome, error) { return s.runHealthCheckInternal(ctx, envID) },
+ func(ctx context.Context) (scheduler.Outcome, error) { return s.runHealthCheck(ctx, envID) },
func(context.Context, scheduler.Run) (scheduler.Outcome, error) {
return scheduler.Outcome{Status: scheduler.Retrying, Message: "Connection checks can safely resume"}, nil
},
)
}
-func (s *EnvironmentService) removeHealthJobInternal(ctx context.Context, envID string) {
- s.jobs.Unregister(ctx, envID)
-}
-
func (s *EnvironmentService) ListEnabledEnvironmentIDs(ctx context.Context) ([]string, error) {
var ids []string
if err := s.db.WithContext(ctx).
@@ -1048,35 +1005,21 @@ func (s *EnvironmentService) ListEnabledEnvironmentIDs(ctx context.Context) ([]s
return ids, nil
}
-func (s *EnvironmentService) registerAllEnabledHealthJobsInternal(ctx context.Context) int {
+// RescheduleHealthJobs re-registers every enabled environment's health job,
+// picking up a changed global interval.
+func (s *EnvironmentService) RescheduleHealthJobs(ctx context.Context) {
+ if !s.jobs.Enabled() {
+ return
+ }
ids, err := s.ListEnabledEnvironmentIDs(ctx)
if err != nil {
slog.ErrorContext(ctx, "failed to list environments for health jobs", "error", err)
- return 0
- }
- for _, id := range ids {
- s.registerHealthJobInternal(ctx, id)
- }
- return len(ids)
-}
-
-// RegisterHealthJobsOnStartup registers a health-check job for every enabled
-// environment. Replaces the old global environment-health job.
-func (s *EnvironmentService) RegisterHealthJobsOnStartup(ctx context.Context) {
- if !s.jobs.Enabled() {
return
}
- n := s.registerAllEnabledHealthJobsInternal(ctx)
- slog.InfoContext(ctx, "Registered environment health jobs on startup", "count", n)
-}
-
-// RescheduleHealthJobs re-registers all enabled environments' health jobs, picking
-// up a changed global interval. Wired from the Jobs UI via job.JobService.
-func (s *EnvironmentService) RescheduleHealthJobs(ctx context.Context) {
- if !s.jobs.Enabled() {
- return
+ for _, id := range ids {
+ s.registerHealthJob(ctx, id)
}
- s.registerAllEnabledHealthJobsInternal(ctx)
+ slog.InfoContext(ctx, "Registered environment health jobs", "count", len(ids))
}
// RunHealthChecksNow queues each enabled environment's health check.
@@ -1090,7 +1033,7 @@ func (s *EnvironmentService) RunHealthChecksNow(ctx context.Context) error {
return err
}
parent, hasParent := jobcontext.Run(ctx)
- request := scheduler.Request{EnvironmentID: "0", Trigger: "internal"}
+ request := scheduler.Request{EnvironmentID: LocalEnvironmentID, Trigger: "internal"}
if hasParent {
request.Trigger = parent.Trigger
request.RequestedBy = parent.RequestedBy
@@ -1105,9 +1048,9 @@ func (s *EnvironmentService) RunHealthChecksNow(ctx context.Context) error {
return nil
}
-// runHealthCheckInternal tests one environment's connection (updating its DB status)
-// and, for online remotes, syncs registries and repositories to it.
-func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID string) (scheduler.Outcome, error) {
+// runHealthCheck tests one environment's connection, persisting its status, and
+// syncs manager-owned resources to online remotes.
+func (s *EnvironmentService) runHealthCheck(ctx context.Context, envID string) (scheduler.Outcome, error) {
lease, admitted, err := s.jobs.TryAcquire(ctx, envID)
if err != nil {
slog.ErrorContext(ctx, "environment health check admission failed", "environmentId", envID, "error", err)
@@ -1136,13 +1079,12 @@ func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID s
return scheduler.Outcome{Status: scheduler.Retrying, Message: "Environment is offline"}, errors.New("environment is offline")
}
- // Local environment (ID "0") has no registries/repositories to push.
- if envID == "0" {
+ // The local environment has no resources to push.
+ if envID == LocalEnvironmentID {
return scheduler.Outcome{Status: scheduler.Succeeded}, nil
}
- // An agent that just came back may have restarted with different state;
- // resend everything once instead of trusting the last delivered payload.
+ // An agent that just came back may have restarted with different state, so resend everything.
if !wasOnline {
s.ForgetSyncState(envID)
}
@@ -1150,22 +1092,16 @@ func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID s
var syncErrors []error
syncCtx, cancel := context.WithTimeout(ctx, environmentHealthCheckTimeout)
defer cancel()
- if syncRegistriesToEnvironmentErr := s.SyncRegistriesToEnvironment(syncCtx, envID); syncRegistriesToEnvironmentErr != nil {
- slog.WarnContext(syncCtx, "failed to sync registries during health check", "environmentId", envID, "error", syncRegistriesToEnvironmentErr)
- syncErrors = append(syncErrors, syncRegistriesToEnvironmentErr)
- }
- if syncS3DestinationsToEnvironmentErr := s.SyncS3DestinationsToEnvironment(syncCtx, envID); syncS3DestinationsToEnvironmentErr != nil {
- slog.WarnContext(syncCtx, "failed to sync S3 destinations during health check", "environmentId", envID, "error", syncS3DestinationsToEnvironmentErr)
- syncErrors = append(syncErrors, syncS3DestinationsToEnvironmentErr)
- }
- if syncRepositoriesToEnvironmentErr := s.SyncRepositoriesToEnvironment(syncCtx, envID); syncRepositoriesToEnvironmentErr != nil {
- slog.WarnContext(syncCtx, "failed to sync git repositories during health check", "environmentId", envID, "error", syncRepositoriesToEnvironmentErr)
- syncErrors = append(syncErrors, syncRepositoriesToEnvironmentErr)
+ for _, resource := range s.resourceSyncs() {
+ if syncErr := resource.sync(syncCtx, envID); syncErr != nil {
+ slog.WarnContext(syncCtx, "failed to sync resources during health check", "kind", resource.kind, "environmentId", envID, "error", syncErr)
+ syncErrors = append(syncErrors, syncErr)
+ }
}
if s.variableSyncer != nil {
- if syncEnvironmentErr := s.variableSyncer.SyncEnvironment(syncCtx, envID); syncEnvironmentErr != nil {
- slog.WarnContext(syncCtx, "failed to sync global variables during health check", "environmentId", envID, "error", syncEnvironmentErr)
- syncErrors = append(syncErrors, syncEnvironmentErr)
+ if syncErr := s.variableSyncer.SyncEnvironment(syncCtx, envID); syncErr != nil {
+ slog.WarnContext(syncCtx, "failed to sync global variables during health check", "environmentId", envID, "error", syncErr)
+ syncErrors = append(syncErrors, syncErr)
}
}
if joinErr := errors.Join(syncErrors...); joinErr != nil {
@@ -1176,7 +1112,7 @@ func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID s
// SyncRegistriesToEnvironment syncs all registries from this manager to a remote environment.
func (s *EnvironmentService) SyncRegistriesToEnvironment(ctx context.Context, environmentID string) error {
- target, err := s.resolveRemoteEnvironmentTargetInternal(ctx, environmentID)
+ target, err := s.resolveRemoteEnvironmentTarget(ctx, environmentID)
if err != nil {
return err
}
@@ -1185,7 +1121,7 @@ func (s *EnvironmentService) SyncRegistriesToEnvironment(ctx context.Context, en
// SyncS3DestinationsToEnvironment sends manager-owned destinations to one remote environment.
func (s *EnvironmentService) SyncS3DestinationsToEnvironment(ctx context.Context, environmentID string) error {
- target, err := s.resolveRemoteEnvironmentTargetInternal(ctx, environmentID)
+ target, err := s.resolveRemoteEnvironmentTarget(ctx, environmentID)
if err != nil {
return err
}
@@ -1194,7 +1130,7 @@ func (s *EnvironmentService) SyncS3DestinationsToEnvironment(ctx context.Context
// SyncRepositoriesToEnvironment syncs all git repositories from this manager to a remote environment.
func (s *EnvironmentService) SyncRepositoriesToEnvironment(ctx context.Context, environmentID string) error {
- target, err := s.resolveRemoteEnvironmentTargetInternal(ctx, environmentID)
+ target, err := s.resolveRemoteEnvironmentTarget(ctx, environmentID)
if err != nil {
return err
}
@@ -1213,7 +1149,7 @@ func (s *EnvironmentService) TestConnection(ctx context.Context, id string, cust
}
status, err := s.health.Probe(ctx, id, apiURL, envRecord.IsEdge, customApiUrl != nil)
if customApiUrl == nil {
- if updateErr := s.updateEnvironmentStatusInternal(ctx, id, status); updateErr != nil {
+ if updateErr := s.updateEnvironmentStatus(ctx, id, status); updateErr != nil {
slog.WarnContext(ctx, "Failed to persist environment status", "environmentId", id, "status", status, "error", updateErr)
}
return status, err
@@ -1225,18 +1161,15 @@ func (s *EnvironmentService) TestConnection(ctx context.Context, id string, cust
return status, nil
}
-func (s *EnvironmentService) updateEnvironmentStatusInternal(ctx context.Context, id, status string) error {
+func (s *EnvironmentService) updateEnvironmentStatus(ctx context.Context, id, status string) error {
var currentEnv Environment
if err := s.db.WithContext(ctx).Select("status", "is_edge").Where("id = ?", id).First(&currentEnv).Error; err != nil {
return fmt.Errorf("failed to check environment status: %w", err)
}
if currentEnv.Status == string(EnvironmentStatusPending) {
- // Edge envs must complete pairing via the agent's outbound tunnel — manager
- // can't dial them directly, so a manager-side reachability check means nothing.
- // Direct envs are reachable from the manager, so a successful health check IS
- // the pairing signal. Don't promote on offline/error ticks though, or a transient
- // blip during initial setup would flip the env out of pending.
+ // Only a successful check pairs a direct environment; edge environments pair
+ // through the agent's tunnel, so manager reachability means nothing for them.
if currentEnv.IsEdge || status != string(EnvironmentStatusOnline) {
slog.DebugContext(ctx, "skipping status update for pending environment", "environmentId", id)
return nil
@@ -1253,8 +1186,7 @@ func (s *EnvironmentService) updateEnvironmentStatusInternal(ctx context.Context
if err := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", id).Updates(updates).Error; err != nil {
return fmt.Errorf("failed to update environment status: %w", err)
}
- // Direct environments have no tunnel callback, so this health-check write is
- // the only moment their liveness changes.
+ // Direct environments have no tunnel callback, so this is their only liveness change.
s.NotifyRuntimeStateChanged()
return nil
}
@@ -1262,8 +1194,7 @@ func (s *EnvironmentService) updateEnvironmentStatusInternal(ctx context.Context
func (s *EnvironmentService) UpdateEnvironmentHeartbeat(ctx context.Context, id string) error {
now := time.Now()
- // Use Exec with raw SQL for better performance
- // Throttle unchanged online status to one write every 30 seconds.
+ // Throttle unchanged online status to one write every 30 seconds; this doubles as the notify throttle.
result := s.db.WithContext(ctx).Exec(`
UPDATE environments
SET last_seen = ?, status = ?, updated_at = ?
@@ -1272,8 +1203,6 @@ func (s *EnvironmentService) UpdateEnvironmentHeartbeat(ctx context.Context, id
`, new(now), string(EnvironmentStatusOnline), new(now), id, string(EnvironmentStatusOnline), now.Add(-30*time.Second))
if result.Error != nil {
- // The 30s throttle above doubles as the notify throttle: a no-op heartbeat
- // changed nothing worth waking a stream for.
return fmt.Errorf("failed to update environment heartbeat: %w", result.Error)
}
@@ -1284,8 +1213,8 @@ func (s *EnvironmentService) UpdateEnvironmentHeartbeat(ctx context.Context, id
return nil
}
-// UpdateEnvironmentConnectionState updates runtime connectivity status without creating
-// a generic "environment updated" event. This is used for edge tunnel connect/disconnect.
+// UpdateEnvironmentConnectionState records an edge tunnel connect or disconnect
+// without emitting an "environment updated" event.
func (s *EnvironmentService) UpdateEnvironmentConnectionState(ctx context.Context, id string, connected bool) error {
now := time.Now()
@@ -1295,8 +1224,7 @@ func (s *EnvironmentService) UpdateEnvironmentConnectionState(ctx context.Contex
if connected {
updates["status"] = string(EnvironmentStatusOnline)
updates["last_seen"] = &now
- // Remember the tunnel transport so the UI can keep showing it after
- // the tunnel drops or while the agent is poll-only.
+ // Remember the transport so the UI can show it after the tunnel drops or while poll-only.
if state, ok := edge.GetTunnelRuntimeState(id).Get(); ok && state.Transport != "" {
updates["last_edge_transport"] = state.Transport
}
@@ -1313,8 +1241,8 @@ func (s *EnvironmentService) UpdateEnvironmentConnectionState(ctx context.Contex
return nil
}
-// ApplyEnvironmentRuntimeState normalizes edge environment runtime status using
-// in-memory tunnel and poll registries without mutating persisted state.
+// ApplyEnvironmentRuntimeState overlays in-memory tunnel and poll state onto an
+// edge environment without mutating persisted state.
func ApplyEnvironmentRuntimeState(env *environment.Environment) {
if env == nil || !env.IsEdge {
return
@@ -1371,9 +1299,8 @@ func ApplyEnvironmentRuntimeState(env *environment.Environment) {
}
}
-// ReconcileEdgeStatusesOnStartup resets edge environments to offline when the manager starts.
-// Live edge tunnels are process-local runtime state, so persisted "online" flags can be stale
-// after a restart until agents reconnect. Pending environments are left untouched.
+// ReconcileEdgeStatusesOnStartup marks non-pending edge environments offline at
+// startup, since tunnels are process-local and persisted "online" flags are stale.
func (s *EnvironmentService) ReconcileEdgeStatusesOnStartup(ctx context.Context) error {
result := s.db.WithContext(ctx).Model(&Environment{}).
Where("is_edge = ?", true).
@@ -1394,97 +1321,58 @@ func (s *EnvironmentService) ReconcileEdgeStatusesOnStartup(ctx context.Context)
return nil
}
-// SyncRegistriesToRemoteEnvironments syncs container registries to all eligible remote environments.
-// Eligibility requires a non-local, enabled environment with a configured access token.
+// SyncRegistriesToRemoteEnvironments syncs container registries to every remote environment with an access token.
func (s *EnvironmentService) SyncRegistriesToRemoteEnvironments(ctx context.Context) error {
- envs, err := s.ListRemoteEnvironments(ctx)
- if err != nil {
- return fmt.Errorf("failed to list remote environments for registry sync: %w", err)
- }
-
- if len(envs) == 0 {
- return nil
- }
-
- var failedCount int
- for _, env := range envs {
- if env.AccessToken == nil || *env.AccessToken == "" {
- slog.DebugContext(ctx, "Skipping registry sync for environment without access token",
- "environmentId", env.ID,
- "environmentName", env.Name)
- continue
- }
-
- if syncRegistriesToEnvironmentErr := s.SyncRegistriesToEnvironment(ctx, env.ID); syncRegistriesToEnvironmentErr != nil {
- failedCount++
- slog.WarnContext(ctx, "Failed to sync registries to remote environment",
- "environmentId", env.ID,
- "environmentName", env.Name,
- "error", syncRegistriesToEnvironmentErr.Error())
- }
- }
-
- if failedCount > 0 {
- return fmt.Errorf("failed to sync registries to %d remote environment(s)", failedCount)
- }
-
- return nil
+ return s.syncToRemoteEnvironments(ctx, "registries", s.SyncRegistriesToEnvironment)
}
-// SyncS3DestinationsToRemoteEnvironments refreshes the destination cache on every enabled remote environment.
+// SyncS3DestinationsToRemoteEnvironments refreshes the destination cache on every remote environment with an access token.
func (s *EnvironmentService) SyncS3DestinationsToRemoteEnvironments(ctx context.Context) error {
+ return s.syncToRemoteEnvironments(ctx, "S3 destinations", s.SyncS3DestinationsToEnvironment)
+}
+
+func (s *EnvironmentService) syncToRemoteEnvironments(ctx context.Context, kind string, sync func(context.Context, string) error) error {
envs, err := s.ListRemoteEnvironments(ctx)
if err != nil {
- return fmt.Errorf("failed to list remote environments for S3 destination sync: %w", err)
+ return fmt.Errorf("failed to list remote environments for %s sync: %w", kind, err)
}
var failedCount int
for _, env := range envs {
if env.AccessToken == nil || strings.TrimSpace(*env.AccessToken) == "" {
- slog.DebugContext(ctx, "Skipping S3 destination sync for environment without access token", "environmentId", env.ID, "environmentName", env.Name)
+ slog.DebugContext(ctx, "Skipping sync for environment without access token", "kind", kind, "environmentId", env.ID, "environmentName", env.Name)
continue
}
- if syncS3DestinationsToEnvironmentErr := s.SyncS3DestinationsToEnvironment(ctx, env.ID); syncS3DestinationsToEnvironmentErr != nil {
+ if syncErr := sync(ctx, env.ID); syncErr != nil {
failedCount++
- slog.WarnContext(ctx, "Failed to sync S3 destinations to remote environment", "environmentId", env.ID, "environmentName", env.Name, "error", syncS3DestinationsToEnvironmentErr)
+ slog.WarnContext(ctx, "Failed to sync to remote environment", "kind", kind, "environmentId", env.ID, "environmentName", env.Name, "error", syncErr)
}
}
if failedCount > 0 {
- return fmt.Errorf("failed to sync S3 destinations to %d remote environment(s)", failedCount)
+ return fmt.Errorf("failed to sync %s to %d remote environment(s)", kind, failedCount)
}
return nil
}
-// CheckS3DestinationReferences returns an error while any managed environment
-// still references the destination, or when a synced environment cannot be
-// checked conclusively. Deleting credentials that a remote environment still
-// needs would strand its policies and retained backups.
+// CheckS3DestinationReferences fails while any synced environment references the
+// destination or cannot be checked, since deleting it would strand remote backups.
func (s *EnvironmentService) CheckS3DestinationReferences(ctx context.Context, destinationID string) error {
envs, err := s.ListRemoteEnvironments(ctx)
if err != nil {
return fmt.Errorf("failed to list remote environments for S3 destination reference check: %w", err)
}
for _, env := range envs {
+ // Environments without an access token never receive destination syncs.
if env.AccessToken == nil || strings.TrimSpace(*env.AccessToken) == "" {
- // Environments without an access token never receive destination
- // syncs, so they cannot hold a reference.
continue
}
var result struct {
InUse bool `json:"inUse"`
}
- if proxyJSONRequestForEnvironmentErr := s.ProxyJSONRequestForEnvironment(
- ctx,
- env,
- http.MethodGet,
- "/api/backups/s3/"+url.PathEscape(
- destinationID,
- )+"/in-use",
- nil,
- &result,
- ); proxyJSONRequestForEnvironmentErr != nil {
- return fmt.Errorf("cannot verify S3 destination references on environment %s; restore connectivity before deleting: %w", env.Name, proxyJSONRequestForEnvironmentErr)
+ path := "/api/backups/s3/" + url.PathEscape(destinationID) + "/in-use"
+ if proxyErr := s.ProxyJSONRequestForEnvironment(ctx, env, http.MethodGet, path, nil, &result); proxyErr != nil {
+ return fmt.Errorf("cannot verify S3 destination references on environment %s; restore connectivity before deleting: %w", env.Name, proxyErr)
}
if result.InUse {
return fmt.Errorf("still referenced by environment %s", env.Name)
@@ -1493,7 +1381,7 @@ func (s *EnvironmentService) CheckS3DestinationReferences(ctx context.Context, d
return nil
}
-func (s *EnvironmentService) resolveRemoteEnvironmentTargetInternal(ctx context.Context, envID string) (*proxy.Target, error) {
+func (s *EnvironmentService) resolveRemoteEnvironmentTarget(ctx context.Context, envID string) (*proxy.Target, error) {
envRecord, err := s.GetEnvironmentByID(ctx, envID)
if err != nil {
return nil, fmt.Errorf("failed to get environment: %w", err)
@@ -1503,7 +1391,7 @@ func (s *EnvironmentService) resolveRemoteEnvironmentTargetInternal(ctx context.
}
func (s *EnvironmentService) ExecuteRemoteRequest(ctx context.Context, envID, method, path string, body []byte) (*remenv.Response, error) {
- target, err := s.resolveRemoteEnvironmentTargetInternal(ctx, envID)
+ target, err := s.resolveRemoteEnvironmentTarget(ctx, envID)
if err != nil {
return nil, err
}
@@ -1515,7 +1403,7 @@ func (s *EnvironmentService) ProxyJSONRequest(ctx context.Context, envID, method
proxyCtx, cancel := s.proxy.Context(ctx)
defer cancel()
- target, err := s.resolveRemoteEnvironmentTargetInternal(proxyCtx, envID)
+ target, err := s.resolveRemoteEnvironmentTarget(proxyCtx, envID)
if err != nil {
return err
}
@@ -1523,8 +1411,7 @@ func (s *EnvironmentService) ProxyJSONRequest(ctx context.Context, envID, method
return s.proxy.JSON(proxyCtx, target, method, path, body, out)
}
-// ProxyJSONRequestForEnvironment sends a JSON request using an already-loaded
-// environment row, avoiding an extra environment lookup on hot stream paths.
+// ProxyJSONRequestForEnvironment sends a JSON request for an already-loaded environment row.
func (s *EnvironmentService) ProxyJSONRequestForEnvironment(ctx context.Context, env Environment, method, path string, body []byte, out any) error {
proxyCtx, cancel := s.proxy.Context(ctx)
defer cancel()
@@ -1582,8 +1469,8 @@ func (s *EnvironmentService) GenerateDeploymentSnippets(ctx context.Context, env
}, nil
}
-// GenerateEdgeDeploymentSnippets generates Docker deployment snippets for an edge agent.
-// Edge agents connect outbound to the manager and don't require exposed ports.
+// GenerateEdgeDeploymentSnippets generates Docker deployment snippets for an
+// edge agent, which dials the manager and exposes no ports.
func (s *EnvironmentService) GenerateEdgeDeploymentSnippets(ctx context.Context, envID, managerURL, apiKey string, edgeCfg *edge.Config) (*DeploymentSnippets, error) {
managerURL = strings.TrimRight(managerURL, "/")
dockerRun, dockerCompose := snippets.Edge(managerURL, apiKey)
@@ -1636,15 +1523,14 @@ func (s *EnvironmentService) ListSwarmNodeAgentEnvironments(ctx context.Context,
return envs, nil
}
-// ListSwarmNodeCandidateEnvironments returns enabled visible environments that
-// can provide swarm-node coverage for a manager environment.
+// ListSwarmNodeCandidateEnvironments returns enabled visible environments that can cover a swarm node.
func (s *EnvironmentService) ListSwarmNodeCandidateEnvironments(ctx context.Context) ([]Environment, error) {
var envs []Environment
if err := s.db.WithContext(ctx).
Model(&Environment{}).
Where("hidden = ?", false).
Where("enabled = ?", true).
- Where("id <> ?", "0").
+ Where("id <> ?", LocalEnvironmentID).
Order("name ASC").
Find(&envs).Error; err != nil {
return nil, fmt.Errorf("failed to list swarm node candidate environments: %w", err)
@@ -1652,8 +1538,7 @@ func (s *EnvironmentService) ListSwarmNodeCandidateEnvironments(ctx context.Cont
return envs, nil
}
-// BindSwarmNodeEnvironment binds an existing visible environment to a swarm
-// node without modifying its connection details or agent token.
+// BindSwarmNodeEnvironment binds a visible environment to a swarm node, keeping its connection and token.
func (s *EnvironmentService) BindSwarmNodeEnvironment(
ctx context.Context,
parentEnvironmentID, nodeID, environmentID string,
@@ -1701,11 +1586,14 @@ func (s *EnvironmentService) BindSwarmNodeEnvironment(
}
s.remoteEnvs.put(envRecord)
- // A rebind also clears the binding on other environments.
+ // A rebind also clears the binding on other environments, whose IDs are unknown.
if rebind {
- s.invalidateAllEnvironmentCacheInternal()
+ for i := range s.environmentGens {
+ s.environmentGens[i].Add(1)
+ }
+ s.environmentCache.Purge()
} else {
- s.invalidateEnvironmentCacheInternal(envRecord.ID)
+ s.invalidateEnvironmentCache(envRecord.ID)
}
s.NotifyRuntimeStateChanged()
return &envRecord, nil
@@ -1719,14 +1607,17 @@ func (s *EnvironmentService) DetachSwarmNodeEnvironment(ctx context.Context, par
Updates(map[string]any{"parent_environment_id": nil, "swarm_node_id": nil, "updated_at": &now}).Error; err != nil {
return fmt.Errorf("failed to detach swarm node environment: %w", err)
}
- s.invalidateAllEnvironmentCacheInternal()
+ // The detached environment's ID is unknown, so invalidate every stripe.
+ for i := range s.environmentGens {
+ s.environmentGens[i].Add(1)
+ }
+ s.environmentCache.Purge()
s.NotifyRuntimeStateChanged()
return nil
}
-// DeleteSwarmNodeAgentDeployment removes a dedicated hidden agent registration
-// while leaving visible remote environments untouched.
+// DeleteSwarmNodeAgentDeployment removes a dedicated hidden agent registration, leaving visible ones untouched.
func (s *EnvironmentService) DeleteSwarmNodeAgentDeployment(ctx context.Context, parentEnvironmentID, nodeID string, userID, username *string) error {
var envRecord Environment
if err := s.db.WithContext(ctx).
@@ -1752,9 +1643,8 @@ func (s *EnvironmentService) EnsureSwarmNodeAgentEnvironment(
}
var env Environment
- // Prefer an existing visible binding. Legacy hidden registrations remain
- // reusable, but all newly provisioned node agents are normal Remote
- // Environments so one token and one agent can serve both use cases.
+ // Prefer a visible binding; legacy hidden registrations stay reusable, but new
+ // node agents are normal remote environments so one token serves both uses.
err = s.db.WithContext(ctx).
Where("parent_environment_id = ?", parentEnvironmentID).
Where("swarm_node_id = ?", nodeID).
@@ -1806,7 +1696,7 @@ func (s *EnvironmentService) UpdateSwarmNodeIdentity(ctx context.Context, envID,
if err := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", envID).Updates(updates).Error; err != nil {
return fmt.Errorf("failed to update swarm node identity: %w", err)
}
- s.invalidateEnvironmentCacheInternal(envID)
+ s.invalidateEnvironmentCache(envID)
s.NotifyRuntimeStateChanged()
return nil
diff --git a/backend/internal/environment/service_test.go b/backend/internal/environment/service_test.go
index eb31ce23a..36b6496c2 100644
--- a/backend/internal/environment/service_test.go
+++ b/backend/internal/environment/service_test.go
@@ -45,42 +45,42 @@ import (
francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
)
-func TestEnvironmentService_OverlappingHealthCheckIsSkippedInternal(t *testing.T) {
- gate := newAdmissionGateForEnvironmentTestInternal(t)
+func TestEnvironmentService_OverlappingHealthCheckIsSkipped(t *testing.T) {
+ gate := newAdmissionGateForEnvironmentTest(t)
- key := scheduler.AdmissionKey{Scope: environmentHealthAdmissionScopeInternal, ID: "environment-id"}
+ key := scheduler.AdmissionKey{Scope: environmentHealthAdmissionScope, ID: "environment-id"}
lease, admitted, err := gate.TryAcquire(t.Context(), key)
require.NoError(t, err)
require.True(t, admitted)
- service := &EnvironmentService{jobs: entityjobs.New(environmentHealthJobPrefix, environmentHealthAdmissionScopeInternal)}
- require.NoError(t, service.SetScheduler(t.Context(), &environmentTestSchedulerInternal{}, gate))
- outcome, runErr := service.runHealthCheckInternal(t.Context(), "environment-id")
+ service := &EnvironmentService{jobs: entityjobs.New(environmentHealthJobPrefix, environmentHealthAdmissionScope)}
+ require.NoError(t, service.SetScheduler(t.Context(), &environmentTestScheduler{}, gate))
+ outcome, runErr := service.runHealthCheck(t.Context(), "environment-id")
require.NoError(t, runErr)
require.Equal(t, scheduler.Skipped, outcome.Status)
lease.Release(t.Context())
}
-type environmentTestSchedulerInternal struct {
+type environmentTestScheduler struct {
submitted []scheduler.Request
added []string
removed []string
}
-func (s *environmentTestSchedulerInternal) AddJob(_ context.Context, job scheduler.Job) error {
+func (s *environmentTestScheduler) AddJob(_ context.Context, job scheduler.Job) error {
s.added = append(s.added, job.Name())
return nil
}
-func (s *environmentTestSchedulerInternal) RemoveJob(_ context.Context, name string) {
+func (s *environmentTestScheduler) RemoveJob(_ context.Context, name string) {
s.removed = append(s.removed, name)
}
-func (s *environmentTestSchedulerInternal) HasJob(_ string) bool {
+func (s *environmentTestScheduler) HasJob(_ string) bool {
return false
}
-func newAdmissionGateForEnvironmentTestInternal(t testing.TB) *runs.Admission {
+func newAdmissionGateForEnvironmentTest(t testing.TB) *runs.Admission {
t.Helper()
runtime := francistest.New(t)
gate := runs.NewAdmission(runtime.Service(), t.Name())
@@ -162,10 +162,10 @@ func createTestEnvironmentServiceUser(t *testing.T, ctx context.Context, userSer
func createTestEnvironment(t *testing.T, db *database.DB, id, apiURL string, accessToken *string) {
t.Helper()
- createNamedTestEnvironmentInternal(t, db, id, "env-"+id, apiURL, accessToken)
+ createNamedTestEnvironment(t, db, id, "env-"+id, apiURL, accessToken)
}
-func createNamedTestEnvironmentInternal(t *testing.T, db *database.DB, id, name, apiURL string, accessToken *string) {
+func createNamedTestEnvironment(t *testing.T, db *database.DB, id, name, apiURL string, accessToken *string) {
t.Helper()
now := time.Now()
@@ -226,9 +226,9 @@ func TestEnvironmentService_DeleteEnvironment_CascadesGitOpsSyncs(t *testing.T)
GitOpsManagedBy: &syncID,
}).Error)
- jobScheduler := &environmentTestSchedulerInternal{}
+ jobScheduler := &environmentTestScheduler{}
svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)
- require.NoError(t, svc.SetScheduler(ctx, jobScheduler, newAdmissionGateForEnvironmentTestInternal(t)))
+ require.NoError(t, svc.SetScheduler(ctx, jobScheduler, newAdmissionGateForEnvironmentTest(t)))
require.NoError(t, svc.DeleteEnvironment(ctx, "env-delete-gitops", nil, nil))
@@ -661,14 +661,14 @@ func TestEnvironmentService_UpdateEnvironmentConnectionState(t *testing.T) {
require.Equal(t, *lastSeen, *env.LastSeen)
}
-func TestEnvironmentService_UpdateEnvironmentStatusInternal_PromotesPendingDirectEnv(t *testing.T) {
+func TestEnvironmentService_UpdateEnvironmentStatus_PromotesPendingDirectEnv(t *testing.T) {
ctx := t.Context()
db := setupEnvironmentServiceTestDB(t)
svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)
createTestEnvironmentWithState(t, db, "direct-pending", "http://agent:3553", string(EnvironmentStatusPending), false, nil)
- require.NoError(t, svc.updateEnvironmentStatusInternal(ctx, "direct-pending", string(EnvironmentStatusOnline)))
+ require.NoError(t, svc.updateEnvironmentStatus(ctx, "direct-pending", string(EnvironmentStatusOnline)))
var env Environment
require.NoError(t, db.WithContext(ctx).Where("id = ?", "direct-pending").First(&env).Error)
@@ -676,18 +676,16 @@ func TestEnvironmentService_UpdateEnvironmentStatusInternal_PromotesPendingDirec
require.NotNil(t, env.LastSeen)
}
-func TestEnvironmentService_UpdateEnvironmentStatusInternal_DoesNotDemotePendingDirectEnvOnFailedTick(t *testing.T) {
+func TestEnvironmentService_UpdateEnvironmentStatus_DoesNotDemotePendingDirectEnvOnFailedTick(t *testing.T) {
ctx := t.Context()
db := setupEnvironmentServiceTestDB(t)
svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)
createTestEnvironmentWithState(t, db, "direct-pending", "http://agent:3553", string(EnvironmentStatusPending), false, nil)
- // A transient health-check failure before pairing completes must NOT flip a
- // pending Direct env to offline/error — the env should stay pending so a later
- // successful tick can still promote it to online.
- require.NoError(t, svc.updateEnvironmentStatusInternal(ctx, "direct-pending", string(EnvironmentStatusOffline)))
- require.NoError(t, svc.updateEnvironmentStatusInternal(ctx, "direct-pending", string(EnvironmentStatusError)))
+ // A transient failure before pairing must leave a pending direct env pending.
+ require.NoError(t, svc.updateEnvironmentStatus(ctx, "direct-pending", string(EnvironmentStatusOffline)))
+ require.NoError(t, svc.updateEnvironmentStatus(ctx, "direct-pending", string(EnvironmentStatusError)))
var env Environment
require.NoError(t, db.WithContext(ctx).Where("id = ?", "direct-pending").First(&env).Error)
@@ -695,7 +693,7 @@ func TestEnvironmentService_UpdateEnvironmentStatusInternal_DoesNotDemotePending
require.Nil(t, env.LastSeen)
}
-func TestEnvironmentService_UpdateEnvironmentStatusInternal_LeavesPendingEdgeEnvAlone(t *testing.T) {
+func TestEnvironmentService_UpdateEnvironmentStatus_LeavesPendingEdgeEnvAlone(t *testing.T) {
ctx := t.Context()
db := setupEnvironmentServiceTestDB(t)
svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)
@@ -704,7 +702,7 @@ func TestEnvironmentService_UpdateEnvironmentStatusInternal_LeavesPendingEdgeEnv
// Edge envs in pending must complete pairing via the agent's outbound tunnel;
// a manager-side reachability tick must NOT promote them.
- require.NoError(t, svc.updateEnvironmentStatusInternal(ctx, "edge-pending", string(EnvironmentStatusOnline)))
+ require.NoError(t, svc.updateEnvironmentStatus(ctx, "edge-pending", string(EnvironmentStatusOnline)))
var env Environment
require.NoError(t, db.WithContext(ctx).Where("id = ?", "edge-pending").First(&env).Error)
@@ -774,7 +772,7 @@ func TestEnvironmentService_UpdateEnvironment_ClearingAccessTokenInvalidatesCach
require.Contains(t, err.Error(), "invalid agent token")
}
-func TestEnvironmentServiceUpdateEnvironmentRejectsTargetChangeWithStoredTokenInternal(t *testing.T) {
+func TestEnvironmentServiceUpdateEnvironmentRejectsTargetChangeWithStoredToken(t *testing.T) {
ctx := t.Context()
db := setupEnvironmentServiceTestDB(t)
svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)
@@ -793,7 +791,7 @@ func TestEnvironmentServiceUpdateEnvironmentRejectsTargetChangeWithStoredTokenIn
require.Equal(t, oldToken, *stored.AccessToken)
}
-func TestEnvironmentServiceUpdateEnvironmentAllowsTargetChangeWithExplicitTokenInternal(t *testing.T) {
+func TestEnvironmentServiceUpdateEnvironmentAllowsTargetChangeWithExplicitToken(t *testing.T) {
for _, tc := range []struct {
name string
token string
@@ -825,7 +823,7 @@ func TestEnvironmentServiceUpdateEnvironmentAllowsTargetChangeWithExplicitTokenI
}
}
-func TestEnvironmentService_getCachedEnvironmentIDForTokenInternal_ExpiresAndCleansReverseIndex(t *testing.T) {
+func TestEnvironmentService_getCachedEnvironmentIDForToken_ExpiresAndCleansReverseIndex(t *testing.T) {
svc := NewEnvironmentService(nil, nil, nil, nil, nil, nil)
svc.edgeTokens.mu.Lock()
svc.edgeTokens.byEnvID["env-expired"] = "expired-token"
@@ -852,7 +850,7 @@ func TestEnvironmentService_ResolveEnvironmentByAccessToken(t *testing.T) {
svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)
accessToken := "remote-token"
- createNamedTestEnvironmentInternal(t, db, "env-remote", "Remote Alpha", "http://remote.example", &accessToken)
+ createNamedTestEnvironment(t, db, "env-remote", "Remote Alpha", "http://remote.example", &accessToken)
env, err := svc.ResolveEnvironmentByAccessToken(ctx, accessToken)
require.NoError(t, err)
@@ -925,7 +923,7 @@ func TestEnvironmentService_GenerateDeploymentSnippets_PublishesAgentURLPort(t *
}
}
-func TestAgentHostPortInternal(t *testing.T) {
+func TestAgentHostPort(t *testing.T) {
t.Parallel()
tests := []struct {
@@ -1082,14 +1080,8 @@ func TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_ReusesLegacyHiddenRe
"environment count = %d, want 1", len(environments))
}
-// TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_TokenResolvesEndToEnd
-// pins the agent token round-trip: the token returned from the swarm node
-// agent provisioning flow must resolve back to the same environment via
-// ResolveEdgeEnvironmentByToken, and rotation must invalidate the previous
-// token while making the new one resolvable. This is the end-to-end gap that
-// would have caught the v1.18.1 "invalid agent token" bug if any silent
-// transformation (trim, encode, hash) were ever introduced between the
-// command-generation path and the poll-validation path.
+// The provisioned agent token must resolve back to its environment, and rotation
+// must retire the old token, with no transformation between issue and validation.
func TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_TokenResolvesEndToEnd(t *testing.T) {
ctx := t.Context()
db := setupEnvironmentServiceTestDB(t)
@@ -1188,8 +1180,8 @@ func TestEnvironmentService_ListMethods_ExcludeHiddenEnvironments(t *testing.T)
svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)
createTestEnvironment(t, db, "0", "http://localhost:3552", nil)
- createNamedTestEnvironmentInternal(t, db, "env-visible", "Visible Remote", "http://visible.example", new("visible-token"))
- createNamedTestEnvironmentInternal(t, db, "env-hidden", "Hidden Node Agent", "edge://swarm-node-hidden", new("hidden-token"))
+ createNamedTestEnvironment(t, db, "env-visible", "Visible Remote", "http://visible.example", new("visible-token"))
+ createNamedTestEnvironment(t, db, "env-hidden", "Hidden Node Agent", "edge://swarm-node-hidden", new("hidden-token"))
require.NoError(t, db.WithContext(ctx).
Model(&Environment{}).
@@ -1223,8 +1215,8 @@ func TestEnvironmentService_ListEnvironmentsPaginated_FiltersByAccessibleEnvIDs(
svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)
createTestEnvironment(t, db, "0", "http://localhost:3552", nil)
- createNamedTestEnvironmentInternal(t, db, "env-a", "Env A", "http://a.example", new("token-a"))
- createNamedTestEnvironmentInternal(t, db, "env-b", "Env B", "http://b.example", new("token-b"))
+ createNamedTestEnvironment(t, db, "env-a", "Env A", "http://a.example", new("token-a"))
+ createNamedTestEnvironment(t, db, "env-b", "Env B", "http://b.example", new("token-b"))
newParams := func(typeFilter string) pagination.QueryParams {
filters := map[string]string{}
@@ -1238,17 +1230,17 @@ func TestEnvironmentService_ListEnvironmentsPaginated_FiltersByAccessibleEnvIDs(
// nil = no restriction: every non-hidden environment is returned.
all, _, err := svc.ListEnvironmentsPaginated(ctx, newParams(""), nil)
require.NoError(t, err)
- require.ElementsMatch(t, []string{"0", "env-a", "env-b"}, environmentIDsInternal(all))
+ require.ElementsMatch(t, []string{"0", "env-a", "env-b"}, environmentIDs(all))
// A non-nil allow-list restricts the result on the DB path.
scoped, _, err := svc.ListEnvironmentsPaginated(ctx, newParams(""), []string{"env-a"})
require.NoError(t, err)
- require.ElementsMatch(t, []string{"env-a"}, environmentIDsInternal(scoped))
+ require.ElementsMatch(t, []string{"env-a"}, environmentIDs(scoped))
// The runtime-filter path (type filter) honors the allow-list too.
scopedTyped, _, err := svc.ListEnvironmentsPaginated(ctx, newParams("http"), []string{"env-a"})
require.NoError(t, err)
- require.ElementsMatch(t, []string{"env-a"}, environmentIDsInternal(scopedTyped))
+ require.ElementsMatch(t, []string{"env-a"}, environmentIDs(scopedTyped))
// An empty (non-nil) allow-list yields no environments on either path.
none, _, err := svc.ListEnvironmentsPaginated(ctx, newParams(""), []string{})
@@ -1330,7 +1322,7 @@ func TestEnvironmentService_ListEnvironmentsPaginated_FiltersByRuntimeType(t *te
grpcTunnel := edge.NewAgentTunnelWithConn("env-grpc", edge.NewGRPCManagerTunnelConn(nil))
edge.GetRegistry().Register("env-grpc", grpcTunnel)
- websocketTunnel, closeWebSocketTunnel := newTestWebSocketTunnelInternal(t, "env-websocket")
+ websocketTunnel, closeWebSocketTunnel := newTestWebSocketTunnel(t, "env-websocket")
defer closeWebSocketTunnel()
edge.GetRegistry().Register("env-websocket", websocketTunnel)
@@ -1357,12 +1349,12 @@ func TestEnvironmentService_ListEnvironmentsPaginated_FiltersByRuntimeType(t *te
Filters: map[string]string{"type": tt.typeFilter},
}, nil)
require.NoError(t, err)
- require.ElementsMatch(t, tt.wantIDs, environmentIDsInternal(listedEnvironments))
+ require.ElementsMatch(t, tt.wantIDs, environmentIDs(listedEnvironments))
})
}
}
-func environmentIDsInternal(environments []environment.Environment) []string {
+func environmentIDs(environments []environment.Environment) []string {
ids := make([]string, 0, len(environments))
for _, env := range environments {
ids = append(ids, env.ID)
@@ -1370,7 +1362,7 @@ func environmentIDsInternal(environments []environment.Environment) []string {
return ids
}
-func newTestWebSocketTunnelInternal(t *testing.T, envID string) (*edge.AgentTunnel, func()) {
+func newTestWebSocketTunnel(t *testing.T, envID string) (*edge.AgentTunnel, func()) {
t.Helper()
connCh := make(chan *websocket.Conn, 1)
@@ -1459,7 +1451,7 @@ func TestEnvironmentService_TestConnection_RejectsInvalidCustomURL(t *testing.T)
require.EqualError(t, err, "Environment connection test failed")
}
-func TestEnvironmentServiceTestConnectionHidesCustomURLFailureInternal(t *testing.T) {
+func TestEnvironmentServiceTestConnectionHidesCustomURLFailure(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusTeapot)
}))
@@ -1476,7 +1468,7 @@ func TestEnvironmentServiceTestConnectionHidesCustomURLFailureInternal(t *testin
require.NotContains(t, err.Error(), "418")
}
-func TestEnvironmentServiceTestConnectionAllowsStoredPrivateURLInternal(t *testing.T) {
+func TestEnvironmentServiceTestConnectionAllowsStoredPrivateURL(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
}))
@@ -1504,12 +1496,12 @@ func TestEnvironmentService_ExecuteRemoteRequest_RejectsInvalidEnvironmentURL(t
require.Contains(t, err.Error(), "invalid environment API URL")
}
-func (s *environmentTestSchedulerInternal) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {
+func (s *environmentTestScheduler) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {
s.submitted = append(s.submitted, request)
return scheduler.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: scheduler.Queued}, nil
}
-func setupSyncDBInternal(t *testing.T) *database.DB {
+func setupSyncDB(t *testing.T) *database.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "sync.db")), &gorm.Config{})
require.NoError(t, err)
@@ -1532,7 +1524,7 @@ func setupSyncDBInternal(t *testing.T) *database.DB {
func TestSyncCredentialDecryptionAbortsSnapshot(t *testing.T) {
for _, kind := range []string{"registry token", "ECR secret", "repository token", "repository SSH key", "S3 secret"} {
t.Run(kind, func(t *testing.T) {
- db := setupSyncDBInternal(t)
+ db := setupSyncDB(t)
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
@@ -1574,7 +1566,7 @@ func TestSyncCredentialDecryptionAbortsSnapshot(t *testing.T) {
}
func TestSyncRepositoriesPreservesEmptyCredentials(t *testing.T) {
- db := setupSyncDBInternal(t)
+ db := setupSyncDB(t)
require.NoError(t, db.Create(&gitrepo.GitRepository{ID: "public-repo", Name: "Public", URL: "https://example.com/public.git", AuthType: "none"}).Error)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var payload gitops.RepositorySyncRequest
@@ -1595,7 +1587,7 @@ func TestSyncRepositoriesPreservesEmptyCredentials(t *testing.T) {
}
func TestSyncSkipsUnchangedPayloadUntilForgotten(t *testing.T) {
- db := setupSyncDBInternal(t)
+ db := setupSyncDB(t)
var calls atomic.Int32
var reject atomic.Bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
@@ -1644,7 +1636,7 @@ func TestSyncSkipsUnchangedPayloadUntilForgotten(t *testing.T) {
// A watcher must see a change signal, and a burst must not block the notifier —
// it runs on the edge tunnel's connect/disconnect path.
-func TestEnvironmentServiceRuntimeChangeSignalCoalescesInternal(t *testing.T) {
+func TestEnvironmentServiceRuntimeChangeSignalCoalesces(t *testing.T) {
s := &EnvironmentService{}
changes, unsubscribe := s.SubscribeRuntimeChanges()
diff --git a/backend/internal/federated/service.go b/backend/internal/federated/service.go
index a0c8a96b5..4ce38e86f 100644
--- a/backend/internal/federated/service.go
+++ b/backend/internal/federated/service.go
@@ -8,6 +8,7 @@ import (
"log/slog"
"net/http"
"regexp"
+ "slices"
"strings"
"sync"
"time"
@@ -18,6 +19,10 @@ import (
httpxtypes "github.com/getarcaneapp/arcane/types/v2/httpx"
"github.com/samber/mo"
"go.getarcane.app/kit/pkg"
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
"golang.org/x/sync/singleflight"
"gorm.io/gorm"
@@ -34,6 +39,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/jwtclaims"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/oidcjwk"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
const (
@@ -41,6 +47,8 @@ const (
defaultFederatedSubjectClaim = "sub"
)
+var errInvalidGrant = common.Classify(common.ErrFederatedCredentialInvalidGrant, errors.New("invalid federated token grant"))
+
type FederatedCredentialService struct {
db *database.DB
authService *auth.AuthService
@@ -53,6 +61,7 @@ type FederatedCredentialService struct {
providerMu sync.RWMutex
keySets map[string]oidc.KeySet
providerGroup singleflight.Group
+ exchanges metric.Int64Counter
}
func NewFederatedCredentialService(
@@ -65,8 +74,17 @@ func NewFederatedCredentialService(
keySetManager *oidcjwk.KeySetManager,
roleService *role.RoleService,
) *FederatedCredentialService {
- if httpClient == nil {
- httpClient = httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: 15 * time.Second, TLSHandshakeTimeout: 10 * time.Second})
+ client := httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: 15 * time.Second, TLSHandshakeTimeout: 10 * time.Second})
+ if httpClient != nil {
+ copied := *httpClient
+ client = &copied
+ }
+ client.Transport = otelhttp.NewTransport(client.Transport)
+
+ exchanges, err := otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.federated.token_exchanges",
+ metric.WithDescription("Federated credential token exchanges by outcome"), metric.WithUnit("{exchange}"))
+ if err != nil {
+ otel.Handle(err)
}
return &FederatedCredentialService{
@@ -75,10 +93,11 @@ func NewFederatedCredentialService(
userService: userService,
settingsService: settingsService,
eventService: eventService,
- httpClient: httpClient,
+ httpClient: client,
keySetManager: keySetManager,
roleService: roleService,
keySets: make(map[string]oidc.KeySet),
+ exchanges: exchanges,
}
}
@@ -139,11 +158,7 @@ func (s *FederatedCredentialService) Create(ctx context.Context, callerUserID st
s.roleService.InvalidateUser(created.IdentityUserID)
}
- reloaded, err := s.Get(ctx, created.ID)
- if err != nil {
- return nil, err
- }
- return reloaded, nil
+ return s.Get(ctx, created.ID)
}
func (s *FederatedCredentialService) List(ctx context.Context, params pagination.QueryParams) ([]federated.FederatedCredential, pagination.Response, error) {
@@ -172,28 +187,28 @@ func (s *FederatedCredentialService) List(ctx context.Context, params pagination
}
func (s *FederatedCredentialService) Get(ctx context.Context, id string) (*federated.FederatedCredential, error) {
- var credential FederatedCredential
- if err := s.db.WithContext(ctx).
- Preload("IdentityUser").
- Preload("Role").
- Preload("Environment").
- Where("id = ?", id).
- First(&credential).Error; err != nil {
- if errors.Is(err, gorm.ErrRecordNotFound) {
- return nil, common.Classify(common.ErrFederatedCredentialNotFound, errors.New("federated credential not found"))
- }
- return nil, fmt.Errorf("failed to get federated credential: %w", err)
+ credential, err := findCredential(s.db.WithContext(ctx).Preload("IdentityUser").Preload("Role").Preload("Environment"), id)
+ if err != nil {
+ return nil, err
}
return new(toFederatedCredentialDTOInternal(&credential)), nil
}
-func (s *FederatedCredentialService) Update(ctx context.Context, callerUserID, id string, req federated.UpdateFederatedCredential) (*federated.FederatedCredential, error) {
+func findCredential(query *gorm.DB, id string) (FederatedCredential, error) {
var credential FederatedCredential
- if err := s.db.WithContext(ctx).Where("id = ?", id).First(&credential).Error; err != nil {
+ if err := query.Where("id = ?", id).First(&credential).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
- return nil, common.Classify(common.ErrFederatedCredentialNotFound, errors.New("federated credential not found"))
+ return credential, common.Classify(common.ErrFederatedCredentialNotFound, errors.New("federated credential not found"))
}
- return nil, fmt.Errorf("failed to load federated credential: %w", err)
+ return credential, fmt.Errorf("failed to load federated credential: %w", err)
+ }
+ return credential, nil
+}
+
+func (s *FederatedCredentialService) Update(ctx context.Context, callerUserID, id string, req federated.UpdateFederatedCredential) (*federated.FederatedCredential, error) {
+ credential, err := findCredential(s.db.WithContext(ctx), id)
+ if err != nil {
+ return nil, err
}
updated, roleChanged, err := applyFederatedCredentialUpdateInternal(credential, req)
@@ -249,20 +264,17 @@ func (s *FederatedCredentialService) Update(ctx context.Context, callerUserID, i
}
func (s *FederatedCredentialService) Delete(ctx context.Context, id string) error {
- var credential FederatedCredential
- if err := s.db.WithContext(ctx).Where("id = ?", id).First(&credential).Error; err != nil {
- if errors.Is(err, gorm.ErrRecordNotFound) {
- return common.Classify(common.ErrFederatedCredentialNotFound, errors.New("federated credential not found"))
- }
- return fmt.Errorf("failed to load federated credential: %w", err)
+ credential, err := findCredential(s.db.WithContext(ctx), id)
+ if err != nil {
+ return err
}
- err := dbutil.WithTx(ctx, s.db.DB, func(tx *gorm.DB) error {
- if err := tx.Delete(&FederatedCredential{}, "id = ?", credential.ID).Error; err != nil {
- return fmt.Errorf("failed to delete federated credential: %w", err)
+ err = dbutil.WithTx(ctx, s.db.DB, func(tx *gorm.DB) error {
+ if deleteErr := tx.Delete(&FederatedCredential{}, "id = ?", credential.ID).Error; deleteErr != nil {
+ return fmt.Errorf("failed to delete federated credential: %w", deleteErr)
}
- if err := tx.Delete(&user.User{}, "id = ?", credential.IdentityUserID).Error; err != nil {
- return fmt.Errorf("failed to delete federated service user: %w", err)
+ if deleteErr := tx.Delete(&user.User{}, "id = ?", credential.IdentityUserID).Error; deleteErr != nil {
+ return fmt.Errorf("failed to delete federated service user: %w", deleteErr)
}
return nil
})
@@ -278,7 +290,8 @@ func (s *FederatedCredentialService) Delete(ctx context.Context, id string) erro
return nil
}
-func (s *FederatedCredentialService) ExchangeToken(ctx context.Context, req federated.TokenExchangeRequest) (*federated.FederatedTokenResponse, error) {
+func (s *FederatedCredentialService) ExchangeToken(ctx context.Context, req federated.TokenExchangeRequest) (_ *federated.FederatedTokenResponse, err error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "federated.token_exchange")
claims := jwtclaims.ParseJWTClaims(req.SubjectToken)
issuer := ""
subject := ""
@@ -291,54 +304,89 @@ func (s *FederatedCredentialService) ExchangeToken(ctx context.Context, req fede
logResult := "failure"
logReason := ""
- var matchedCredential *FederatedCredential
- var matchedUser *user.User
+ var credentialID, credentialName, userID, username string
defer func() {
- s.logExchangeInternal(ctx, logResult, logReason, issuer, subject, audiences, matchedCredential, matchedUser)
+ outcome := []attribute.KeyValue{attribute.String("arcane.federated.outcome", logResult), attribute.String("arcane.federated.reason", logReason)}
+ s.exchanges.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes(outcome...))
+ span.SetAttributes(append(outcome, attribute.String("arcane.federated.issuer", issuer), attribute.String("arcane.federated.credential_id", credentialID))...)
+ tracing.End(span, err)
+
+ slog.InfoContext(ctx, "Federated credential token exchange",
+ "result", logResult,
+ "reason", logReason,
+ "issuer", issuer,
+ "subject", subject,
+ "audiences", audiences,
+ "credentialId", credentialID,
+ )
+ if s.eventService == nil {
+ return
+ }
+ severity := event.EventSeverityInfo
+ title := "Federated credential token exchange"
+ if logResult != "success" {
+ severity = event.EventSeverityWarning
+ title = "Federated credential token exchange rejected"
+ }
+ eventRequest := event.CreateEventRequest{
+ Type: event.EventTypeFederatedExchange,
+ Severity: severity,
+ Title: title,
+ Description: "Workload identity federation token exchange",
+ ResourceType: new("federated_credential"),
+ ResourceID: mo.EmptyableToOption(strings.TrimSpace(credentialID)).ToPointer(),
+ ResourceName: mo.EmptyableToOption(strings.TrimSpace(credentialName)).ToPointer(),
+ UserID: mo.EmptyableToOption(strings.TrimSpace(userID)).ToPointer(),
+ Username: mo.EmptyableToOption(strings.TrimSpace(username)).ToPointer(),
+ Metadata: database.JSON{
+ "action": "federated_token_exchange",
+ "result": logResult,
+ "reason": logReason,
+ "issuer": issuer,
+ "subject": subject,
+ "audiences": audiences,
+ "credentialId": credentialID,
+ },
+ }
+ go func() {
+ bgCtx := context.WithoutCancel(ctx)
+ if _, eventErr := s.eventService.CreateEvent(bgCtx, eventRequest); eventErr != nil {
+ slog.WarnContext(bgCtx, "failed to audit federated credential token exchange", "error", eventErr)
+ }
+ }()
}()
- if req.GrantType != federated.TokenExchangeGrantType || strings.TrimSpace(req.SubjectToken) == "" {
- logReason = "invalid_request"
- return nil, common.Classify(common.ErrFederatedCredentialInvalidRequest, errors.New("invalid federated token exchange request"))
- }
- switch req.SubjectTokenType {
- case federated.SubjectTokenTypeJWT, federated.SubjectTokenTypeIDToken:
- default:
- logReason = "invalid_request"
- return nil, common.Classify(common.ErrFederatedCredentialInvalidRequest, errors.New("invalid federated token exchange request"))
- }
- if req.RequestedTokenType != "" && req.RequestedTokenType != federated.RequestedTokenTypeAccessJWT {
+ invalidRequest := req.GrantType != federated.TokenExchangeGrantType || strings.TrimSpace(req.SubjectToken) == "" ||
+ (req.SubjectTokenType != federated.SubjectTokenTypeJWT && req.SubjectTokenType != federated.SubjectTokenTypeIDToken) ||
+ (req.RequestedTokenType != "" && req.RequestedTokenType != federated.RequestedTokenTypeAccessJWT)
+ if invalidRequest {
logReason = "invalid_request"
return nil, common.Classify(common.ErrFederatedCredentialInvalidRequest, errors.New("invalid federated token exchange request"))
}
if issuer == "" {
logReason = "missing_issuer"
- return nil, common.Classify(common.ErrFederatedCredentialInvalidGrant, errors.New("invalid federated token grant"))
+ return nil, errInvalidGrant
}
var credentials []FederatedCredential
- if err := s.db.WithContext(ctx).
+ if findErr := s.db.WithContext(ctx).
Where("issuer_url = ? AND enabled = ?", issuer, true).
Order("created_at ASC").
Order("id ASC").
- Find(&credentials).Error; err != nil {
+ Find(&credentials).Error; findErr != nil {
logReason = "credential_lookup_failed"
- return nil, fmt.Errorf("failed to list federated credentials for issuer: %w", err)
+ return nil, fmt.Errorf("failed to list federated credentials for issuer: %w", findErr)
}
now := time.Now()
- active := credentials[:0]
- for _, credential := range credentials {
- if credential.ExpiresAt == nil || !now.After(*credential.ExpiresAt) {
- active = append(active, credential)
- }
- }
- credentials = active
+ credentials = slices.DeleteFunc(credentials, func(credential FederatedCredential) bool {
+ return credential.ExpiresAt != nil && now.After(*credential.ExpiresAt)
+ })
if len(credentials) == 0 {
logReason = "issuer_not_allowed"
- return nil, common.Classify(common.ErrFederatedCredentialInvalidGrant, errors.New("invalid federated token grant"))
+ return nil, errInvalidGrant
}
- verifiedToken, verifiedClaims, err := s.verifySubjectTokenInternal(ctx, issuer, req.SubjectToken)
+ verifiedToken, verifiedClaims, err := s.verifySubjectToken(ctx, issuer, req.SubjectToken)
if err != nil {
logReason = "token_verification_failed"
return nil, common.Classify(common.ErrFederatedCredentialInvalidGrant, fmt.Errorf("invalid federated token grant: %w", err))
@@ -350,15 +398,39 @@ func (s *FederatedCredentialService) ExchangeToken(ctx context.Context, req fede
audiences = append([]string{}, verifiedToken.Audience...)
}
- credential := selectMatchingCredentialInternal(credentials, verifiedToken.Audience, verifiedClaims)
- if credential == nil {
+ matchIndex := slices.IndexFunc(credentials, func(credential FederatedCredential) bool {
+ return credentialMatchesToken(&credential, verifiedToken.Audience, verifiedClaims)
+ })
+ if matchIndex < 0 {
logReason = "no_matching_credential"
- return nil, common.Classify(common.ErrFederatedCredentialInvalidGrant, errors.New("invalid federated token grant"))
+ return nil, errInvalidGrant
+ }
+ credential := &credentials[matchIndex]
+ credentialID, credentialName = credential.ID, credential.Name
+ logReason = "token_replay_rejected"
+ if verifiedToken.Expiry.IsZero() || time.Now().After(verifiedToken.Expiry) {
+ return nil, errInvalidGrant
+ }
+ if pruneErr := s.db.WithContext(ctx).Where("expires_at < ?", time.Now()).Delete(&FederatedTokenReplay{}).Error; pruneErr != nil {
+ return nil, fmt.Errorf("failed to prune federated token replay records: %w", pruneErr)
+ }
+ tokenID := strings.TrimSpace(kit.ToString(jwtclaims.GetByPath(verifiedClaims, "jti").OrEmpty()))
+ tokenKind := "jti"
+ if tokenID == "" {
+ tokenID = req.SubjectToken
+ tokenKind = "token"
}
- matchedCredential = credential
- if recordTokenReplayGuardErr := s.recordTokenReplayGuardInternal(ctx, issuer, req.SubjectToken, verifiedClaims, verifiedToken.Expiry); recordTokenReplayGuardErr != nil {
- logReason = "token_replay_rejected"
- return nil, recordTokenReplayGuardErr
+ replay := FederatedTokenReplay{
+ TokenHash: kit.SHA256Hex(issuer + "\x00" + tokenKind + "\x00" + tokenID),
+ IssuerURL: issuer,
+ ExpiresAt: verifiedToken.Expiry,
+ }
+ if replayErr := s.db.WithContext(ctx).Create(&replay).Error; replayErr != nil {
+ message := strings.ToLower(replayErr.Error())
+ if strings.Contains(message, "unique") || strings.Contains(message, "duplicate key") {
+ return nil, errInvalidGrant
+ }
+ return nil, fmt.Errorf("failed to record federated token replay guard: %w", replayErr)
}
identityUser, err := s.userService.GetUserByID(ctx, credential.IdentityUserID)
@@ -366,7 +438,7 @@ func (s *FederatedCredentialService) ExchangeToken(ctx context.Context, req fede
logReason = "identity_user_missing"
return nil, common.Classify(common.ErrFederatedCredentialInvalidGrant, fmt.Errorf("invalid federated token grant: %w", err))
}
- matchedUser = identityUser
+ userID, username = identityUser.ID, identityUser.Username
tokenPair, err := s.authService.IssueFederatedToken(ctx, identityUser, credential.ID, credential.TokenTTLSeconds)
if err != nil {
@@ -396,18 +468,54 @@ func (s *FederatedCredentialService) ExchangeToken(ctx context.Context, req fede
}, nil
}
-func (s *FederatedCredentialService) verifySubjectTokenInternal(ctx context.Context, issuer, rawToken string) (*oidc.IDToken, map[string]any, error) {
- keySet, err := s.keySetForIssuerInternal(ctx, issuer)
- if err != nil {
- return nil, nil, err
+// verifySubjectToken discovers the issuer's JWK set once and verifies rawToken against it.
+func (s *FederatedCredentialService) verifySubjectToken(ctx context.Context, issuer, rawToken string) (*oidc.IDToken, map[string]any, error) {
+ s.providerMu.RLock()
+ keySet := s.keySets[issuer]
+ s.providerMu.RUnlock()
+ if keySet == nil {
+ value, err, _ := s.providerGroup.Do(issuer, func() (any, error) {
+ providerCtx := oidc.ClientContext(context.WithoutCancel(ctx), s.httpClient)
+ provider, err := oidc.NewProvider(providerCtx, issuer)
+ if err != nil {
+ return nil, fmt.Errorf("failed to discover federated issuer: %w", err)
+ }
+
+ var metadata struct {
+ JWKSURL string `json:"jwks_uri"`
+ }
+ if claimsErr := provider.Claims(&metadata); claimsErr != nil {
+ return nil, fmt.Errorf("failed to read federated issuer metadata: %w", claimsErr)
+ }
+ if metadata.JWKSURL == "" {
+ return nil, errors.New("federated issuer metadata is missing jwks_uri")
+ }
+ if s.keySetManager == nil {
+ return nil, errors.New("JWK set manager is not configured")
+ }
+
+ discovered, err := s.keySetManager.KeySet(context.WithoutCancel(ctx), s.httpClient, metadata.JWKSURL)
+ if err != nil {
+ return nil, fmt.Errorf("failed to configure federated issuer JWK set: %w", err)
+ }
+ s.providerMu.Lock()
+ s.keySets[issuer] = discovered
+ s.providerMu.Unlock()
+ return discovered, nil
+ })
+ if err != nil {
+ return nil, nil, err
+ }
+ if keySet, _ = value.(oidc.KeySet); keySet == nil {
+ return nil, nil, errors.New("federated issuer discovery returned invalid key set")
+ }
}
- providerCtx := oidc.ClientContext(ctx, s.httpClient)
verifier := oidc.NewVerifier(issuer, keySet, &oidc.Config{
SkipClientIDCheck: true,
SupportedSigningAlgs: oidcjwk.SupportedSigningAlgs(),
})
- idToken, err := verifier.Verify(providerCtx, rawToken)
+ idToken, err := verifier.Verify(oidc.ClientContext(ctx, s.httpClient), rawToken)
if err != nil {
return nil, nil, err
}
@@ -419,110 +527,11 @@ func (s *FederatedCredentialService) verifySubjectTokenInternal(ctx context.Cont
return idToken, claims, nil
}
-func (s *FederatedCredentialService) recordTokenReplayGuardInternal(ctx context.Context, issuer, rawToken string, claims map[string]any, expiresAt time.Time) error {
- if expiresAt.IsZero() || time.Now().After(expiresAt) {
- return common.Classify(common.ErrFederatedCredentialInvalidGrant, errors.New("invalid federated token grant"))
- }
-
- now := time.Now()
- if err := s.db.WithContext(ctx).
- Where("expires_at < ?", now).
- Delete(&FederatedTokenReplay{}).Error; err != nil {
- return fmt.Errorf("failed to prune federated token replay records: %w", err)
- }
-
- tokenID := strings.TrimSpace(kit.ToString(jwtclaims.GetByPath(claims, "jti").OrEmpty()))
- tokenKind := "jti"
- if tokenID == "" {
- tokenID = rawToken
- tokenKind = "token"
- }
- replay := FederatedTokenReplay{
- TokenHash: kit.SHA256Hex(issuer + "\x00" + tokenKind + "\x00" + tokenID),
- IssuerURL: issuer,
- ExpiresAt: expiresAt,
- }
- if err := s.db.WithContext(ctx).Create(&replay).Error; err != nil {
- message := strings.ToLower(err.Error())
- if strings.Contains(message, "unique") || strings.Contains(message, "duplicate key") {
- return common.Classify(common.ErrFederatedCredentialInvalidGrant, errors.New("invalid federated token grant"))
- }
- return fmt.Errorf("failed to record federated token replay guard: %w", err)
- }
- return nil
-}
-
-func (s *FederatedCredentialService) keySetForIssuerInternal(ctx context.Context, issuer string) (oidc.KeySet, error) {
- s.providerMu.RLock()
- if keySet := s.keySets[issuer]; keySet != nil {
- s.providerMu.RUnlock()
- return keySet, nil
- }
- s.providerMu.RUnlock()
-
- value, err, _ := s.providerGroup.Do(issuer, func() (any, error) {
- providerCtx := oidc.ClientContext(context.WithoutCancel(ctx), s.httpClient)
- provider, err := oidc.NewProvider(providerCtx, issuer)
- if err != nil {
- return nil, fmt.Errorf("failed to discover federated issuer: %w", err)
- }
-
- var metadata struct {
- JWKSURL string `json:"jwks_uri"`
- }
- if claimsErr := provider.Claims(&metadata); claimsErr != nil {
- return nil, fmt.Errorf("failed to read federated issuer metadata: %w", claimsErr)
- }
- if metadata.JWKSURL == "" {
- return nil, errors.New("federated issuer metadata is missing jwks_uri")
- }
- if s.keySetManager == nil {
- return nil, errors.New("JWK set manager is not configured")
- }
-
- keySet, err := s.keySetManager.KeySet(context.WithoutCancel(ctx), s.httpClient, metadata.JWKSURL)
- if err != nil {
- return nil, fmt.Errorf("failed to configure federated issuer JWK set: %w", err)
- }
- s.providerMu.Lock()
- s.keySets[issuer] = keySet
- s.providerMu.Unlock()
- return keySet, nil
+func credentialMatchesToken(credential *FederatedCredential, tokenAudiences []string, claims map[string]any) bool {
+ audienceMatched := slices.ContainsFunc(credential.Audiences, func(audience string) bool {
+ audience = strings.TrimSpace(audience)
+ return audience != "" && slices.Contains(tokenAudiences, audience)
})
- if err != nil {
- return nil, err
- }
-
- keySet, ok := value.(oidc.KeySet)
- if !ok || keySet == nil {
- return nil, errors.New("federated issuer discovery returned invalid key set")
- }
- return keySet, nil
-}
-
-func selectMatchingCredentialInternal(credentials []FederatedCredential, tokenAudiences []string, claims map[string]any) *FederatedCredential {
- for i := range credentials {
- credential := &credentials[i]
- if credentialMatchesTokenInternal(credential, tokenAudiences, claims) {
- return credential
- }
- }
- return nil
-}
-
-func credentialMatchesTokenInternal(credential *FederatedCredential, tokenAudiences []string, claims map[string]any) bool {
- audiences := make(map[string]struct{}, len(credential.Audiences))
- for _, audience := range credential.Audiences {
- if audience = strings.TrimSpace(audience); audience != "" {
- audiences[audience] = struct{}{}
- }
- }
- audienceMatched := false
- for _, audience := range tokenAudiences {
- if _, audienceMatched = audiences[audience]; audienceMatched {
- break
- }
- }
if !audienceMatched {
return false
}
@@ -552,66 +561,3 @@ func credentialMatchesTokenInternal(credential *FederatedCredential, tokenAudien
matched, err := regexp.MatchString(expression.String(), subject)
return err == nil && matched
}
-
-func (s *FederatedCredentialService) logExchangeInternal(ctx context.Context, result, reason, issuer, subject string, audiences []string, credential *FederatedCredential, identityUser *user.User) {
- credentialID := ""
- credentialName := ""
- if credential != nil {
- credentialID = credential.ID
- credentialName = credential.Name
- }
- slog.InfoContext(ctx, "Federated credential token exchange",
- "result", result,
- "reason", reason,
- "issuer", issuer,
- "subject", subject,
- "audiences", audiences,
- "credentialId", credentialID,
- )
-
- if s.eventService == nil {
- return
- }
-
- metadata := database.JSON{
- "action": "federated_token_exchange",
- "result": result,
- "reason": reason,
- "issuer": issuer,
- "subject": subject,
- "audiences": audiences,
- "credentialId": credentialID,
- }
-
- userID := ""
- username := ""
- if identityUser != nil {
- userID = identityUser.ID
- username = identityUser.Username
- }
- severity := event.EventSeverityInfo
- title := "Federated credential token exchange"
- if result != "success" {
- severity = event.EventSeverityWarning
- title = "Federated credential token exchange rejected"
- }
-
- go func() {
- bgCtx := context.WithoutCancel(ctx)
- _, err := s.eventService.CreateEvent(bgCtx, event.CreateEventRequest{
- Type: event.EventTypeFederatedExchange,
- Severity: severity,
- Title: title,
- Description: "Workload identity federation token exchange",
- ResourceType: mo.EmptyableToOption(strings.TrimSpace("federated_credential")).ToPointer(),
- ResourceID: mo.EmptyableToOption(strings.TrimSpace(credentialID)).ToPointer(),
- ResourceName: mo.EmptyableToOption(strings.TrimSpace(credentialName)).ToPointer(),
- UserID: mo.EmptyableToOption(strings.TrimSpace(userID)).ToPointer(),
- Username: mo.EmptyableToOption(strings.TrimSpace(username)).ToPointer(),
- Metadata: metadata,
- })
- if err != nil {
- slog.WarnContext(bgCtx, "failed to audit federated credential token exchange", "error", err)
- }
- }()
-}
diff --git a/backend/internal/federated/service_test.go b/backend/internal/federated/service_test.go
index 32325e873..a0947fc86 100644
--- a/backend/internal/federated/service_test.go
+++ b/backend/internal/federated/service_test.go
@@ -35,20 +35,20 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/oidcjwk"
)
-type federatedTestIssuerInternal struct {
+type federatedTestIssuer struct {
IssuerURL string
private *mldsa.PrivateKey
keyID string
server *httptest.Server
}
-func newFederatedTestIssuerInternal(t *testing.T) *federatedTestIssuerInternal {
+func newFederatedTestIssuer(t *testing.T) *federatedTestIssuer {
t.Helper()
privateKey, err := mldsa.GenerateKey(mldsa.MLDSA87())
require.NoError(t, err)
- issuer := &federatedTestIssuerInternal{
+ issuer := &federatedTestIssuer{
private: privateKey,
keyID: "federated-test-key",
}
@@ -90,7 +90,7 @@ func newFederatedTestIssuerInternal(t *testing.T) *federatedTestIssuerInternal {
return issuer
}
-func (i *federatedTestIssuerInternal) tokenInternal(t *testing.T, subject string, audience []string) string {
+func (i *federatedTestIssuer) token(t *testing.T, subject string, audience []string) string {
t.Helper()
now := time.Now()
@@ -110,7 +110,7 @@ func (i *federatedTestIssuerInternal) tokenInternal(t *testing.T, subject string
return string(signed)
}
-func setupFederatedCredentialServiceTestDBInternal(t *testing.T) *database.DB {
+func setupFederatedCredentialServiceTestDB(t *testing.T) *database.DB {
t.Helper()
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.NewReplacer("/", "_", " ", "_").Replace(t.Name()))
@@ -135,15 +135,15 @@ func setupFederatedCredentialServiceTestDBInternal(t *testing.T) *database.DB {
return &database.DB{DB: db}
}
-func setupFederatedCredentialServiceInternal(t *testing.T, issuer *federatedTestIssuerInternal) (*FederatedCredentialService, *auth.AuthService, *database.DB) {
+func setupFederatedCredentialService(t *testing.T, issuer *federatedTestIssuer) (*FederatedCredentialService, *auth.AuthService, *database.DB) {
t.Helper()
ctx := t.Context()
- db := setupFederatedCredentialServiceTestDBInternal(t)
+ db := setupFederatedCredentialServiceTestDB(t)
roleSvc := role.NewRoleService(db)
userSvc := user.NewUserService(db, roleSvc, session.RevokeAllUserSessionsExceptInDB)
sessionSvc := session.NewSessionService(db)
- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
eventSvc := event.NewEventService(db, &config.Config{}, nil)
signingKey, err := mldsa.GenerateKey(mldsa.MLDSA87())
@@ -197,8 +197,8 @@ func setupFederatedCredentialServiceInternal(t *testing.T, issuer *federatedTest
}
func TestFederatedCredentialServiceExchangeToken(t *testing.T) {
- issuer := newFederatedTestIssuerInternal(t)
- service, authSvc, db := setupFederatedCredentialServiceInternal(t, issuer)
+ issuer := newFederatedTestIssuer(t)
+ service, authSvc, db := setupFederatedCredentialService(t, issuer)
ctx := t.Context()
tests := []struct {
@@ -208,18 +208,18 @@ func TestFederatedCredentialServiceExchangeToken(t *testing.T) {
}{
{
name: "issues an Arcane bearer token for a matching issuer audience and subject",
- token: issuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),
+ token: issuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),
},
{
name: "rejects audience mismatch",
- token: issuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"other-audience"}),
+ token: issuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"other-audience"}),
wantError: func(err error) bool {
return errors.Is(err, common.ErrFederatedCredentialInvalidGrant)
},
},
{
name: "rejects subject mismatch",
- token: issuer.tokenInternal(t, "repo:other/repo:ref:refs/heads/main", []string{"arcane-ci"}),
+ token: issuer.token(t, "repo:other/repo:ref:refs/heads/main", []string{"arcane-ci"}),
wantError: func(err error) bool {
return errors.Is(err, common.ErrFederatedCredentialInvalidGrant)
},
@@ -261,14 +261,14 @@ func TestFederatedCredentialServiceExchangeToken(t *testing.T) {
}
}
-func TestFederatedCredentialServiceExchangeTokenRejectsIssuerWithoutCredentialInternal(t *testing.T) {
- issuer := newFederatedTestIssuerInternal(t)
- otherIssuer := newFederatedTestIssuerInternal(t)
- service, _, _ := setupFederatedCredentialServiceInternal(t, issuer)
+func TestFederatedCredentialServiceExchangeTokenRejectsIssuerWithoutCredential(t *testing.T) {
+ issuer := newFederatedTestIssuer(t)
+ otherIssuer := newFederatedTestIssuer(t)
+ service, _, _ := setupFederatedCredentialService(t, issuer)
resp, err := service.ExchangeToken(t.Context(), federated.TokenExchangeRequest{
GrantType: federated.TokenExchangeGrantType,
- SubjectToken: otherIssuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),
+ SubjectToken: otherIssuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),
SubjectTokenType: federated.SubjectTokenTypeJWT,
Audience: "https://arcane.example.com",
})
@@ -278,14 +278,14 @@ func TestFederatedCredentialServiceExchangeTokenRejectsIssuerWithoutCredentialIn
require.Nil(t, resp)
}
-func TestFederatedCredentialServiceExchangeTokenDoesNotRequireGlobalFeatureFlagInternal(t *testing.T) {
- issuer := newFederatedTestIssuerInternal(t)
- service, _, _ := setupFederatedCredentialServiceInternal(t, issuer)
+func TestFederatedCredentialServiceExchangeTokenDoesNotRequireGlobalFeatureFlag(t *testing.T) {
+ issuer := newFederatedTestIssuer(t)
+ service, _, _ := setupFederatedCredentialService(t, issuer)
service.settingsService = nil
resp, err := service.ExchangeToken(t.Context(), federated.TokenExchangeRequest{
GrantType: federated.TokenExchangeGrantType,
- SubjectToken: issuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),
+ SubjectToken: issuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),
SubjectTokenType: federated.SubjectTokenTypeJWT,
Audience: "https://arcane.example.com",
})
@@ -295,9 +295,9 @@ func TestFederatedCredentialServiceExchangeTokenDoesNotRequireGlobalFeatureFlagI
require.NotEmpty(t, resp.AccessToken)
}
-func TestFederatedCredentialServiceExchangeTokenRejectsExpiredCredentialInternal(t *testing.T) {
- issuer := newFederatedTestIssuerInternal(t)
- service, _, db := setupFederatedCredentialServiceInternal(t, issuer)
+func TestFederatedCredentialServiceExchangeTokenRejectsExpiredCredential(t *testing.T) {
+ issuer := newFederatedTestIssuer(t)
+ service, _, db := setupFederatedCredentialService(t, issuer)
expiredAt := time.Now().Add(-time.Minute)
require.NoError(t, db.WithContext(t.Context()).
Model(&FederatedCredential{}).
@@ -306,7 +306,7 @@ func TestFederatedCredentialServiceExchangeTokenRejectsExpiredCredentialInternal
resp, err := service.ExchangeToken(t.Context(), federated.TokenExchangeRequest{
GrantType: federated.TokenExchangeGrantType,
- SubjectToken: issuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),
+ SubjectToken: issuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),
SubjectTokenType: federated.SubjectTokenTypeJWT,
Audience: "https://arcane.example.com",
})
@@ -316,14 +316,14 @@ func TestFederatedCredentialServiceExchangeTokenRejectsExpiredCredentialInternal
require.Nil(t, resp)
}
-func TestFederatedCredentialServiceUpdateDisableRevokesIssuedSessionsInternal(t *testing.T) {
- issuer := newFederatedTestIssuerInternal(t)
- service, authSvc, _ := setupFederatedCredentialServiceInternal(t, issuer)
+func TestFederatedCredentialServiceUpdateDisableRevokesIssuedSessions(t *testing.T) {
+ issuer := newFederatedTestIssuer(t)
+ service, authSvc, _ := setupFederatedCredentialService(t, issuer)
ctx := t.Context()
resp, err := service.ExchangeToken(ctx, federated.TokenExchangeRequest{
GrantType: federated.TokenExchangeGrantType,
- SubjectToken: issuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),
+ SubjectToken: issuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),
SubjectTokenType: federated.SubjectTokenTypeJWT,
Audience: "https://arcane.example.com",
})
@@ -340,11 +340,11 @@ func TestFederatedCredentialServiceUpdateDisableRevokesIssuedSessionsInternal(t
require.ErrorIs(t, err, common.ErrSessionRevoked, "unexpected error: %v", err)
}
-func TestFederatedCredentialServiceRejectsReplayedSubjectTokenInternal(t *testing.T) {
- issuer := newFederatedTestIssuerInternal(t)
- service, _, _ := setupFederatedCredentialServiceInternal(t, issuer)
+func TestFederatedCredentialServiceRejectsReplayedSubjectToken(t *testing.T) {
+ issuer := newFederatedTestIssuer(t)
+ service, _, _ := setupFederatedCredentialService(t, issuer)
ctx := t.Context()
- subjectToken := issuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"})
+ subjectToken := issuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"})
req := federated.TokenExchangeRequest{
GrantType: federated.TokenExchangeGrantType,
SubjectToken: subjectToken,
@@ -363,8 +363,8 @@ func TestFederatedCredentialServiceRejectsReplayedSubjectTokenInternal(t *testin
}
func TestFederatedCredentialServiceCreateRejectsBareWildcardGlob(t *testing.T) {
- issuer := newFederatedTestIssuerInternal(t)
- service, _, _ := setupFederatedCredentialServiceInternal(t, issuer)
+ issuer := newFederatedTestIssuer(t)
+ service, _, _ := setupFederatedCredentialService(t, issuer)
_, err := service.Create(t.Context(), "admin-user", federated.CreateFederatedCredential{
Name: "Unsafe wildcard",
@@ -380,7 +380,7 @@ func TestFederatedCredentialServiceCreateRejectsBareWildcardGlob(t *testing.T) {
require.ErrorIs(t, err, common.ErrFederatedCredentialInvalid, "unexpected error: %v", err)
}
-func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
+func newSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
svc, err := settings.NewSettingsService(ctx, db)
if err == nil {
diff --git a/backend/internal/gitops/service.go b/backend/internal/gitops/service.go
index 81fbe4d10..e36e58eeb 100644
--- a/backend/internal/gitops/service.go
+++ b/backend/internal/gitops/service.go
@@ -23,6 +23,10 @@ import (
"go.getarcane.app/acfs"
"go.getarcane.app/kit/pkg"
"go.getarcane.app/kit/pkg/mapping"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
+ "go.opentelemetry.io/otel/trace"
"gorm.io/gorm"
"gorm.io/gorm/clause"
@@ -35,14 +39,25 @@ import (
projectpkg "github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/gitutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
+var syncRuns = func() metric.Int64Counter {
+ counter, err := otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.gitops.syncs",
+ metric.WithDescription("GitOps sync runs by outcome"), metric.WithUnit("{sync}"))
+ if err != nil {
+ otel.Handle(err)
+ }
+ return counter
+}()
+
const (
defaultGitSyncTimeout = 5 * time.Minute
@@ -749,7 +764,22 @@ func (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id s
// performSyncAdmitted runs one sync under its admission lease and dispatches by mode. backupAdopt makes a
// backup replace whatever the remote holds in its backup directory.
-func (s *GitOpsSyncService) performSyncAdmitted(ctx context.Context, environmentID, id string, actor user.Actor, backupAdopt bool) (*gitops.SyncResult, error) {
+func (s *GitOpsSyncService) performSyncAdmitted(ctx context.Context, environmentID, id string, actor user.Actor, backupAdopt bool) (syncResult *gitops.SyncResult, syncErr error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "gitops.sync", trace.WithAttributes(
+ attribute.String("arcane.gitops.sync_id", id),
+ attribute.String("arcane.environment.id", environmentID),
+ ))
+ counted := false
+ defer func() {
+ success := syncErr == nil && syncResult != nil && syncResult.Success
+ if counted {
+ outcome := attribute.String("arcane.gitops.outcome", kit.Ternary(success, "success", "failure"))
+ span.SetAttributes(outcome)
+ syncRuns.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes(outcome))
+ }
+ tracing.End(span, syncErr)
+ }()
+
// Overlapping runs of one sync (schedule, kick, manual, webhook) must not race the clone and redeploy.
lease, admitted, err := s.jobs.TryAcquire(ctx, id)
if err != nil {
@@ -757,9 +787,11 @@ func (s *GitOpsSyncService) performSyncAdmitted(ctx context.Context, environment
}
if !admitted {
slog.InfoContext(ctx, "GitOps sync already in progress; skipping", "syncId", id)
+ span.SetAttributes(attribute.Bool("arcane.gitops.skipped", true))
return &gitops.SyncResult{Success: false, Message: "sync already in progress", SyncedAt: time.Now()}, nil
}
defer lease.Release(ctx)
+ counted = true
syncCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)
defer cancel()
@@ -768,6 +800,14 @@ func (s *GitOpsSyncService) performSyncAdmitted(ctx context.Context, environment
if err != nil {
return nil, err
}
+ span.SetAttributes(
+ attribute.String("arcane.git.branch", syncRecord.Branch),
+ attribute.String("arcane.gitops.mode", syncRecord.Mode),
+ attribute.String("arcane.gitops.target_type", syncRecord.TargetType),
+ )
+ if syncRecord.Repository != nil {
+ span.SetAttributes(git.RepositoryAttribute(syncRecord.Repository.URL))
+ }
result := &gitops.SyncResult{
Success: false,
diff --git a/backend/internal/image/children/patch/service.go b/backend/internal/image/children/patch/service.go
index 226449fd7..e90233796 100644
--- a/backend/internal/image/children/patch/service.go
+++ b/backend/internal/image/children/patch/service.go
@@ -32,6 +32,10 @@ import (
"github.com/project-copacetic/copacetic/pkg/types"
"github.com/samber/mo"
"go.getarcane.app/acfs"
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -45,6 +49,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/logging"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
// scheduledPatchUser is the actor recorded for scheduled patches and their verification scans.
@@ -52,6 +57,8 @@ var (
scheduledPatchUser = usertypes.Actor{Username: "System"}
// errPatchInterrupted fails a patch cut off mid-run; its record keeps the message so it never reruns.
errPatchInterrupted = errors.New("image patch was interrupted before its result was recorded; start a new patch")
+ // registryTransport traces platform-manifest lookups; ggcr adds its retry wrappers on top.
+ registryTransport = otelhttp.NewTransport(remote.DefaultTransport)
)
// Target is one image a scheduled patch run covers.
@@ -276,7 +283,16 @@ func (s *Service) startPatch(
}
// runPatch executes one recorded patch with Copacetic and settles its record and activity.
-func (s *Service) runPatch(ctx context.Context, record *ImagePatchRecord, opts imagepatch.PatchOptions, reportData []byte, copaImageRef, activityID string) error {
+func (s *Service) runPatch(ctx context.Context, record *ImagePatchRecord, opts imagepatch.PatchOptions, reportData []byte, copaImageRef, activityID string) (runErr error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "image.patch", trace.WithAttributes(
+ attribute.String("arcane.environment.id", record.EnvironmentID),
+ attribute.String("arcane.image.patch_id", record.ID),
+ attribute.String("arcane.image.ref", record.OriginalRef),
+ attribute.String("arcane.image.patched_ref", record.PatchedRef),
+ attribute.String("arcane.image.patch_mode", record.Mode),
+ attribute.String("arcane.activity.id", activityID),
+ ))
+ defer func() { tracing.End(span, runErr) }()
fail := func(err error, durationMs int64) error {
s.finishPatchRecord(ctx, record, imagepatch.PatchStatusFailed, err.Error(), nil, durationMs)
s.completePatchActivity(ctx, activityID, false, err.Error())
@@ -428,7 +444,7 @@ func platformPinnedRef(ctx context.Context, imageRef string, target v1.Platform)
if err != nil {
return ""
}
- desc, err := remote.Get(ref, remote.WithContext(ctx), remote.WithAuthFromKeychain(authn.DefaultKeychain))
+ desc, err := remote.Get(ref, remote.WithContext(ctx), remote.WithTransport(registryTransport), remote.WithAuthFromKeychain(authn.DefaultKeychain))
if err != nil || !desc.MediaType.IsIndex() {
return ""
}
diff --git a/backend/internal/imageupdate/module.go b/backend/internal/imageupdate/module.go
index 9bb2f3037..c6698180d 100644
--- a/backend/internal/imageupdate/module.go
+++ b/backend/internal/imageupdate/module.go
@@ -3,14 +3,20 @@ package imageupdate
import (
"context"
+ "errors"
+ "log/slog"
"net/http"
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/types/v2/image"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
type Module struct {
@@ -18,7 +24,32 @@ type Module struct {
getUpdateInfoByImageRefs func(context.Context, []string) (map[string]*image.UpdateInfo, error)
}
+// New wires image update routes and registers the arcane.image.updates gauges once.
func New(service *ImageUpdateService, getUpdateInfo func(context.Context, []string) (map[string]*image.UpdateInfo, error)) *Module {
+ if service != nil {
+ meter := otel.Meter(tracing.InstrumentationName)
+ updates, updatesErr := meter.Int64ObservableGauge("arcane.image.updates",
+ metric.WithDescription("Local images with an available update by update type"), metric.WithUnit("{image}"))
+ checkErrors, checkErrorsErr := meter.Int64ObservableGauge("arcane.image.update.check_errors",
+ metric.WithDescription("Local images whose last update check failed"), metric.WithUnit("{image}"))
+ err := errors.Join(updatesErr, checkErrorsErr)
+ if err == nil {
+ _, err = meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
+ summary, summaryErr := service.GetUpdateSummary(ctx)
+ if summaryErr != nil {
+ slog.DebugContext(ctx, "Skipping image update metrics", "error", summaryErr)
+ return nil
+ }
+ o.ObserveInt64(updates, int64(summary.DigestUpdates), metric.WithAttributes(attribute.String("image.update.type", "digest")))
+ o.ObserveInt64(updates, int64(summary.ImagesWithUpdates-summary.DigestUpdates), metric.WithAttributes(attribute.String("image.update.type", "tag")))
+ o.ObserveInt64(checkErrors, int64(summary.ErrorsCount))
+ return nil
+ }, updates, checkErrors)
+ }
+ if err != nil {
+ otel.Handle(err)
+ }
+ }
return &Module{service: service, getUpdateInfoByImageRefs: getUpdateInfo}
}
@@ -41,13 +72,7 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppCont
func RegisterImageUpdates(
api huma.API,
imageUpdateSvc *ImageUpdateService,
- getUpdateInfoByImageRefs func(
- context.Context,
- []string,
- ) (
- map[string]*image.UpdateInfo,
- error,
- ),
+ getUpdateInfoByImageRefs func(context.Context, []string) (map[string]*image.UpdateInfo, error),
appCtx handlerutil.ActivityAppContext,
) {
h := &ImageUpdateHandler{
diff --git a/backend/internal/job/service.go b/backend/internal/job/service.go
index c3620bde7..7f80a2634 100644
--- a/backend/internal/job/service.go
+++ b/backend/internal/job/service.go
@@ -20,6 +20,10 @@ import (
"github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/robfig/cron/v3"
"go.getarcane.app/kit/pkg"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
+ "go.opentelemetry.io/otel/trace"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -35,6 +39,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
// JobService manages configuration for background job schedules.
@@ -58,6 +63,9 @@ type JobService struct {
scheduler scheduler.JobController
lifecycleCtx context.Context
location *time.Location // Timezone for cron schedule calculations
+ runCount metric.Int64Counter
+ runDuration metric.Float64Histogram
+ activeRuns metric.Int64UpDownCounter
// environment-health is no longer a single scheduler job — it fans out to one
// dynamic job per environment owned by environment.EnvironmentService. These bridge the Jobs
@@ -88,6 +96,19 @@ func NewJobService(
location: cfg.GetLocation(),
}
+ meter := otel.Meter(tracing.InstrumentationName)
+ var errs [3]error
+ service.runCount, errs[0] = meter.Int64Counter("arcane.job.runs",
+ metric.WithDescription("Executed job runs by outcome"), metric.WithUnit("{run}"))
+ service.runDuration, errs[1] = meter.Float64Histogram("arcane.job.duration",
+ metric.WithDescription("Job run duration by outcome"), metric.WithUnit("s"),
+ metric.WithExplicitBucketBoundaries(0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 120, 300, 600, 1800, 3600))
+ service.activeRuns, errs[2] = meter.Int64UpDownCounter("arcane.job.active",
+ metric.WithDescription("Job runs currently executing"), metric.WithUnit("{run}"))
+ if err := errors.Join(errs[:]...); err != nil {
+ otel.Handle(err)
+ }
+
if coordinator != nil {
coordinator.SetExecutor(service.executeRunInternal, service.reconcileRunInternal)
if localActivity != nil {
@@ -609,7 +630,33 @@ func (s *JobService) authorizeRunInternal(ctx context.Context, run scheduler.Run
return nil
}
-func (s *JobService) executeRunInternal(ctx context.Context, run scheduler.Run) (scheduler.Outcome, error) {
+func (s *JobService) executeRunInternal(ctx context.Context, run scheduler.Run) (result scheduler.Outcome, runErr error) {
+ // Entity-scoped jobs ("gitops-sync:<id>") share one span name and metric series per job type.
+ jobType, _, _ := strings.Cut(run.JobID, ":")
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "job "+jobType, trace.WithAttributes(
+ attribute.String("arcane.job.id", run.JobID),
+ attribute.String("arcane.job.run_id", run.ID),
+ attribute.String("arcane.environment.id", run.EnvironmentID),
+ ))
+ jobAttr := attribute.String("arcane.job.id", jobType)
+ started := time.Now()
+ if s.activeRuns != nil {
+ s.activeRuns.Add(ctx, 1, metric.WithAttributes(jobAttr))
+ }
+ defer func() {
+ outcomeAttr := attribute.String("arcane.job.outcome", string(result.Status))
+ span.SetAttributes(outcomeAttr)
+ tracing.End(span, runErr)
+ if s.activeRuns != nil {
+ // Record against a non-canceled context so shutdown-interrupted runs still count.
+ metricCtx := context.WithoutCancel(ctx)
+ attrs := metric.WithAttributes(jobAttr, outcomeAttr)
+ s.activeRuns.Add(metricCtx, -1, metric.WithAttributes(jobAttr))
+ s.runCount.Add(metricCtx, 1, attrs)
+ s.runDuration.Record(metricCtx, time.Since(started).Seconds(), attrs)
+ }
+ }()
+
if err := s.authorizeRunInternal(ctx, run); err != nil {
return scheduler.Outcome{Status: scheduler.Failed}, err
}
diff --git a/backend/internal/middleware/environment.go b/backend/internal/middleware/environment.go
index d24466ae3..14dd1cabd 100644
--- a/backend/internal/middleware/environment.go
+++ b/backend/internal/middleware/environment.go
@@ -2,6 +2,7 @@ package middleware
import (
"bytes"
+ "cmp"
"context"
"encoding/json/v2"
"errors"
@@ -25,8 +26,8 @@ import (
"github.com/labstack/echo/v5"
"github.com/samber/mo"
"go.getarcane.app/kit/pkg"
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
- "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"
@@ -37,8 +38,7 @@ const (
apiEnvironmentsPrefix = "/api/environments/"
environmentsPathMarker = "/environments/"
- // proxyTimeout is intentionally generous because some proxied operations
- // (e.g., image pulls with progress streaming) can take multiple minutes.
+ // proxyTimeout is generous because proxied operations such as image pulls can stream for minutes.
proxyTimeout = 30 * time.Minute
maxProxiedWorkspaceManifestBytes = 1024 * 1024
@@ -63,14 +63,8 @@ var managementEndpointSet = map[string]struct{}{
// Returns: apiURL, accessToken, enabled, error
type EnvResolver func(ctx context.Context, id string) (string, *string, bool, error)
-// AuthValidator validates authentication for a request and resolves the
-// caller's effective permission set. The boolean result reports whether the
-// request is authenticated; the permission set is used to authorize proxied
-// requests against the target environment. Sudo permission sets (internal
-// agent proxies) bypass authorization. The resolved user is returned so
-// per-user context (e.g. the icon catalog preference) can travel with the
-// proxied request; it is nil for callers that are not a user (environment
-// bootstrap keys).
+// AuthValidator authenticates a request and returns the caller's permission set and user (nil for non-user callers).
+// Sudo permission sets (internal agent proxies) bypass authorization.
type AuthValidator func(ctx context.Context, c *echo.Context) (*authz.PermissionSet, *usertypes.Actor, bool)
// EnvironmentMiddleware proxies requests for remote environments to their respective agents.
@@ -82,10 +76,7 @@ type EnvironmentMiddleware struct {
httpClient *http.Client
registry *edge.TunnelRegistry
matcher *authz.PermissionMatcher
- // checkOrigin is the same Origin validator the local WebSocket endpoints
- // use. Proxied upgrades previously accepted any Origin, so a cross-origin
- // page could ride the caller's session cookie into a remote environment's
- // terminal or log stream.
+ // checkOrigin is the local WebSocket Origin validator, so proxied upgrades cannot ride a cross-origin session.
checkOrigin func(*http.Request) bool
}
@@ -103,12 +94,16 @@ func NewEnvProxyMiddlewareWithParamAndRegistry(
registry = edge.NewTunnelRegistry()
}
+ // Traced so direct agents continue the manager's trace.
+ httpClient := httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: proxyTimeout, TLSHandshakeTimeout: 10 * time.Second})
+ httpClient.Transport = otelhttp.NewTransport(httpClient.Transport)
+
m := &EnvironmentMiddleware{
localID: localID,
paramName: paramName,
resolver: resolver,
authValidator: authValidator,
- httpClient: httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: proxyTimeout, TLSHandshakeTimeout: 10 * time.Second}),
+ httpClient: httpClient,
registry: registry,
matcher: matcher,
checkOrigin: checkOrigin,
@@ -122,13 +117,18 @@ func NewEnvProxyMiddlewareWithParamAndRegistry(
// Handle is the main middleware handler.
func (m *EnvironmentMiddleware) Handle(c *echo.Context, next echo.HandlerFunc) error {
- envID := m.extractEnvironmentID(c)
+ req := c.Request()
+ ctx := req.Context()
- if envID == "" || envID == m.localID {
+ _, rest, hasMarker := strings.Cut(req.URL.Path, environmentsPathMarker)
+ pathEnvID, _, _ := strings.Cut(rest, "/")
+ envID := cmp.Or(c.Param(m.paramName), pathEnvID)
+ if !hasMarker || envID == "" || envID == m.localID {
return next(c)
}
- if !m.hasResourcePath(c, envID) {
+ suffix, ok := strings.CutPrefix(req.URL.Path, apiEnvironmentsPrefix+envID)
+ if !ok || len(suffix) <= 1 || suffix[0] != '/' || isManagementPath(req.Method, suffix) {
return next(c)
}
@@ -136,168 +136,196 @@ func (m *EnvironmentMiddleware) Handle(c *echo.Context, next echo.HandlerFunc) e
var perms *authz.PermissionSet
var user *usertypes.Actor
if m.authValidator != nil {
- ps, u, ok := m.authValidator(c.Request().Context(), c)
- if !ok {
- return c.JSON(http.StatusUnauthorized, map[string]any{
- "success": false,
- "data": map[string]any{"error": "Authentication required to access remote environments"},
- })
+ var authenticated bool
+ perms, user, authenticated = m.authValidator(ctx, c)
+ if !authenticated {
+ return writeProxyError(c, http.StatusUnauthorized, "Authentication required to access remote environments")
+ }
+ if user != nil {
+ RecordAuthenticatedRequest(ctx, req.Header, user)
}
- perms, user = ps, u
}
- m.setIconCatalogHeaderInternal(c, user)
- m.setUpdateInitiatorHeadersInternal(c, user)
+ // SECURITY: forwarded identity headers are always cleared first; only server-resolved values are sent.
+ req.Header.Del(HeaderIconCatalog)
+ if user != nil && user.Preferences.IconCatalog != nil && *user.Preferences.IconCatalog != "" {
+ req.Header.Set(HeaderIconCatalog, *user.Preferences.IconCatalog)
+ }
+ req.Header.Del(HeaderUpdateInitiatorID)
+ req.Header.Del(HeaderUpdateInitiatorName)
+ req.Header.Del(HeaderUpdateInitiatorDisplayName)
+ isContainerUpdate := req.Method == http.MethodPost && strings.Contains(req.URL.Path, "/containers/") && strings.HasSuffix(req.URL.Path, "/update")
+ if isContainerUpdate && user != nil && user.ID != "" {
+ req.Header.Set(HeaderUpdateInitiatorID, user.ID)
+ req.Header.Set(HeaderUpdateInitiatorName, user.Username)
+ if user.DisplayName != nil {
+ req.Header.Set(HeaderUpdateInitiatorDisplayName, *user.DisplayName)
+ }
+ }
- apiURL, accessToken, enabled, err := m.resolver(c.Request().Context(), envID)
+ apiURL, accessToken, enabled, err := m.resolver(ctx, envID)
if err != nil || apiURL == "" {
- return c.JSON(http.StatusNotFound, map[string]any{
- "success": false,
- "data": map[string]any{"error": "Environment not found"},
- })
+ return writeProxyError(c, http.StatusNotFound, "Environment not found")
}
-
if !enabled {
- return c.JSON(http.StatusBadRequest, map[string]any{
- "success": false,
- "data": map[string]any{"error": errors.New("Environment is disabled").Error()}, //nolint:staticcheck // Preserve the existing error message.
- })
+ return writeProxyError(c, http.StatusBadRequest, "Environment is disabled")
}
- // SECURITY: Enforce the caller's per-environment permission BEFORE proxying.
- // Remote agents run with a sudo permission set and perform no authorization
- // of their own, so authorization for proxied requests must happen here,
- // mirroring the per-operation RequirePermission checks used for the local
- // environment.
- if m.proxyPermissionDenied(c, perms, envID) {
- return c.JSON(http.StatusForbidden, map[string]any{
- "success": false,
- "data": map[string]any{"error": "You don't have permission to perform this action on this environment"},
- })
+ // SECURITY: Remote agents trust the manager as sudo, so per-environment authorization happens here.
+ if m.proxyPermissionDenied(c, perms, envID, suffix) {
+ return writeProxyError(c, http.StatusForbidden, "You don't have permission to perform this action on this environment")
}
- isEdgeEnvironment := isEdgeEnvironmentURLInternal(apiURL)
-
- if handled, proxyActiveEdgeTunnelErr := m.proxyActiveEdgeTunnelInternal(c, envID, accessToken); handled {
- return proxyActiveEdgeTunnelErr
+ tunnel, hasTunnel := m.getActiveEdgeTunnel(envID).Get()
+ if !hasTunnel && strings.HasPrefix(strings.ToLower(strings.TrimSpace(apiURL)), "edge://") {
+ if tunnel, hasTunnel = m.waitForActiveEdgeTunnel(ctx, envID).Get(); !hasTunnel {
+ slog.WarnContext(ctx, "No active edge tunnel for environment", "environmentId", envID)
+ return writeProxyError(c, http.StatusBadGateway, "Edge agent is not connected")
+ }
+ slog.InfoContext(ctx, "Recovered edge tunnel during request", "environmentId", envID)
}
- if isEdgeEnvironment {
- if handled, proxyRecoveredEdgeTunnelErr := m.proxyRecoveredEdgeTunnelInternal(c, envID, accessToken); handled {
- return proxyRecoveredEdgeTunnelErr
+ proxyPath := path.Join(apiEnvironmentsPrefix, m.localID) + suffix
+ isWebSocket := httpx.IsWebSocketUpgradeRequest(req)
+ if hasTunnel {
+ slog.DebugContext(ctx, "Routing request through edge tunnel", "environmentId", envID, "path", req.URL.Path)
+ edge.SetAgentToken(req, accessToken)
+ if isWebSocket {
+ return edge.ProxyWebSocketRequest(c, tunnel, proxyPath, m.checkOrigin)
}
+ return edge.ProxyHTTPRequest(c, tunnel, proxyPath)
+ }
- slog.WarnContext(c.Request().Context(), "No active edge tunnel for environment", "environmentId", envID)
- return m.abortEdgeTunnelUnavailable(c)
+ target := strings.TrimRight(apiURL, "/") + proxyPath
+ if req.URL.RawQuery != "" {
+ target += "?" + req.URL.RawQuery
+ }
+ if isWebSocket {
+ if proxyErr := ws.ProxyHTTP(c.Response(), req, edge.HTTPToWebSocketURL(target), edge.BuildWebSocketHeaders(c, accessToken), m.checkOrigin); proxyErr != nil {
+ slog.ErrorContext(ctx, "websocket proxy failed", "err", proxyErr)
+ }
+ return nil
}
- target := m.buildTargetURL(c, envID, apiURL)
+ targetURL, err := httpx.ValidateOutboundHTTPURL(target)
+ if err != nil {
+ return writeProxyError(c, http.StatusInternalServerError, "Invalid proxy target URL: "+err.Error())
+ }
- if httpx.IsWebSocketUpgradeRequest(c.Request()) {
- return m.proxyWebSocket(c, target, accessToken, envID)
+ // Stream the body unbuffered (uploads reach gigabytes); ContentLength 0 means no body, -1 means chunked.
+ var requestBody io.ReadCloser
+ var contentLength int64
+ switch {
+ case req.Body == nil:
+ case req.ContentLength != 0:
+ requestBody, contentLength = req.Body, req.ContentLength
+ default:
+ _ = req.Body.Close()
}
- return m.proxyHTTP(c, target, accessToken)
-}
-// setIconCatalogHeaderInternal stamps the authenticated user's icon catalog
-// preference onto the outgoing proxied request. Remote agents authenticate the
-// proxy as a synthetic user with no preferences, so without this every remote
-// environment resolves icons against the default catalog.
-//
-// SECURITY: the header is always cleared first, so a browser-supplied value
-// never rides through; only the server-resolved preference is forwarded.
-func (m *EnvironmentMiddleware) setIconCatalogHeaderInternal(c *echo.Context, user *usertypes.Actor) {
- c.Request().Header.Del(HeaderIconCatalog)
- if user == nil || user.Preferences.IconCatalog == nil || *user.Preferences.IconCatalog == "" {
- return
- }
- c.Request().Header.Set(HeaderIconCatalog, *user.Preferences.IconCatalog)
-}
+ // The body is never logged: it can carry compose files and registry credentials.
+ slog.DebugContext(ctx, "Creating proxy request", "method", req.Method, "target", target, "contentLength", req.ContentLength, "contentType", req.Header.Get("Content-Type"))
+
+ proxyReq := (&http.Request{
+ Method: req.Method,
+ URL: targetURL,
+ Host: targetURL.Host,
+ Header: make(http.Header),
+ Body: requestBody,
+ ContentLength: contentLength,
+ }).WithContext(ctx)
+
+ edge.CopyRequestHeaders(req.Header, proxyReq.Header)
+ edge.SetAuthHeader(proxyReq, c)
+ edge.SetAgentToken(proxyReq, accessToken)
+ edge.SetForwardedHeaders(proxyReq, c.RealIP(), req.Host)
-func (m *EnvironmentMiddleware) setUpdateInitiatorHeadersInternal(c *echo.Context, user *usertypes.Actor) {
- headers := c.Request().Header
- headers.Del(HeaderUpdateInitiatorID)
- headers.Del(HeaderUpdateInitiatorName)
- headers.Del(HeaderUpdateInitiatorDisplayName)
- updatePath := strings.Contains(c.Request().URL.Path, "/containers/") && strings.HasSuffix(c.Request().URL.Path, "/update")
- if c.Request().Method != http.MethodPost || !updatePath {
- return
+ resp, err := m.httpClient.Do(proxyReq)
+ if err != nil {
+ return writeProxyError(c, http.StatusBadGateway, "Proxy request failed: "+err.Error())
}
- if user == nil || user.ID == "" {
- return
+ defer func() { _ = resp.Body.Close() }()
+
+ // An upstream 401 means the agent rejected the manager's credentials; a 502 keeps browsers from dropping the session.
+ if resp.StatusCode == http.StatusUnauthorized {
+ slog.WarnContext(ctx, "Remote environment rejected the manager's credentials", "target", resp.Request.URL.Host)
+ return writeProxyError(c, http.StatusBadGateway, "Remote environment rejected the manager's credentials")
}
- headers.Set(HeaderUpdateInitiatorID, user.ID)
- headers.Set(HeaderUpdateInitiatorName, user.Username)
- if user.DisplayName != nil {
- headers.Set(HeaderUpdateInitiatorDisplayName, *user.DisplayName)
+ w := c.Response()
+ edge.CopyResponseHeaders(resp.Header, w.Header(), edge.BuildHopByHopHeaders(resp.Header))
+ w.WriteHeader(resp.StatusCode)
+ if req.Method != http.MethodHead {
+ edge.CopyBodyWithFlush(w, resp.Body)
}
+ return nil
}
-// proxyPermissionDenied reports whether the caller lacks permission to perform
-// the proxied request against environment envID. It mirrors the per-operation
-// RequirePermission checks enforced for the local environment: the permission
-// required for the (method, path) is looked up in the matcher and evaluated
-// against the caller's permission set for the target environment.
-//
-// Sudo callers bypass the check. Requests whose (method, path) has no known
-// permission mapping are denied (default-deny), so a newly added proxied route
-// cannot silently bypass authorization before it is mapped.
-func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authz.PermissionSet, envID string) bool {
- if m.matcher == nil {
- return false
- }
- if ps != nil && ps.Sudo {
+// proxyPermissionDenied reports whether the caller lacks permission for the proxied request, mirroring local
+// RequirePermission checks. Sudo bypasses it; routes without a permission mapping are denied by default.
+func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authz.PermissionSet, envID, suffix string) bool {
+ if m.matcher == nil || (ps != nil && ps.Sudo) {
return false
}
- method := c.Request().Method
- suffix := m.buildResourceSuffix(c.Request().URL.Path, envID)
- if kind, ok := uploadSessionKindInternal(method, suffix); ok {
- // Upload-session routes derive their permission from the {kind} path
- // segment, so they carry no static mapping in the matcher; resolve the
- // kind here and fail closed on unknown kinds.
- perm, known := authz.UploadKindPermission(kind)
- if !known || !ps.Allows(perm, envID) {
- slog.DebugContext(c.Request().Context(), "Denying proxied upload session request: permission denied",
- "method", method, "path", suffix, "kind", kind, "environmentId", envID)
- return true
+ req := c.Request()
+ ctx := req.Context()
+ method := req.Method
+ segments := strings.Split(strings.Trim(suffix, "/"), "/")
+
+ // Upload-session routes derive their permission from the {kind} segment; unknown kinds fail closed.
+ if len(segments) >= 2 && segments[0] == "uploads" && segments[1] != "" {
+ var isUploadSession bool
+ switch len(segments) {
+ case 2:
+ isUploadSession = method == http.MethodPost
+ case 3:
+ isUploadSession = method == http.MethodGet || method == http.MethodDelete
+ case 5:
+ isUploadSession = method == http.MethodPut && segments[3] == "chunks"
+ }
+ if isUploadSession {
+ perm, known := authz.UploadKindPermission(segments[1])
+ denied := !known || !ps.Allows(perm, envID)
+ if denied {
+ slog.DebugContext(ctx, "Denying proxied upload session request: permission denied",
+ "method", method, "path", suffix, "kind", segments[1], "environmentId", envID)
+ }
+ return denied
}
- return false
}
+
perm, ok := m.matcher.Lookup(method, suffix).Get()
if !ok {
- slog.WarnContext(c.Request().Context(), "Denying proxied request with no known permission mapping",
- "method", method, "path", suffix, "environmentId", envID)
+ slog.WarnContext(ctx, "Denying proxied request with no known permission mapping", "method", method, "path", suffix, "environmentId", envID)
return true
}
if perm == "" {
- // Explicitly public proxied route (e.g. public settings): allowed for
- // any authenticated caller, matching local enforcement.
+ // Explicitly public route: allowed for any authenticated caller.
return false
}
-
- scopeEnvID := kit.Ternary(authz.IsEnvScoped(perm), envID, "")
- if !ps.Allows(perm, scopeEnvID) {
- slog.DebugContext(c.Request().Context(), "Denying proxied request: permission denied",
- "method", method, "path", suffix, "permission", perm, "environmentId", envID)
+ if !ps.Allows(perm, kit.Ternary(authz.IsEnvScoped(perm), envID, "")) {
+ slog.DebugContext(ctx, "Denying proxied request: permission denied", "method", method, "path", suffix, "permission", perm, "environmentId", envID)
return true
}
+
var required []string
var err error
- if isVolumeWorkspaceUpdateRequestInternal(method, suffix) {
- required, err = proxiedVolumeWorkspacePermissionsInternal(c.Request())
+ if method == http.MethodPut && len(segments) == 3 && segments[0] == "volumes" && segments[1] != "" && segments[2] == "workspace" {
+ required, err = proxiedVolumeWorkspacePermissions(req)
} else {
- required, err = proxiedGitOpsSyncPermissionsInternal(c.Request(), method, suffix)
+ required, err = proxiedGitOpsSyncPermissions(req, method, segments)
}
if err != nil {
- slog.DebugContext(c.Request().Context(), "Denying proxied request with invalid body",
- "path", suffix, "environmentId", envID, "error", err)
+ slog.DebugContext(ctx, "Denying proxied request with invalid body", "path", suffix, "environmentId", envID, "error", err)
return true
}
- required = withContainerResourceSortPermissionsInternal(required, c.Request(), method, suffix)
+ // Resource-sorted container lists collect per-container stats, so they also need containers:read.
+ if method == http.MethodGet && len(segments) == 1 && segments[0] == "containers" && container.IsResourceSort(req.URL.Query().Get("sort")) {
+ required = append(required, authz.PermContainersRead)
+ }
for _, operationPermission := range required {
if !ps.Allows(operationPermission, envID) {
- slog.DebugContext(c.Request().Context(), "Denying proxied request: body-derived permission denied",
+ slog.DebugContext(ctx, "Denying proxied request: body-derived permission denied",
"path", suffix, "permission", operationPermission, "environmentId", envID)
return true
}
@@ -305,35 +333,8 @@ func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authz
return false
}
-// uploadSessionKindInternal extracts the upload kind from the exact
-// upload-session route shapes: POST /uploads/{kind}, GET or DELETE
-// /uploads/{kind}/{uploadId}, and PUT /uploads/{kind}/{uploadId}/chunks/{n}.
-func uploadSessionKindInternal(method, suffix string) (string, bool) {
- segments := strings.Split(strings.Trim(suffix, "/"), "/")
- if len(segments) < 2 || segments[0] != "uploads" || segments[1] == "" {
- return "", false
- }
- switch {
- case len(segments) == 2 && method == http.MethodPost:
- return segments[1], true
- case len(segments) == 3 && (method == http.MethodGet || method == http.MethodDelete):
- return segments[1], true
- case len(segments) == 5 && method == http.MethodPut && segments[3] == "chunks":
- return segments[1], true
- }
- return "", false
-}
-
-func isVolumeWorkspaceUpdateRequestInternal(method, suffix string) bool {
- if method != http.MethodPut {
- return false
- }
- segments := strings.Split(strings.Trim(suffix, "/"), "/")
- return len(segments) == 3 && segments[0] == "volumes" && segments[1] != "" && segments[2] == "workspace"
-}
-
-func proxiedVolumeWorkspacePermissionsInternal(request *http.Request) ([]string, error) {
- if request == nil || request.Body == nil {
+func proxiedVolumeWorkspacePermissions(request *http.Request) ([]string, error) {
+ if request.Body == nil {
return nil, errors.New("missing multipart request body")
}
mediaType, params, err := mime.ParseMediaType(request.Header.Get("Content-Type"))
@@ -349,12 +350,7 @@ func proxiedVolumeWorkspacePermissionsInternal(request *http.Request) ([]string,
}
defer func() {
_, _ = captured.Seek(0, io.SeekStart)
- request.Body = &proxiedReplayBodyInternal{
- Reader: io.MultiReader(captured, originalBody),
- captured: captured,
- original: originalBody,
- path: captured.Name(),
- }
+ request.Body = &proxiedReplayBody{Reader: io.MultiReader(captured, originalBody), captured: captured, original: originalBody}
}()
reader := multipart.NewReader(io.TeeReader(originalBody, captured), params["boundary"])
@@ -392,17 +388,13 @@ func proxiedVolumeWorkspacePermissionsInternal(request *http.Request) ([]string,
}
}
-// proxiedGitOpsSyncPermissionsInternal mirrors the handler-level lifecycle and
-// backup gates for POST /gitops-syncs, POST /gitops-syncs/import, and PUT
-// /gitops-syncs/{syncId}. The agent authenticates forwarded requests as sudo,
-// so these body-dependent checks must run at the proxy. Returns nil for any
-// other route.
-func proxiedGitOpsSyncPermissionsInternal(request *http.Request, method, suffix string) ([]string, error) {
- segments := strings.Split(strings.Trim(suffix, "/"), "/")
+// proxiedGitOpsSyncPermissions mirrors the handler lifecycle and backup gates for gitops sync create, import,
+// and update, since the agent trusts forwarded requests as sudo. Returns nil for any other route.
+func proxiedGitOpsSyncPermissions(request *http.Request, method string, segments []string) ([]string, error) {
isCreate := len(segments) == 1 && segments[0] == "gitops-syncs" && method == http.MethodPost
isImport := len(segments) == 2 && segments[0] == "gitops-syncs" && segments[1] == "import" && method == http.MethodPost
isUpdate := len(segments) == 2 && segments[0] == "gitops-syncs" && segments[1] != "" && method == http.MethodPut
- if !isCreate && !isImport && !isUpdate || request == nil || request.Body == nil {
+ if !isCreate && !isImport && !isUpdate || request.Body == nil {
return nil, nil
}
@@ -416,228 +408,87 @@ func proxiedGitOpsSyncPermissionsInternal(request *http.Request, method, suffix
}
request.Body = io.NopCloser(bytes.NewReader(body))
- type syncBodyInternal struct {
+ type syncBody struct {
gitops.PreDeployConfigRequest
Mode string `json:"mode"`
}
- var items []syncBodyInternal
+ var items []syncBody
+ var decodeErr error
if isImport {
- if err := json.Unmarshal(body, &items); err != nil {
- return nil, fmt.Errorf("decode gitops sync import body: %w", err)
- }
+ decodeErr = json.Unmarshal(body, &items)
} else {
- var single syncBodyInternal
- if err := json.Unmarshal(body, &single); err != nil {
- return nil, fmt.Errorf("decode gitops sync body: %w", err)
- }
- items = []syncBodyInternal{single}
+ items = make([]syncBody, 1)
+ decodeErr = json.Unmarshal(body, &items[0])
}
-
- var required []string
- for _, item := range items {
- if item.HasPreDeployConfig() && !slices.Contains(required, authz.PermGitOpsLifecycle) {
- required = append(required, authz.PermGitOpsLifecycle)
- }
- if isCreate && item.Mode == gitops.SyncModeBackup && !slices.Contains(required, authz.PermGitOpsBackup) {
- required = append(required, authz.PermGitOpsBackup)
- }
+ if decodeErr != nil {
+ return nil, fmt.Errorf("decode gitops sync body: %w", decodeErr)
}
- return required, nil
-}
-// withContainerResourceSortPermissionsInternal adds containers:read for
-// resource-sorted container lists, which collect per-container stats.
-func withContainerResourceSortPermissionsInternal(required []string, request *http.Request, method, suffix string) []string {
- if request == nil || method != http.MethodGet || strings.Trim(suffix, "/") != "containers" {
- return required
+ var required []string
+ if slices.ContainsFunc(items, func(item syncBody) bool { return item.HasPreDeployConfig() }) {
+ required = append(required, authz.PermGitOpsLifecycle)
}
- if container.IsResourceSort(request.URL.Query().Get("sort")) {
- return append(required, authz.PermContainersRead)
+ if isCreate && slices.ContainsFunc(items, func(item syncBody) bool { return item.Mode == gitops.SyncModeBackup }) {
+ required = append(required, authz.PermGitOpsBackup)
}
- return required
+ return required, nil
}
-type proxiedReplayBodyInternal struct {
+type proxiedReplayBody struct {
io.Reader
captured *os.File
original io.ReadCloser
- path string
}
-func (b *proxiedReplayBodyInternal) Close() error {
+func (b *proxiedReplayBody) Close() error {
capturedErr := b.captured.Close()
originalErr := b.original.Close()
// System temp scratch: no acfs root exists for it.
- removeErr := os.Remove(b.path)
+ removeErr := os.Remove(b.captured.Name())
return errors.Join(capturedErr, originalErr, removeErr)
}
-func (m *EnvironmentMiddleware) proxyActiveEdgeTunnelInternal(c *echo.Context, envID string, accessToken *string) (bool, error) {
- tunnel, ok := m.getActiveEdgeTunnelInternal(envID).Get()
- if !ok {
- return false, nil
- }
-
- slog.DebugContext(c.Request().Context(), "Routing request through edge tunnel", "environmentId", envID, "path", c.Request().URL.Path)
- m.setProxyContextHeadersInternal(c, accessToken)
- return true, m.proxyThroughTunnelInternal(c, tunnel, envID)
-}
-
-func (m *EnvironmentMiddleware) proxyRecoveredEdgeTunnelInternal(c *echo.Context, envID string, accessToken *string) (bool, error) {
- edge.TouchTunnelDemand(envID, edge.DefaultTunnelDemandTTL)
-
- tunnel, ok := m.waitForActiveEdgeTunnelInternal(c.Request().Context(), envID, edge.DefaultTunnelAcquireTimeout()).Get()
- if !ok {
- return false, nil
- }
-
- slog.InfoContext(c.Request().Context(), "Recovered edge tunnel during request", "environmentId", envID)
- m.setProxyContextHeadersInternal(c, accessToken)
- return true, m.proxyThroughTunnelInternal(c, tunnel, envID)
-}
-
-func (m *EnvironmentMiddleware) setProxyContextHeadersInternal(c *echo.Context, accessToken *string) {
- if accessToken != nil && *accessToken != "" {
- c.Request().Header.Set(edge.HeaderAgentToken, *accessToken)
- c.Request().Header.Set(edge.HeaderAPIKey, *accessToken)
- }
-}
-
-func (m *EnvironmentMiddleware) proxyThroughTunnelInternal(c *echo.Context, tunnel *edge.AgentTunnel, envID string) error {
- proxyPath := m.buildProxyPath(c, envID)
- if httpx.IsWebSocketUpgradeRequest(c.Request()) {
- return edge.ProxyWebSocketRequest(c, tunnel, proxyPath, m.checkOrigin)
- }
- return edge.ProxyHTTPRequest(c, tunnel, proxyPath)
-}
-
-// hasResourcePath reports whether the request targets a proxiable resource path.
-func (m *EnvironmentMiddleware) hasResourcePath(c *echo.Context, envID string) bool {
- suffix, ok := strings.CutPrefix(c.Request().URL.Path, apiEnvironmentsPrefix+envID)
- if !ok || len(suffix) <= 1 || suffix[0] != '/' {
- return false
- }
- return !isManagementPathInternal(c.Request().Method, suffix)
-}
-
-func isManagementPathInternal(method, suffix string) bool {
- if suffix == "/jobs" || strings.HasPrefix(suffix, "/jobs/") {
- return true
- }
- if isCentralSwarmManagementPathInternal(method, suffix) {
- return true
- }
- if isCentralVolumeBackupPolicyPathInternal(method, suffix) {
- return true
- }
- if suffix == "/activities" || strings.HasPrefix(suffix, "/activities/") {
+// isManagementPath reports whether an environment-scoped path is served by the manager instead of being proxied.
+func isManagementPath(method, suffix string) bool {
+ if _, ok := managementEndpointSet[suffix]; ok {
return true
}
-
- if strings.HasPrefix(suffix, "/notifications") {
- return true
+ // Webhooks live in the manager DB and their public trigger endpoint resolves tokens there.
+ for _, prefix := range []string{"/jobs", "/activities", "/webhooks"} {
+ if suffix == prefix || strings.HasPrefix(suffix, prefix+"/") {
+ return true
+ }
}
-
- // Webhooks are managed centrally: rows live in the manager DB (keyed by the
- // real environment ID) and the public trigger endpoint resolves tokens there.
- if suffix == "/webhooks" || strings.HasPrefix(suffix, "/webhooks/") {
+ if strings.HasPrefix(suffix, "/notifications") || strings.HasPrefix(suffix, "/deployment/mtls/") {
return true
}
- if strings.HasPrefix(suffix, "/deployment/mtls/") {
+ switch method + " " + suffix {
+ case "GET /swarm/join-candidates", "GET /swarm/nodes", "POST /swarm/join-environments", "POST /swarm/nodes/agents/reconcile":
return true
}
- _, isManagement := managementEndpointSet[suffix]
- return isManagement
-}
-
-func isCentralVolumeBackupPolicyPathInternal(method, suffix string) bool {
- if method != http.MethodGet && method != http.MethodPut {
- return false
- }
parts := strings.Split(strings.Trim(suffix, "/"), "/")
- return len(parts) == 3 && parts[0] == "volumes" && parts[1] != "" && parts[2] == "backup-policy"
-}
-
-func isCentralSwarmManagementPathInternal(method, suffix string) bool {
- if method == http.MethodGet && (suffix == "/swarm/join-candidates" || suffix == "/swarm/nodes") {
- return true
- }
- if method == http.MethodPost && (suffix == "/swarm/join-environments" || suffix == "/swarm/nodes/agents/reconcile") {
- return true
+ if len(parts) == 3 && parts[0] == "volumes" && parts[1] != "" && parts[2] == "backup-policy" {
+ return method == http.MethodGet || method == http.MethodPut
}
-
- parts := strings.Split(strings.Trim(suffix, "/"), "/")
- if len(parts) == 3 && method == http.MethodGet && parts[0] == "swarm" && parts[1] == "nodes" {
- return true
+ if len(parts) < 3 || parts[0] != "swarm" || parts[1] != "nodes" {
+ return false
}
- if len(parts) == 5 && parts[0] == "swarm" && parts[1] == "nodes" && parts[3] == "agent" {
- if parts[4] == "binding" {
- return method == http.MethodPut || method == http.MethodDelete
- }
- if parts[4] == "deployment" {
- return method == http.MethodPost || method == http.MethodDelete
- }
+ switch {
+ case len(parts) == 3:
+ return method == http.MethodGet
+ case len(parts) == 5 && parts[3] == "agent" && parts[4] == "binding":
+ return method == http.MethodPut || method == http.MethodDelete
+ case len(parts) == 5 && parts[3] == "agent" && parts[4] == "deployment":
+ return method == http.MethodPost || method == http.MethodDelete
}
-
return false
}
-// extractEnvironmentID gets the environment ID from the request.
-func (m *EnvironmentMiddleware) extractEnvironmentID(c *echo.Context) string {
- requestPath := c.Request().URL.Path
-
- if !strings.Contains(requestPath, environmentsPathMarker) {
- return ""
- }
-
- if envID := c.Param(m.paramName); envID != "" {
- return envID
- }
-
- if _, rest, ok := strings.Cut(requestPath, environmentsPathMarker); ok {
- if envID, _, _ := strings.Cut(rest, "/"); envID != "" {
- return envID
- }
- }
-
- return ""
-}
-
-// buildResourceSuffix extracts the resource path after stripping the environment ID prefix.
-func (m *EnvironmentMiddleware) buildResourceSuffix(requestPath, envID string) string {
- suffix, _ := strings.CutPrefix(requestPath, apiEnvironmentsPrefix+envID)
- if suffix != "" && suffix[0] != '/' {
- suffix = "/" + suffix
- }
- return suffix
-}
-
-// buildTargetURL constructs the full proxy target URL for a remote environment.
-func (m *EnvironmentMiddleware) buildTargetURL(c *echo.Context, envID, apiURL string) string {
- req := c.Request()
- suffix := m.buildResourceSuffix(req.URL.Path, envID)
- target := strings.TrimRight(apiURL, "/") + path.Join(apiEnvironmentsPrefix, m.localID) + suffix
- if qs := req.URL.RawQuery; qs != "" {
- target += "?" + qs
- }
- return target
-}
-
-// buildProxyPath constructs the path sent through the edge tunnel.
-func (m *EnvironmentMiddleware) buildProxyPath(c *echo.Context, envID string) string {
- return path.Join(apiEnvironmentsPrefix, m.localID) + m.buildResourceSuffix(c.Request().URL.Path, envID)
-}
-
-func isEdgeEnvironmentURLInternal(apiURL string) bool {
- normalized := strings.ToLower(strings.TrimSpace(apiURL))
- return strings.HasPrefix(normalized, "edge://")
-}
-
-func (m *EnvironmentMiddleware) getActiveEdgeTunnelInternal(envID string) mo.Option[*edge.AgentTunnel] {
+func (m *EnvironmentMiddleware) getActiveEdgeTunnel(envID string) mo.Option[*edge.AgentTunnel] {
if m.registry == nil {
return mo.None[*edge.AgentTunnel]()
}
@@ -649,158 +500,31 @@ func (m *EnvironmentMiddleware) getActiveEdgeTunnelInternal(envID string) mo.Opt
return mo.Some(tunnel)
}
-func (m *EnvironmentMiddleware) waitForActiveEdgeTunnelInternal(ctx context.Context, envID string, timeout time.Duration) mo.Option[*edge.AgentTunnel] {
- if timeout <= 0 {
- return m.getActiveEdgeTunnelInternal(envID)
- }
-
- if tunnel, ok := m.getActiveEdgeTunnelInternal(envID).Get(); ok {
- return mo.Some(tunnel)
- }
+// waitForActiveEdgeTunnel signals tunnel demand and polls until the edge agent connects or the acquire timeout elapses.
+func (m *EnvironmentMiddleware) waitForActiveEdgeTunnel(ctx context.Context, envID string) mo.Option[*edge.AgentTunnel] {
+ edge.TouchTunnelDemand(envID, edge.DefaultTunnelDemandTTL)
- waitCtx, cancel := context.WithTimeout(ctx, timeout)
+ waitCtx, cancel := context.WithTimeout(ctx, edge.DefaultTunnelAcquireTimeout())
defer cancel()
ticker := time.NewTicker(edge.DefaultTunnelAcquirePollEvery)
defer ticker.Stop()
for {
+ if tunnel := m.getActiveEdgeTunnel(envID); tunnel.IsPresent() {
+ return tunnel
+ }
select {
case <-waitCtx.Done():
return mo.None[*edge.AgentTunnel]()
case <-ticker.C:
- if tunnel, ok := m.getActiveEdgeTunnelInternal(envID).Get(); ok {
- return mo.Some(tunnel)
- }
}
}
}
-func (m *EnvironmentMiddleware) abortEdgeTunnelUnavailable(c *echo.Context) error {
- return c.JSON(http.StatusBadGateway, map[string]any{
+func writeProxyError(c *echo.Context, status int, message string) error {
+ return c.JSON(status, map[string]any{
"success": false,
- "data": map[string]any{
- "error": "Edge agent is not connected",
- },
+ "data": map[string]any{"error": message},
})
}
-
-// proxyWebSocket handles WebSocket proxy requests.
-func (m *EnvironmentMiddleware) proxyWebSocket(c *echo.Context, target string, accessToken *string, envID string) error {
- if isEdgeEnvironmentURLInternal(target) {
- slog.WarnContext(c.Request().Context(), "Refusing direct websocket proxy to edge environment without active tunnel", "environmentId", envID, "target", target)
- return m.abortEdgeTunnelUnavailable(c)
- }
-
- wsTarget := edge.HTTPToWebSocketURL(target)
- headers := edge.BuildWebSocketHeaders(c, accessToken)
-
- if err := ws.ProxyHTTP(c.Response(), c.Request(), wsTarget, headers, m.checkOrigin); err != nil {
- slog.ErrorContext(c.Request().Context(), "websocket proxy failed", "err", err)
- }
- return nil
-}
-
-// proxyHTTP handles standard HTTP proxy requests.
-func (m *EnvironmentMiddleware) proxyHTTP(c *echo.Context, target string, accessToken *string) error {
- if isEdgeEnvironmentURLInternal(target) {
- slog.WarnContext(c.Request().Context(), "Refusing direct HTTP proxy to edge environment without active tunnel", "target", target)
- return m.abortEdgeTunnelUnavailable(c)
- }
-
- req, err := m.createProxyRequest(c, target, accessToken)
- if err != nil {
- errMessage := "Failed to create proxy request: " + err.Error()
- if errors.Is(err, common.ErrEnvironmentInvalidProxyTarget) {
- errMessage = err.Error()
- }
- return c.JSON(http.StatusInternalServerError, map[string]any{
- "success": false,
- "data": map[string]any{"error": errMessage},
- })
- }
-
- resp, err := m.httpClient.Do(req)
- if err != nil {
- return c.JSON(http.StatusBadGateway, map[string]any{
- "success": false,
- "data": map[string]any{"error": "Proxy request failed: " + err.Error()},
- })
- }
- defer func() { _ = resp.Body.Close() }()
-
- return m.writeProxyResponse(c, resp)
-}
-
-// createProxyRequest builds the HTTP request to forward to the remote environment.
-func (m *EnvironmentMiddleware) createProxyRequest(c *echo.Context, target string, accessToken *string) (*http.Request, error) {
- srcReq := c.Request()
- validatedTarget, err := httpx.ValidateOutboundHTTPURL(target)
- if err != nil {
- return nil, common.Classify(common.ErrEnvironmentInvalidProxyTarget, fmt.Errorf("Invalid proxy target URL: %w", err)) //nolint:staticcheck // Preserve the existing error message.
- }
-
- // The body is streamed straight through rather than buffered: volume backup
- // and image import uploads run to gigabytes, and reading them into memory
- // cost roughly twice their size per in-flight request. This drops GetBody,
- // so the transport can no longer replay the body across a redirect or an
- // idle-connection retry; a proxied API call has no legitimate need for
- // either, and a failure surfaces as a 502 rather than silent corruption.
- //
- // A server request's ContentLength is 0 only when there is genuinely no
- // body (-1 means chunked/unknown), so it is the safe signal for whether to
- // forward one at all.
- var requestBody io.ReadCloser
- var contentLength int64
- switch {
- case srcReq.Body == nil:
- case srcReq.ContentLength != 0:
- requestBody, contentLength = srcReq.Body, srcReq.ContentLength
- default:
- _ = srcReq.Body.Close()
- }
-
- // The body is deliberately not logged: at debug level it would put compose
- // files and registry credentials into the ring buffer served by
- // /api/diagnostics/logs, and stringifying it allocated even when debug was off.
- slog.DebugContext(srcReq.Context(), "Creating proxy request", "method", srcReq.Method, "target", target, "contentLength", srcReq.ContentLength, "contentType", srcReq.Header.Get("Content-Type"))
-
- requestURL := *validatedTarget
- req := (&http.Request{
- Method: srcReq.Method,
- URL: &requestURL,
- Host: requestURL.Host,
- Header: make(http.Header),
- Body: requestBody,
- ContentLength: contentLength,
- }).WithContext(srcReq.Context())
-
- edge.CopyRequestHeaders(srcReq.Header, req.Header)
- edge.SetAuthHeader(req, c)
- edge.SetAgentToken(req, accessToken)
- edge.SetForwardedHeaders(req, c.RealIP(), srcReq.Host)
-
- return req, nil
-}
-
-// writeProxyResponse copies the proxy response back to the client. An upstream
-// 401 means the agent rejected the manager's credentials; passing it through
-// makes browser clients treat it as a lost session, so it becomes a 502.
-func (m *EnvironmentMiddleware) writeProxyResponse(c *echo.Context, resp *http.Response) error {
- if resp.StatusCode == http.StatusUnauthorized {
- slog.WarnContext(c.Request().Context(), "Remote environment rejected the manager's credentials", "target", resp.Request.URL.Host)
- return c.JSON(http.StatusBadGateway, map[string]any{
- "success": false,
- "data": map[string]any{"error": "Remote environment rejected the manager's credentials"},
- })
- }
- w := c.Response()
- hopByHop := edge.BuildHopByHopHeaders(resp.Header)
- edge.CopyResponseHeaders(resp.Header, w.Header(), hopByHop)
-
- w.WriteHeader(resp.StatusCode)
- if c.Request().Method != http.MethodHead {
- edge.CopyBodyWithFlush(w, resp.Body)
- }
- return nil
-}
diff --git a/backend/internal/middleware/environment_test.go b/backend/internal/middleware/environment_test.go
index 36030b77b..d7148f89e 100644
--- a/backend/internal/middleware/environment_test.go
+++ b/backend/internal/middleware/environment_test.go
@@ -277,42 +277,21 @@ func TestEnvironmentMiddleware_LocalEnvironmentSkipsProxyPermissionCheck(t *test
assert.True(t, localHandlerHit, "local environment request must reach the local handler, not be proxy-authorized")
}
-func TestEnvironmentMiddleware_ProxyWebSocketRejectsEdgeTargetsWithoutTunnel(t *testing.T) {
+func TestEnvironmentMiddleware_RejectsInvalidProxyTarget(t *testing.T) {
middleware := newTestEnvironmentMiddleware()
- e := echo.New()
- recorder := httptest.NewRecorder()
- req := httptest.NewRequest(http.MethodGet, "/api/environments/env-edge/ws/system/stats", http.NoBody)
- c := e.NewContext(req, recorder)
-
- _ = middleware.proxyWebSocket(c, "edge://oracle-1/api/environments/0/ws/system/stats", nil, "env-edge")
-
- assert.Equal(t, http.StatusBadGateway, recorder.Code)
- assert.Contains(t, recorder.Body.String(), "Edge agent is not connected")
-}
-
-func TestEnvironmentMiddleware_ProxyHTTPRejectsEdgeTargetsWithoutTunnel(t *testing.T) {
- middleware := newTestEnvironmentMiddleware()
- e := echo.New()
- recorder := httptest.NewRecorder()
- req := httptest.NewRequest(http.MethodGet, "/api/environments/env-edge/containers", http.NoBody)
- c := e.NewContext(req, recorder)
-
- _ = middleware.proxyHTTP(c, "edge://oracle-1/api/environments/0/containers", nil)
-
- assert.Equal(t, http.StatusBadGateway, recorder.Code)
- assert.Contains(t, recorder.Body.String(), "Edge agent is not connected")
-}
+ middleware.resolver = func(ctx context.Context, id string) (string, *string, bool, error) {
+ _, _ = ctx, id
+ return "ftp://example.com", nil, true, nil
+ }
+ router := echo.New()
+ attachMiddleware(router, middleware)
-func TestEnvironmentMiddleware_CreateProxyRequest_RejectsInvalidProxyTarget(t *testing.T) {
- middleware := newTestEnvironmentMiddleware()
- e := echo.New()
+ req := httptest.NewRequest(http.MethodGet, "/api/environments/env-remote/containers", http.NoBody)
recorder := httptest.NewRecorder()
- req := httptest.NewRequest(http.MethodGet, "/api/environments/env-edge/containers", http.NoBody)
- c := e.NewContext(req, recorder)
+ router.ServeHTTP(recorder, req)
- _, err := middleware.createProxyRequest(c, "ftp://example.com/containers", nil)
- require.Error(t, err)
- assert.Contains(t, err.Error(), "Invalid proxy target URL")
+ assert.Equal(t, http.StatusInternalServerError, recorder.Code)
+ assert.Contains(t, recorder.Body.String(), "Invalid proxy target URL")
}
func TestEnvironmentMiddleware_KeepsNodeAgentDeploymentCreationLocal(t *testing.T) {
@@ -404,21 +383,21 @@ func TestEnvironmentMiddleware_ForwardsResolvedIconCatalogHeaderOnly(t *testing.
}
}
-func TestIsCentralSwarmManagementPathInternal_IsMethodAware(t *testing.T) {
- assert.True(t, isCentralSwarmManagementPathInternal(http.MethodGet, "/swarm/nodes"))
- assert.True(t, isCentralSwarmManagementPathInternal(http.MethodGet, "/swarm/nodes/node-1"))
- assert.False(t, isCentralSwarmManagementPathInternal(http.MethodPatch, "/swarm/nodes/node-1"))
- assert.True(t, isCentralSwarmManagementPathInternal(http.MethodPost, "/swarm/nodes/node-1/agent/deployment"))
- assert.True(t, isCentralSwarmManagementPathInternal(http.MethodDelete, "/swarm/nodes/node-1/agent/deployment"))
- assert.True(t, isCentralSwarmManagementPathInternal(http.MethodPut, "/swarm/nodes/node-1/agent/binding"))
- assert.True(t, isCentralSwarmManagementPathInternal(http.MethodDelete, "/swarm/nodes/node-1/agent/binding"))
+func TestIsManagementPath_SwarmIsMethodAware(t *testing.T) {
+ assert.True(t, isManagementPath(http.MethodGet, "/swarm/nodes"))
+ assert.True(t, isManagementPath(http.MethodGet, "/swarm/nodes/node-1"))
+ assert.False(t, isManagementPath(http.MethodPatch, "/swarm/nodes/node-1"))
+ assert.True(t, isManagementPath(http.MethodPost, "/swarm/nodes/node-1/agent/deployment"))
+ assert.True(t, isManagementPath(http.MethodDelete, "/swarm/nodes/node-1/agent/deployment"))
+ assert.True(t, isManagementPath(http.MethodPut, "/swarm/nodes/node-1/agent/binding"))
+ assert.True(t, isManagementPath(http.MethodDelete, "/swarm/nodes/node-1/agent/binding"))
}
-func TestIsCentralVolumeBackupPolicyPathInternal_IsMethodAware(t *testing.T) {
- assert.True(t, isCentralVolumeBackupPolicyPathInternal(http.MethodGet, "/volumes/app-data/backup-policy"))
- assert.True(t, isCentralVolumeBackupPolicyPathInternal(http.MethodPut, "/volumes/app-data/backup-policy"))
- assert.False(t, isCentralVolumeBackupPolicyPathInternal(http.MethodPost, "/volumes/app-data/backup-policy"))
- assert.False(t, isCentralVolumeBackupPolicyPathInternal(http.MethodGet, "/volumes/app-data/backups"))
+func TestIsManagementPath_VolumeBackupPolicyIsMethodAware(t *testing.T) {
+ assert.True(t, isManagementPath(http.MethodGet, "/volumes/app-data/backup-policy"))
+ assert.True(t, isManagementPath(http.MethodPut, "/volumes/app-data/backup-policy"))
+ assert.False(t, isManagementPath(http.MethodPost, "/volumes/app-data/backup-policy"))
+ assert.False(t, isManagementPath(http.MethodGet, "/volumes/app-data/backups"))
}
const proxyTestEnvID = "remote-1"
@@ -448,7 +427,7 @@ func TestProxyPermissionDeniedBlocksWriteForReadOnlyUser(t *testing.T) {
c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")
- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),
+ require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers/abc/restart"),
"expected restart to be denied for a read-only user")
}
@@ -459,7 +438,7 @@ func TestProxyPermissionDeniedAllowsWriteForPermittedUser(t *testing.T) {
c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")
- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),
+ require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers/abc/restart"),
"expected restart to be allowed for a user with containers:restart")
}
@@ -470,7 +449,7 @@ func TestProxyPermissionDeniedAllowsRead(t *testing.T) {
c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers")
- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),
+ require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers"),
"expected list to be allowed for a user with containers:list")
}
@@ -482,7 +461,7 @@ func TestProxyPermissionDeniedDeniesPermissionFromDifferentEnv(t *testing.T) {
c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")
- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),
+ require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers/abc/restart"),
"expected denial: permission is scoped to a different environment")
}
@@ -490,7 +469,7 @@ func TestProxyPermissionDeniedSudoBypasses(t *testing.T) {
m := newProxyAuthzMiddleware(containerMatcher())
c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")
- require.False(t, m.proxyPermissionDenied(c, authz.SudoPermissionSet(), proxyTestEnvID),
+ require.False(t, m.proxyPermissionDenied(c, authz.SudoPermissionSet(), proxyTestEnvID, "/containers/abc/restart"),
"expected sudo permission set to bypass the permission check")
}
@@ -501,7 +480,7 @@ func TestProxyPermissionDeniedDefaultDeniesUnmappedRoute(t *testing.T) {
c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/unknown/resource")
- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),
+ require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/unknown/resource"),
"expected an unmapped proxied route to be denied by default")
}
@@ -511,16 +490,10 @@ func TestProxyPermissionDeniedAllowsPublicRoute(t *testing.T) {
c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/settings/public")
- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),
+ require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/settings/public"),
"expected an explicitly public route to be allowed for any authenticated caller")
}
-func volumeWorkspaceMatcher() *authz.PermissionMatcher {
- m := authz.NewPermissionMatcher()
- m.Add(http.MethodPut, "/volumes/{volumeName}/workspace", authz.PermVolumesRead)
- return m
-}
-
func newProxyVolumeWorkspaceContext(t *testing.T, changes []volume.WorkspaceFileChange, fileFirst ...bool) (*echo.Context, []byte) {
t.Helper()
var body bytes.Buffer
@@ -548,40 +521,36 @@ func newProxyVolumeWorkspaceContext(t *testing.T, changes []volume.WorkspaceFile
}
func TestProxyPermissionDeniedChecksVolumeWorkspaceManifestPermissions(t *testing.T) {
- m := newProxyAuthzMiddleware(volumeWorkspaceMatcher())
+ matcher := authz.NewPermissionMatcher()
+ matcher.Add(http.MethodPut, "/volumes/{volumeName}/workspace", authz.PermVolumesRead)
+ m := newProxyAuthzMiddleware(matcher)
changes := []volume.WorkspaceFileChange{{Operation: volume.FileOpRename}}
readOnly := authz.NewPermissionSet()
readOnly.AddEnv(proxyTestEnvID, authz.PermVolumesRead)
c, _ := newProxyVolumeWorkspaceContext(t, changes)
- require.True(t, m.proxyPermissionDenied(c, readOnly, proxyTestEnvID))
+ require.True(t, m.proxyPermissionDenied(c, readOnly, proxyTestEnvID, "/volumes/data/workspace"))
uploadOnly := authz.NewPermissionSet()
uploadOnly.AddEnv(proxyTestEnvID, authz.PermVolumesRead, authz.PermVolumesUpload)
c, _ = newProxyVolumeWorkspaceContext(t, changes)
- require.True(t, m.proxyPermissionDenied(c, uploadOnly, proxyTestEnvID))
+ require.True(t, m.proxyPermissionDenied(c, uploadOnly, proxyTestEnvID, "/volumes/data/workspace"))
permitted := authz.NewPermissionSet()
permitted.AddEnv(proxyTestEnvID, authz.PermVolumesRead, authz.PermVolumesUpload, authz.PermVolumesDelete)
c, rawBody := newProxyVolumeWorkspaceContext(t, changes)
- require.False(t, m.proxyPermissionDenied(c, permitted, proxyTestEnvID))
+ require.False(t, m.proxyPermissionDenied(c, permitted, proxyTestEnvID, "/volumes/data/workspace"))
replayed, err := io.ReadAll(c.Request().Body)
require.NoError(t, err)
require.Equal(t, rawBody, replayed)
c, rawBody = newProxyVolumeWorkspaceContext(t, changes, true)
- require.False(t, m.proxyPermissionDenied(c, permitted, proxyTestEnvID))
+ require.False(t, m.proxyPermissionDenied(c, permitted, proxyTestEnvID, "/volumes/data/workspace"))
replayed, err = io.ReadAll(c.Request().Body)
require.NoError(t, err)
require.Equal(t, rawBody, replayed)
}
-func gitOpsSyncMatcher() *authz.PermissionMatcher {
- m := authz.NewPermissionMatcher()
- m.Add(http.MethodPost, "/gitops-syncs/import", authz.PermGitOpsCreate)
- return m
-}
-
func newProxyGitOpsImportContext(body string) *echo.Context {
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/gitops-syncs/import", bytes.NewReader([]byte(body)))
@@ -590,35 +559,30 @@ func newProxyGitOpsImportContext(body string) *echo.Context {
}
func TestProxyPermissionDeniedChecksGitOpsImportLifecyclePermission(t *testing.T) {
- m := newProxyAuthzMiddleware(gitOpsSyncMatcher())
+ matcher := authz.NewPermissionMatcher()
+ matcher.Add(http.MethodPost, "/gitops-syncs/import", authz.PermGitOpsCreate)
+ m := newProxyAuthzMiddleware(matcher)
hookBody := `[{"syncName":"a","gitRepo":"r","branch":"main","dockerComposePath":"compose.yaml","preDeployScriptPath":"scripts/run.sh"}]`
plainBody := `[{"syncName":"a","gitRepo":"r","branch":"main","dockerComposePath":"compose.yaml"}]`
createOnly := authz.NewPermissionSet()
createOnly.AddEnv(proxyTestEnvID, authz.PermGitOpsCreate)
- require.True(t, m.proxyPermissionDenied(newProxyGitOpsImportContext(hookBody), createOnly, proxyTestEnvID))
- require.False(t, m.proxyPermissionDenied(newProxyGitOpsImportContext(plainBody), createOnly, proxyTestEnvID))
+ require.True(t, m.proxyPermissionDenied(newProxyGitOpsImportContext(hookBody), createOnly, proxyTestEnvID, "/gitops-syncs/import"))
+ require.False(t, m.proxyPermissionDenied(newProxyGitOpsImportContext(plainBody), createOnly, proxyTestEnvID, "/gitops-syncs/import"))
withLifecycle := authz.NewPermissionSet()
withLifecycle.AddEnv(proxyTestEnvID, authz.PermGitOpsCreate, authz.PermGitOpsLifecycle)
c := newProxyGitOpsImportContext(hookBody)
- require.False(t, m.proxyPermissionDenied(c, withLifecycle, proxyTestEnvID))
+ require.False(t, m.proxyPermissionDenied(c, withLifecycle, proxyTestEnvID, "/gitops-syncs/import"))
replayed, err := io.ReadAll(c.Request().Body)
require.NoError(t, err)
require.Equal(t, hookBody, string(replayed))
}
-// wsTerminalMatcher mirrors ws.AddProxiedPermissions for the container terminal
-// stream: the proxy computes the suffix "/ws/containers/{id}/terminal" for a
-// forwarded WebSocket request, and the matcher requires containers:exec for it.
-func wsTerminalMatcher() *authz.PermissionMatcher {
- m := authz.NewPermissionMatcher()
- m.Add(http.MethodGet, "/ws/containers/{containerId}/terminal", authz.PermContainersExec)
- return m
-}
-
func TestProxyPermissionDeniedWSTerminalRequiresExec(t *testing.T) {
- m := newProxyAuthzMiddleware(wsTerminalMatcher())
+ matcher := authz.NewPermissionMatcher()
+ matcher.Add(http.MethodGet, "/ws/containers/{containerId}/terminal", authz.PermContainersExec)
+ m := newProxyAuthzMiddleware(matcher)
// A caller who can read and list containers but lacks containers:exec must
// not be able to open a terminal stream on the remote environment.
@@ -627,13 +591,13 @@ func TestProxyPermissionDeniedWSTerminalRequiresExec(t *testing.T) {
c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/ws/containers/abc/terminal")
- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),
+ require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/ws/containers/abc/terminal"),
"expected WS terminal to be denied without containers:exec")
// Granting containers:exec allows the same stream.
ps.AddEnv(proxyTestEnvID, authz.PermContainersExec)
- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),
+ require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/ws/containers/abc/terminal"),
"expected WS terminal to be allowed with containers:exec")
}
@@ -645,21 +609,21 @@ func TestProxyPermissionDeniedResourceSortRequiresRead(t *testing.T) {
for _, sort := range []string{"cpuUsage", "memoryUsage"} {
c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers?sort="+sort)
- require.True(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID),
+ require.True(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID, "/containers"),
"expected resource sort %q to be denied with only containers:list", sort)
}
c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers?sort=name")
- require.False(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID),
+ require.False(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID, "/containers"),
"expected non-resource sorts to stay allowed with containers:list")
c = newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers")
- require.False(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID),
+ require.False(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID, "/containers"),
"expected the plain list to stay allowed with containers:list")
withRead := authz.NewPermissionSet()
withRead.AddEnv(proxyTestEnvID, authz.PermContainersList, authz.PermContainersRead)
c = newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers?sort=memoryUsage&order=desc")
- require.False(t, m.proxyPermissionDenied(c, withRead, proxyTestEnvID),
+ require.False(t, m.proxyPermissionDenied(c, withRead, proxyTestEnvID, "/containers"),
"expected resource sort to be allowed with containers:list and containers:read")
}
diff --git a/backend/internal/middleware/telemetry.go b/backend/internal/middleware/telemetry.go
new file mode 100644
index 000000000..efd9e05ca
--- /dev/null
+++ b/backend/internal/middleware/telemetry.go
@@ -0,0 +1,66 @@
+package middleware
+
+import (
+ "context"
+ "net/http"
+ "strings"
+
+ usertypes "github.com/getarcaneapp/arcane/types/v2/user"
+ "github.com/labstack/echo/v5"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
+ "go.opentelemetry.io/otel/trace"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
+)
+
+var authenticatedRequests = func() metric.Int64Counter {
+ counter, err := otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.auth.requests",
+ metric.WithDescription("Authenticated API requests by credential type"),
+ metric.WithUnit("{request}"),
+ )
+ if err != nil {
+ otel.Handle(err)
+ }
+ return counter
+}()
+
+// RecordAuthenticatedRequest tags the request span with the acting user and
+// counts the request by credential type.
+func RecordAuthenticatedRequest(ctx context.Context, header http.Header, user *usertypes.Actor) {
+ method := "session"
+ switch {
+ case header.Get(HeaderApiKey) != "":
+ method = "api_key"
+ case strings.HasPrefix(header.Get("Authorization"), "Bearer "):
+ method = "bearer"
+ }
+ trace.SpanFromContext(ctx).SetAttributes(
+ attribute.String("enduser.id", user.ID),
+ attribute.String("user.name", user.Username),
+ attribute.String("arcane.auth.method", method),
+ )
+ authenticatedRequests.Add(ctx, 1, metric.WithAttributes(attribute.String("arcane.auth.method", method)))
+}
+
+// SpanEnvironment tags the request span with the routed environment's ID, and with its name once the request
+// succeeds, so unauthenticated or unknown-environment requests never trigger a lookup.
+func SpanEnvironment(environmentName func(ctx context.Context, id string) string) echo.MiddlewareFunc {
+ return func(next echo.HandlerFunc) echo.HandlerFunc {
+ return func(c *echo.Context) error {
+ ctx := c.Request().Context()
+ span := trace.SpanFromContext(ctx)
+ id := c.Param("id")
+ if id == "" || !strings.HasPrefix(c.Path(), "/api/environments/:id") || !span.IsRecording() {
+ return next(c)
+ }
+ span.SetAttributes(attribute.String("arcane.environment.id", id))
+ err := next(c)
+ if resp, unwrapErr := echo.UnwrapResponse(c.Response()); err == nil && unwrapErr == nil && resp.Status < http.StatusBadRequest {
+ span.SetAttributes(attribute.String("arcane.environment.name", environmentName(ctx, id)))
+ }
+ return err
+ }
+ }
+}
diff --git a/backend/internal/notification/handler.go b/backend/internal/notification/handler.go
index 0ffacc4b2..8ed2fe903 100644
--- a/backend/internal/notification/handler.go
+++ b/backend/internal/notification/handler.go
@@ -71,7 +71,7 @@ func normalizeNotificationTestType(testType string) string {
}
func isSupportedNotificationTestType(testType string) bool {
- _, ok := supportedNotificationTestTypes[testType]
+ _, ok := notificationTestEventTypes[testType]
return ok
}
diff --git a/backend/internal/notification/service.go b/backend/internal/notification/service.go
index 4f137db25..ead660179 100644
--- a/backend/internal/notification/service.go
+++ b/backend/internal/notification/service.go
@@ -20,6 +20,11 @@ import (
"github.com/getarcaneapp/arcane/types/v2/system"
"go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/crypto"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/codes"
+ "go.opentelemetry.io/otel/metric"
+ "go.opentelemetry.io/otel/trace"
"golang.org/x/sync/errgroup"
"github.com/getarcaneapp/arcane/backend/v2/internal/apns"
@@ -34,10 +39,11 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation"
)
-var notificationCredentialFieldsByProviderInternal = map[notifications.NotificationProvider][]string{
+var notificationCredentialFieldsByProvider = map[notifications.NotificationProvider][]string{
notifications.NotificationProviderDiscord: {"token"},
notifications.NotificationProviderEmail: {"smtpPassword"},
notifications.NotificationProviderTelegram: {"botToken"},
@@ -47,12 +53,11 @@ var notificationCredentialFieldsByProviderInternal = map[notifications.Notificat
notifications.NotificationProviderPushover: {"token"},
notifications.NotificationProviderGotify: {"token"},
notifications.NotificationProviderMatrix: {"password"},
- // The Google Chat incoming webhook URL embeds the key and token query
- // parameters, so the whole URL is treated as a credential.
+ // The Google Chat webhook URL embeds its key and token, so the whole URL is a credential.
notifications.NotificationProviderGoogleChat: {"webhookUrl"},
}
-var notificationTargetFieldByProviderInternal = map[notifications.NotificationProvider]string{
+var notificationTargetFieldByProvider = map[notifications.NotificationProvider]string{
notifications.NotificationProviderEmail: "smtpHost",
notifications.NotificationProviderSignal: "host",
notifications.NotificationProviderNtfy: "host",
@@ -61,7 +66,7 @@ var notificationTargetFieldByProviderInternal = map[notifications.NotificationPr
}
const (
- notificationDispatchConcurrencyInternal = 4
+ notificationDispatchConcurrency = 4
notificationTestTypeSimple = "simple"
notificationTestTypeImageUpdate = "image-update"
@@ -71,11 +76,30 @@ const (
notificationTestTypeAutoHeal = "auto-heal"
)
+// notificationTestEventTypes lists supported test types and the event a real notification of that kind is gated on ("" = none).
+var notificationTestEventTypes = map[string]notifications.NotificationEventType{
+ notificationTestTypeSimple: "",
+ notificationTestTypeImageUpdate: notifications.NotificationEventImageUpdate,
+ notificationTestTypeBatchImageUpdate: notifications.NotificationEventImageUpdate,
+ notificationTestTypeVulnerability: notifications.NotificationEventVulnerabilityFound,
+ notificationTestTypePruneReport: notifications.NotificationEventPruneReport,
+ notificationTestTypeAutoHeal: notifications.NotificationEventAutoHeal,
+}
+
var (
ErrUnauthorizedNotificationDispatch = errors.New("unauthorized notification dispatch")
ErrUnsupportedDispatchKind = errors.New("unsupported notification dispatch kind")
)
+var notificationsSent = func() metric.Int64Counter {
+ counter, err := otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.notifications.sent",
+ metric.WithDescription("Notification provider sends by outcome"), metric.WithUnit("{notification}"))
+ if err != nil {
+ otel.Handle(err)
+ }
+ return counter
+}()
+
type NotificationService struct {
db *database.DB
config *config.Config
@@ -91,7 +115,7 @@ type NotificationTarget struct {
EnvironmentName string
}
-func logManagerDispatchNotificationInternal(ctx context.Context, target NotificationTarget, kind notification.DispatchKind) {
+func logManagerDispatchNotification(ctx context.Context, target NotificationTarget, kind notification.DispatchKind) {
slog.InfoContext(ctx,
"Manager dispatching notification on behalf of agent",
"environmentId", target.EnvironmentID,
@@ -100,10 +124,6 @@ func logManagerDispatchNotificationInternal(ctx context.Context, target Notifica
)
}
-func (s *NotificationService) ResolveNotificationTarget(ctx context.Context, environmentID string) (NotificationTarget, error) {
- return s.resolveNotificationTargetInternal(ctx, environmentID)
-}
-
func NewNotificationService(db *database.DB, cfg *config.Config, environmentSvc *environment.EnvironmentService, eventSvc *event.EventService, apnsSvc *apns.ApnsService) *NotificationService {
return &NotificationService{
db: db,
@@ -116,19 +136,15 @@ func NewNotificationService(db *database.DB, cfg *config.Config, environmentSvc
}
}
-func (s *NotificationService) resolveNotificationTargetInternal(ctx context.Context, environmentID string) (NotificationTarget, error) {
+func (s *NotificationService) ResolveNotificationTarget(ctx context.Context, environmentID string) (NotificationTarget, error) {
trimmedEnvironmentID := cmp.Or(strings.TrimSpace(environmentID), "0")
if s.environmentSvc != nil {
env, err := s.environmentSvc.GetEnvironmentByID(ctx, trimmedEnvironmentID)
if err == nil && env != nil {
- environmentName := strings.TrimSpace(env.Name)
- if environmentName == "" && trimmedEnvironmentID == "0" {
- environmentName = "Local Docker"
- }
return NotificationTarget{
EnvironmentID: env.ID,
- EnvironmentName: environmentName,
+ EnvironmentName: cmp.Or(strings.TrimSpace(env.Name), kit.Ternary(env.ID == "0", "Local Docker", "")),
}, nil
}
if trimmedEnvironmentID != "0" {
@@ -148,31 +164,7 @@ func (s *NotificationService) resolveNotificationTargetInternal(ctx context.Cont
}, nil
}
-func (s *NotificationService) resolveNotificationTargetForAccessTokenInternal(ctx context.Context, accessToken string) (NotificationTarget, error) {
- if s.environmentSvc == nil {
- return NotificationTarget{}, errors.New("environment service not initialized")
- }
-
- env, err := s.environmentSvc.ResolveEnvironmentByAccessToken(ctx, accessToken)
- if err != nil {
- if errors.Is(err, environment.ErrEnvironmentAccessTokenRequired) || errors.Is(err, environment.ErrInvalidEnvironmentAccessToken) {
- return NotificationTarget{}, ErrUnauthorizedNotificationDispatch
- }
- return NotificationTarget{}, err
- }
-
- environmentName := strings.TrimSpace(env.Name)
- if environmentName == "" && env.ID == "0" {
- environmentName = "Local Docker"
- }
-
- return NotificationTarget{
- EnvironmentID: env.ID,
- EnvironmentName: environmentName,
- }, nil
-}
-
-func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.Context, payload notification.DispatchRequest) (notification.DispatchResponse, error) {
+func (s *NotificationService) dispatchNotificationToManager(ctx context.Context, payload notification.DispatchRequest) (notification.DispatchResponse, error) {
body, err := json.Marshal(payload)
if err != nil {
return notification.DispatchResponse{}, fmt.Errorf("failed to marshal notification dispatch payload: %w", err)
@@ -186,9 +178,7 @@ func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.
})
}
- // Tunnel-connected edge agents typically have neither MANAGER_API_URL nor
- // AGENT_TOKEN configured for HTTP; ride the agent-to-manager event channel
- // instead so their notifications are not silently lost (#3002).
+ // Tunnel-connected edge agents often lack MANAGER_API_URL/AGENT_TOKEN, so they dispatch over the tunnel (#3002).
if s.config == nil || strings.TrimSpace(httpx.ManagerBaseURL(s.config.ManagerApiUrl)) == "" || strings.TrimSpace(s.config.AgentToken) == "" {
if publishViaTunnelErr := publishViaTunnel(); publishViaTunnelErr != nil {
return notification.DispatchResponse{}, fmt.Errorf("notification dispatch needs either MANAGER_API_URL + AGENT_TOKEN or a connected edge tunnel: %w", publishViaTunnelErr)
@@ -206,8 +196,7 @@ func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.
resp, err := s.httpClient.Do(req)
if err != nil {
- // HTTP unreachable does not mean the manager is: an agent configured for
- // HTTP may still hold a healthy edge tunnel, so ride that before giving up.
+ // An agent configured for HTTP may still hold a healthy edge tunnel, so try it before giving up.
if tunnelErr := publishViaTunnel(); tunnelErr == nil {
return notification.DispatchResponse{Message: "notification dispatched via edge tunnel"}, nil
}
@@ -225,10 +214,7 @@ func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.
return apiResponse.Data, nil
}
- // A non-2xx also means the notification was not dispatched, and the HTTP
- // path can fail independently of the tunnel (stale AGENT_TOKEN, wrong
- // MANAGER_API_URL, intermediate proxy), so the tunnel fallback applies here
- // too before surfacing the status error.
+ // A non-2xx (stale token, wrong URL, proxy) can fail independently of the tunnel, so fall back here too.
responseBody, _ := io.ReadAll(resp.Body)
if tunnelErr := publishViaTunnel(); tunnelErr == nil {
return notification.DispatchResponse{Message: "notification dispatched via edge tunnel"}, nil
@@ -240,32 +226,40 @@ func (s *NotificationService) DispatchNotification(ctx context.Context, accessTo
if s.config != nil && s.config.AgentMode {
return notification.DispatchResponse{}, errors.New("notification dispatch is manager-only")
}
+ if s.environmentSvc == nil {
+ return notification.DispatchResponse{}, errors.New("environment service not initialized")
+ }
- target, err := s.resolveNotificationTargetForAccessTokenInternal(ctx, accessToken)
+ env, err := s.environmentSvc.ResolveEnvironmentByAccessToken(ctx, accessToken)
if err != nil {
+ if errors.Is(err, environment.ErrEnvironmentAccessTokenRequired) || errors.Is(err, environment.ErrInvalidEnvironmentAccessToken) {
+ return notification.DispatchResponse{}, ErrUnauthorizedNotificationDispatch
+ }
return notification.DispatchResponse{}, err
}
- return s.dispatchForTargetInternal(ctx, target, payload)
+ target := NotificationTarget{
+ EnvironmentID: env.ID,
+ EnvironmentName: cmp.Or(strings.TrimSpace(env.Name), kit.Ternary(env.ID == "0", "Local Docker", "")),
+ }
+ return s.dispatchForTarget(ctx, target, payload)
}
-// DispatchNotificationForEnvironment dispatches on behalf of an agent whose
-// identity was already established by its edge tunnel session, so no access
-// token is exchanged (#3002).
+// DispatchNotificationForEnvironment dispatches for an agent already identified by its edge tunnel session (#3002).
func (s *NotificationService) DispatchNotificationForEnvironment(ctx context.Context, environmentID string, payload notification.DispatchRequest) (notification.DispatchResponse, error) {
if s.config != nil && s.config.AgentMode {
return notification.DispatchResponse{}, errors.New("notification dispatch is manager-only")
}
- target, err := s.resolveNotificationTargetInternal(ctx, environmentID)
+ target, err := s.ResolveNotificationTarget(ctx, environmentID)
if err != nil {
return notification.DispatchResponse{}, err
}
- return s.dispatchForTargetInternal(ctx, target, payload)
+ return s.dispatchForTarget(ctx, target, payload)
}
-func (s *NotificationService) dispatchForTargetInternal(ctx context.Context, target NotificationTarget, payload notification.DispatchRequest) (notification.DispatchResponse, error) {
+func (s *NotificationService) dispatchForTarget(ctx context.Context, target NotificationTarget, payload notification.DispatchRequest) (notification.DispatchResponse, error) {
var err error
dispatchResponse := notification.DispatchResponse{Message: "Notification dispatched successfully"}
switch payload.Kind {
@@ -273,40 +267,29 @@ func (s *NotificationService) dispatchForTargetInternal(ctx context.Context, tar
if payload.ImageUpdate == nil {
return notification.DispatchResponse{}, errors.New("image update payload is required")
}
- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)
- dispatchResponse.Delivered, err = s.sendImageUpdateNotificationForTargetInternal(
- ctx,
- target,
- payload.ImageUpdate.ImageRef,
- &payload.ImageUpdate.UpdateInfo,
- notifications.NotificationEventImageUpdate,
- )
+ logManagerDispatchNotification(ctx, target, payload.Kind)
+ imageUpdate := payload.ImageUpdate
+ dispatchResponse.Delivered, err = s.sendImageUpdateNotificationForTarget(ctx, target, imageUpdate.ImageRef, &imageUpdate.UpdateInfo, notifications.NotificationEventImageUpdate)
return dispatchResponse, err
case notification.DispatchKindBatchImageUpdate:
if payload.BatchImageUpdate == nil {
return notification.DispatchResponse{}, errors.New("batch image update payload is required")
}
- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)
- dispatchResponse.Delivered, err = s.sendBatchImageUpdateNotificationForTargetInternal(ctx, target, payload.BatchImageUpdate.Updates)
+ logManagerDispatchNotification(ctx, target, payload.Kind)
+ dispatchResponse.Delivered, err = s.sendBatchImageUpdateNotificationForTarget(ctx, target, payload.BatchImageUpdate.Updates)
return dispatchResponse, err
case notification.DispatchKindContainerUpdate:
if payload.ContainerUpdate == nil {
return notification.DispatchResponse{}, errors.New("container update payload is required")
}
- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)
- return dispatchResponse, s.sendContainerUpdateNotificationForTargetInternal(
- ctx,
- target,
- payload.ContainerUpdate.ContainerName,
- payload.ContainerUpdate.ImageRef,
- payload.ContainerUpdate.OldDigest,
- payload.ContainerUpdate.NewDigest,
- )
+ logManagerDispatchNotification(ctx, target, payload.Kind)
+ update := payload.ContainerUpdate
+ return dispatchResponse, s.sendContainerUpdateNotificationForTarget(ctx, target, update.ContainerName, update.ImageRef, update.OldDigest, update.NewDigest)
case notification.DispatchKindBatchContainerUpdate:
if payload.BatchContainerUpdate == nil {
return notification.DispatchResponse{}, errors.New("batch container update payload is required")
}
- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)
+ logManagerDispatchNotification(ctx, target, payload.Kind)
entries := make([]notifications.ContainerUpdateBatchEntry, 0, len(payload.BatchContainerUpdate.Updates))
for _, update := range payload.BatchContainerUpdate.Updates {
entries = append(entries, notifications.ContainerUpdateBatchEntry{
@@ -316,33 +299,25 @@ func (s *NotificationService) dispatchForTargetInternal(ctx context.Context, tar
NewDigest: update.NewDigest,
})
}
- return dispatchResponse, s.sendBatchContainerUpdateNotificationForTargetInternal(ctx, target, entries)
+ return dispatchResponse, s.sendBatchContainerUpdateNotificationForTarget(ctx, target, entries)
case notification.DispatchKindVulnerabilityFound:
if payload.VulnerabilityFound == nil {
return notification.DispatchResponse{}, errors.New("vulnerability payload is required")
}
- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)
- return dispatchResponse, s.sendVulnerabilityNotificationForTargetInternal(ctx, target, VulnerabilityNotificationPayload{
- CVEID: payload.VulnerabilityFound.CVEID,
- CVELink: payload.VulnerabilityFound.CVELink,
- Severity: payload.VulnerabilityFound.Severity,
- ImageName: payload.VulnerabilityFound.ImageName,
- FixedVersion: payload.VulnerabilityFound.FixedVersion,
- PkgName: payload.VulnerabilityFound.PkgName,
- InstalledVersion: payload.VulnerabilityFound.InstalledVersion,
- })
+ logManagerDispatchNotification(ctx, target, payload.Kind)
+ return dispatchResponse, s.sendVulnerabilityNotificationForTarget(ctx, target, VulnerabilityNotificationPayload(*payload.VulnerabilityFound))
case notification.DispatchKindPruneReport:
if payload.PruneReport == nil {
return notification.DispatchResponse{}, errors.New("prune report payload is required")
}
- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)
- return dispatchResponse, s.sendPruneReportNotificationForTargetInternal(ctx, target, &payload.PruneReport.Result)
+ logManagerDispatchNotification(ctx, target, payload.Kind)
+ return dispatchResponse, s.sendPruneReportNotificationForTarget(ctx, target, &payload.PruneReport.Result)
case notification.DispatchKindAutoHeal:
if payload.AutoHeal == nil {
return notification.DispatchResponse{}, errors.New("auto-heal payload is required")
}
- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)
- return dispatchResponse, s.sendAutoHealNotificationForTargetInternal(ctx, target, payload.AutoHeal.ContainerName, payload.AutoHeal.ContainerID)
+ logManagerDispatchNotification(ctx, target, payload.Kind)
+ return dispatchResponse, s.sendAutoHealNotificationForTarget(ctx, target, payload.AutoHeal.ContainerName, payload.AutoHeal.ContainerID)
default:
return notification.DispatchResponse{}, fmt.Errorf("%s: %w", payload.Kind, ErrUnsupportedDispatchKind)
}
@@ -383,7 +358,7 @@ func (s *NotificationService) CreateOrUpdateSettings(ctx context.Context, provid
err := s.db.WithContext(ctx).Where("provider = ?", provider).First(&setting).Error
existingConfig := kit.Ternary(err == nil, setting.Config, database.JSON(nil))
- encryptedConfig, encryptErr := encryptNotificationConfigCredentialsInternal(provider, localConfig, existingConfig)
+ encryptedConfig, encryptErr := encryptNotificationConfigCredentials(provider, localConfig, existingConfig)
if encryptErr != nil {
return nil, encryptErr
}
@@ -416,8 +391,8 @@ func (s *NotificationService) CreateOrUpdateSettings(ctx context.Context, provid
// RedactNotificationConfigCredentials returns a copy of config with provider credential fields blanked for API responses.
func RedactNotificationConfigCredentials(provider notifications.NotificationProvider, localConfig database.JSON) database.JSON {
- redacted := cloneNotificationConfigInternal(localConfig)
- for _, field := range notificationCredentialFieldsByProviderInternal[provider] {
+ redacted := cloneNotificationConfig(localConfig)
+ for _, field := range notificationCredentialFieldsByProvider[provider] {
value, ok := redacted[field]
if !ok {
continue
@@ -431,16 +406,31 @@ func RedactNotificationConfigCredentials(provider notifications.NotificationProv
return redacted
}
-func encryptNotificationConfigCredentialsInternal(provider notifications.NotificationProvider, localConfig, existingConfig database.JSON) (database.JSON, error) {
- encryptedConfig := cloneNotificationConfigInternal(localConfig)
- preserveConfig := existingConfig
- if provider == notifications.NotificationProviderSignal {
- preserveConfig = signalCredentialPreservationConfigInternal(localConfig, existingConfig)
- }
- if provider == notifications.NotificationProviderEmail {
- preserveConfig = emailCredentialPreservationConfigInternal(localConfig, existingConfig)
+// encryptNotificationConfigCredentials encrypts credential fields, preserving stored ones left blank.
+// Kept separate from CreateOrUpdateSettings to stay under the gocognit limit.
+func encryptNotificationConfigCredentials(provider notifications.NotificationProvider, localConfig, existingConfig database.JSON) (database.JSON, error) {
+ encryptedConfig := cloneNotificationConfig(localConfig)
+
+ // Switching auth mode drops the stored credential of the other mode.
+ preserveConfig := cloneNotificationConfig(existingConfig)
+ switch provider {
+ case notifications.NotificationProviderSignal:
+ user, _ := localConfig["user"].(string)
+ password, _ := localConfig["password"].(string)
+ token, _ := localConfig["token"].(string)
+ if strings.TrimSpace(token) != "" {
+ delete(preserveConfig, "password")
+ }
+ if strings.TrimSpace(user) != "" || strings.TrimSpace(password) != "" {
+ delete(preserveConfig, "token")
+ }
+ case notifications.NotificationProviderEmail:
+ if authMode, _ := localConfig["authMode"].(string); authMode == string(notifications.EmailAuthModeNone) {
+ delete(preserveConfig, "smtpPassword")
+ }
}
- if targetField := notificationTargetFieldByProviderInternal[provider]; targetField != "" {
+
+ if targetField := notificationTargetFieldByProvider[provider]; targetField != "" {
currentTarget, _ := existingConfig[targetField].(string)
nextTarget, _ := encryptedConfig[targetField].(string)
if strings.TrimSpace(nextTarget) == "" && strings.TrimSpace(currentTarget) != "" {
@@ -448,9 +438,9 @@ func encryptNotificationConfigCredentialsInternal(provider notifications.Notific
encryptedConfig[targetField] = currentTarget
}
- storedCredentials := make(map[string]bool, len(notificationCredentialFieldsByProviderInternal[provider]))
- updatedCredentials := make(map[string]bool, len(notificationCredentialFieldsByProviderInternal[provider]))
- for _, field := range notificationCredentialFieldsByProviderInternal[provider] {
+ storedCredentials := make(map[string]bool, len(notificationCredentialFieldsByProvider[provider]))
+ updatedCredentials := make(map[string]bool, len(notificationCredentialFieldsByProvider[provider]))
+ for _, field := range notificationCredentialFieldsByProvider[provider] {
preservedValue, _ := preserveConfig[field].(string)
updatedValue, _ := encryptedConfig[field].(string)
storedCredentials[field] = preservedValue != ""
@@ -470,7 +460,8 @@ func encryptNotificationConfigCredentialsInternal(provider notifications.Notific
return nil, err
}
}
- for _, field := range notificationCredentialFieldsByProviderInternal[provider] {
+
+ for _, field := range notificationCredentialFieldsByProvider[provider] {
value, _ := encryptedConfig[field].(string)
if value == "" {
if existingValue, ok := preserveConfig[field].(string); ok && existingValue != "" {
@@ -479,7 +470,11 @@ func encryptNotificationConfigCredentialsInternal(provider notifications.Notific
continue
}
- encrypted, err := encryptNotificationCredentialInternal(value)
+ // Already-encrypted values are kept as is.
+ if _, decryptErr := crypto.Decrypt(value); decryptErr == nil {
+ continue
+ }
+ encrypted, err := crypto.Encrypt(value)
if err != nil {
return nil, fmt.Errorf("failed to encrypt notification credential %q: %w", field, err)
}
@@ -488,41 +483,8 @@ func encryptNotificationConfigCredentialsInternal(provider notifications.Notific
return encryptedConfig, nil
}
-func signalCredentialPreservationConfigInternal(localConfig, existingConfig database.JSON) database.JSON {
- preserveConfig := cloneNotificationConfigInternal(existingConfig)
- user, _ := localConfig["user"].(string)
- password, _ := localConfig["password"].(string)
- token, _ := localConfig["token"].(string)
-
- if strings.TrimSpace(token) != "" {
- delete(preserveConfig, "password")
- }
- if strings.TrimSpace(user) != "" || strings.TrimSpace(password) != "" {
- delete(preserveConfig, "token")
- }
-
- return preserveConfig
-}
-
-func emailCredentialPreservationConfigInternal(localConfig, existingConfig database.JSON) database.JSON {
- preserveConfig := cloneNotificationConfigInternal(existingConfig)
- if authMode, _ := localConfig["authMode"].(string); authMode == string(notifications.EmailAuthModeNone) {
- delete(preserveConfig, "smtpPassword")
- }
- return preserveConfig
-}
-
-func encryptNotificationCredentialInternal(value string) (string, error) {
- if value == "" {
- return "", nil
- }
- if _, err := crypto.Decrypt(value); err == nil {
- return value, nil
- }
- return crypto.Encrypt(value)
-}
-
-func cloneNotificationConfigInternal(localConfig database.JSON) database.JSON {
+// cloneNotificationConfig copies config, returning an empty non-nil map for nil.
+func cloneNotificationConfig(localConfig database.JSON) database.JSON {
if localConfig == nil {
return database.JSON{}
}
@@ -538,92 +500,57 @@ func (s *NotificationService) DeleteSettings(ctx context.Context, provider notif
return nil
}
+// isEventEnabled defaults to enabled when the config has no entry for eventType.
func (s *NotificationService) isEventEnabled(localConfig database.JSON, eventType notifications.NotificationEventType) bool {
events, ok := localConfig["events"].(map[string]any)
if !ok {
- return true // If no events config, default to enabled
+ return true
}
-
enabled, ok := events[string(eventType)].(bool)
- if !ok {
- return true // If event type not specified, default to enabled
- }
-
- return enabled
+ return !ok || enabled
}
-// logNotificationInternal records a delivery attempt in the event log so sends and
-// failures are visible alongside every other Arcane event.
-func (
- s *NotificationService,
-) logNotificationInternal(
+// send delivers content to one provider inside a notification.send span and counts the outcome.
+func (s *NotificationService) send(
ctx context.Context,
- environmentID string,
- provider notifications.NotificationProvider,
- subject, status string,
- errMsg *string,
- metadata database.JSON,
-) {
- if s.eventSvc == nil {
- return
- }
-
- severity := event.EventSeveritySuccess
- title := fmt.Sprintf("Notification sent via %s", provider)
- description := subject
- if errMsg != nil {
- severity = event.EventSeverityError
- title = fmt.Sprintf("Notification failed via %s", provider)
- description = fmt.Sprintf("%s: %s", subject, *errMsg)
- }
-
- eventMetadata := cloneNotificationConfigInternal(metadata)
- eventMetadata["provider"] = string(provider)
- eventMetadata["status"] = status
-
- resourceType := "notification"
- providerName := string(provider)
- req := event.CreateEventRequest{
- Type: event.EventTypeNotificationSend,
- Severity: severity,
- Title: title,
- Description: description,
- ResourceType: &resourceType,
- ResourceName: &providerName,
- EnvironmentID: &environmentID,
- Metadata: eventMetadata,
- }
-
- done := make(chan struct{})
- go func() {
- defer close(done)
- defer utils.RecoverToError(nil, "notification event logging", "provider", providerName)
- logCtx, cancelLog := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
- defer cancelLog()
- if _, err := s.eventSvc.CreateEvent(logCtx, req); err != nil {
- slog.WarnContext(logCtx, "Failed to log notification event", "provider", providerName, "error", err.Error())
+ target NotificationTarget,
+ eventType notifications.NotificationEventType,
+ setting *NotificationSettings,
+ content notifications.Content,
+) (handled bool, err error) {
+ providerAttr := attribute.String("arcane.notification.provider", string(setting.Provider))
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "notification.send", trace.WithAttributes(
+ providerAttr,
+ attribute.String("arcane.notification.kind", string(eventType)),
+ attribute.String("arcane.environment.id", target.EnvironmentID),
+ attribute.Int64("arcane.notification.setting_id", int64(setting.ID)),
+ ))
+ defer func() {
+ // Delivery errors can embed webhook URLs whose paths hold secrets, so spans record only the error type.
+ if err != nil {
+ span.SetStatus(codes.Error, "notification delivery failed")
+ span.SetAttributes(attribute.String("error.type", fmt.Sprintf("%T", err)))
}
+ span.End()
}()
- // Cancellation ends the caller's wait, not persistence of a completed attempt.
- select {
- case <-done:
- case <-ctx.Done():
+ handled, err = notifications.Deliver(ctx, setting.Provider, setting.Config, content)
+ if handled {
+ outcomeAttr := attribute.String("arcane.notification.outcome", kit.Ternary(err == nil, "success", "failure"))
+ notificationsSent.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes(providerAttr, outcomeAttr))
}
+ return handled, err
}
-// notifyEnabledProvidersInternal is the single fan-out loop behind every
-// notification event: it walks all provider settings, skips disabled providers
-// and providers with the event unsubscribed, dispatches to the rest, logs each
-// attempt, and aggregates send errors. It returns how many providers the
-// notification was actually delivered to.
-func (s *NotificationService) notifyEnabledProvidersInternal(
+// notifyEnabledProviders sends content to every enabled provider subscribed to eventType, logs each attempt,
+// and returns how many providers it was delivered to along with the aggregated send errors.
+func (s *NotificationService) notifyEnabledProviders(
ctx context.Context,
target NotificationTarget,
eventType notifications.NotificationEventType,
logRef string,
metadata database.JSON,
- dispatch func(ctx context.Context, provider notifications.NotificationProvider, config database.JSON) (handled bool, err error),
+ content notifications.Content,
) (int, error) {
settings, err := s.GetAllSettings(ctx)
if err != nil {
@@ -640,12 +567,12 @@ func (s *NotificationService) notifyEnabledProvidersInternal(
handled := make([]bool, len(eligible))
sendErrs := make([]error, len(eligible))
var g errgroup.Group
- g.SetLimit(notificationDispatchConcurrencyInternal)
- for i, setting := range eligible {
+ g.SetLimit(notificationDispatchConcurrency)
+ for i := range eligible {
g.Go(func() error {
- defer utils.RecoverToError(&sendErrs[i], "notification dispatch", "provider", setting.Provider)
+ defer utils.RecoverToError(&sendErrs[i], "notification dispatch", "provider", eligible[i].Provider)
handled[i] = true
- handled[i], sendErrs[i] = dispatch(ctx, setting.Provider, setting.Config)
+ handled[i], sendErrs[i] = s.send(ctx, target, eventType, &eligible[i], content)
return nil
})
}
@@ -660,11 +587,49 @@ func (s *NotificationService) notifyEnabledProvidersInternal(
slog.WarnContext(ctx, "Unknown notification provider", "provider", setting.Provider)
continue
}
+
+ providerName := string(setting.Provider)
+ req := event.CreateEventRequest{
+ Type: event.EventTypeNotificationSend,
+ Severity: event.EventSeveritySuccess,
+ Title: "Notification sent via " + providerName,
+ Description: logRef,
+ ResourceType: new("notification"),
+ ResourceName: &providerName,
+ EnvironmentID: &target.EnvironmentID,
+ Metadata: cloneNotificationConfig(metadata),
+ }
+ req.Metadata["provider"] = providerName
+ req.Metadata["status"] = "success"
if sendErrs[i] == nil {
delivered++
+ } else {
+ msg := sendErrs[i].Error()
+ errs = append(errs, fmt.Sprintf("%s: %s", providerName, msg))
+ req.Severity = event.EventSeverityError
+ req.Title = "Notification failed via " + providerName
+ req.Description = fmt.Sprintf("%s: %s", logRef, msg)
+ req.Metadata["status"] = "failed"
+ }
+
+ if s.eventSvc == nil {
+ continue
+ }
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ defer utils.RecoverToError(nil, "notification event logging", "provider", providerName)
+ logCtx, cancelLog := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+ defer cancelLog()
+ if _, createErr := s.eventSvc.CreateEvent(logCtx, req); createErr != nil {
+ slog.WarnContext(logCtx, "Failed to log notification event", "provider", providerName, "error", createErr.Error())
+ }
+ }()
+ // Cancellation ends the caller's wait, not persistence of a completed attempt.
+ select {
+ case <-done:
+ case <-ctx.Done():
}
- status, errMsg := collectNotificationSendResultInternal(&errs, setting.Provider, sendErrs[i])
- s.logNotificationInternal(ctx, target.EnvironmentID, setting.Provider, logRef, status, errMsg, metadata)
}
if s.apnsSvc != nil {
@@ -679,25 +644,6 @@ func (s *NotificationService) notifyEnabledProvidersInternal(
return delivered, nil
}
-func collectNotificationSendResultInternal(localErrors *[]string, provider notifications.NotificationProvider, sendErr error) (string, *string) {
- if sendErr == nil {
- return "success", nil
- }
-
- msg := sendErr.Error()
- *localErrors = append(*localErrors, fmt.Sprintf("%s: %s", provider, msg))
- return "failed", &msg
-}
-
-var supportedNotificationTestTypes = map[string]struct{}{
- notificationTestTypeSimple: {},
- notificationTestTypeImageUpdate: {},
- notificationTestTypeBatchImageUpdate: {},
- notificationTestTypeVulnerability: {},
- notificationTestTypePruneReport: {},
- notificationTestTypeAutoHeal: {},
-}
-
// VulnerabilityNotificationPayload is the data sent to all providers for vulnerability_found events.
// Only vulnerabilities with a fixed version should trigger this notification.
type VulnerabilityNotificationPayload struct {
@@ -710,8 +656,9 @@ type VulnerabilityNotificationPayload struct {
InstalledVersion string // optional
}
-// SendBatchContainerUpdateNotification dispatches a single batched
-// notification covering all containers updated in one auto-update run.
+// --- Event entry points ---
+
+// SendBatchContainerUpdateNotification dispatches one notification covering all containers updated in an auto-update run.
func (s *NotificationService) SendBatchContainerUpdateNotification(ctx context.Context, entries []notifications.ContainerUpdateBatchEntry) error {
if len(entries) == 0 {
return nil
@@ -727,22 +674,22 @@ func (s *NotificationService) SendBatchContainerUpdateNotification(ctx context.C
NewDigest: entry.NewDigest,
})
}
- _, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{
+ _, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindBatchContainerUpdate,
BatchContainerUpdate: &notification.DispatchBatchContainerUpdate{Updates: updates},
})
return err
}
- target, err := s.resolveNotificationTargetInternal(ctx, "")
+ target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return err
}
- return s.sendBatchContainerUpdateNotificationForTargetInternal(ctx, target, entries)
+ return s.sendBatchContainerUpdateNotificationForTarget(ctx, target, entries)
}
-func (s *NotificationService) sendBatchContainerUpdateNotificationForTargetInternal(ctx context.Context, target NotificationTarget, entries []notifications.ContainerUpdateBatchEntry) error {
+func (s *NotificationService) sendBatchContainerUpdateNotificationForTarget(ctx context.Context, target NotificationTarget, entries []notifications.ContainerUpdateBatchEntry) error {
containerNames := make([]string, 0, len(entries))
for _, entry := range entries {
containerNames = append(containerNames, entry.ContainerName)
@@ -756,34 +703,20 @@ func (s *NotificationService) sendBatchContainerUpdateNotificationForTargetInter
}
content := s.templates.BatchContainerUpdate(target.EnvironmentName, entries)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventContainerUpdate)
- _, err := s.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventContainerUpdate, strings.Join(
- containerNames,
- ", ",
- ), metadata, func(
- ctx context.Context,
- provider notifications.NotificationProvider,
- config database.JSON,
- ) (
- bool,
- error,
- ) {
- return notifications.Deliver(ctx, provider, config, content)
- })
+ logRef := strings.Join(containerNames, ", ")
+ _, err := s.notifyEnabledProviders(ctx, target, notifications.NotificationEventContainerUpdate, logRef, metadata, content)
return err
}
-// --- Event entry points ---
-
-// SendImageUpdateNotification dispatches a single-image update notification and
-// returns the number of eligible providers it was delivered to (0 means no
-// provider has this event enabled, so callers must not mark the update notified).
+// SendImageUpdateNotification returns how many providers the notification was delivered to.
+// Zero means no provider has this event enabled, so callers must not mark the update notified.
func (s *NotificationService) SendImageUpdateNotification(ctx context.Context, imageRef string, updateInfo *imageupdate.Response, eventType notifications.NotificationEventType) (int, error) {
if updateInfo == nil {
return 0, errors.New("updateInfo is required")
}
if s.config != nil && s.config.AgentMode {
- dispatchResponse, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{
+ dispatchResponse, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindImageUpdate,
ImageUpdate: &notification.DispatchImageUpdate{
ImageRef: imageRef,
@@ -796,26 +729,21 @@ func (s *NotificationService) SendImageUpdateNotification(ctx context.Context, i
return dispatchResponse.Delivered, nil
}
- target, err := s.resolveNotificationTargetInternal(ctx, "")
+ target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return 0, err
}
- return s.sendImageUpdateNotificationForTargetInternal(ctx, target, imageRef, updateInfo, eventType)
+ return s.sendImageUpdateNotificationForTarget(ctx, target, imageRef, updateInfo, eventType)
}
-func (
- s *NotificationService,
-) sendImageUpdateNotificationForTargetInternal(
+func (s *NotificationService) sendImageUpdateNotificationForTarget(
ctx context.Context,
target NotificationTarget,
imageRef string,
updateInfo *imageupdate.Response,
eventType notifications.NotificationEventType,
-) (
- int,
- error,
-) {
+) (int, error) {
metadata := database.JSON{
"hasUpdate": updateInfo.HasUpdate,
"currentDigest": updateInfo.CurrentDigest,
@@ -825,14 +753,12 @@ func (
}
content := s.templates.ImageUpdate(target.EnvironmentName, imageRef, updateInfo)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, eventType)
- return s.notifyEnabledProvidersInternal(ctx, target, eventType, imageRef, metadata, func(ctx context.Context, provider notifications.NotificationProvider, config database.JSON) (bool, error) {
- return notifications.Deliver(ctx, provider, config, content)
- })
+ return s.notifyEnabledProviders(ctx, target, eventType, imageRef, metadata, content)
}
func (s *NotificationService) SendContainerUpdateNotification(ctx context.Context, containerName, imageRef, oldDigest, newDigest string) error {
if s.config != nil && s.config.AgentMode {
- _, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{
+ _, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindContainerUpdate,
ContainerUpdate: &notification.DispatchContainerUpdate{
ContainerName: containerName,
@@ -844,15 +770,15 @@ func (s *NotificationService) SendContainerUpdateNotification(ctx context.Contex
return err
}
- target, err := s.resolveNotificationTargetInternal(ctx, "")
+ target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return err
}
- return s.sendContainerUpdateNotificationForTargetInternal(ctx, target, containerName, imageRef, oldDigest, newDigest)
+ return s.sendContainerUpdateNotificationForTarget(ctx, target, containerName, imageRef, oldDigest, newDigest)
}
-func (s *NotificationService) sendContainerUpdateNotificationForTargetInternal(ctx context.Context, target NotificationTarget, containerName, imageRef, oldDigest, newDigest string) error {
+func (s *NotificationService) sendContainerUpdateNotificationForTarget(ctx context.Context, target NotificationTarget, containerName, imageRef, oldDigest, newDigest string) error {
metadata := database.JSON{
"containerName": containerName,
"oldDigest": oldDigest,
@@ -861,33 +787,20 @@ func (s *NotificationService) sendContainerUpdateNotificationForTargetInternal(c
}
content := s.templates.ContainerUpdate(target.EnvironmentName, containerName, imageRef, oldDigest, newDigest)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventContainerUpdate)
- _, err := s.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventContainerUpdate, imageRef, metadata, func(
- ctx context.Context,
- provider notifications.NotificationProvider,
- config database.JSON,
- ) (
- bool,
- error,
- ) {
- return notifications.Deliver(ctx, provider, config, content)
- })
+ _, err := s.notifyEnabledProviders(ctx, target, notifications.NotificationEventContainerUpdate, imageRef, metadata, content)
return err
}
-func isVulnerabilitySummaryPayload(payload VulnerabilityNotificationPayload) bool {
- return strings.HasPrefix(strings.ToUpper(strings.TrimSpace(payload.CVEID)), "DAILY SUMMARY")
-}
-
-// SendVulnerabilityNotification notifies all enabled providers that have vulnerability_found event enabled.
+// SendVulnerabilityNotification notifies providers subscribed to vulnerability_found events.
// Only daily summary payloads are sent; legacy per-CVE payloads are ignored.
func (s *NotificationService) SendVulnerabilityNotification(ctx context.Context, payload VulnerabilityNotificationPayload) error {
- if !isVulnerabilitySummaryPayload(payload) {
+ if !strings.HasPrefix(strings.ToUpper(strings.TrimSpace(payload.CVEID)), "DAILY SUMMARY") {
slog.InfoContext(ctx, "skipping legacy individual vulnerability notification payload", "cve", payload.CVEID)
return nil
}
if s.config != nil && s.config.AgentMode {
- _, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{
+ _, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindVulnerabilityFound,
VulnerabilityFound: &notification.DispatchVulnerabilityFound{
CVEID: payload.CVEID,
@@ -902,15 +815,15 @@ func (s *NotificationService) SendVulnerabilityNotification(ctx context.Context,
return err
}
- target, err := s.resolveNotificationTargetInternal(ctx, "")
+ target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return err
}
- return s.sendVulnerabilityNotificationForTargetInternal(ctx, target, payload)
+ return s.sendVulnerabilityNotificationForTarget(ctx, target, payload)
}
-func (s *NotificationService) sendVulnerabilityNotificationForTargetInternal(ctx context.Context, target NotificationTarget, payload VulnerabilityNotificationPayload) error {
+func (s *NotificationService) sendVulnerabilityNotificationForTarget(ctx context.Context, target NotificationTarget, payload VulnerabilityNotificationPayload) error {
metadata := database.JSON{
"cveId": payload.CVEID,
"severity": payload.Severity,
@@ -919,30 +832,21 @@ func (s *NotificationService) sendVulnerabilityNotificationForTargetInternal(ctx
}
content := s.templates.Vulnerability(target.EnvironmentName, notification.DispatchVulnerabilityFound(payload))
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventVulnerabilityFound)
- _, err := s.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventVulnerabilityFound, payload.ImageName, metadata, func(
- ctx context.Context,
- provider notifications.NotificationProvider,
- config database.JSON,
- ) (
- bool,
- error,
- ) {
- return notifications.Deliver(ctx, provider, config, content)
- })
+ _, err := s.notifyEnabledProviders(ctx, target, notifications.NotificationEventVulnerabilityFound, payload.ImageName, metadata, content)
return err
}
-// SendBatchImageUpdateNotification dispatches a batched image-update notification
-// and returns the number of eligible providers it was delivered to (0 means no
-// provider has this event enabled, so callers must not mark the updates notified).
+// SendBatchImageUpdateNotification returns how many providers the notification was delivered to.
+// Zero means no provider has this event enabled, so callers must not mark the updates notified.
func (s *NotificationService) SendBatchImageUpdateNotification(ctx context.Context, updates map[string]*imageupdate.Response) (int, error) {
- updatesWithChanges := filterUpdatesWithChangesInternal(updates)
+ updatesWithChanges := maps.Clone(updates)
+ maps.DeleteFunc(updatesWithChanges, func(_ string, update *imageupdate.Response) bool { return update == nil || !update.HasUpdate })
if len(updatesWithChanges) == 0 {
return 0, nil
}
if s.config != nil && s.config.AgentMode {
- dispatchResponse, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{
+ dispatchResponse, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindBatchImageUpdate,
BatchImageUpdate: &notification.DispatchBatchImageUpdate{
Updates: updatesWithChanges,
@@ -954,33 +858,21 @@ func (s *NotificationService) SendBatchImageUpdateNotification(ctx context.Conte
return dispatchResponse.Delivered, nil
}
- target, err := s.resolveNotificationTargetInternal(ctx, "")
+ target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return 0, err
}
- return s.sendBatchImageUpdateNotificationForTargetInternal(ctx, target, updatesWithChanges)
+ return s.sendBatchImageUpdateNotificationForTarget(ctx, target, updatesWithChanges)
}
-func filterUpdatesWithChangesInternal(updates map[string]*imageupdate.Response) map[string]*imageupdate.Response {
- updatesWithChanges := make(map[string]*imageupdate.Response, len(updates))
- for imageRef, update := range updates {
- if update != nil && update.HasUpdate {
- updatesWithChanges[imageRef] = update
- }
- }
- return updatesWithChanges
-}
-
-func (s *NotificationService) sendBatchImageUpdateNotificationForTargetInternal(ctx context.Context, target NotificationTarget, updates map[string]*imageupdate.Response) (int, error) {
- updatesWithChanges := filterUpdatesWithChangesInternal(updates)
-
+func (s *NotificationService) sendBatchImageUpdateNotificationForTarget(ctx context.Context, target NotificationTarget, updates map[string]*imageupdate.Response) (int, error) {
+ updatesWithChanges := maps.Clone(updates)
+ maps.DeleteFunc(updatesWithChanges, func(_ string, update *imageupdate.Response) bool { return update == nil || !update.HasUpdate })
if len(updatesWithChanges) == 0 {
return 0, nil
}
- imageRefs := slices.Collect(maps.Keys(updatesWithChanges))
-
metadata := database.JSON{
"updateCount": len(updatesWithChanges),
"eventType": string(notifications.NotificationEventImageUpdate),
@@ -988,19 +880,8 @@ func (s *NotificationService) sendBatchImageUpdateNotificationForTargetInternal(
}
content := s.templates.BatchImageUpdate(target.EnvironmentName, updatesWithChanges)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventImageUpdate)
- return s.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventImageUpdate, strings.Join(
- imageRefs,
- ", ",
- ), metadata, func(
- ctx context.Context,
- provider notifications.NotificationProvider,
- config database.JSON,
- ) (
- bool,
- error,
- ) {
- return notifications.Deliver(ctx, provider, config, content)
- })
+ logRef := strings.Join(slices.Collect(maps.Keys(updatesWithChanges)), ", ")
+ return s.notifyEnabledProviders(ctx, target, notifications.NotificationEventImageUpdate, logRef, metadata, content)
}
func (s *NotificationService) SendPruneReportNotification(ctx context.Context, result *system.PruneAllResult) error {
@@ -1009,15 +890,22 @@ func (s *NotificationService) SendPruneReportNotification(ctx context.Context, r
return nil
}
- hasChanges := pruneResultHasChangesInternal(result)
+ hasChanges := result.SpaceReclaimed > 0 ||
+ len(result.ContainersPruned) > 0 ||
+ len(result.ImagesDeleted) > 0 ||
+ len(result.VolumesDeleted) > 0 ||
+ len(result.NetworksDeleted) > 0
hasErrors := len(result.Errors) > 0
if !hasChanges && !hasErrors {
slog.InfoContext(ctx, "skipping prune report notification because no resources were pruned and no errors were reported")
return nil
}
+ if !hasChanges {
+ slog.WarnContext(ctx, "sending prune report notification with errors but no resources were pruned", "errorCount", len(result.Errors))
+ }
if s.config != nil && s.config.AgentMode {
- _, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{
+ _, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindPruneReport,
PruneReport: &notification.DispatchPruneReport{
Result: *result,
@@ -1026,63 +914,29 @@ func (s *NotificationService) SendPruneReportNotification(ctx context.Context, r
return err
}
- target, err := s.resolveNotificationTargetInternal(ctx, "")
+ target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return err
}
- return s.sendPruneReportNotificationForTargetInternal(ctx, target, result)
+ return s.sendPruneReportNotificationForTarget(ctx, target, result)
}
-func (s *NotificationService) sendPruneReportNotificationForTargetInternal(ctx context.Context, target NotificationTarget, result *system.PruneAllResult) error {
- hasChanges := pruneResultHasChangesInternal(result)
- hasErrors := len(result.Errors) > 0
-
+func (s *NotificationService) sendPruneReportNotificationForTarget(ctx context.Context, target NotificationTarget, result *system.PruneAllResult) error {
metadata := database.JSON{
"spaceReclaimed": result.SpaceReclaimed,
"eventType": string(notifications.NotificationEventPruneReport),
}
content := s.templates.PruneReport(target.EnvironmentName, result)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventPruneReport)
- _, err := s.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventPruneReport, "System Prune Report", metadata, func(
- ctx context.Context,
- provider notifications.NotificationProvider,
- config database.JSON,
- ) (
- bool,
- error,
- ) {
- return notifications.Deliver(ctx, provider, config, content)
- })
- if err != nil {
- return err
- }
- if hasErrors && !hasChanges {
- slog.WarnContext(ctx, "sending prune report notification with errors but no resources were pruned", "errorCount", len(result.Errors))
- }
-
- return nil
-}
-
-func pruneResultHasChangesInternal(result *system.PruneAllResult) bool {
- if result == nil {
- return false
- }
-
- if result.SpaceReclaimed > 0 {
- return true
- }
-
- return len(result.ContainersPruned) > 0 ||
- len(result.ImagesDeleted) > 0 ||
- len(result.VolumesDeleted) > 0 ||
- len(result.NetworksDeleted) > 0
+ _, err := s.notifyEnabledProviders(ctx, target, notifications.NotificationEventPruneReport, "System Prune Report", metadata, content)
+ return err
}
// SendAutoHealNotification sends a notification when a container is auto-healed.
func (s *NotificationService) SendAutoHealNotification(ctx context.Context, containerName, containerID string) error {
if s.config != nil && s.config.AgentMode {
- _, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{
+ _, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindAutoHeal,
AutoHeal: &notification.DispatchAutoHeal{
ContainerName: containerName,
@@ -1092,86 +946,73 @@ func (s *NotificationService) SendAutoHealNotification(ctx context.Context, cont
return err
}
- target, err := s.resolveNotificationTargetInternal(ctx, "")
+ target, err := s.ResolveNotificationTarget(ctx, "")
if err != nil {
return err
}
- return s.sendAutoHealNotificationForTargetInternal(ctx, target, containerName, containerID)
+ return s.sendAutoHealNotificationForTarget(ctx, target, containerName, containerID)
}
-func (s *NotificationService) sendAutoHealNotificationForTargetInternal(ctx context.Context, target NotificationTarget, containerName, containerID string) error {
+func (s *NotificationService) sendAutoHealNotificationForTarget(ctx context.Context, target NotificationTarget, containerName, containerID string) error {
metadata := database.JSON{
"containerID": containerID,
"eventType": string(notifications.NotificationEventAutoHeal),
}
content := s.templates.AutoHeal(target.EnvironmentName, containerName)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventAutoHeal)
- _, err := s.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventAutoHeal, containerName, metadata, func(
- ctx context.Context,
- provider notifications.NotificationProvider,
- config database.JSON,
- ) (
- bool,
- error,
- ) {
- return notifications.Deliver(ctx, provider, config, content)
- })
+ _, err := s.notifyEnabledProviders(ctx, target, notifications.NotificationEventAutoHeal, containerName, metadata, content)
return err
}
// --- Test notifications ---
-// notificationEventTypeForTestTypeInternal maps a test type to the event type a
-// real notification of that kind would be gated on ("" = no event gate).
-func notificationEventTypeForTestTypeInternal(testType string) notifications.NotificationEventType {
- switch testType {
- case notificationTestTypeImageUpdate, notificationTestTypeBatchImageUpdate:
- return notifications.NotificationEventImageUpdate
- case notificationTestTypeVulnerability:
- return notifications.NotificationEventVulnerabilityFound
- case notificationTestTypePruneReport:
- return notifications.NotificationEventPruneReport
- case notificationTestTypeAutoHeal:
- return notifications.NotificationEventAutoHeal
- default:
- return ""
- }
-}
-
-// testNotificationWarningInternal reports why a real notification would not send
-// even though the test did: the provider is disabled, or the tested event type is
-// unsubscribed. Empty means real notifications would send.
-func (s *NotificationService) testNotificationWarningInternal(setting *NotificationSettings, testType string) string {
- if !setting.Enabled {
- return fmt.Sprintf("%s is disabled, so real notifications will not send", setting.Provider)
+// TestNotification sends a test message even to a disabled provider (testing before enabling is legitimate)
+// and returns a warning when a real notification of the tested kind would not send.
+func (s *NotificationService) TestNotification(ctx context.Context, environmentID string, provider notifications.NotificationProvider, testType string) (string, error) {
+ setting, err := s.GetSettingsByProvider(ctx, provider)
+ if err != nil {
+ return "", fmt.Errorf("please save your %s settings before testing", provider)
}
- eventType := notificationEventTypeForTestTypeInternal(testType)
- if eventType != "" && !s.isEventEnabled(setting.Config, eventType) {
- return fmt.Sprintf("%s events are disabled for %s, so real notifications will not send", eventType, setting.Provider)
+ testType = cmp.Or(strings.TrimSpace(testType), notificationTestTypeSimple)
+ eventType, ok := notificationTestEventTypes[testType]
+ if !ok {
+ return "", fmt.Errorf("unsupported notification test type: %s", testType)
}
- if eventType == "" && !slices.ContainsFunc(notifications.AllNotificationEventTypes, func(candidate notifications.NotificationEventType) bool {
+
+ var warning string
+ switch {
+ case !setting.Enabled:
+ warning = fmt.Sprintf("%s is disabled, so real notifications will not send", setting.Provider)
+ case eventType != "" && !s.isEventEnabled(setting.Config, eventType):
+ warning = fmt.Sprintf("%s events are disabled for %s, so real notifications will not send", eventType, setting.Provider)
+ case eventType == "" && !slices.ContainsFunc(notifications.AllNotificationEventTypes, func(candidate notifications.NotificationEventType) bool {
return s.isEventEnabled(setting.Config, candidate)
- }) {
- return fmt.Sprintf("no events are subscribed for %s, so real notifications will not send", setting.Provider)
+ }):
+ warning = fmt.Sprintf("no events are subscribed for %s, so real notifications will not send", setting.Provider)
}
- return ""
-}
-func (s *NotificationService) testNotificationContentInternal(environmentName, testType string) notifications.Content {
- switch testType {
- case notificationTestTypeVulnerability:
- return s.templates.Vulnerability(environmentName, notification.DispatchVulnerabilityFound{
+ target, err := s.ResolveNotificationTarget(ctx, environmentID)
+ if err != nil {
+ return "", err
+ }
+
+ var content notifications.Content
+ switch {
+ case provider == notifications.NotificationProviderEmail && testType == notificationTestTypeSimple:
+ content = s.templates.TestEmail(target.EnvironmentName)
+ case testType == notificationTestTypeVulnerability:
+ content = s.templates.Vulnerability(target.EnvironmentName, notification.DispatchVulnerabilityFound{
CVEID: "Daily Summary - " + time.Now().UTC().Format("2006-01-02"),
Severity: "Critical:1 High:3 Medium:2 Low:1 Unknown:0",
ImageName: "5 image(s) scanned, 2 with fixable vulnerabilities",
FixedVersion: "7 fixable vulnerability record(s)",
PkgName: "CVE-2025-1234, CVE-2025-5678, CVE-2026-0001",
})
- case notificationTestTypeAutoHeal:
- return s.templates.AutoHeal(environmentName, "test-container")
- case notificationTestTypePruneReport:
- return s.templates.PruneReport(environmentName, &system.PruneAllResult{
+ case testType == notificationTestTypeAutoHeal:
+ content = s.templates.AutoHeal(target.EnvironmentName, "test-container")
+ case testType == notificationTestTypePruneReport:
+ content = s.templates.PruneReport(target.EnvironmentName, &system.PruneAllResult{
Success: true,
ContainersPruned: []string{"a1b2c3d4e5f6", "f6e5d4c3b2a1"},
ImagesDeleted: []string{"sha256:1111111111111111111111111111111111111111111111111111111111111111"},
@@ -1184,8 +1025,8 @@ func (s *NotificationService) testNotificationContentInternal(environmentName, t
BuildCacheSpaceReclaimed: 230162432,
Errors: []string{},
})
- case notificationTestTypeBatchImageUpdate:
- return s.templates.BatchImageUpdate(environmentName, map[string]*imageupdate.Response{
+ case testType == notificationTestTypeBatchImageUpdate:
+ content = s.templates.BatchImageUpdate(target.EnvironmentName, map[string]*imageupdate.Response{
"nginx:latest": {
HasUpdate: true,
UpdateType: "digest",
@@ -1213,7 +1054,7 @@ func (s *NotificationService) testNotificationContentInternal(environmentName, t
})
default: // simple and image-update
imageRef := kit.Ternary(testType == notificationTestTypeSimple, "test/image:latest", "nginx:latest")
- return s.templates.ImageUpdate(environmentName, imageRef, &imageupdate.Response{
+ content = s.templates.ImageUpdate(target.EnvironmentName, imageRef, &imageupdate.Response{
HasUpdate: true,
UpdateType: "digest",
CurrentDigest: "sha256:abc123def456789012345678901234567890",
@@ -1222,39 +1063,11 @@ func (s *NotificationService) testNotificationContentInternal(environmentName, t
ResponseTimeMs: 100,
})
}
-}
-
-// TestNotification sends a test message to the provider regardless of its enabled
-// state (testing before enabling is legitimate) and returns a warning when a real
-// notification of the tested kind would not send.
-func (s *NotificationService) TestNotification(ctx context.Context, environmentID string, provider notifications.NotificationProvider, testType string) (string, error) {
- setting, err := s.GetSettingsByProvider(ctx, provider)
- if err != nil {
- return "", fmt.Errorf("please save your %s settings before testing", provider)
- }
- testType = cmp.Or(strings.TrimSpace(testType), notificationTestTypeSimple)
- if _, ok := supportedNotificationTestTypes[testType]; !ok {
- return "", fmt.Errorf("unsupported notification test type: %s", testType)
- }
- warning := s.testNotificationWarningInternal(setting, testType)
-
- target, err := s.resolveNotificationTargetInternal(ctx, environmentID)
- if err != nil {
- return "", err
- }
-
- if provider == notifications.NotificationProviderEmail && testType == notificationTestTypeSimple {
- _, sendErr := notifications.Deliver(ctx, notifications.NotificationProviderEmail, setting.Config, s.templates.TestEmail(target.EnvironmentName))
- return warning, sendErr
- }
- content := s.testNotificationContentInternal(target.EnvironmentName, testType)
- // Stamp the event vars so the Test button exercises a configured generic
- // payload template. The simple test renders image-update content, so it
- // reports that event type.
- testEventType := cmp.Or(notificationEventTypeForTestTypeInternal(testType), notifications.NotificationEventImageUpdate)
+ // Event vars let the test exercise a generic payload template; the simple test renders image-update content.
+ testEventType := cmp.Or(eventType, notifications.NotificationEventImageUpdate)
content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, testEventType)
- handled, sendErr := notifications.Deliver(ctx, provider, setting.Config, content)
+ handled, sendErr := s.send(ctx, target, testEventType, setting, content)
if !handled {
return "", fmt.Errorf("unknown provider: %s", provider)
}
diff --git a/backend/internal/notification/service_test.go b/backend/internal/notification/service_test.go
index a6326c369..555d4ead7 100644
--- a/backend/internal/notification/service_test.go
+++ b/backend/internal/notification/service_test.go
@@ -5,7 +5,6 @@ import (
"context"
"encoding/base64"
"encoding/json"
- "errors"
"log/slog"
"net/http"
"net/http/httptest"
@@ -53,7 +52,7 @@ func setupNotificationTestDB(t *testing.T) *database.DB {
return &database.DB{DB: db}
}
-func setupNotificationTestServiceInternal(t *testing.T) (*database.DB, *NotificationService) {
+func setupNotificationTestService(t *testing.T) (*database.DB, *NotificationService) {
t.Helper()
db := setupNotificationTestDB(t)
@@ -66,7 +65,7 @@ func setupNotificationTestServiceInternal(t *testing.T) (*database.DB, *Notifica
return db, NewNotificationService(db, cfg, envSvc, event.NewEventService(db, cfg, nil), nil)
}
-func newNotificationTestUpdateInfoInternal() *imageupdate.Response {
+func newNotificationTestUpdateInfo() *imageupdate.Response {
return &imageupdate.Response{
HasUpdate: true,
UpdateType: "digest",
@@ -76,7 +75,7 @@ func newNotificationTestUpdateInfoInternal() *imageupdate.Response {
}
}
-func captureNotificationServiceLogsInternal(t *testing.T) *bytes.Buffer {
+func captureNotificationServiceLogs(t *testing.T) *bytes.Buffer {
t.Helper()
var buf bytes.Buffer
@@ -90,11 +89,11 @@ func captureNotificationServiceLogsInternal(t *testing.T) *bytes.Buffer {
return &buf
}
-func TestNotificationService_ResolveNotificationTargetInternal_UsesEnvironmentRecordAndFallback(t *testing.T) {
+func TestNotificationService_ResolveNotificationTarget_UsesEnvironmentRecordAndFallback(t *testing.T) {
ctx := t.Context()
- db, svc := setupNotificationTestServiceInternal(t)
+ db, svc := setupNotificationTestService(t)
- target, err := svc.resolveNotificationTargetInternal(ctx, "")
+ target, err := svc.ResolveNotificationTarget(ctx, "")
require.NoError(t, err)
require.Equal(t, "0", target.EnvironmentID)
require.Equal(t, "Local Docker", target.EnvironmentName)
@@ -108,42 +107,21 @@ func TestNotificationService_ResolveNotificationTargetInternal_UsesEnvironmentRe
Status: string(environment.EnvironmentStatusOnline),
}).Error)
- target, err = svc.resolveNotificationTargetInternal(ctx, "env-remote")
+ target, err = svc.ResolveNotificationTarget(ctx, "env-remote")
require.NoError(t, err)
require.Equal(t, "env-remote", target.EnvironmentID)
require.Equal(t, "Remote Alpha", target.EnvironmentName)
}
-func TestNotificationService_ResolveNotificationTargetForAccessTokenInternal_UsesStoredEnvironmentName(t *testing.T) {
- ctx := t.Context()
- db, svc := setupNotificationTestServiceInternal(t)
-
- token := "remote-token"
- now := time.Now()
- require.NoError(t, db.WithContext(ctx).Create(&environment.Environment{
- ID: "env-remote", CreatedAt: now, UpdatedAt: &now,
- Name: "Remote Edge",
- ApiUrl: "http://remote.example",
- Enabled: true,
- Status: string(environment.EnvironmentStatusOnline),
- AccessToken: &token,
- }).Error)
-
- target, err := svc.resolveNotificationTargetForAccessTokenInternal(ctx, token)
- require.NoError(t, err)
- require.Equal(t, "env-remote", target.EnvironmentID)
- require.Equal(t, "Remote Edge", target.EnvironmentName)
-}
-
func TestNotificationService_DispatchNotification_InvalidAccessTokenReturnsUnauthorizedSentinel(t *testing.T) {
ctx := t.Context()
- _, svc := setupNotificationTestServiceInternal(t)
+ _, svc := setupNotificationTestService(t)
_, err := svc.DispatchNotification(ctx, "missing-token", notification.DispatchRequest{
Kind: notification.DispatchKindImageUpdate,
ImageUpdate: &notification.DispatchImageUpdate{
ImageRef: "nginx:latest",
- UpdateInfo: *newNotificationTestUpdateInfoInternal(),
+ UpdateInfo: *newNotificationTestUpdateInfo(),
},
})
@@ -153,7 +131,7 @@ func TestNotificationService_DispatchNotification_InvalidAccessTokenReturnsUnaut
func TestNotificationService_DispatchNotification_UnsupportedKindReturnsSentinel(t *testing.T) {
ctx := t.Context()
- db, svc := setupNotificationTestServiceInternal(t)
+ db, svc := setupNotificationTestService(t)
token := "remote-token"
now := time.Now()
@@ -179,8 +157,8 @@ func TestNotificationService_DispatchNotification_UnsupportedKindReturnsSentinel
func TestNotificationService_DispatchNotification_LogsManagerDispatchForAgent(t *testing.T) {
ctx := t.Context()
- db, svc := setupNotificationTestServiceInternal(t)
- logBuffer := captureNotificationServiceLogsInternal(t)
+ db, svc := setupNotificationTestService(t)
+ logBuffer := captureNotificationServiceLogs(t)
token := "remote-token"
now := time.Now()
@@ -197,7 +175,7 @@ func TestNotificationService_DispatchNotification_LogsManagerDispatchForAgent(t
Kind: notification.DispatchKindImageUpdate,
ImageUpdate: &notification.DispatchImageUpdate{
ImageRef: "nginx:latest",
- UpdateInfo: *newNotificationTestUpdateInfoInternal(),
+ UpdateInfo: *newNotificationTestUpdateInfo(),
},
})
@@ -251,7 +229,7 @@ func TestNotificationService_SendImageUpdateNotification_AgentModeDispatchesToMa
ManagerApiUrl: server.URL,
}, envSvc, nil, nil)
- delivered, err := svc.SendImageUpdateNotification(ctx, "nginx:latest", newNotificationTestUpdateInfoInternal(), notifications.NotificationEventImageUpdate)
+ delivered, err := svc.SendImageUpdateNotification(ctx, "nginx:latest", newNotificationTestUpdateInfo(), notifications.NotificationEventImageUpdate)
require.NoError(t, err)
require.Equal(t, 2, delivered)
require.EqualValues(t, 1, calls.Load())
@@ -260,7 +238,7 @@ func TestNotificationService_SendImageUpdateNotification_AgentModeDispatchesToMa
require.Equal(t, "nginx:latest", dispatched.ImageUpdate.ImageRef)
}
-func TestNotificationService_SendBatchImageUpdateNotification_AgentModeUsesManagerDeliveredCountInternal(t *testing.T) {
+func TestNotificationService_SendBatchImageUpdateNotification_AgentModeUsesManagerDeliveredCount(t *testing.T) {
ctx := t.Context()
db := setupNotificationTestDB(t)
envSvc := environment.NewEnvironmentService(db, nil, nil, nil, nil, nil)
@@ -299,7 +277,7 @@ func TestNotificationService_SendBatchImageUpdateNotification_AgentModeUsesManag
}, envSvc, nil, nil)
delivered, err := svc.SendBatchImageUpdateNotification(ctx, map[string]*imageupdate.Response{
- "nginx:latest": newNotificationTestUpdateInfoInternal(),
+ "nginx:latest": newNotificationTestUpdateInfo(),
})
require.NoError(t, err)
require.Equal(t, 0, delivered)
@@ -323,7 +301,7 @@ func TestNotificationService_SendImageUpdateNotification_AgentModeRequiresUpdate
require.Contains(t, err.Error(), "updateInfo is required")
}
-func TestNotificationService_SendBatchImageUpdateNotification_AgentModeSkipsNoOpDispatchInternal(t *testing.T) {
+func TestNotificationService_SendBatchImageUpdateNotification_AgentModeSkipsNoOpDispatch(t *testing.T) {
ctx := t.Context()
db := setupNotificationTestDB(t)
envSvc := environment.NewEnvironmentService(db, nil, nil, nil, nil, nil)
@@ -364,8 +342,8 @@ func TestNotificationService_SendBatchImageUpdateNotification_AgentModeSkipsNoOp
})
}
-func TestBuildImageUpdateNotificationMessageInternal_IncludesEnvironment(t *testing.T) {
- updateInfo := newNotificationTestUpdateInfoInternal()
+func TestBuildImageUpdateNotificationMessage_IncludesEnvironment(t *testing.T) {
+ updateInfo := newNotificationTestUpdateInfo()
message := notifications.BuildImageUpdateNotificationMessage(notifications.MessageFormatMarkdown, "Remote Alpha", "nginx:latest", updateInfo)
require.Contains(t, message, "**Environment:** Remote Alpha")
@@ -375,16 +353,16 @@ func TestBuildImageUpdateNotificationMessageInternal_IncludesEnvironment(t *test
require.Contains(t, plainMessage, "Environment: Remote Alpha")
}
-func TestBuildContainerUpdateNotificationMessageInternal_IncludesEnvironment(t *testing.T) {
+func TestBuildContainerUpdateNotificationMessage_IncludesEnvironment(t *testing.T) {
message := notifications.BuildContainerUpdateNotificationMessage(notifications.MessageFormatMarkdown, "Local Lab", "nginx", "nginx:latest", "sha256:old", "sha256:new")
require.Contains(t, message, "**Environment:** Local Lab")
require.Equal(t, 1, strings.Count(message, "Environment"))
}
-func TestBuildBatchImageUpdateNotificationMessageInternal_EnvironmentAppearsOnce(t *testing.T) {
+func TestBuildBatchImageUpdateNotificationMessage_EnvironmentAppearsOnce(t *testing.T) {
updates := map[string]*imageupdate.Response{
- "nginx:latest": newNotificationTestUpdateInfoInternal(),
+ "nginx:latest": newNotificationTestUpdateInfo(),
"redis:latest": {
HasUpdate: true,
UpdateType: "minor",
@@ -399,7 +377,7 @@ func TestBuildBatchImageUpdateNotificationMessageInternal_EnvironmentAppearsOnce
require.Equal(t, 1, strings.Count(message, "Environment"))
}
-func TestBuildVulnerabilitySummaryNotificationMessageInternal_IncludesEnvironment(t *testing.T) {
+func TestBuildVulnerabilitySummaryNotificationMessage_IncludesEnvironment(t *testing.T) {
message := notifications.BuildVulnerabilitySummaryNotificationMessage(
notifications.MessageFormatMarkdown,
"Remote Alpha",
@@ -414,7 +392,7 @@ func TestBuildVulnerabilitySummaryNotificationMessageInternal_IncludesEnvironmen
require.Equal(t, 1, strings.Count(message, "Environment"))
}
-func TestBuildPruneReportNotificationMessageInternal_IncludesEnvironment(t *testing.T) {
+func TestBuildPruneReportNotificationMessage_IncludesEnvironment(t *testing.T) {
message := notifications.BuildPruneReportNotificationMessage(notifications.MessageFormatMarkdown, "Cluster One", &system.PruneAllResult{
SpaceReclaimed: 3825205248,
ContainerSpaceReclaimed: 503316480,
@@ -427,14 +405,14 @@ func TestBuildPruneReportNotificationMessageInternal_IncludesEnvironment(t *test
require.Equal(t, 1, strings.Count(message, "Environment"))
}
-func TestBuildAutoHealNotificationMessageInternal_IncludesEnvironment(t *testing.T) {
+func TestBuildAutoHealNotificationMessage_IncludesEnvironment(t *testing.T) {
message := notifications.BuildAutoHealNotificationMessage(notifications.MessageFormatMarkdown, "Cluster One", "nginx")
require.Contains(t, message, "**Environment:** Cluster One")
require.Equal(t, 1, strings.Count(message, "Environment"))
}
-func TestNotificationCredentialInternal_KeepsPlaintextLegacyValues(t *testing.T) {
+func TestNotificationCredential_KeepsPlaintextLegacyValues(t *testing.T) {
setupNotificationTestDB(t)
value := "discord-webhook-token/plaintext"
@@ -443,7 +421,7 @@ func TestNotificationCredentialInternal_KeepsPlaintextLegacyValues(t *testing.T)
require.Equal(t, "discord-webhook-token/plaintext", value)
}
-func TestNotificationCredentialInternal_DecryptsEncryptedValues(t *testing.T) {
+func TestNotificationCredential_DecryptsEncryptedValues(t *testing.T) {
setupNotificationTestDB(t)
encrypted, err := crypto.Encrypt("gotify-application-token")
@@ -453,7 +431,7 @@ func TestNotificationCredentialInternal_DecryptsEncryptedValues(t *testing.T) {
require.Equal(t, "gotify-application-token", encrypted)
}
-func TestNotificationCredentialInternal_ReturnsErrorForCorruptedCiphertext(t *testing.T) {
+func TestNotificationCredential_ReturnsErrorForCorruptedCiphertext(t *testing.T) {
setupNotificationTestDB(t)
encrypted, err := crypto.Encrypt("gotify-application-token")
@@ -465,7 +443,7 @@ func TestNotificationCredentialInternal_ReturnsErrorForCorruptedCiphertext(t *te
require.Error(t, notifications.DecryptStringCredential(new(base64.StdEncoding.EncodeToString(ciphertext))))
}
-func TestNotificationCredentialInternal_LeavesEmptyValuesEmpty(t *testing.T) {
+func TestNotificationCredential_LeavesEmptyValuesEmpty(t *testing.T) {
setupNotificationTestDB(t)
value := ""
@@ -474,7 +452,7 @@ func TestNotificationCredentialInternal_LeavesEmptyValuesEmpty(t *testing.T) {
require.Empty(t, value)
}
-func TestNotificationService_CreateOrUpdateSettingsEncryptsCredentialFieldsInternal(t *testing.T) {
+func TestNotificationService_CreateOrUpdateSettingsEncryptsCredentialFields(t *testing.T) {
ctx := t.Context()
db := setupNotificationTestDB(t)
svc := NewNotificationService(db, &config.Config{}, nil, nil, nil)
@@ -497,7 +475,7 @@ func TestNotificationService_CreateOrUpdateSettingsEncryptsCredentialFieldsInter
require.Equal(t, "discord-secret-token", decrypted)
}
-func TestNotificationService_CreateOrUpdateSettingsPreservesStoredCredentialWhenEmptyInternal(t *testing.T) {
+func TestNotificationService_CreateOrUpdateSettingsPreservesStoredCredentialWhenEmpty(t *testing.T) {
ctx := t.Context()
db := setupNotificationTestDB(t)
svc := NewNotificationService(db, &config.Config{}, nil, nil, nil)
@@ -525,7 +503,7 @@ func TestNotificationService_CreateOrUpdateSettingsPreservesStoredCredentialWhen
require.Equal(t, "initial-gotify-token", decrypted)
}
-func TestNotificationService_CreateOrUpdateSettingsRejectsTargetChangeWithStoredCredentialInternal(t *testing.T) {
+func TestNotificationService_CreateOrUpdateSettingsRejectsTargetChangeWithStoredCredential(t *testing.T) {
ctx := t.Context()
db := setupNotificationTestDB(t)
svc := NewNotificationService(db, &config.Config{}, nil, nil, nil)
@@ -560,7 +538,7 @@ func TestNotificationService_CreateOrUpdateSettingsRejectsTargetChangeWithStored
require.Equal(t, originalToken, updated.Config["token"])
}
-func TestNotificationService_CreateOrUpdateSettingsClearsEmailPasswordWhenAuthModeNoneInternal(t *testing.T) {
+func TestNotificationService_CreateOrUpdateSettingsClearsEmailPasswordWhenAuthModeNone(t *testing.T) {
ctx := t.Context()
db := setupNotificationTestDB(t)
svc := NewNotificationService(db, &config.Config{}, nil, nil, nil)
@@ -584,7 +562,7 @@ func TestNotificationService_CreateOrUpdateSettingsClearsEmailPasswordWhenAuthMo
require.Empty(t, stored.Config["smtpPassword"])
}
-func TestNotificationService_CreateOrUpdateSettingsPreservesCredentialAcrossDisableInternal(t *testing.T) {
+func TestNotificationService_CreateOrUpdateSettingsPreservesCredentialAcrossDisable(t *testing.T) {
ctx := t.Context()
db := setupNotificationTestDB(t)
svc := NewNotificationService(db, &config.Config{}, nil, nil, nil)
@@ -615,7 +593,7 @@ func TestNotificationService_CreateOrUpdateSettingsPreservesCredentialAcrossDisa
require.Equal(t, "initial-gotify-token", decrypted)
}
-func TestNotificationService_CreateOrUpdateSettingsKeepsConfigWhenDisabledInternal(t *testing.T) {
+func TestNotificationService_CreateOrUpdateSettingsKeepsConfigWhenDisabled(t *testing.T) {
ctx := t.Context()
db := setupNotificationTestDB(t)
svc := NewNotificationService(db, &config.Config{}, nil, nil, nil)
@@ -651,53 +629,55 @@ func TestNotificationService_CreateOrUpdateSettingsKeepsConfigWhenDisabledIntern
require.Equal(t, false, events["prune_report"])
}
-func TestNotificationService_NotifyEnabledProvidersInternal_SkipsFiltersAndAggregatesInternal(t *testing.T) {
+func TestNotificationService_NotifyEnabledProviders_SkipsFiltersAndAggregates(t *testing.T) {
ctx := t.Context()
db := setupNotificationTestDB(t)
svc := NewNotificationService(db, &config.Config{}, nil, event.NewEventService(db, nil, nil), nil)
+ var webhookCalls atomic.Int32
+ var cancelOnWebhook atomic.Pointer[context.CancelFunc]
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ webhookCalls.Add(1)
+ if cancelWebhook := cancelOnWebhook.Load(); cancelWebhook != nil {
+ (*cancelWebhook)()
+ }
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer server.Close()
+
rows := []NotificationSettings{
{Provider: notifications.NotificationProviderDiscord, Enabled: false, Config: database.JSON{}},
{Provider: notifications.NotificationProviderSlack, Enabled: true, Config: database.JSON{
"events": map[string]any{string(notifications.NotificationEventPruneReport): false},
}},
- {Provider: notifications.NotificationProviderGotify, Enabled: true, Config: database.JSON{}},
- {Provider: notifications.NotificationProviderNtfy, Enabled: true, Config: database.JSON{}},
+ {Provider: notifications.NotificationProviderGeneric, Enabled: true, Config: database.JSON{"webhookUrl": server.URL}},
+ // An unconfigured Telegram provider fails before any network call.
+ {Provider: notifications.NotificationProviderTelegram, Enabled: true, Config: database.JSON{}},
}
for i := range rows {
require.NoError(t, db.WithContext(ctx).Create(&rows[i]).Error)
}
- var dispatchedMu sync.Mutex
- var dispatched []notifications.NotificationProvider
+ content := notifications.Content{Text: notifications.TextByFormat(func(notifications.MessageFormat) string { return "loop-test" })}
target := NotificationTarget{EnvironmentID: "0", EnvironmentName: "Local Docker"}
- delivered, err := svc.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventPruneReport, "loop-test", database.JSON{"eventType": "prune_report"},
- func(_ context.Context, provider notifications.NotificationProvider, _ database.JSON) (bool, error) {
- dispatchedMu.Lock()
- dispatched = append(dispatched, provider)
- dispatchedMu.Unlock()
- if provider == notifications.NotificationProviderNtfy {
- return true, errors.New("boom")
- }
- return true, nil
- })
+ delivered, err := svc.notifyEnabledProviders(ctx, target, notifications.NotificationEventPruneReport, "loop-test", database.JSON{"eventType": "prune_report"}, content)
// Disabled and event-disabled rows are never dispatched.
- require.ElementsMatch(t, []notifications.NotificationProvider{notifications.NotificationProviderGotify, notifications.NotificationProviderNtfy}, dispatched)
+ require.EqualValues(t, 1, webhookCalls.Load())
require.Equal(t, 1, delivered)
require.Error(t, err)
- require.Contains(t, err.Error(), "notification errors: ntfy: boom")
+ require.Contains(t, err.Error(), "notification errors: telegram: telegram bot token not configured")
// Each dispatched attempt lands in the event log.
var events []event.Event
require.NoError(t, db.WithContext(ctx).Where("type = ?", event.EventTypeNotificationSend).Order("created_at").Find(&events).Error)
require.Len(t, events, 2)
require.Equal(t, event.EventSeveritySuccess, events[0].Severity)
- require.Equal(t, "Notification sent via gotify", events[0].Title)
+ require.Equal(t, "Notification sent via generic", events[0].Title)
require.Equal(t, "loop-test", events[0].Description)
require.Equal(t, event.EventSeverityError, events[1].Severity)
- require.Equal(t, "Notification failed via ntfy", events[1].Title)
- require.Contains(t, events[1].Description, "boom")
+ require.Equal(t, "Notification failed via telegram", events[1].Title)
+ require.Contains(t, events[1].Description, "telegram bot token not configured")
// Completed attempts persist even when cancellation ends the caller's wait.
sqlDB, err := db.DB.DB()
@@ -713,14 +693,11 @@ func TestNotificationService_NotifyEnabledProvidersInternal_SkipsFiltersAndAggre
}))
canceledCtx, cancel := context.WithCancel(ctx)
defer cancel()
+ cancelOnWebhook.Store(&cancel)
returned := make(chan struct{})
go func() {
defer close(returned)
- _, _ = svc.notifyEnabledProvidersInternal(canceledCtx, target, notifications.NotificationEventPruneReport, "canceled-loop-test", nil,
- func(context.Context, notifications.NotificationProvider, database.JSON) (bool, error) {
- cancel()
- return true, nil
- })
+ _, _ = svc.notifyEnabledProviders(canceledCtx, target, notifications.NotificationEventPruneReport, "canceled-loop-test", nil, content)
}()
select {
case <-returned:
@@ -730,7 +707,7 @@ func TestNotificationService_NotifyEnabledProvidersInternal_SkipsFiltersAndAggre
releaseOnce.Do(func() { close(releaseLogging) })
require.Eventually(t, func() bool {
var count int64
- return db.WithContext(ctx).Model(&event.Event{}).Where("description = ?", "canceled-loop-test").Count(&count).Error == nil && count == 2
+ return db.WithContext(ctx).Model(&event.Event{}).Where("description LIKE ?", "canceled-loop-test%").Count(&count).Error == nil && count == 2
}, time.Second, 10*time.Millisecond)
}
@@ -745,17 +722,17 @@ func TestSupportedNotificationTestTypes_IncludesAutoHeal(t *testing.T) {
}
for _, tt := range expected {
- _, ok := supportedNotificationTestTypes[tt]
- require.True(t, ok, "expected %q to be in supportedNotificationTestTypes", tt)
+ _, ok := notificationTestEventTypes[tt]
+ require.True(t, ok, "expected %q to be in notificationTestEventTypes", tt)
}
- require.Len(t, supportedNotificationTestTypes, len(expected),
- "supportedNotificationTestTypes has unexpected entries")
+ require.Len(t, notificationTestEventTypes, len(expected),
+ "notificationTestEventTypes has unexpected entries")
}
func TestNotificationService_DispatchNotificationForEnvironment_ResolvesTunnelSessionEnvironment(t *testing.T) {
ctx := t.Context()
- db, svc := setupNotificationTestServiceInternal(t)
+ db, svc := setupNotificationTestService(t)
now := time.Now()
require.NoError(t, db.WithContext(ctx).Create(&environment.Environment{
@@ -770,7 +747,7 @@ func TestNotificationService_DispatchNotificationForEnvironment_ResolvesTunnelSe
Kind: notification.DispatchKindImageUpdate,
ImageUpdate: &notification.DispatchImageUpdate{
ImageRef: "nginx:latest",
- UpdateInfo: *newNotificationTestUpdateInfoInternal(),
+ UpdateInfo: *newNotificationTestUpdateInfo(),
},
})
require.NoError(t, err)
@@ -787,11 +764,11 @@ func TestNotificationService_AgentDispatchWithoutHTTPConfigFallsBackToTunnel(t *
// No MANAGER_API_URL/AGENT_TOKEN and no active tunnel: the error must point
// at both options instead of only the HTTP env vars (#3002).
- _, err := svc.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{
+ _, err := svc.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindImageUpdate,
ImageUpdate: &notification.DispatchImageUpdate{
ImageRef: "nginx:latest",
- UpdateInfo: *newNotificationTestUpdateInfoInternal(),
+ UpdateInfo: *newNotificationTestUpdateInfo(),
},
})
require.Error(t, err)
@@ -806,11 +783,11 @@ func TestNotificationService_AgentDispatchHTTPFailureFallsBackToTunnel(t *testin
// HTTP dispatch fails and no tunnel is connected either: the fallback must
// not mask the HTTP transport error.
- _, err := svc.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{
+ _, err := svc.dispatchNotificationToManager(ctx, notification.DispatchRequest{
Kind: notification.DispatchKindImageUpdate,
ImageUpdate: &notification.DispatchImageUpdate{
ImageRef: "nginx:latest",
- UpdateInfo: *newNotificationTestUpdateInfoInternal(),
+ UpdateInfo: *newNotificationTestUpdateInfo(),
},
})
require.Error(t, err)
diff --git a/backend/internal/oidc/service.go b/backend/internal/oidc/service.go
index bd147d5ab..aa32f7ac9 100644
--- a/backend/internal/oidc/service.go
+++ b/backend/internal/oidc/service.go
@@ -9,17 +9,21 @@ import (
"errors"
"fmt"
"log/slog"
+ "maps"
"net/http"
"net/url"
"slices"
"strings"
- "sync"
"time"
"github.com/coreos/go-oidc/v3/oidc"
authtypes "github.com/getarcaneapp/arcane/types/v2/auth"
"github.com/samber/hot"
"go.getarcane.app/kit/pkg"
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
"golang.org/x/oauth2"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
@@ -27,6 +31,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/jwtclaims"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/oidcjwk"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
type OidcService struct {
@@ -35,7 +40,6 @@ type OidcService struct {
config *config.Config
httpClient *http.Client
insecureHttpClient *http.Client
- providerMutex sync.RWMutex
providerCache *hot.HotCache[oidcProviderKey, *oidc.Provider]
keySetManager *oidcjwk.KeySetManager
}
@@ -58,77 +62,18 @@ func NewOidcService(authService *auth.AuthService, settingsService *settings.Set
httpClient = http.DefaultClient
}
- // Create a copy of the client to avoid modifying the shared one
- // and remove the timeout so we can control it via context
+ // Copy the shared client and drop its timeout so requests are bounded by context.
oidcClient := *httpClient
oidcClient.Timeout = 0
+ insecureClient := oidcClient
- service := &OidcService{
- authService: authService,
- settingsService: settingsService,
- config: cfg,
- httpClient: &oidcClient,
- keySetManager: keySetManager,
- }
- service.providerCache = hot.NewHotCache[oidcProviderKey, *oidc.Provider](hot.LRU, 4).Build()
- return service
-}
-
-func (s *OidcService) keySetInternal(ctx context.Context, jwksURL string, skipTLS bool) (oidc.KeySet, error) {
- if s.keySetManager == nil {
- return nil, errors.New("JWK set manager is not configured")
- }
- return s.keySetManager.KeySet(context.WithoutCancel(ctx), s.getHttpClientInternal(skipTLS), jwksURL)
-}
-
-func (s *OidcService) getEffectiveConfigInternal(ctx context.Context) (*settings.OidcConfig, error) {
- oidcConfig, err := s.authService.GetOidcConfig(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to get OIDC config: %w", err)
- }
- if oidcConfig.IssuerURL == "" {
- return nil, errors.New("issuer URL must be configured")
- }
- return oidcConfig, nil
-}
-
-func (s *OidcService) getHttpClientInternal(skipTlsVerify bool) *http.Client {
- if skipTlsVerify {
- return s.getInsecureHttpClientInternal()
- }
- return s.httpClient
-}
-
-func (s *OidcService) getInsecureHttpClientInternal() *http.Client {
- s.providerMutex.RLock()
- if s.insecureHttpClient != nil {
- s.providerMutex.RUnlock()
- return s.insecureHttpClient
- }
- s.providerMutex.RUnlock()
-
- s.providerMutex.Lock()
- defer s.providerMutex.Unlock()
-
- if s.insecureHttpClient != nil {
- return s.insecureHttpClient
- }
-
- // Create insecure client
- insecureClient := *s.httpClient
-
- var insecureTransport *http.Transport
- if transport, ok := insecureClient.Transport.(*http.Transport); ok {
+ // Derive the insecure transport from the caller's raw transport, before either client is instrumented.
+ insecureTransport := &http.Transport{}
+ if transport, ok := httpClient.Transport.(*http.Transport); ok {
insecureTransport = transport.Clone()
- } else {
- // Transport is nil or not *http.Transport - create a new default transport
- if defaultTransport, localOk := http.DefaultTransport.(*http.Transport); localOk {
- insecureTransport = defaultTransport.Clone()
- } else {
- insecureTransport = &http.Transport{}
- }
+ } else if defaultTransport, isTransport := http.DefaultTransport.(*http.Transport); isTransport {
+ insecureTransport = defaultTransport.Clone()
}
-
if insecureTransport.TLSClientConfig == nil {
// #nosec G402 - This is explicitly an insecure client for OIDC discovery when TLS verification is skipped
insecureTransport.TLSClientConfig = &tls.Config{
@@ -144,38 +89,56 @@ func (s *OidcService) getInsecureHttpClientInternal() *http.Client {
insecureTransport.Protocols.SetHTTP1(true)
}
insecureTransport.Protocols.SetHTTP2(true)
- insecureClient.Transport = insecureTransport
- s.insecureHttpClient = &insecureClient
- return s.insecureHttpClient
+
+ oidcClient.Transport = otelhttp.NewTransport(httpClient.Transport)
+ insecureClient.Transport = otelhttp.NewTransport(insecureTransport)
+
+ return &OidcService{
+ authService: authService,
+ settingsService: settingsService,
+ config: cfg,
+ httpClient: &oidcClient,
+ insecureHttpClient: &insecureClient,
+ keySetManager: keySetManager,
+ providerCache: hot.NewHotCache[oidcProviderKey, *oidc.Provider](hot.LRU, 4).Build(),
+ }
}
-func (s *OidcService) ensureOpenIDScopeInternal(scopes []string) []string {
- hasOpenID := slices.Contains(scopes, oidc.ScopeOpenID)
- if !hasOpenID {
- scopes = append([]string{oidc.ScopeOpenID}, scopes...)
+func (s *OidcService) effectiveConfig(ctx context.Context) (*settings.OidcConfig, error) {
+ oidcConfig, err := s.authService.GetOidcConfig(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get OIDC config: %w", err)
}
- return scopes
+ if oidcConfig.IssuerURL == "" {
+ return nil, errors.New("issuer URL must be configured")
+ }
+ return oidcConfig, nil
+}
+
+func (s *OidcService) httpClientFor(skipTLSVerify bool) *http.Client {
+ return kit.Ternary(skipTLSVerify, s.insecureHttpClient, s.httpClient)
}
-func (s *OidcService) hasManualEndpointsInternal(cfg *settings.OidcConfig) bool {
+func hasManualEndpoints(cfg *settings.OidcConfig) bool {
return cfg.AuthorizationEndpoint != "" || cfg.TokenEndpoint != "" || cfg.UserinfoEndpoint != ""
}
-func (s *OidcService) getOauth2ConfigInternal(cfg *settings.OidcConfig, provider *oidc.Provider, origin, mobileRedirectURI string) (oauth2.Config, error) {
+// requestedScopes returns the configured scopes (defaulting to email and profile), always including openid.
+func requestedScopes(cfg *settings.OidcConfig) []string {
scopes := strings.Fields(cfg.Scopes)
if len(scopes) == 0 {
scopes = []string{"email", "profile"}
}
- scopes = s.ensureOpenIDScopeInternal(scopes)
+ if !slices.Contains(scopes, oidc.ScopeOpenID) {
+ scopes = append([]string{oidc.ScopeOpenID}, scopes...)
+ }
+ return scopes
+}
- var endpoint oauth2.Endpoint
+func (s *OidcService) buildOAuth2Config(cfg *settings.OidcConfig, provider *oidc.Provider, origin, mobileRedirectURI string) (oauth2.Config, error) {
+ endpoint := oauth2.Endpoint{AuthURL: cfg.AuthorizationEndpoint, TokenURL: cfg.TokenEndpoint}
if provider != nil {
endpoint = provider.Endpoint()
- } else {
- endpoint = oauth2.Endpoint{
- AuthURL: cfg.AuthorizationEndpoint,
- TokenURL: cfg.TokenEndpoint,
- }
}
if endpoint.AuthURL == "" || endpoint.TokenURL == "" {
return oauth2.Config{}, errors.New("authorization and token endpoints must be configured")
@@ -191,10 +154,18 @@ func (s *OidcService) getOauth2ConfigInternal(cfg *settings.OidcConfig, provider
ClientSecret: cfg.ClientSecret,
Endpoint: endpoint,
RedirectURL: redirectURL,
- Scopes: scopes,
+ Scopes: requestedScopes(cfg),
}, nil
}
+func configAttributes(cfg *settings.OidcConfig) []attribute.KeyValue {
+ return []attribute.KeyValue{
+ attribute.String("arcane.oidc.issuer", cfg.IssuerURL),
+ attribute.Bool("arcane.oidc.skip_tls_verify", cfg.SkipTlsVerify),
+ attribute.Bool("arcane.oidc.manual_endpoints", hasManualEndpoints(cfg)),
+ }
+}
+
// GetMobileRedirectAllowlist returns the configured list of acceptable mobile OAuth redirect URIs.
func (s *OidcService) GetMobileRedirectAllowlist(ctx context.Context) []string {
raw := ""
@@ -207,9 +178,7 @@ func (s *OidcService) GetMobileRedirectAllowlist(ctx context.Context) []string {
return kit.TrimNonEmpty(strings.Split(raw, ","))
}
-// ValidateMobileRedirectURI returns nil if uri exactly matches one of the
-// configured mobile redirect URIs. Full-string match is required — partial
-// matches on scheme or host could be abused for open-redirect attacks.
+// ValidateMobileRedirectURI requires an exact allowlist match; partial matches would allow open redirects.
func (s *OidcService) ValidateMobileRedirectURI(ctx context.Context, uri string) error {
if uri == "" {
return errors.New("mobile redirect URI is empty")
@@ -221,16 +190,16 @@ func (s *OidcService) ValidateMobileRedirectURI(ctx context.Context, uri string)
}
func (s *OidcService) GenerateAuthURL(ctx context.Context, redirectTo, origin, mobileRedirectURI string) (string, string, error) {
- oidcConfig, err := s.getEffectiveConfigInternal(ctx)
+ oidcConfig, err := s.effectiveConfig(ctx)
if err != nil {
slog.ErrorContext(ctx, "GenerateAuthURL: failed to get OIDC config", "error", err)
return "", "", err
}
var provider *oidc.Provider
- if !s.hasManualEndpointsInternal(oidcConfig) {
+ if !hasManualEndpoints(oidcConfig) {
var discoverProviderErr error
- provider, discoverProviderErr = s.getOrDiscoverProviderInternal(ctx, oidcConfig)
+ provider, discoverProviderErr = s.getOrDiscoverProvider(ctx, oidcConfig)
if discoverProviderErr != nil {
slog.ErrorContext(ctx, "GenerateAuthURL: provider discovery failed", "issuer", oidcConfig.IssuerURL, "error", discoverProviderErr)
return "", "", fmt.Errorf("failed to discover provider: %w", discoverProviderErr)
@@ -241,7 +210,7 @@ func (s *OidcService) GenerateAuthURL(ctx context.Context, redirectTo, origin, m
nonce := kit.RandomString(32)
codeVerifier := kit.RandomString(128)
- oauth2Config, err := s.getOauth2ConfigInternal(oidcConfig, provider, origin, mobileRedirectURI)
+ oauth2Config, err := s.buildOAuth2Config(oidcConfig, provider, origin, mobileRedirectURI)
if err != nil {
slog.ErrorContext(ctx, "GenerateAuthURL: invalid OIDC endpoints", "error", err)
return "", "", err
@@ -252,23 +221,20 @@ func (s *OidcService) GenerateAuthURL(ctx context.Context, redirectTo, origin, m
oauth2.S256ChallengeOption(codeVerifier),
)
- stateData := OidcState{
+ stateJSON, err := json.Marshal(OidcState{
State: state,
Nonce: nonce,
CodeVerifier: codeVerifier,
RedirectTo: redirectTo,
CreatedAt: time.Now(),
- }
-
- stateJSON, err := json.Marshal(stateData)
+ })
if err != nil {
slog.ErrorContext(ctx, "GenerateAuthURL: failed to marshal state", "error", err)
return "", "", fmt.Errorf("failed to encode state: %w", err)
}
- encodedState := base64.URLEncoding.EncodeToString(stateJSON)
slog.DebugContext(ctx, "GenerateAuthURL: generated authorization URL", "issuer", oidcConfig.IssuerURL, "scopes", oauth2Config.Scopes)
- return authURL, encodedState, nil
+ return authURL, base64.URLEncoding.EncodeToString(stateJSON), nil
}
func (s *OidcService) GetOidcRedirectURL(origin string) string {
@@ -279,24 +245,17 @@ func (s *OidcService) GetOidcRedirectURL(origin string) string {
return baseUrl + "/auth/oidc/callback"
}
-func (s *OidcService) getOrDiscoverProviderInternal(ctx context.Context, cfg *settings.OidcConfig) (*oidc.Provider, error) {
- if s.providerCache == nil {
- s.providerCache = hot.NewHotCache[oidcProviderKey, *oidc.Provider](hot.LRU, 4).Build()
- }
+func (s *OidcService) getOrDiscoverProvider(ctx context.Context, cfg *settings.OidcConfig) (*oidc.Provider, error) {
key := oidcProviderKey{issuer: cfg.IssuerURL, skipTLS: cfg.SkipTlsVerify}
provider, found, err := s.providerCache.GetWithLoaders(key, func(keys []oidcProviderKey) (map[oidcProviderKey]*oidc.Provider, error) {
providers := make(map[oidcProviderKey]*oidc.Provider, len(keys))
for _, providerKey := range keys {
- discoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
- providerCtx := oidc.ClientContext(discoveryCtx, s.getHttpClientInternal(providerKey.skipTLS))
- discovered, discoveredIssuer, discoverErr := s.discoverProviderInternal(providerCtx, providerKey.issuer)
- cancel()
+ discovered, discoverErr := s.discoverProvider(ctx, providerKey)
if discoverErr != nil {
- slog.ErrorContext(ctx, "getOrDiscoverProviderInternal: discovery failed", "issuer", providerKey.issuer, "skipTls", providerKey.skipTLS, "error", discoverErr)
+ slog.ErrorContext(ctx, "getOrDiscoverProvider: discovery failed", "issuer", providerKey.issuer, "skipTls", providerKey.skipTLS, "error", discoverErr)
return nil, fmt.Errorf("failed to discover provider at %s: %w", providerKey.issuer, discoverErr)
}
providers[providerKey] = discovered
- slog.DebugContext(ctx, "getOrDiscoverProviderInternal: provider cached", "issuer", providerKey.issuer, "effectiveIssuer", discoveredIssuer, "skipTls", providerKey.skipTLS)
}
return providers, nil
})
@@ -309,213 +268,103 @@ func (s *OidcService) getOrDiscoverProviderInternal(ctx context.Context, cfg *se
return provider, nil
}
-func (s *OidcService) discoverProviderInternal(ctx context.Context, issuer string) (*oidc.Provider, string, error) {
- provider, err := oidc.NewProvider(ctx, issuer)
+// discoverProvider fetches provider metadata, retrying once with the issuer's trailing slash toggled.
+func (s *OidcService) discoverProvider(ctx context.Context, key oidcProviderKey) (_ *oidc.Provider, err error) {
+ discoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+ defer cancel()
+ discoveryCtx, span := otel.Tracer(tracing.InstrumentationName).Start(discoveryCtx, "oidc.discover_provider", trace.WithAttributes(
+ attribute.String("arcane.oidc.issuer", key.issuer),
+ attribute.Bool("arcane.oidc.skip_tls_verify", key.skipTLS),
+ ))
+ defer func() { tracing.End(span, err) }()
+ providerCtx := oidc.ClientContext(discoveryCtx, s.httpClientFor(key.skipTLS))
+
+ provider, err := oidc.NewProvider(providerCtx, key.issuer)
if err == nil {
- return provider, issuer, nil
- }
-
- altIssuer := strings.TrimRight(issuer, "/")
- if altIssuer == issuer {
- altIssuer = issuer + "/"
+ slog.DebugContext(ctx, "getOrDiscoverProvider: provider cached", "issuer", key.issuer, "skipTls", key.skipTLS)
+ return provider, nil
}
- slog.WarnContext(ctx, "getOrDiscoverProviderInternal: retrying discovery with alternate issuer", "configured", issuer, "alternate", altIssuer)
- provider, altErr := oidc.NewProvider(ctx, altIssuer)
- if altErr == nil {
- return provider, altIssuer, nil
+ altIssuer := strings.TrimRight(key.issuer, "/")
+ if altIssuer == key.issuer {
+ altIssuer = key.issuer + "/"
}
-
- slog.ErrorContext(ctx, "getOrDiscoverProviderInternal: discovery failed with alternate issuer", "issuer", altIssuer, "error", altErr)
- return nil, issuer, err
-}
-
-func (s *OidcService) exchangeTokenInternal(ctx context.Context, cfg *settings.OidcConfig, provider *oidc.Provider, code, verifier, origin, mobileRedirectURI string) (*oauth2.Token, error) {
- oauth2Config, err := s.getOauth2ConfigInternal(cfg, provider, origin, mobileRedirectURI)
- if err != nil {
+ slog.WarnContext(ctx, "getOrDiscoverProvider: retrying discovery with alternate issuer", "configured", key.issuer, "alternate", altIssuer)
+ provider, altErr := oidc.NewProvider(providerCtx, altIssuer)
+ if altErr != nil {
+ slog.ErrorContext(ctx, "getOrDiscoverProvider: discovery failed with alternate issuer", "issuer", altIssuer, "error", altErr)
return nil, err
}
-
- providerCtx := oidc.ClientContext(ctx, s.getHttpClientInternal(cfg.SkipTlsVerify))
- token, err := oauth2Config.Exchange(providerCtx, code, oauth2.VerifierOption(verifier))
- if err != nil {
- slog.ErrorContext(ctx, "exchangeTokenInternal: token exchange failed", "tokenEndpoint", oauth2Config.Endpoint.TokenURL, "error", err)
- return nil, fmt.Errorf("failed to exchange authorization code: %w", err)
- }
-
- slog.DebugContext(ctx, "exchangeTokenInternal: token exchange successful", "hasAccessToken", token.AccessToken != "", "hasRefreshToken", token.RefreshToken != "")
- return token, nil
-}
-
-func (s *OidcService) fetchClaimsInternal(ctx context.Context, cfg *settings.OidcConfig, provider *oidc.Provider, token *oauth2.Token, idToken *oidc.IDToken) (map[string]any, error) {
- providerCtx := oidc.ClientContext(ctx, s.getHttpClientInternal(cfg.SkipTlsVerify))
- var claims map[string]any
-
- if idToken != nil {
- if err := idToken.Claims(&claims); err != nil {
- slog.WarnContext(ctx, "fetchClaimsInternal: failed to extract claims from ID token", "error", err)
- } else {
- slog.DebugContext(ctx, "fetchClaimsInternal: extracted claims from ID token")
- }
- }
-
- var userInfoClaims map[string]any
- switch {
- case provider != nil:
- userInfo, err := provider.UserInfo(providerCtx, oauth2.StaticTokenSource(token))
- if err != nil {
- slog.DebugContext(ctx, "fetchClaimsInternal: userinfo endpoint call failed", "error", err)
- if claims != nil {
- return claims, nil
- }
- return nil, fmt.Errorf("failed to fetch userinfo: %w", err)
- }
- if claimsErr := userInfo.Claims(&userInfoClaims); claimsErr != nil {
- slog.WarnContext(ctx, "fetchClaimsInternal: failed to decode userinfo claims", "error", claimsErr)
- if claims != nil {
- return claims, nil
- }
- return nil, fmt.Errorf("failed to decode userinfo claims: %w", claimsErr)
- }
- slog.DebugContext(ctx, "fetchClaimsInternal: fetched userinfo claims successfully")
- case cfg.UserinfoEndpoint != "":
- manualClaims, err := s.fetchUserInfoClaimsInternal(providerCtx, cfg, token)
- if err != nil {
- slog.DebugContext(ctx, "fetchClaimsInternal: userinfo endpoint call failed", "error", err)
- if claims != nil {
- return claims, nil
- }
- return nil, fmt.Errorf("failed to fetch userinfo: %w", err)
- }
- userInfoClaims = manualClaims
- slog.DebugContext(ctx, "fetchClaimsInternal: fetched userinfo claims successfully")
- case claims != nil:
- return claims, nil
- default:
- return nil, errors.New("userinfo endpoint not configured")
- }
-
- if claims == nil {
- claims = make(map[string]any)
- }
- for k, v := range userInfoClaims {
- if _, exists := claims[k]; !exists {
- claims[k] = v
- }
- }
-
- return claims, nil
+ span.SetAttributes(attribute.String("arcane.oidc.effective_issuer", altIssuer))
+ slog.DebugContext(ctx, "getOrDiscoverProvider: provider cached", "issuer", key.issuer, "effectiveIssuer", altIssuer, "skipTls", key.skipTLS)
+ return provider, nil
}
-func (s *OidcService) fetchUserInfoClaimsInternal(ctx context.Context, cfg *settings.OidcConfig, token *oauth2.Token) (map[string]any, error) {
- if cfg.UserinfoEndpoint == "" {
- return nil, errors.New("userinfo endpoint not configured")
- }
- if token == nil || token.AccessToken == "" {
- return nil, errors.New("missing access token for userinfo request")
- }
+func (s *OidcService) HandleCallback(ctx context.Context, code, state, storedState, origin, mobileRedirectURI string) (_ *authtypes.OidcUserInfo, _ *authtypes.OidcTokenResponse, err error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "oidc.callback")
+ defer func() { tracing.End(span, err) }()
+ slog.DebugContext(ctx, "HandleCallback: processing callback", "codePresent", code != "", "statePresent", state != "")
- request, err := http.NewRequestWithContext(ctx, http.MethodGet, cfg.UserinfoEndpoint, http.NoBody)
- if err != nil {
- return nil, err
+ var stateData OidcState
+ stateJSON, err := base64.URLEncoding.DecodeString(storedState)
+ if err == nil {
+ err = json.Unmarshal(stateJSON, &stateData)
}
-
- tokenType := cmp.Or(token.TokenType, "Bearer")
- request.Header.Set("Authorization", fmt.Sprintf("%s %s", tokenType, token.AccessToken))
- request.Header.Set("Accept", "application/json")
-
- client := s.getHttpClientInternal(cfg.SkipTlsVerify)
- resp, err := client.Do(request)
if err != nil {
- return nil, err
- }
- defer func() { _ = resp.Body.Close() }()
-
- if resp.StatusCode < 200 || resp.StatusCode >= 300 {
- return nil, fmt.Errorf("userinfo endpoint returned status %d", resp.StatusCode)
+ slog.ErrorContext(ctx, "HandleCallback: failed to decode stored state", "error", err)
+ return nil, nil, fmt.Errorf("invalid state parameter: %w", err)
}
-
- var claims map[string]any
- if unmarshalReadErr := json.UnmarshalRead(resp.Body, &claims); unmarshalReadErr != nil {
- return nil, unmarshalReadErr
+ if state != stateData.State {
+ slog.ErrorContext(ctx, "HandleCallback: state mismatch", "receivedLen", len(state), "expectedLen", len(stateData.State))
+ return nil, nil, errors.New("state parameter mismatch")
}
-
- return claims, nil
-}
-
-func (s *OidcService) HandleCallback(ctx context.Context, code, state, storedState, origin, mobileRedirectURI string) (*authtypes.OidcUserInfo, *authtypes.OidcTokenResponse, error) {
- slog.DebugContext(ctx, "HandleCallback: processing callback", "codePresent", code != "", "statePresent", state != "")
-
- stateData, err := s.validateStateInternal(ctx, state, storedState)
- if err != nil {
- return nil, nil, err
+ if time.Since(stateData.CreatedAt) > 10*time.Minute {
+ slog.ErrorContext(ctx, "HandleCallback: state expired", "age", time.Since(stateData.CreatedAt))
+ return nil, nil, errors.New("authentication state has expired")
}
- cfg, err := s.getEffectiveConfigInternal(ctx)
+ cfg, err := s.effectiveConfig(ctx)
if err != nil {
slog.ErrorContext(ctx, "HandleCallback: failed to get OIDC config", "error", err)
return nil, nil, err
}
+ span.SetAttributes(configAttributes(cfg)...)
var provider *oidc.Provider
- if !s.hasManualEndpointsInternal(cfg) {
- provider, err = s.getOrDiscoverProviderInternal(ctx, cfg)
+ if !hasManualEndpoints(cfg) {
+ provider, err = s.getOrDiscoverProvider(ctx, cfg)
if err != nil {
return nil, nil, err
}
}
- token, err := s.exchangeTokenInternal(ctx, cfg, provider, code, stateData.CodeVerifier, origin, mobileRedirectURI)
+ oauth2Config, err := s.buildOAuth2Config(cfg, provider, origin, mobileRedirectURI)
if err != nil {
return nil, nil, err
}
-
- idToken, rawIDToken, err := s.verifyIDTokenInternal(ctx, provider, cfg, token, stateData.Nonce)
+ token, err := oauth2Config.Exchange(oidc.ClientContext(ctx, s.httpClientFor(cfg.SkipTlsVerify)), code, oauth2.VerifierOption(stateData.CodeVerifier))
if err != nil {
- return nil, nil, err
+ slog.ErrorContext(ctx, "HandleCallback: token exchange failed", "tokenEndpoint", oauth2Config.Endpoint.TokenURL, "error", err)
+ return nil, nil, fmt.Errorf("failed to exchange authorization code: %w", err)
}
+ slog.DebugContext(ctx, "HandleCallback: token exchange successful", "hasAccessToken", token.AccessToken != "", "hasRefreshToken", token.RefreshToken != "")
- return s.buildUserInfoInternal(ctx, provider, cfg, token, idToken, rawIDToken)
-}
-
-func (s *OidcService) validateStateInternal(ctx context.Context, state, storedState string) (*OidcState, error) {
- stateData, err := s.decodeStateInternal(ctx, storedState)
+ idToken, rawIDToken, err := s.verifyIDToken(ctx, provider, cfg, token, stateData.Nonce)
if err != nil {
- slog.ErrorContext(ctx, "HandleCallback: failed to decode stored state", "error", err)
- return nil, fmt.Errorf("invalid state parameter: %w", err)
- }
-
- if state != stateData.State {
- slog.ErrorContext(ctx, "HandleCallback: state mismatch", "receivedLen", len(state), "expectedLen", len(stateData.State))
- return nil, errors.New("state parameter mismatch")
+ return nil, nil, err
}
- if time.Since(stateData.CreatedAt) > 10*time.Minute {
- slog.ErrorContext(ctx, "HandleCallback: state expired", "age", time.Since(stateData.CreatedAt))
- return nil, errors.New("authentication state has expired")
- }
- return stateData, nil
+ return s.buildUserInfo(ctx, provider, cfg, token, idToken, rawIDToken)
}
-func (s *OidcService) verifyIDTokenInternal(ctx context.Context, provider *oidc.Provider, cfg *settings.OidcConfig, token *oauth2.Token, nonce string) (*oidc.IDToken, string, error) {
- var rawIDToken string
- if idTokenValue := token.Extra("id_token"); idTokenValue != nil {
- if idTokenStr, ok := idTokenValue.(string); ok {
- rawIDToken = idTokenStr
- }
- }
-
+func (s *OidcService) verifyIDToken(ctx context.Context, provider *oidc.Provider, cfg *settings.OidcConfig, token *oauth2.Token, nonce string) (*oidc.IDToken, string, error) {
+ rawIDToken, _ := token.Extra("id_token").(string)
if rawIDToken == "" {
slog.WarnContext(ctx, "HandleCallback: no ID token in response (non-compliant OIDC response)")
return nil, "", nil
}
- verifierConfig := &oidc.Config{
- ClientID: cfg.ClientID,
- SupportedSigningAlgs: oidcjwk.SupportedSigningAlgs(),
- }
-
- var issuer, jwksURL string
+ issuer, jwksURL := cfg.IssuerURL, cfg.JwksURI
if provider != nil {
var meta struct {
Issuer string `json:"issuer"`
@@ -525,21 +374,24 @@ func (s *OidcService) verifyIDTokenInternal(ctx context.Context, provider *oidc.
return nil, "", fmt.Errorf("failed to read provider metadata: %w", err)
}
issuer, jwksURL = meta.Issuer, meta.JWKSURL
- } else {
- issuer, jwksURL = cfg.IssuerURL, cfg.JwksURI
}
if jwksURL == "" {
return nil, "", errors.New("jwks URI must be configured when using manual OIDC endpoints")
}
+ if s.keySetManager == nil {
+ return nil, "", errors.New("failed to configure provider JWK set: JWK set manager is not configured")
+ }
- keySet, err := s.keySetInternal(ctx, jwksURL, cfg.SkipTlsVerify)
+ client := s.httpClientFor(cfg.SkipTlsVerify)
+ keySet, err := s.keySetManager.KeySet(context.WithoutCancel(ctx), client, jwksURL)
if err != nil {
return nil, "", fmt.Errorf("failed to configure provider JWK set: %w", err)
}
- providerCtx := oidc.ClientContext(ctx, s.getHttpClientInternal(cfg.SkipTlsVerify))
- verifier := oidc.NewVerifier(issuer, keySet, verifierConfig)
-
- idToken, err := verifier.Verify(providerCtx, rawIDToken)
+ verifier := oidc.NewVerifier(issuer, keySet, &oidc.Config{
+ ClientID: cfg.ClientID,
+ SupportedSigningAlgs: oidcjwk.SupportedSigningAlgs(),
+ })
+ idToken, err := verifier.Verify(oidc.ClientContext(ctx, client), rawIDToken)
if err != nil {
slog.ErrorContext(ctx, "HandleCallback: ID token verification failed", "error", err)
return nil, "", fmt.Errorf("failed to verify ID token: %w", err)
@@ -563,24 +415,63 @@ func (s *OidcService) verifyIDTokenInternal(ctx context.Context, provider *oidc.
return idToken, rawIDToken, nil
}
-func (
- s *OidcService,
-) buildUserInfoInternal(
- ctx context.Context,
- provider *oidc.Provider,
- cfg *settings.OidcConfig,
- token *oauth2.Token,
- idToken *oidc.IDToken,
- rawIDToken string,
-) (
- *authtypes.OidcUserInfo,
- *authtypes.OidcTokenResponse,
- error,
-) {
- claims, err := s.fetchClaimsInternal(ctx, cfg, provider, token, idToken)
- if err != nil {
- slog.ErrorContext(ctx, "HandleCallback: failed to fetch claims", "error", err)
- return nil, nil, fmt.Errorf("failed to fetch user claims: %w", err)
+func (s *OidcService) buildUserInfo(
+ ctx context.Context, provider *oidc.Provider, cfg *settings.OidcConfig, token *oauth2.Token, idToken *oidc.IDToken, rawIDToken string,
+) (*authtypes.OidcUserInfo, *authtypes.OidcTokenResponse, error) {
+ var claims map[string]any
+ if idToken != nil {
+ if claimsErr := idToken.Claims(&claims); claimsErr != nil {
+ slog.WarnContext(ctx, "HandleCallback: failed to extract claims from ID token", "error", claimsErr)
+ }
+ }
+
+ // Userinfo claims fill gaps; ID token claims win on conflict.
+ var userInfoClaims map[string]any
+ var userInfoErr error
+ client := s.httpClientFor(cfg.SkipTlsVerify)
+ switch {
+ case provider != nil:
+ userInfo, fetchErr := provider.UserInfo(oidc.ClientContext(ctx, client), oauth2.StaticTokenSource(token))
+ if fetchErr != nil {
+ userInfoErr = fmt.Errorf("failed to fetch userinfo: %w", fetchErr)
+ } else if decodeErr := userInfo.Claims(&userInfoClaims); decodeErr != nil {
+ userInfoErr = fmt.Errorf("failed to decode userinfo claims: %w", decodeErr)
+ }
+ case cfg.UserinfoEndpoint != "" && token.AccessToken == "":
+ userInfoErr = errors.New("failed to fetch userinfo: missing access token for userinfo request")
+ case cfg.UserinfoEndpoint != "":
+ request, requestErr := http.NewRequestWithContext(ctx, http.MethodGet, cfg.UserinfoEndpoint, http.NoBody)
+ if requestErr != nil {
+ userInfoErr = fmt.Errorf("failed to fetch userinfo: %w", requestErr)
+ break
+ }
+ request.Header.Set("Authorization", cmp.Or(token.TokenType, "Bearer")+" "+token.AccessToken)
+ request.Header.Set("Accept", "application/json")
+ resp, doErr := client.Do(request)
+ if doErr != nil {
+ userInfoErr = fmt.Errorf("failed to fetch userinfo: %w", doErr)
+ break
+ }
+ defer func() { _ = resp.Body.Close() }()
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ userInfoErr = fmt.Errorf("failed to fetch userinfo: userinfo endpoint returned status %d", resp.StatusCode)
+ } else if decodeErr := json.UnmarshalRead(resp.Body, &userInfoClaims); decodeErr != nil {
+ userInfoErr = fmt.Errorf("failed to fetch userinfo: %w", decodeErr)
+ }
+ case claims == nil:
+ userInfoErr = errors.New("userinfo endpoint not configured")
+ }
+ if userInfoErr != nil {
+ slog.DebugContext(ctx, "HandleCallback: userinfo request failed", "error", userInfoErr)
+ if claims == nil {
+ slog.ErrorContext(ctx, "HandleCallback: failed to fetch claims", "error", userInfoErr)
+ return nil, nil, fmt.Errorf("failed to fetch user claims: %w", userInfoErr)
+ }
+ userInfoClaims = nil
+ }
+ if merged := maps.Clone(userInfoClaims); merged != nil {
+ maps.Copy(merged, claims)
+ claims = merged
}
subject := jwtclaims.GetStringClaim(claims, "sub")
@@ -603,81 +494,102 @@ func (
Extra: claims,
}
- tokenType := cmp.Or(token.TokenType, "Bearer")
-
tokenResp := &authtypes.OidcTokenResponse{
AccessToken: token.AccessToken,
- TokenType: tokenType,
+ TokenType: cmp.Or(token.TokenType, "Bearer"),
RefreshToken: token.RefreshToken,
IDToken: rawIDToken,
}
if !token.Expiry.IsZero() {
- expiresIn := max(int(time.Until(token.Expiry).Seconds()), 0)
- tokenResp.ExpiresIn = expiresIn
+ tokenResp.ExpiresIn = max(int(time.Until(token.Expiry).Seconds()), 0)
}
+ trace.SpanFromContext(ctx).SetAttributes(attribute.Int("arcane.oidc.groups", len(userInfoDto.Groups)), attribute.Int("arcane.oidc.roles", len(userInfoDto.Roles)))
slog.InfoContext(ctx, "HandleCallback: authentication successful", "subject", userInfoDto.Subject, "email", userInfoDto.Email)
return &userInfoDto, tokenResp, nil
}
-func (s *OidcService) decodeStateInternal(ctx context.Context, encodedState string) (*OidcState, error) {
- stateJSON, err := base64.URLEncoding.DecodeString(encodedState)
+// postForm sends an OAuth form POST and decodes the JSON response body, along with the HTTP status code.
+func postForm(ctx context.Context, client *http.Client, endpoint string, values url.Values) (map[string]any, int, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(values.Encode()))
if err != nil {
- slog.ErrorContext(ctx, "decodeStateInternal: failed to decode base64 state", "error", err)
- return nil, err
+ return nil, 0, err
}
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ req.Header.Set("Accept", "application/json")
- var stateData OidcState
- if unmarshalErr := json.Unmarshal(stateJSON, &stateData); unmarshalErr != nil {
- slog.ErrorContext(ctx, "decodeStateInternal: failed to unmarshal state JSON", "error", unmarshalErr)
- return nil, unmarshalErr
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, 0, err
}
+ defer func() { _ = resp.Body.Close() }()
- return &stateData, nil
+ var body map[string]any
+ decodeErr := json.UnmarshalRead(resp.Body, &body)
+ return body, resp.StatusCode, decodeErr
}
// InitiateDeviceAuth initiates the OIDC device authorization flow.
-func (s *OidcService) InitiateDeviceAuth(ctx context.Context) (*authtypes.OidcDeviceAuthResponse, error) {
- cfg, err := s.getEffectiveConfigInternal(ctx)
- if err != nil {
- slog.ErrorContext(ctx, "InitiateDeviceAuth: failed to get OIDC config", "error", err)
- return nil, err
- }
+func (s *OidcService) InitiateDeviceAuth(ctx context.Context) (_ *authtypes.OidcDeviceAuthResponse, err error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "oidc.device_authorize")
+ defer func() { tracing.End(span, err) }()
- deviceEndpoint, err := s.getDeviceAuthorizationEndpointInternal(ctx, cfg)
+ cfg, err := s.effectiveConfig(ctx)
if err != nil {
- slog.ErrorContext(ctx, "InitiateDeviceAuth: failed to get device endpoint", "error", err)
+ slog.ErrorContext(ctx, "InitiateDeviceAuth: failed to get OIDC config", "error", err)
return nil, err
}
+ span.SetAttributes(configAttributes(cfg)...)
- scopes := strings.Fields(cfg.Scopes)
- if len(scopes) == 0 {
- scopes = []string{"email", "profile"}
+ deviceEndpoint := cfg.DeviceAuthorizationEndpoint
+ if deviceEndpoint == "" {
+ provider, discoverErr := s.getOrDiscoverProvider(ctx, cfg)
+ if discoverErr != nil {
+ return nil, fmt.Errorf("failed to discover provider: %w", discoverErr)
+ }
+ var claims struct {
+ DeviceAuthorizationEndpoint string `json:"device_authorization_endpoint"`
+ }
+ if claimsErr := provider.Claims(&claims); claimsErr != nil {
+ return nil, fmt.Errorf("failed to get device authorization endpoint from provider: %w", claimsErr)
+ }
+ if claims.DeviceAuthorizationEndpoint == "" {
+ return nil, errors.New("device authorization endpoint not found in provider configuration")
+ }
+ deviceEndpoint = claims.DeviceAuthorizationEndpoint
}
- scopes = s.ensureOpenIDScopeInternal(scopes)
values := url.Values{}
values.Set("client_id", cfg.ClientID)
- values.Set("scope", strings.Join(scopes, " "))
+ values.Set("scope", strings.Join(requestedScopes(cfg), " "))
if cfg.ClientSecret != "" {
values.Set("client_secret", cfg.ClientSecret)
}
- respData, err := s.makeDeviceAuthRequestInternal(ctx, deviceEndpoint, values, cfg.SkipTlsVerify)
- if err != nil {
- return nil, err
+ respData, status, err := postForm(ctx, s.httpClientFor(cfg.SkipTlsVerify), deviceEndpoint, values)
+ switch {
+ case status == 0:
+ slog.ErrorContext(ctx, "InitiateDeviceAuth: request failed", "error", err)
+ return nil, fmt.Errorf("device authorization request failed: %w", err)
+ case status < 200 || status >= 300:
+ if errMsg, ok := respData["error"].(string); ok && err == nil {
+ return nil, fmt.Errorf("device authorization failed: %s", errMsg)
+ }
+ return nil, fmt.Errorf("device authorization endpoint returned status %d", status)
+ case err != nil:
+ return nil, fmt.Errorf("failed to decode device authorization response: %w", err)
}
- deviceCode, ok := respData["device_code"].(string)
- if !ok || deviceCode == "" {
+ deviceCode, _ := respData["device_code"].(string)
+ if deviceCode == "" {
return nil, errors.New("invalid device_code in response")
}
- userCode, ok := respData["user_code"].(string)
- if !ok || userCode == "" {
+ userCode, _ := respData["user_code"].(string)
+ if userCode == "" {
return nil, errors.New("invalid user_code in response")
}
- verificationUri, ok := respData["verification_uri"].(string)
- if !ok || verificationUri == "" {
+ verificationUri, _ := respData["verification_uri"].(string)
+ if verificationUri == "" {
return nil, errors.New("invalid verification_uri in response")
}
expiresIn, ok := respData["expires_in"].(float64)
@@ -686,200 +598,100 @@ func (s *OidcService) InitiateDeviceAuth(ctx context.Context) (*authtypes.OidcDe
}
response := &authtypes.OidcDeviceAuthResponse{
- DeviceCode: deviceCode,
- UserCode: userCode,
- VerificationUri: verificationUri,
- ExpiresIn: int(expiresIn),
- }
-
- if uri, localOk := respData["verification_uri_complete"].(string); localOk {
- response.VerificationUriComplete = uri
- }
- if interval, localOk2 := respData["interval"].(float64); localOk2 {
- response.Interval = int(interval)
- } else {
- response.Interval = 5
+ DeviceCode: deviceCode,
+ UserCode: userCode,
+ VerificationUri: verificationUri,
+ VerificationUriComplete: kit.As(respData["verification_uri_complete"], ""),
+ ExpiresIn: int(expiresIn),
+ Interval: int(kit.As(respData["interval"], float64(5))),
}
slog.DebugContext(ctx, "InitiateDeviceAuth: device authorization initiated", "userCode", response.UserCode, "expiresIn", response.ExpiresIn)
return response, nil
}
-// getDeviceAuthorizationEndpointInternal discovers or returns the configured device authorization endpoint.
-func (s *OidcService) getDeviceAuthorizationEndpointInternal(ctx context.Context, cfg *settings.OidcConfig) (string, error) {
- if cfg.DeviceAuthorizationEndpoint != "" {
- return cfg.DeviceAuthorizationEndpoint, nil
- }
-
- provider, err := s.getOrDiscoverProviderInternal(ctx, cfg)
- if err != nil {
- return "", fmt.Errorf("failed to discover provider: %w", err)
- }
-
- var claims struct {
- DeviceAuthorizationEndpoint string `json:"device_authorization_endpoint"`
- }
- if claimsErr := provider.Claims(&claims); claimsErr != nil {
- return "", fmt.Errorf("failed to get device authorization endpoint from provider: %w", claimsErr)
- }
-
- if claims.DeviceAuthorizationEndpoint == "" {
- return "", errors.New("device authorization endpoint not found in provider configuration")
- }
-
- return claims.DeviceAuthorizationEndpoint, nil
-}
-
-// makeDeviceAuthRequestInternal makes a device authorization request.
-func (s *OidcService) makeDeviceAuthRequestInternal(ctx context.Context, endpoint string, params url.Values, skipTls bool) (map[string]any, error) {
- body := params.Encode()
-
- req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(body))
- if err != nil {
- return nil, err
- }
- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- req.Header.Set("Accept", "application/json")
-
- client := s.getHttpClientInternal(skipTls)
- resp, err := client.Do(req)
- if err != nil {
- slog.ErrorContext(ctx, "makeDeviceAuthRequestInternal: request failed", "error", err)
- return nil, fmt.Errorf("device authorization request failed: %w", err)
- }
- defer func() { _ = resp.Body.Close() }()
-
- if resp.StatusCode < 200 || resp.StatusCode >= 300 {
- var errorResp map[string]any
- if unmarshalReadErr := json.UnmarshalRead(resp.Body, &errorResp); unmarshalReadErr == nil {
- if errMsg, ok := errorResp["error"].(string); ok {
- return nil, fmt.Errorf("device authorization failed: %s", errMsg)
- }
- }
- return nil, fmt.Errorf("device authorization endpoint returned status %d", resp.StatusCode)
- }
-
- var respData map[string]any
- if decodeDeviceAuthorizationErr := json.UnmarshalRead(resp.Body, &respData); decodeDeviceAuthorizationErr != nil {
- return nil, fmt.Errorf("failed to decode device authorization response: %w", decodeDeviceAuthorizationErr)
- }
-
- return respData, nil
-}
-
// ExchangeDeviceToken exchanges a device code for tokens.
-func (s *OidcService) ExchangeDeviceToken(ctx context.Context, deviceCode string) (*authtypes.OidcUserInfo, *authtypes.OidcTokenResponse, error) {
- cfg, err := s.getEffectiveConfigInternal(ctx)
+func (s *OidcService) ExchangeDeviceToken(ctx context.Context, deviceCode string) (_ *authtypes.OidcUserInfo, _ *authtypes.OidcTokenResponse, err error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "oidc.device_token")
+ defer func() {
+ // Pending and slow_down are normal polling outcomes, not failures.
+ pending := err != nil && (err.Error() == "authorization_pending" || err.Error() == "slow_down")
+ tracing.End(span, kit.Ternary(pending, nil, err))
+ }()
+
+ cfg, err := s.effectiveConfig(ctx)
if err != nil {
slog.ErrorContext(ctx, "ExchangeDeviceToken: failed to get OIDC config", "error", err)
return nil, nil, err
}
+ span.SetAttributes(configAttributes(cfg)...)
- var tokenEndpoint string
- if cfg.TokenEndpoint != "" {
- tokenEndpoint = cfg.TokenEndpoint
- } else {
- provider, getOrDiscoverProviderErr := s.getOrDiscoverProviderInternal(ctx, cfg)
- if getOrDiscoverProviderErr != nil {
- return nil, nil, getOrDiscoverProviderErr
+ var provider *oidc.Provider
+ if !hasManualEndpoints(cfg) {
+ if provider, err = s.getOrDiscoverProvider(ctx, cfg); err != nil {
+ return nil, nil, err
}
- tokenEndpoint = provider.Endpoint().TokenURL
}
-
- params := map[string]string{
- "grant_type": "urn:ietf:params:oauth:grant-type:device_code",
- "device_code": deviceCode,
- "client_id": cfg.ClientID,
+ tokenEndpoint := cfg.TokenEndpoint
+ if tokenEndpoint == "" {
+ endpointProvider := provider
+ if endpointProvider == nil {
+ if endpointProvider, err = s.getOrDiscoverProvider(ctx, cfg); err != nil {
+ return nil, nil, err
+ }
+ }
+ tokenEndpoint = endpointProvider.Endpoint().TokenURL
}
+
+ values := url.Values{}
+ values.Set("grant_type", "urn:ietf:params:oauth:grant-type:device_code")
+ values.Set("device_code", deviceCode)
+ values.Set("client_id", cfg.ClientID)
if cfg.ClientSecret != "" {
- params["client_secret"] = cfg.ClientSecret
+ values.Set("client_secret", cfg.ClientSecret)
}
- tokenResp, err := s.makeTokenRequestInternal(ctx, tokenEndpoint, params, cfg.SkipTlsVerify)
- if err != nil {
- return nil, nil, err
+ tokenResp, status, err := postForm(ctx, s.httpClientFor(cfg.SkipTlsVerify), tokenEndpoint, values)
+ if status == 0 {
+ slog.ErrorContext(ctx, "ExchangeDeviceToken: token request failed", "error", err)
+ return nil, nil, fmt.Errorf("token request failed: %w", err)
}
-
- var provider *oidc.Provider
- if !s.hasManualEndpointsInternal(cfg) {
- provider, err = s.getOrDiscoverProviderInternal(ctx, cfg)
- if err != nil {
- return nil, nil, err
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to decode token response: %w", err)
+ }
+ if status < 200 || status >= 300 {
+ errMsg, ok := tokenResp["error"].(string)
+ switch {
+ case !ok:
+ return nil, nil, fmt.Errorf("token endpoint returned status %d", status)
+ case errMsg == "authorization_pending", errMsg == "slow_down", errMsg == "expired_token", errMsg == "access_denied":
+ return nil, nil, errors.New(errMsg)
+ default:
+ return nil, nil, fmt.Errorf("token exchange failed: %s", errMsg)
}
}
- accessToken, ok := tokenResp["access_token"].(string)
- if !ok || accessToken == "" {
+ accessToken, _ := tokenResp["access_token"].(string)
+ if accessToken == "" {
return nil, nil, errors.New("invalid access_token in response")
}
token := &oauth2.Token{
- AccessToken: accessToken,
- TokenType: kit.As(tokenResp["token_type"], "Bearer"),
- }
- if refreshToken, localOk := tokenResp["refresh_token"].(string); localOk {
- token.RefreshToken = refreshToken
+ AccessToken: accessToken,
+ TokenType: kit.As(tokenResp["token_type"], "Bearer"),
+ RefreshToken: kit.As(tokenResp["refresh_token"], ""),
}
- if expiresIn, localOk2 := tokenResp["expires_in"].(float64); localOk2 {
+ if expiresIn, ok := tokenResp["expires_in"].(float64); ok {
token.Expiry = time.Now().Add(time.Duration(expiresIn) * time.Second)
}
- if idToken, localOk3 := tokenResp["id_token"].(string); localOk3 {
+ if idToken, ok := tokenResp["id_token"].(string); ok {
token = token.WithExtra(map[string]any{"id_token": idToken})
}
- idToken, rawIDToken, err := s.verifyIDTokenInternal(ctx, provider, cfg, token, "")
+ idToken, rawIDToken, err := s.verifyIDToken(ctx, provider, cfg, token, "")
if err != nil {
return nil, nil, err
}
- return s.buildUserInfoInternal(ctx, provider, cfg, token, idToken, rawIDToken)
-}
-
-// makeTokenRequestInternal makes a token exchange request.
-func (s *OidcService) makeTokenRequestInternal(ctx context.Context, endpoint string, params map[string]string, skipTls bool) (map[string]any, error) {
- values := url.Values{}
- for k, v := range params {
- values.Set(k, v)
- }
- body := values.Encode()
-
- req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(body))
- if err != nil {
- return nil, err
- }
- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- req.Header.Set("Accept", "application/json")
-
- client := s.getHttpClientInternal(skipTls)
- resp, err := client.Do(req)
- if err != nil {
- slog.ErrorContext(ctx, "makeTokenRequestInternal: request failed", "error", err)
- return nil, fmt.Errorf("token request failed: %w", err)
- }
- defer func() { _ = resp.Body.Close() }()
-
- var tokenResp map[string]any
- if unmarshalReadErr := json.UnmarshalRead(resp.Body, &tokenResp); unmarshalReadErr != nil {
- return nil, fmt.Errorf("failed to decode token response: %w", unmarshalReadErr)
- }
-
- if resp.StatusCode < 200 || resp.StatusCode >= 300 {
- if errMsg, ok := tokenResp["error"].(string); ok {
- switch errMsg {
- case "authorization_pending":
- return nil, errors.New("authorization_pending")
- case "slow_down":
- return nil, errors.New("slow_down")
- case "expired_token":
- return nil, errors.New("expired_token")
- case "access_denied":
- return nil, errors.New("access_denied")
- default:
- return nil, fmt.Errorf("token exchange failed: %s", errMsg)
- }
- }
- return nil, fmt.Errorf("token endpoint returned status %d", resp.StatusCode)
- }
-
- return tokenResp, nil
+ return s.buildUserInfo(ctx, provider, cfg, token, idToken, rawIDToken)
}
diff --git a/backend/internal/oidc/service_test.go b/backend/internal/oidc/service_test.go
index 0024dcce3..8289428c4 100644
--- a/backend/internal/oidc/service_test.go
+++ b/backend/internal/oidc/service_test.go
@@ -67,7 +67,7 @@ func TestGetMobileRedirectAllowlistTrimsWhitespace(t *testing.T) {
func TestGetMobileRedirectAllowlistUsesSettings(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
+ settingsService, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err,
"settings.NewSettingsService: %v", err)
@@ -108,7 +108,7 @@ func setupSettingsTestDB(t *testing.T) *database.DB {
return &database.DB{DB: db}
}
-func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
+func newSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
svc, err := settings.NewSettingsService(ctx, db)
if err == nil {
diff --git a/backend/internal/registry/children/browse/service.go b/backend/internal/registry/children/browse/service.go
index 46476ed72..d759a256a 100644
--- a/backend/internal/registry/children/browse/service.go
+++ b/backend/internal/registry/children/browse/service.go
@@ -17,11 +17,15 @@ import (
"github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
"golang.org/x/sync/errgroup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/registryauth"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
const tagDetailsConcurrency = 4
@@ -52,7 +56,13 @@ func NewService(
// ListRepositories lists the repositories of a configured registry through the
// catalog API, limited to the namespace of the registry URL.
-func (s *Service) ListRepositories(ctx context.Context, id string, params pagination.QueryParams) ([]containerregistry.Repository, pagination.Response, error) {
+func (s *Service) ListRepositories(ctx context.Context, id string, params pagination.QueryParams) (listed []containerregistry.Repository, paging pagination.Response, listErr error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "registry.list_repositories", trace.WithAttributes(attribute.String("arcane.registry.id", id)))
+ defer func() {
+ span.SetAttributes(attribute.Int("arcane.result.count", len(listed)))
+ tracing.End(span, listErr)
+ }()
+
lookupCtx, cancel := s.lookupContext(ctx)
defer cancel()
@@ -82,7 +92,16 @@ func (s *Service) ListRepositories(ctx context.Context, id string, params pagina
// ListRepositoryTags lists the tags of a repository. Manifest details are only
// fetched for the requested page, and a failure is recorded on the tag itself
// so one unreadable manifest does not hide the rest of the page.
-func (s *Service) ListRepositoryTags(ctx context.Context, id, repository string, params pagination.QueryParams) ([]containerregistry.RepositoryTag, pagination.Response, error) {
+func (s *Service) ListRepositoryTags(ctx context.Context, id, repository string, params pagination.QueryParams) (listed []containerregistry.RepositoryTag, paging pagination.Response, listErr error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "registry.list_tags", trace.WithAttributes(
+ attribute.String("arcane.registry.id", id),
+ attribute.String("arcane.registry.repository", repository),
+ ))
+ defer func() {
+ span.SetAttributes(attribute.Int("arcane.result.count", len(listed)))
+ tracing.End(span, listErr)
+ }()
+
lookupCtx, cancel := s.lookupContext(ctx)
defer cancel()
@@ -121,7 +140,17 @@ func (s *Service) ListRepositoryTags(ctx context.Context, id, repository string,
// DeleteRepositoryTag deletes the manifest a tag points to. Registries delete
// manifests by digest, so every tag sharing that digest is removed too.
-func (s *Service) DeleteRepositoryTag(ctx context.Context, id, repository, tag string) (string, error) {
+func (s *Service) DeleteRepositoryTag(ctx context.Context, id, repository, tag string) (deleted string, opErr error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "registry.delete_tag", trace.WithAttributes(
+ attribute.String("arcane.registry.id", id),
+ attribute.String("arcane.registry.repository", repository),
+ attribute.String("arcane.image.tag", tag),
+ ))
+ defer func() {
+ span.SetAttributes(attribute.String("arcane.image.digest", deleted))
+ tracing.End(span, opErr)
+ }()
+
lookupCtx, cancel := s.lookupContext(ctx)
defer cancel()
diff --git a/backend/internal/registry/service.go b/backend/internal/registry/service.go
index 65bfef964..608720e39 100644
--- a/backend/internal/registry/service.go
+++ b/backend/internal/registry/service.go
@@ -36,6 +36,10 @@ import (
"go.getarcane.app/updater/digest"
"go.getarcane.app/updater/refs"
"go.getarcane.app/updater/registry"
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
"golang.org/x/sync/singleflight"
"gorm.io/gorm"
@@ -48,6 +52,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation"
)
@@ -110,6 +115,10 @@ func NewContainerRegistryService(
if len(distributionHTTPClients) > 0 && distributionHTTPClients[0] != nil {
distributionHTTPClient = distributionHTTPClients[0]
}
+ // Trace a copy so the injected client keeps its raw transport.
+ traced := *distributionHTTPClient
+ traced.Transport = otelhttp.NewTransport(distributionHTTPClient.Transport)
+ distributionHTTPClient = &traced
service := &ContainerRegistryService{
db: db,
dockerClient: dockerClient,
@@ -795,7 +804,11 @@ func registryDisplayNameInternal(registryHost, registryType string) string {
}
}
-func (s *ContainerRegistryService) TestRegistry(ctx context.Context, registryURL, username, token string) error {
+func (s *ContainerRegistryService) TestRegistry(ctx context.Context, registryURL, username, token string) (testErr error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "registry.test_credentials", trace.WithAttributes(
+ attribute.String("arcane.registry.host", registryauth.NormalizeRegistryForComparison(registryURL)),
+ ))
+ defer func() { tracing.End(span, testErr) }()
if strings.TrimSpace(username) == "" && strings.TrimSpace(token) == "" {
// No credentials configured — skip the credential test.
return nil
@@ -870,7 +883,14 @@ func (s *ContainerRegistryService) ImageDigest(ctx context.Context, imageRef str
return digestValue, nil
}
-func (s *ContainerRegistryService) InspectImageDigest(ctx context.Context, imageRef string, externalCreds []containerregistry.Credential) (*containerregistry.DigestResult, error) {
+func (s *ContainerRegistryService) InspectImageDigest(ctx context.Context, imageRef string, externalCreds []containerregistry.Credential) (inspected *containerregistry.DigestResult, digestErr error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "registry.inspect_digest", trace.WithAttributes(attribute.String("arcane.image.ref", imageRef)))
+ defer func() {
+ if inspected != nil && inspected.Digest != "" {
+ span.SetAttributes(attribute.String("arcane.image.digest", inspected.Digest))
+ }
+ tracing.End(span, digestErr)
+ }()
parts, err := refs.NormalizeReference(imageRef)
if err != nil {
return nil, common.Classify(common.ErrValidation, err)
@@ -1556,7 +1576,12 @@ func (s *ContainerRegistryService) fetchDigestFromRegistryInternal(ctx context.C
// ListImageTags discovers repository tags with the same credential precedence as
// digest checks. External credentials replace local credentials when supplied.
-func (s *ContainerRegistryService) ListImageTags(ctx context.Context, imageRef string, externalCreds []containerregistry.Credential) ([]string, error) {
+func (s *ContainerRegistryService) ListImageTags(ctx context.Context, imageRef string, externalCreds []containerregistry.Credential) (tagList []string, tagsErr error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "registry.list_tags", trace.WithAttributes(attribute.String("arcane.image.ref", imageRef)))
+ defer func() {
+ span.SetAttributes(attribute.Int("arcane.result.count", len(tagList)))
+ tracing.End(span, tagsErr)
+ }()
if refs.IsDigestPinnedReference(imageRef) || refs.IsImageIDLikeReference(imageRef) {
return nil, errors.New("cannot discover tags for an immutable image reference")
}
diff --git a/backend/internal/settings/handler.go b/backend/internal/settings/handler.go
index a16d49612..c978b53a6 100644
--- a/backend/internal/settings/handler.go
+++ b/backend/internal/settings/handler.go
@@ -1,11 +1,13 @@
package settings
import (
+ "cmp"
"context"
- "errors"
"fmt"
+ "maps"
"net/http"
"os"
+ "slices"
"strconv"
"strings"
@@ -22,6 +24,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/search"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/telemetry"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
@@ -75,110 +78,18 @@ type GetCategoriesOutput struct {
Body []category.Category
}
-// validateProjectsDirectoryValueInternal validates a projects directory value allowing:
-// - Unix absolute paths (/...)
-// - Windows drive paths (C:/..., C:\...)
-// - Mapping format "container:host" where container is absolute Unix or Windows path
-func validateProjectsDirectoryValueInternal(path string) error {
- switch {
- case projects.IsWindowsDrivePath(path):
- return nil
- case strings.Contains(path, ":"):
- parts := strings.SplitN(path, ":", 2)
- if len(parts) != 2 {
- return errors.New("projectsDirectory must be an absolute path or valid mapping format")
- }
- container := parts[0]
- if !strings.HasPrefix(container, "/") && !projects.IsWindowsDrivePath(container) {
- return errors.New("projectsDirectory mapping format: container path must be absolute")
- }
- return nil
- default:
- if !strings.HasPrefix(path, "/") {
- return errors.New("projectsDirectory must be an absolute path starting with '/'")
- }
- return nil
- }
-}
-
-// validateAbsoluteDirectoryPathInternal validates a plain absolute directory path allowing:
-// - Unix absolute paths (/...)
-// - Windows drive paths (C:/..., C:\...)
-func validateAbsoluteDirectoryPathInternal(path string) error {
- switch {
- case projects.IsWindowsDrivePath(path):
- return nil
- case strings.HasPrefix(path, "/"):
- return nil
- default:
- return errors.New("must be an absolute path")
- }
-}
-
-func filterSettingsCategoriesInternal(ps *authz.PermissionSet, categories []category.Category) []category.Category {
- if ps == nil {
- return []category.Category{}
- }
- filtered := make([]category.Category, 0, len(categories))
- for _, cat := range categories {
- if canAccessSettingsCategoryAtAnyScopeInternal(ps, cat.ID) {
- filtered = append(filtered, cat)
- }
- }
- return filtered
-}
-
-func canAccessSettingsCategoryAtAnyScopeInternal(ps *authz.PermissionSet, categoryID string) bool {
- if ps == nil {
- return false
- }
- if authz.CanAccessSettingsCategory(ps, categoryID, "") {
- return true
- }
- for envID := range ps.PerEnv {
- if authz.CanAccessSettingsCategory(ps, categoryID, envID) {
- return true
- }
- }
- return false
-}
-
-func (h *SettingsHandler) appendRuntimeSettingsInternal(settingsDto []settings.PublicSetting, includeAuthenticatedOnly bool) []settings.PublicSetting {
- if !includeAuthenticatedOnly {
- return settingsDto
- }
-
- uiConfigDisabled := false
- if h.cfg != nil {
- uiConfigDisabled = h.cfg.UIConfigurationDisabled
- }
- settingsDto = append(settingsDto, settings.PublicSetting{
- Key: "uiConfigDisabled",
- Value: strconv.FormatBool(uiConfigDisabled),
- Type: "boolean",
- })
-
- projectWorkspaceMaxFileSizeMB := 10
- volumeWorkspaceMaxFileSizeMB := 10
- if h.cfg != nil {
- projectWorkspaceMaxFileSizeMB = workspace.EffectiveMaxFileSizeMB(h.cfg.ProjectWorkspaceMaxFileSizeMB)
- volumeWorkspaceMaxFileSizeMB = workspace.EffectiveMaxFileSizeMB(h.cfg.VolumeWorkspaceMaxFileSizeMB)
- }
+func (h *SettingsHandler) appendRuntimeSettings(settingsDto []settings.PublicSetting, includeAuthenticatedOnly bool) []settings.PublicSetting {
+ relay := h.cfg != nil && !h.cfg.AgentMode
settingsDto = append(settingsDto,
- settings.PublicSetting{Key: projectWorkspaceMaxFileSizeSettingKey, Value: strconv.Itoa(projectWorkspaceMaxFileSizeMB), Type: "number"},
- settings.PublicSetting{Key: volumeWorkspaceMaxFileSizeSettingKey, Value: strconv.Itoa(volumeWorkspaceMaxFileSizeMB), Type: "number"},
+ settings.PublicSetting{Key: "frontendTracingEnabled", Value: strconv.FormatBool(relay && telemetry.Endpoint("traces") != ""), Type: "boolean"},
+ settings.PublicSetting{Key: "frontendMetricsEnabled", Value: strconv.FormatBool(relay && telemetry.Endpoint("metrics") != ""), Type: "boolean"},
+ settings.PublicSetting{Key: "frontendLogsEnabled", Value: strconv.FormatBool(relay && telemetry.Endpoint("logs") != ""), Type: "boolean"},
)
-
- backupVolumeName := "arcane-backups"
- if h.cfg != nil && strings.TrimSpace(h.cfg.BackupVolumeName) != "" {
- backupVolumeName = h.cfg.BackupVolumeName
+ if !includeAuthenticatedOnly {
+ return settingsDto
}
- settingsDto = append(settingsDto, settings.PublicSetting{
- Key: "backupVolumeName",
- Value: backupVolumeName,
- Type: "string",
- })
+ cfg := cmp.Or(h.cfg, &config.Config{})
var edgeCfg *edge.Config
if h.cfg != nil {
edgeCfg = &edge.Config{
@@ -188,15 +99,23 @@ func (h *SettingsHandler) appendRuntimeSettingsInternal(settingsDto []settings.P
}
}
_, edgeMTLSCAErr := edge.AvailableManagerMTLSCAPath(edgeCfg)
- settingsDto = append(settingsDto, settings.PublicSetting{
- Key: "edgeMTLSManagerCAAvailable",
- Value: strconv.FormatBool(edgeMTLSCAErr == nil),
- Type: "boolean",
- })
+ projectWorkspaceMaxFileSizeMB := workspace.EffectiveMaxFileSizeMB(cfg.ProjectWorkspaceMaxFileSizeMB)
+ volumeWorkspaceMaxFileSizeMB := workspace.EffectiveMaxFileSizeMB(cfg.VolumeWorkspaceMaxFileSizeMB)
+ settingsDto = append(settingsDto,
+ settings.PublicSetting{Key: "uiConfigDisabled", Value: strconv.FormatBool(cfg.UIConfigurationDisabled), Type: "boolean"},
+ settings.PublicSetting{Key: projectWorkspaceMaxFileSizeSettingKey, Value: strconv.Itoa(projectWorkspaceMaxFileSizeMB), Type: "number"},
+ settings.PublicSetting{Key: volumeWorkspaceMaxFileSizeSettingKey, Value: strconv.Itoa(volumeWorkspaceMaxFileSizeMB), Type: "number"},
+ settings.PublicSetting{
+ Key: "backupVolumeName",
+ Value: kit.Ternary(strings.TrimSpace(cfg.BackupVolumeName) != "", cfg.BackupVolumeName, "arcane-backups"),
+ Type: "string",
+ },
+ settings.PublicSetting{Key: "edgeMTLSManagerCAAvailable", Value: strconv.FormatBool(edgeMTLSCAErr == nil), Type: "boolean"},
+ )
if h.settingsService != nil {
- cfg := h.settingsService.GetSettingsConfig()
- depotConfigured := strings.TrimSpace(cfg.DepotProjectId.Value) != "" && strings.TrimSpace(cfg.DepotToken.Value) != ""
+ settingsCfg := h.settingsService.GetSettingsConfig()
+ depotConfigured := strings.TrimSpace(settingsCfg.DepotProjectId.Value) != "" && strings.TrimSpace(settingsCfg.DepotToken.Value) != ""
settingsDto = append(settingsDto,
settings.PublicSetting{Key: "depotConfigured", Value: strconv.FormatBool(depotConfigured), Type: "boolean"},
settings.PublicSetting{Key: "enableGravatarEnvForced", Value: strconv.FormatBool(h.settingsService.IsEnvOverrideActive("enableGravatar")), Type: "boolean"},
@@ -223,7 +142,7 @@ func (h *SettingsHandler) GetPublicSettings(ctx context.Context, input *GetPubli
return nil, huma.Error500InternalServerError("Failed to map settings")
}
- return &GetPublicSettingsOutput{Body: h.appendRuntimeSettingsInternal(settingsDto, false)}, nil
+ return &GetPublicSettingsOutput{Body: h.appendRuntimeSettings(settingsDto, false)}, nil
}
// GetSettings returns all settings for an environment.
@@ -244,13 +163,10 @@ func (h *SettingsHandler) GetSettings(ctx context.Context, input *GetSettingsInp
}
allowedKeys[projectWorkspaceMaxFileSizeSettingKey] = struct{}{}
allowedKeys[volumeWorkspaceMaxFileSizeSettingKey] = struct{}{}
- filtered := make([]settings.PublicSetting, 0, len(*settingsDto))
- for _, setting := range *settingsDto {
- if _, ok := allowedKeys[setting.Key]; ok {
- filtered = append(filtered, setting)
- }
- }
- settingsDto = &filtered
+ *settingsDto = slices.DeleteFunc(*settingsDto, func(setting settings.PublicSetting) bool {
+ _, ok := allowedKeys[setting.Key]
+ return !ok
+ })
}
return &GetSettingsOutput{Body: *settingsDto}, nil
}
@@ -262,99 +178,79 @@ func (h *SettingsHandler) GetSettings(ctx context.Context, input *GetSettingsInp
return nil, huma.Error500InternalServerError("Failed to map settings")
}
- return &GetSettingsOutput{Body: h.appendRuntimeSettingsInternal(settingsDto, true)}, nil
+ return &GetSettingsOutput{Body: h.appendRuntimeSettings(settingsDto, true)}, nil
}
// UpdateSettings updates settings for an environment.
func (h *SettingsHandler) UpdateSettings(ctx context.Context, input *UpdateSettingsInput) (*handlerutil.Out[[]settings.SettingDto], error) {
- if err := h.validateSettingsUpdateInput(input.Body); err != nil {
- return nil, err
+ update := input.Body
+ // Only validate changed directories so env-provided values do not block unrelated saves.
+ currentCfg := h.settingsService.GetSettingsConfig()
+ projectsDir := ""
+ if update.ProjectsDirectory != nil && *update.ProjectsDirectory != currentCfg.ProjectsDirectory.Value {
+ projectsDir = *update.ProjectsDirectory
}
-
- if input.EnvironmentID != "0" {
- return h.updateSettingsForRemoteEnvironment(ctx, input)
- }
-
- return h.updateSettingsForLocalEnvironment(ctx, input.Body)
-}
-
-func (h *SettingsHandler) validateSettingsUpdateInput(input settings.Update) error {
- // Validate projects directory if provided and changed from current value.
- // Skip validation when the value matches the current (possibly env-overridden) setting
- // so that saving unrelated settings doesn't fail due to env-provided directory formats.
- if input.ProjectsDirectory != nil && *input.ProjectsDirectory != "" {
- currentDir := h.settingsService.GetSettingsConfig().ProjectsDirectory.Value
- if *input.ProjectsDirectory != currentDir {
- if err := validateProjectsDirectoryValueInternal(*input.ProjectsDirectory); err != nil {
- return huma.Error400BadRequest(err.Error())
- }
+ if projectsDir != "" && !projects.IsWindowsDrivePath(projectsDir) {
+ // Mapping format is "container:host"; the container path must be absolute.
+ if container, _, isMapping := strings.Cut(projectsDir, ":"); !strings.HasPrefix(container, "/") {
+ return nil, huma.Error400BadRequest(kit.Ternary(isMapping,
+ "projectsDirectory mapping format: container path must be absolute",
+ "projectsDirectory must be an absolute path starting with '/'"))
}
}
- if input.SwarmStackSourcesDirectory != nil && *input.SwarmStackSourcesDirectory != "" {
- currentDir := h.settingsService.GetSettingsConfig().SwarmStackSourcesDirectory.Value
- if *input.SwarmStackSourcesDirectory != currentDir {
- if err := validateAbsoluteDirectoryPathInternal(*input.SwarmStackSourcesDirectory); err != nil {
- return huma.Error400BadRequest("swarmStackSourcesDirectory " + err.Error())
- }
- }
+ if dir := update.SwarmStackSourcesDirectory; dir != nil && *dir != "" && *dir != currentCfg.SwarmStackSourcesDirectory.Value &&
+ !projects.IsWindowsDrivePath(*dir) && !strings.HasPrefix(*dir, "/") {
+ return nil, huma.Error400BadRequest("swarmStackSourcesDirectory must be an absolute path")
}
- if input.AvatarMaxUploadSizeMb != nil && strings.TrimSpace(*input.AvatarMaxUploadSizeMb) != "" {
- value, err := strconv.Atoi(strings.TrimSpace(*input.AvatarMaxUploadSizeMb))
+ if update.AvatarMaxUploadSizeMb != nil && strings.TrimSpace(*update.AvatarMaxUploadSizeMb) != "" {
+ value, err := strconv.Atoi(strings.TrimSpace(*update.AvatarMaxUploadSizeMb))
if err != nil || value < 1 || value > 50 {
- return huma.Error400BadRequest("avatarMaxUploadSizeMb must be between 1 and 50")
+ return nil, huma.Error400BadRequest("avatarMaxUploadSizeMb must be between 1 and 50")
}
}
- if input.TrivyConfig != nil && strings.TrimSpace(*input.TrivyConfig) != "" {
+ if update.TrivyConfig != nil && strings.TrimSpace(*update.TrivyConfig) != "" {
var doc map[string]any
- if err := yaml.Unmarshal([]byte(*input.TrivyConfig), &doc); err != nil {
- return huma.Error400BadRequest("trivyConfig must be a YAML mapping: " + err.Error())
+ if err := yaml.Unmarshal([]byte(*update.TrivyConfig), &doc); err != nil {
+ return nil, huma.Error400BadRequest("trivyConfig must be a YAML mapping: " + err.Error())
}
if doc == nil {
- return huma.Error400BadRequest("trivyConfig must be a YAML mapping")
+ return nil, huma.Error400BadRequest("trivyConfig must be a YAML mapping")
}
}
- return nil
-}
-
-func (h *SettingsHandler) updateSettingsForRemoteEnvironment(ctx context.Context, input *UpdateSettingsInput) (*handlerutil.Out[[]settings.SettingDto], error) {
- // Check if trying to update auth settings on non-local environment.
- if hasAuthSettingsUpdateInternal(input.Body) {
- return nil, huma.Error403Forbidden("Authentication settings can only be updated from the main environment")
- }
-
- apiResp, err := h.proxyRemoteJSON.JSON[base.ApiResponse[[]settings.SettingDto]](ctx, input.EnvironmentID, http.MethodPut, "/api/environments/0/settings", input.Body)
- if err != nil {
- return nil, err
+ if input.EnvironmentID != "0" {
+ if update.AuthLocalEnabled != nil || update.OidcEnabled != nil || update.AuthSessionTimeout != nil || update.AuthPasswordPolicy != nil ||
+ update.OidcClientId != nil || update.OidcClientSecret != nil || update.OidcIssuerUrl != nil || update.OidcScopes != nil ||
+ update.OidcMergeAccounts != nil || update.OidcSkipTlsVerify != nil || update.OidcAutoRedirectToProvider != nil ||
+ update.OidcProviderName != nil || update.OidcProviderLogoUrl != nil || update.OidcGroupsClaim != nil {
+ return nil, huma.Error403Forbidden("Authentication settings can only be updated from the main environment")
+ }
+ apiResp, err := h.proxyRemoteJSON.JSON[base.ApiResponse[[]settings.SettingDto]](ctx, input.EnvironmentID, http.MethodPut, "/api/environments/0/settings", update)
+ if err != nil {
+ return nil, err
+ }
+ return &handlerutil.Out[[]settings.SettingDto]{Body: *apiResp}, nil
}
- return &handlerutil.Out[[]settings.SettingDto]{Body: *apiResp}, nil
-}
-
-func (h *SettingsHandler) updateSettingsForLocalEnvironment(ctx context.Context, input settings.Update) (*handlerutil.Out[[]settings.SettingDto], error) {
- if input.ProjectsDirectory != nil && *input.ProjectsDirectory != "" {
- currentDir := h.settingsService.GetSettingsConfig().ProjectsDirectory.Value
- if *input.ProjectsDirectory != currentDir {
- resolved, err := projects.GetProjectsDirectory(ctx, strings.TrimSpace(*input.ProjectsDirectory))
- if err != nil {
- return nil, huma.Error400BadRequest(fmt.Sprintf("cannot use projects directory %q: %v", *input.ProjectsDirectory, err))
- }
- // os.* rather than acfs: this validates a user-typed directory before
- // it becomes a confinement root, so no root exists for it yet.
- f, err := os.Open(resolved)
- if err != nil {
- return nil, huma.Error400BadRequest(fmt.Sprintf("cannot read projects directory %q: %v", resolved, err))
- }
- if closeErr := f.Close(); closeErr != nil {
- return nil, huma.Error400BadRequest(fmt.Sprintf("cannot read projects directory %q: %v", resolved, closeErr))
- }
+ if projectsDir != "" {
+ resolved, err := projects.GetProjectsDirectory(ctx, strings.TrimSpace(projectsDir))
+ if err != nil {
+ return nil, huma.Error400BadRequest(fmt.Sprintf("cannot use projects directory %q: %v", projectsDir, err))
+ }
+ // os rather than acfs: the directory is validated before it becomes a confinement root.
+ f, err := os.Open(resolved)
+ if err == nil {
+ err = f.Close()
+ }
+ if err != nil {
+ return nil, huma.Error400BadRequest(fmt.Sprintf("cannot read projects directory %q: %v", resolved, err))
}
}
- updatedSettings, err := h.settingsService.UpdateSettings(ctx, input)
+ updatedSettings, err := h.settingsService.UpdateSettings(ctx, update)
if err != nil {
apiErr := common.ToAPIError(err)
if apiErr.HTTPStatus() == http.StatusInternalServerError {
@@ -365,49 +261,43 @@ func (h *SettingsHandler) updateSettingsForLocalEnvironment(ctx context.Context,
settingDtos := make([]settings.SettingDto, 0, len(updatedSettings))
for _, setting := range updatedSettings {
- settingDtos = append(settingDtos, settings.SettingDto{
- Key: setting.Key,
- Type: "string",
- Value: setting.Value,
- })
+ settingDtos = append(settingDtos, settings.SettingDto{Key: setting.Key, Type: "string", Value: setting.Value})
}
return &handlerutil.Out[[]settings.SettingDto]{
- Body: base.ApiResponse[[]settings.SettingDto]{
- Success: true,
- Data: settingDtos,
- },
+ Body: base.ApiResponse[[]settings.SettingDto]{Success: true, Data: settingDtos},
}, nil
}
-func hasAuthSettingsUpdateInternal(req settings.Update) bool {
- return req.AuthLocalEnabled != nil || req.OidcEnabled != nil ||
- req.AuthSessionTimeout != nil || req.AuthPasswordPolicy != nil ||
- req.OidcClientId != nil ||
- req.OidcClientSecret != nil || req.OidcIssuerUrl != nil ||
- req.OidcScopes != nil ||
- req.OidcMergeAccounts != nil ||
- req.OidcSkipTlsVerify != nil || req.OidcAutoRedirectToProvider != nil ||
- req.OidcProviderName != nil || req.OidcProviderLogoUrl != nil ||
- req.OidcGroupsClaim != nil
-}
-
// Search searches settings by query.
func (h *SettingsHandler) Search(ctx context.Context, input *SearchSettingsInput) (*SearchSettingsOutput, error) {
if strings.TrimSpace(input.Body.Query) == "" {
return nil, huma.Error400BadRequest("Query parameter is required")
}
- ps, _ := middleware.PermissionsFromContext(ctx)
- results := search.Search(h.settingsSearchService.GetSettingsCategories(), input.Body.Query, searchtypes.SettingsProfile)
- results.Results = filterSettingsCategoriesInternal(ps, results.Results)
- results.Count = len(results.Results)
+ categories, err := h.GetCategories(ctx, nil)
+ if err != nil {
+ return nil, err
+ }
+ results := search.Search(categories.Body, input.Body.Query, searchtypes.SettingsProfile)
+ if results.Results == nil {
+ results.Results = []category.Category{}
+ }
return &SearchSettingsOutput{Body: results}, nil
}
-// GetCategories returns all available settings categories.
+// GetCategories returns the settings categories reachable at any scope.
func (h *SettingsHandler) GetCategories(ctx context.Context, input *struct{}) (*GetCategoriesOutput, error) {
+ categories := []category.Category{}
ps, _ := middleware.PermissionsFromContext(ctx)
- categories := filterSettingsCategoriesInternal(ps, h.settingsSearchService.GetSettingsCategories())
+ if ps == nil {
+ return &GetCategoriesOutput{Body: categories}, nil
+ }
+ scopes := append([]string{""}, slices.Collect(maps.Keys(ps.PerEnv))...)
+ for _, cat := range h.settingsSearchService.GetSettingsCategories() {
+ if slices.ContainsFunc(scopes, func(envID string) bool { return authz.CanAccessSettingsCategory(ps, cat.ID, envID) }) {
+ categories = append(categories, cat)
+ }
+ }
return &GetCategoriesOutput{Body: categories}, nil
}
diff --git a/backend/internal/settings/handler_test.go b/backend/internal/settings/handler_test.go
index 312e4420c..562d9f61e 100644
--- a/backend/internal/settings/handler_test.go
+++ b/backend/internal/settings/handler_test.go
@@ -19,7 +19,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
)
-func TestSettingsHandlerAppendRuntimeSettingsInternal(t *testing.T) {
+func TestSettingsHandlerAppendRuntimeSettings(t *testing.T) {
handler := &SettingsHandler{cfg: &config.Config{
UIConfigurationDisabled: true,
BackupVolumeName: "custom-backups",
@@ -27,13 +27,13 @@ func TestSettingsHandlerAppendRuntimeSettingsInternal(t *testing.T) {
VolumeWorkspaceMaxFileSizeMB: 18,
}}
- publicKeys := runtimeSettingKeysInternal(handler.appendRuntimeSettingsInternal(nil, false))
+ publicKeys := runtimeSettingKeys(handler.appendRuntimeSettings(nil, false))
require.NotContains(t, publicKeys, "uiConfigDisabled")
require.NotContains(t, publicKeys, "backupVolumeName")
require.NotContains(t, publicKeys, "depotConfigured")
require.NotContains(t, publicKeys, "edgeMTLSManagerCAAvailable")
- authenticatedKeys := runtimeSettingKeysInternal(handler.appendRuntimeSettingsInternal(nil, true))
+ authenticatedKeys := runtimeSettingKeys(handler.appendRuntimeSettings(nil, true))
require.Equal(t, "12", authenticatedKeys[projectWorkspaceMaxFileSizeSettingKey])
require.Equal(t, "18", authenticatedKeys[volumeWorkspaceMaxFileSizeSettingKey])
require.Equal(t, "true", authenticatedKeys["uiConfigDisabled"])
@@ -41,20 +41,20 @@ func TestSettingsHandlerAppendRuntimeSettingsInternal(t *testing.T) {
require.Equal(t, "false", authenticatedKeys["edgeMTLSManagerCAAvailable"])
}
-func TestSettingsHandlerAppendRuntimeSettingsDoesNotGenerateEdgeMTLSCAInternal(t *testing.T) {
+func TestSettingsHandlerAppendRuntimeSettingsDoesNotGenerateEdgeMTLSCA(t *testing.T) {
assetsDir := t.TempDir()
handler := &SettingsHandler{cfg: &config.Config{
EdgeMTLSMode: edge.EdgeMTLSModeRequired,
EdgeMTLSAssetsDir: assetsDir,
}}
- authenticatedKeys := runtimeSettingKeysInternal(handler.appendRuntimeSettingsInternal(nil, true))
+ authenticatedKeys := runtimeSettingKeys(handler.appendRuntimeSettings(nil, true))
require.Equal(t, "false", authenticatedKeys["edgeMTLSManagerCAAvailable"])
require.NoFileExists(t, filepath.Join(assetsDir, "ca.crt"))
require.NoFileExists(t, filepath.Join(assetsDir, "ca.key"))
}
-func TestSettingsHandlerUpdateLocalEnvironmentRejectsUnreadableProjectsDirectoryInternal(t *testing.T) {
+func TestSettingsHandlerUpdateLocalEnvironmentRejectsUnreadableProjectsDirectory(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("POSIX permission-denied behavior is not portable to Windows")
}
@@ -63,7 +63,7 @@ func TestSettingsHandlerUpdateLocalEnvironmentRejectsUnreadableProjectsDirectory
}
ctx := t.Context()
- settingsService, err := newSettingsServiceForTestInternal(t, ctx, setupSettingsTestDB(t))
+ settingsService, err := newSettingsServiceForTest(t, ctx, setupSettingsTestDB(t))
require.NoError(t, err)
originalDir := settingsService.GetSettingsConfig().ProjectsDirectory.Value
@@ -72,7 +72,7 @@ func TestSettingsHandlerUpdateLocalEnvironmentRejectsUnreadableProjectsDirectory
t.Cleanup(func() { _ = os.Chmod(unreadable, 0o700) })
handler := &SettingsHandler{settingsService: settingsService, cfg: &config.Config{}}
- _, err = handler.updateSettingsForLocalEnvironment(ctx, settings.Update{ProjectsDirectory: new(unreadable)})
+ _, err = handler.UpdateSettings(ctx, &UpdateSettingsInput{EnvironmentID: "0", Body: settings.Update{ProjectsDirectory: new(unreadable)}})
require.Error(t, err)
var statusErr huma.StatusError
@@ -82,7 +82,7 @@ func TestSettingsHandlerUpdateLocalEnvironmentRejectsUnreadableProjectsDirectory
require.Equal(t, originalDir, settingsService.GetSettingsConfig().ProjectsDirectory.Value)
}
-func TestSettingsHandlerRemoteWorkspaceSettingsVisibilityInternal(t *testing.T) {
+func TestSettingsHandlerRemoteWorkspaceSettingsVisibility(t *testing.T) {
remoteSettings := []settings.PublicSetting{
{Key: "dockerHost", Type: "string", Value: "unix:///var/run/docker.sock"},
{Key: "baseServerUrl", Type: "string", Value: "https://manager.example"},
@@ -99,7 +99,7 @@ func TestSettingsHandlerRemoteWorkspaceSettingsVisibilityInternal(t *testing.T)
return json.Unmarshal(payload, output)
}
- settingsService, err := newSettingsServiceForTestInternal(t, t.Context(), setupSettingsTestDB(t))
+ settingsService, err := newSettingsServiceForTest(t, t.Context(), setupSettingsTestDB(t))
require.NoError(t, err)
handler := &SettingsHandler{settingsService: settingsService, proxyRemoteJSON: proxy}
@@ -116,7 +116,7 @@ func TestSettingsHandlerRemoteWorkspaceSettingsVisibilityInternal(t *testing.T)
require.Equal(t, remoteSettings, output.Body)
}
-func runtimeSettingKeysInternal(publicSettings []settings.PublicSetting) map[string]string {
+func runtimeSettingKeys(publicSettings []settings.PublicSetting) map[string]string {
keys := make(map[string]string, len(publicSettings))
for _, setting := range publicSettings {
keys[setting.Key] = setting.Value
diff --git a/backend/internal/settings/helpers.go b/backend/internal/settings/helpers.go
new file mode 100644
index 000000000..9aa2c7798
--- /dev/null
+++ b/backend/internal/settings/helpers.go
@@ -0,0 +1,47 @@
+package settings
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "uuid"
+
+ "go.getarcane.app/kit/pkg"
+ "gorm.io/gorm"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+)
+
+// EnsureInstanceID returns this installation's persisted instance ID, creating it on first start.
+func EnsureInstanceID(ctx context.Context, db *database.DB) (string, error) {
+ return ensureSettingValue(ctx, db, "instanceId", "instance ID", func() (string, error) { return uuid.New().String(), nil })
+}
+
+func ensureSettingValue(ctx context.Context, db *database.DB, key, label string, generate func() (string, error)) (string, error) {
+ var value string
+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ var setting SettingVariable
+ if err := tx.Where("key = ?", key).Limit(1).Find(&setting).Error; err != nil {
+ return fmt.Errorf("failed to load %s: %w", label, err)
+ }
+ if setting.Value != "" {
+ value = setting.Value
+ return nil
+ }
+
+ generated, err := generate()
+ if err != nil {
+ return err
+ }
+ if saveErr := tx.Save(&SettingVariable{Key: key, Value: generated}).Error; saveErr != nil {
+ return fmt.Errorf("failed to persist %s: %w", label, saveErr)
+ }
+ value = generated
+ return nil
+ })
+ return value, err
+}
+
+func settingEnvName(key string) string {
+ return strings.ToUpper(kit.SnakeCase(key))
+}
diff --git a/backend/internal/settings/service.go b/backend/internal/settings/service.go
index dc28a73bd..0e2e0fd2c 100644
--- a/backend/internal/settings/service.go
+++ b/backend/internal/settings/service.go
@@ -9,7 +9,6 @@ import (
"errors"
"fmt"
"log/slog"
- "maps"
"os"
"path/filepath"
"reflect"
@@ -51,12 +50,8 @@ const (
type SettingsService struct {
db *database.DB
config atomic.Pointer[Settings]
- // effectiveConfig is the env-override-applied snapshot, rebuilt whenever
- // config is stored. Overrides are resolved once at startup and never
- // change, so materializing them here turns every read (GetSettings and
- // the typed getters — ~100 call sites, some in per-project loops) into a
- // pointer load instead of a full struct Clone plus a reflection pass.
- // The snapshot is shared: readers must never mutate it.
+ // effectiveConfig is the shared, read-only env-override-applied snapshot,
+ // rebuilt whenever config is stored so reads are a pointer load.
effectiveConfig atomic.Pointer[Settings]
envOverrides []settingsEnvOverride
writes sync.Mutex
@@ -69,15 +64,9 @@ type SettingsService struct {
changes *concurrency.Signal[[]libarcane.SettingUpdate]
}
-type settingsUpdateResultInternal struct {
- settings []SettingVariable
- changes []libarcane.SettingUpdate
-}
-
type settingsEnvOverride struct {
fieldIndex int
key string
- envVarName string
value string
}
@@ -92,28 +81,57 @@ func NewSettingsService(ctx context.Context, db *database.DB) (*SettingsService,
lifecycleCtx: ctx,
changes: concurrency.NewSignal[[]libarcane.SettingUpdate](),
}
- svc.envOverrides = resolveSettingsEnvOverridesInternal()
+ fields, _ := getSettingsFieldCacheInternal()
+ for _, field := range fields {
+ if !slices.Contains(splitSettingAttrsInternal(field.attrs), "envOverride") {
+ continue
+ }
+ if val, ok, _ := utils.LookupEnvOrFile(settingEnvName(field.key)); ok && val != "" {
+ svc.envOverrides = append(svc.envOverrides, settingsEnvOverride{fieldIndex: field.index, key: field.key, value: kit.TrimQuotes(val)})
+ }
+ }
if len(svc.envOverrides) > 0 {
slog.InfoContext(ctx, "Loaded Environment Settings Overrides", "count", len(svc.envOverrides))
}
- err := svc.LoadDatabaseSettings(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to load settings: %w", err)
- }
-
- err = svc.setupInstanceID(ctx)
- if err != nil {
+ if _, err := EnsureInstanceID(ctx, db); err != nil {
return nil, fmt.Errorf("failed to setup instance ID: %w", err)
}
+ if err := svc.LoadDatabaseSettings(ctx); err != nil {
+ return nil, fmt.Errorf("failed to load settings: %w", err)
+ }
- go svc.runEffectsInternal()
+ go func() {
+ defer close(svc.effectsDone)
+ for {
+ svc.effectsMu.Lock()
+ if len(svc.effects) > 0 {
+ effect := svc.effects[0]
+ svc.effects[0] = nil
+ svc.effects = svc.effects[1:]
+ svc.effectsMu.Unlock()
+ func() { defer utils.RecoverToError(nil, "settings effect"); effect() }()
+ continue
+ }
+ closed := svc.effectsClosed
+ svc.effectsMu.Unlock()
+ if closed {
+ return
+ }
+ select {
+ case <-svc.effectsWake:
+ case <-svc.lifecycleCtx.Done():
+ svc.effectsMu.Lock()
+ svc.effectsClosed = true
+ svc.effectsMu.Unlock()
+ }
+ }
+ }()
return svc, nil
}
-// SubscribeSettingsChanges invokes callback once when any subscribed key is
-// present in a successful update. The callback receives only matching values.
-// Callbacks run serially on the ordered effects worker, outside the write mutex.
+// SubscribeSettingsChanges invokes callback once with the matching values when any subscribed key
+// is present in a successful update. Callbacks run serially on the effects worker, outside the write mutex.
func (s *SettingsService) SubscribeSettingsChanges(keys []string, callback func([]libarcane.SettingUpdate)) func() {
if callback == nil || len(keys) == 0 {
return func() {}
@@ -130,7 +148,7 @@ func (s *SettingsService) SubscribeSettingsChanges(keys []string, callback func(
}
}
if len(matching) > 0 {
- if err := s.enqueueEffectInternal(func() { callback(matching) }); err != nil && s.lifecycleCtx.Err() == nil {
+ if err := s.enqueueEffect(func() { callback(matching) }); err != nil && s.lifecycleCtx.Err() == nil {
slog.ErrorContext(s.lifecycleCtx, "Failed to queue settings change effects", "error", err)
}
}
@@ -154,7 +172,7 @@ func (s *SettingsService) NotifySettingsChanges(ctx context.Context, keys ...str
}
updates = append(updates, libarcane.SettingUpdate{Key: key, Value: value})
}
- s.publishSettingsChangesInternal(updates)
+ s.publishSettingsChanges(updates)
return nil
}
@@ -167,33 +185,75 @@ func (s *SettingsService) GetSettingsConfig() *Settings {
return v
}
-func (s *SettingsService) LoadDatabaseSettings(ctx context.Context) (err error) {
- dst, err := s.loadDatabaseSettingsInternal(ctx, s.db)
- if err != nil {
- return err
+func (s *SettingsService) LoadDatabaseSettings(ctx context.Context) error {
+ var stored []SettingVariable
+ queryCtx, queryCancel := context.WithTimeout(ctx, 10*time.Second)
+ defer queryCancel()
+ if err := s.db.WithContext(queryCtx).Find(&stored).Error; err != nil {
+ return fmt.Errorf("failed to load configuration from the database: %w", err)
+ }
+
+ dst := DefaultSettingsConfig()
+ appCfg := config.Load()
+ fields, _ := getSettingsFieldCacheInternal()
+ rv := reflect.ValueOf(dst).Elem()
+ if appCfg.UIConfigurationDisabled || appCfg.AgentMode {
+ slog.DebugContext(ctx, "LoadDatabaseSettings: using env path",
+ "uiConfigurationDisabled", appCfg.UIConfigurationDisabled, "agentMode", appCfg.AgentMode, "environment", appCfg.Environment)
+
+ storedValues := make(map[string]string, len(stored))
+ for _, setting := range stored {
+ storedValues[setting.Key] = setting.Value
+ }
+ for _, field := range fields {
+ value := rv.Field(field.index).FieldByName("Value")
+ if slices.Contains(splitSettingAttrsInternal(field.attrs), "internal") {
+ if val, ok := storedValues[field.key]; ok {
+ value.SetString(val)
+ }
+ continue
+ }
+
+ envVarName := settingEnvName(field.key)
+ if val, ok, _ := utils.LookupEnvOrFile(envVarName); ok {
+ mask := kit.Ternary(val == "", "<empty>", fmt.Sprintf("%d chars", len(val)))
+ slog.DebugContext(ctx, "LoadDatabaseSettings: env override found", "key", field.key, "env", envVarName, "valueMasked", mask)
+ value.SetString(kit.TrimQuotes(val))
+ continue
+ }
+ if val, ok := storedValues[field.key]; ok {
+ slog.DebugContext(ctx, "LoadDatabaseSettings: using database fallback", "key", field.key)
+ value.SetString(val)
+ continue
+ }
+ slog.DebugContext(ctx, "LoadDatabaseSettings: env not set and no database value", "key", field.key, "env", envVarName)
+ }
+
+ count := 0
+ for _, field := range fields {
+ if rv.Field(field.index).FieldByName("Value").String() != "" {
+ count++
+ }
+ }
+ slog.DebugContext(ctx, "LoadDatabaseSettings: completed env load", "loadedFields", count)
+ } else {
+ for _, setting := range stored {
+ if err := dst.UpdateField(setting.Key, setting.Value, false); err != nil && !errors.Is(err, SettingKeyNotFoundError{}) {
+ return fmt.Errorf("failed to process settings for key '%s': %w", setting.Key, err)
+ }
+ }
+ s.applyEnvOverrides(dst)
}
s.config.Store(dst)
effective := dst.Clone()
- s.applyEnvOverrides(ctx, effective)
+ s.applyEnvOverrides(effective)
s.effectiveConfig.Store(effective)
return nil
}
-func (s *SettingsService) refreshSettingsCacheInternal(ctx context.Context) error {
- if err := s.LoadDatabaseSettings(ctx); err != nil {
- return fmt.Errorf("failed to refresh settings cache: %w", err)
- }
-
- return nil
-}
-
-func (s *SettingsService) getDefaultSettings() *Settings {
- return DefaultSettingsConfig()
-}
-
// DefaultSettingsConfig returns the canonical default settings model used by Arcane.
func DefaultSettingsConfig() *Settings {
return &Settings{
@@ -317,111 +377,7 @@ func DefaultSettingsConfig() *Settings {
}
}
-func (s *SettingsService) loadDatabaseSettingsInternal(ctx context.Context, db *database.DB) (*Settings, error) {
- if config.Load().UIConfigurationDisabled || config.Load().AgentMode {
- slog.DebugContext(
- ctx,
- "loadDatabaseSettingsInternal: using env path",
- "uiConfigurationDisabled",
- config.Load().UIConfigurationDisabled,
- "agentMode",
- config.Load().AgentMode,
- "environment",
- config.Load().Environment,
- )
- return s.loadDatabaseConfigFromEnv(ctx, db)
- }
-
- dest := s.getDefaultSettings()
-
- var loaded []SettingVariable
- queryCtx, queryCancel := context.WithTimeout(ctx, 10*time.Second)
- defer queryCancel()
- err := db.WithContext(queryCtx).
- Find(&loaded).Error
- if err != nil {
- return nil, fmt.Errorf("failed to load configuration from the database: %w", err)
- }
-
- for _, v := range loaded {
- err = dest.UpdateField(v.Key, v.Value, false)
-
- if err != nil && !errors.Is(err, SettingKeyNotFoundError{}) {
- return nil, fmt.Errorf("failed to process settings for key '%s': %w", v.Key, err)
- }
- }
-
- // Apply environment variable overrides for fields tagged with "envOverride"
- s.applyEnvOverrides(ctx, dest)
-
- return dest, nil
-}
-
-func (s *SettingsService) loadDatabaseConfigFromEnv(ctx context.Context, db *database.DB) (*Settings, error) {
- dest := s.getDefaultSettings()
-
- // Fetch all settings once to avoid N+1 queries for internal keys
- var allSettings []SettingVariable
- if err := db.WithContext(ctx).Find(&allSettings).Error; err != nil {
- return nil, fmt.Errorf("failed to load settings for env config: %w", err)
- }
- settingsMap := make(map[string]string, len(allSettings))
- for _, s := range allSettings {
- settingsMap[s.Key] = s.Value
- }
-
- rt := reflect.ValueOf(dest).Elem().Type()
- rv := reflect.ValueOf(dest).Elem()
- for i := range rt.NumField() {
- field := rt.Field(i)
-
- tagParts := strings.Split(field.Tag.Get("key"), ",")
- key := tagParts[0]
- isInternal := slices.Contains(tagParts[1:], "internal")
-
- if isInternal {
- if val, ok := settingsMap[key]; ok {
- rv.Field(i).FieldByName("Value").SetString(val)
- }
- continue
- }
-
- envVarName := strings.ToUpper(kit.SnakeCase(key))
-
- // debug: log each env name checked and whether a value exists
- if val, ok, _ := utils.LookupEnvOrFile(envVarName); ok {
- mask := "<empty>"
- if val != "" {
- mask = fmt.Sprintf("%d chars", len(val))
- }
- slog.DebugContext(ctx, "loadDatabaseConfigFromEnv: env override found", "key", key, "env", envVarName, "valueMasked", mask)
- rv.Field(i).FieldByName("Value").SetString(kit.TrimQuotes(val))
- continue
- }
- if val, ok := settingsMap[key]; ok {
- // Fallback to database if environment variable is not set
- slog.DebugContext(ctx, "loadDatabaseConfigFromEnv: using database fallback", "key", key)
- rv.Field(i).FieldByName("Value").SetString(val)
- continue
- }
- slog.DebugContext(ctx, "loadDatabaseConfigFromEnv: env not set and no database value", "key", key, "env", envVarName)
- }
-
- // debug: final snapshot (only show which fields are non-empty)
- count := 0
- for i := range rt.NumField() {
- v := rv.Field(i).FieldByName("Value").String()
- if v != "" {
- count++
- }
- }
- slog.DebugContext(ctx, "loadDatabaseConfigFromEnv: completed env load", "loadedFields", count)
-
- return dest, nil
-}
-
-func (s *SettingsService) applyEnvOverrides(ctx context.Context, dest *Settings) {
- _ = ctx
+func (s *SettingsService) applyEnvOverrides(dest *Settings) {
rv := reflect.ValueOf(dest).Elem()
for _, override := range s.envOverrides {
@@ -429,64 +385,27 @@ func (s *SettingsService) applyEnvOverrides(ctx context.Context, dest *Settings)
}
}
-func resolveSettingsEnvOverridesInternal() []settingsEnvOverride {
- rt := reflect.TypeFor[Settings]()
- overrides := make([]settingsEnvOverride, 0)
-
- for i := range rt.NumField() {
- field := rt.Field(i)
- tagValue := field.Tag.Get("key")
- if tagValue == "" {
- continue
- }
-
- // Parse tag attributes (e.g., "dockerHost,public,envOverride")
- parts := strings.Split(tagValue, ",")
- key := parts[0]
- hasEnvOverride := slices.Contains(parts[1:], "envOverride")
-
- if !hasEnvOverride {
- continue
- }
-
- envVarName := strings.ToUpper(kit.SnakeCase(key))
- if val, ok, _ := utils.LookupEnvOrFile(envVarName); ok && val != "" {
- overrides = append(overrides, settingsEnvOverride{
- fieldIndex: i,
- key: key,
- envVarName: envVarName,
- value: kit.TrimQuotes(val),
- })
- }
- }
-
- return overrides
+// IsEnvOverrideActive reports whether a non-empty environment override controls the setting.
+func (s *SettingsService) IsEnvOverrideActive(key string) bool {
+ return slices.ContainsFunc(s.envOverrides, func(override settingsEnvOverride) bool { return override.key == key })
}
-// IsEnvOverrideActive reports whether an environment override currently controls the setting.
-// and its corresponding environment variable is currently set to a non-empty value.
-func (s *SettingsService) IsEnvOverrideActive(key string) bool {
- for _, override := range s.envOverrides {
- if override.key == key {
- return true
- }
+// GetSettings returns the shared, read-only effective (env-override-applied) settings snapshot.
+// Mutation flows go through UpdateSettings, which works on an explicit clone.
+func (s *SettingsService) GetSettings(_ context.Context) (*Settings, error) {
+ if effective := s.effectiveConfig.Load(); effective != nil {
+ return effective, nil
}
- return false
-}
-
-// GetSettings returns the effective (env-override-applied) settings snapshot.
-// The snapshot is shared across callers and must be treated as read-only;
-// mutation flows go through UpdateSettings, which works on an explicit clone.
-func (s *SettingsService) GetSettings(ctx context.Context) (*Settings, error) {
- settingsCfg := s.getEffectiveSettingsConfigInternal(ctx)
+ // Only reachable before LoadDatabaseSettings has completed in the
+ // constructor; fall back to building the snapshot per call.
+ settingsCfg := s.GetSettingsConfig().Clone()
+ s.applyEnvOverrides(settingsCfg)
return settingsCfg, nil
}
-// GetSettingsOrDefaults is a convenience for hot paths that need a snapshot but cannot
-// meaningfully recover from a settings load failure. It logs any error and guarantees a
-// non-nil *Settings (defaults: a zero-valued struct, which the SettingVariable helpers
-// like kit.ParseOrDefault treat as "use the caller's default").
+// GetSettingsOrDefaults logs any settings load failure and always returns a non-nil *Settings;
+// a zero-valued struct makes SettingVariable helpers fall back to the caller's default.
func (s *SettingsService) GetSettingsOrDefaults(ctx context.Context) *Settings {
cfg, err := s.GetSettings(ctx)
if err != nil {
@@ -495,19 +414,7 @@ func (s *SettingsService) GetSettingsOrDefaults(ctx context.Context) *Settings {
return kit.Ternary(cfg == nil, &Settings{}, cfg)
}
-func (s *SettingsService) getEffectiveSettingsConfigInternal(ctx context.Context) *Settings {
- if effective := s.effectiveConfig.Load(); effective != nil {
- return effective
- }
-
- // Only reachable before LoadDatabaseSettings has completed in the
- // constructor; fall back to building the snapshot per call.
- settingsCfg := s.GetSettingsConfig().Clone()
- s.applyEnvOverrides(ctx, settingsCfg)
- return settingsCfg
-}
-
-func validateSettingValueInternal(key, value string) error {
+func validateSettingValue(key, value string) error {
if key == "upgradeLogRetentionDays" && value != "" {
days, err := strconv.Atoi(value)
if err != nil || days < 0 || days > 3650 {
@@ -523,7 +430,7 @@ func validateSettingValueInternal(key, value string) error {
}
func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string) error {
- if err := validateSettingValueInternal(key, value); err != nil {
+ if err := validateSettingValue(key, value); err != nil {
return err
}
if err := libarcane.ValidateCronSetting(key, value); err != nil {
@@ -535,10 +442,7 @@ func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string)
return err
}
- if err := s.updateSettingValueNoRefreshInternal(ctx, key, value); err != nil {
- return err
- }
- return s.refreshSettingsCacheInternal(context.WithoutCancel(ctx))
+ return s.persistSettings(ctx, []SettingVariable{{Key: key, Value: value}})
}
// UpdateSettingValues persists a group of internal setting values atomically
@@ -546,7 +450,7 @@ func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string)
func (s *SettingsService) UpdateSettingValues(ctx context.Context, updates []libarcane.SettingUpdate) error {
values := make([]SettingVariable, 0, len(updates))
for _, update := range updates {
- if err := validateSettingValueInternal(update.Key, update.Value); err != nil {
+ if err := validateSettingValue(update.Key, update.Value); err != nil {
return err
}
values = append(values, SettingVariable{Key: update.Key, Value: update.Value})
@@ -557,23 +461,7 @@ func (s *SettingsService) UpdateSettingValues(ctx context.Context, updates []lib
return err
}
- if err := s.persistSettings(ctx, values); err != nil {
- return err
- }
- return s.refreshSettingsCacheInternal(context.WithoutCancel(ctx))
-}
-
-func (s *SettingsService) updateSettingValueNoRefreshInternal(ctx context.Context, key, value string) error {
- if key == "oidcProviderName" {
- value = normalization.Text(value, true, true)
- }
- return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- settingVar := &SettingVariable{
- Key: key,
- Value: value,
- }
- return tx.Save(settingVar).Error
- })
+ return s.persistSettings(ctx, values)
}
// UpdateSettings publishes the refreshed snapshot before returning. Each
@@ -587,33 +475,7 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.U
if err := ctx.Err(); err != nil {
return nil, err
}
- result, err := s.updateSettingsInternal(ctx, updates)
- if err == nil {
- s.publishSettingsChangesInternal(result.changes)
- }
- return result.settings, err
-}
-
-func (s *SettingsService) publishSettingsChangesInternal(updates []libarcane.SettingUpdate) {
- safe := make([]libarcane.SettingUpdate, 0, len(updates))
- cfg := s.GetSettingsConfig()
- for _, update := range updates {
- _, _, sensitive, err := cfg.FieldByKey(update.Key)
- if err != nil {
- slog.WarnContext(s.lifecycleCtx, "Skipping settings change notification for unknown key", "key", update.Key, "error", err)
- continue
- }
- if !sensitive {
- safe = append(safe, update)
- }
- }
- if len(safe) > 0 {
- s.changes.Publish(safe)
- }
-}
-func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates settings.Update) (settingsUpdateResultInternal, error) {
- defaultCfg := s.getDefaultSettings()
cfg := s.GetSettingsConfig().Clone()
oidcClientSecretUpdated := updates.OidcClientSecret != nil && !s.IsEnvOverrideActive("oidcClientSecret")
trivyServerTokenUpdated := updates.TrivyServerToken != nil && *updates.TrivyServerToken != "" && !s.IsEnvOverrideActive("trivyServerToken")
@@ -633,7 +495,7 @@ func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates se
map[string]bool{"oidcClientSecret": cfg.OidcClientSecret.Value != ""},
map[string]bool{"oidcClientSecret": updates.OidcClientSecret != nil},
); err != nil {
- return settingsUpdateResultInternal{}, err
+ return nil, err
}
effectiveValue := func(current SettingVariable, next *string) string {
@@ -642,10 +504,7 @@ func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates se
}
return current.Value
}
- oidcClientSecret := cfg.OidcClientSecret.Value
- if oidcClientSecretUpdated {
- oidcClientSecret = *updates.OidcClientSecret
- }
+ oidcClientSecret := kit.Ternary(oidcClientSecretUpdated, kit.FromPtr(updates.OidcClientSecret), cfg.OidcClientSecret.Value)
if effectiveValue(cfg.OidcEnabled, updates.OidcEnabled) == "true" {
required := []struct{ key, value string }{
{"oidcClientId", effectiveValue(cfg.OidcClientId, updates.OidcClientId)},
@@ -654,16 +513,7 @@ func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates se
}
for _, field := range required {
if strings.TrimSpace(field.value) == "" {
- return settingsUpdateResultInternal{}, common.Classify(
- common.ErrValidation,
- &base.FieldError{
- Field: field.key,
- Err: fmt.Errorf(
- "enabling OIDC requires %s",
- field.key,
- ),
- },
- )
+ return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: field.key, Err: fmt.Errorf("enabling OIDC requires %s", field.key)})
}
}
}
@@ -676,12 +526,12 @@ func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates se
map[string]bool{"trivyServerToken": cfg.TrivyServerToken.Value != ""},
map[string]bool{"trivyServerToken": trivyServerTokenUpdated},
); err != nil {
- return settingsUpdateResultInternal{}, err
+ return nil, err
}
- valuesToUpdate, err := s.prepareUpdateValues(updates, cfg, defaultCfg)
+ valuesToUpdate, err := s.prepareUpdateValues(updates, cfg, DefaultSettingsConfig())
if err != nil {
- return settingsUpdateResultInternal{}, err
+ return nil, err
}
if oidcClientSecretUpdated {
valuesToUpdate = append(valuesToUpdate, SettingVariable{Key: "oidcClientSecret", Value: *updates.OidcClientSecret})
@@ -691,21 +541,34 @@ func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates se
}
if persistSettingsErr := s.persistSettings(ctx, valuesToUpdate); persistSettingsErr != nil {
- return settingsUpdateResultInternal{}, persistSettingsErr
+ return nil, persistSettingsErr
}
- if refreshSettingsCacheErr := s.refreshSettingsCacheInternal(context.WithoutCancel(ctx)); refreshSettingsCacheErr != nil {
- return settingsUpdateResultInternal{}, refreshSettingsCacheErr
- }
- settingsCfg := s.GetSettingsConfig()
changes := make([]libarcane.SettingUpdate, 0, len(valuesToUpdate))
for _, value := range valuesToUpdate {
changes = append(changes, libarcane.SettingUpdate{Key: value.Key, Value: value.Value})
}
- return settingsUpdateResultInternal{
- settings: settingsCfg.ToSettingVariableSlice(SettingVisibilityNonAdmin, false),
- changes: changes,
- }, nil
+ result := s.GetSettingsConfig().ToSettingVariableSlice(SettingVisibilityNonAdmin, false)
+ s.publishSettingsChanges(changes)
+ return result, nil
+}
+
+func (s *SettingsService) publishSettingsChanges(updates []libarcane.SettingUpdate) {
+ safe := make([]libarcane.SettingUpdate, 0, len(updates))
+ cfg := s.GetSettingsConfig()
+ for _, update := range updates {
+ _, _, sensitive, err := cfg.FieldByKey(update.Key)
+ if err != nil {
+ slog.WarnContext(s.lifecycleCtx, "Skipping settings change notification for unknown key", "key", update.Key, "error", err)
+ continue
+ }
+ if !sensitive {
+ safe = append(safe, update)
+ }
+ }
+ if len(safe) > 0 {
+ s.changes.Publish(safe)
+ }
}
func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defaultCfg *Settings) ([]SettingVariable, error) {
@@ -714,15 +577,20 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa
valuesToUpdate := make([]SettingVariable, 0)
for i := range rt.NumField() {
- field := rt.Field(i)
fieldValue := rv.Field(i)
-
- key, value, ok := extractUpdateValue(field, fieldValue)
- if !ok {
+ if fieldValue.Kind() == reflect.Pointer && fieldValue.IsNil() {
continue
}
+ key, _, _ := strings.Cut(rt.Field(i).Tag.Get("json"), ",")
+ if key == "" {
+ continue
+ }
+ var value string
+ if fieldValue.Kind() == reflect.Pointer {
+ value = fieldValue.Elem().String()
+ }
- if err := validateSettingValueInternal(key, value); err != nil {
+ if err := validateSettingValue(key, value); err != nil {
return nil, err
}
@@ -746,18 +614,11 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa
return nil, fmt.Errorf("invalid cron expression for %s: %w", key, err)
}
- var valueToSave string
- var err error
-
- if value == "" {
- defaultValue, _, _, _ := defaultCfg.FieldByKey(key)
- valueToSave = defaultValue
- err = cfg.UpdateField(key, defaultValue, true)
- } else {
- valueToSave = value
- err = cfg.UpdateField(key, value, true)
+ valueToSave := value
+ if valueToSave == "" {
+ valueToSave, _, _, _ = defaultCfg.FieldByKey(key)
}
-
+ err := cfg.UpdateField(key, valueToSave, true)
if errors.Is(err, SettingSensitiveForbiddenError{}) {
continue
}
@@ -771,26 +632,8 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa
return valuesToUpdate, nil
}
-func extractUpdateValue(field reflect.StructField, fieldValue reflect.Value) (string, string, bool) {
- if fieldValue.Kind() == reflect.Pointer && fieldValue.IsNil() {
- return "", "", false
- }
-
- key, _, _ := strings.Cut(field.Tag.Get("json"), ",")
- if key == "" {
- return "", "", false
- }
-
- var value string
- if fieldValue.Kind() == reflect.Pointer {
- value = fieldValue.Elem().String()
- }
-
- return key, value, true
-}
-
func (s *SettingsService) persistSettings(ctx context.Context, values []SettingVariable) error {
- return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
for _, setting := range values {
if setting.Key == "oidcProviderName" {
setting.Value = normalization.Text(setting.Value, true, true)
@@ -800,12 +643,19 @@ func (s *SettingsService) persistSettings(ctx context.Context, values []SettingV
}
}
return nil
- })
+ }); err != nil {
+ return err
+ }
+
+ if err := s.LoadDatabaseSettings(context.WithoutCancel(ctx)); err != nil {
+ return fmt.Errorf("failed to refresh settings cache: %w", err)
+ }
+ return nil
}
-// retiredSettingValuesInternal lists former defaults that startup replaces
+// retiredSettingValues lists former defaults that startup replaces
// with the current default when an install still holds them verbatim.
-var retiredSettingValuesInternal = map[string][]string{
+var retiredSettingValues = map[string][]string{
"dockerClientRefreshInterval": {"*/30 * * * * *"},
"autoHealInterval": {"*/30 * * * * *"},
}
@@ -817,10 +667,9 @@ func (s *SettingsService) EnsureDefaultSettings(ctx context.Context) error {
return err
}
- defaultSettings := s.getDefaultSettings()
- defaultSettingVars := defaultSettings.ToSettingVariableSlice(SettingVisibilityAll, false)
+ defaultSettingVars := DefaultSettingsConfig().ToSettingVariableSlice(SettingVisibilityAll, false)
- if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
for _, defaultSetting := range defaultSettingVars {
var existing SettingVariable
err := tx.Where("key = ?", defaultSetting.Key).First(&existing).Error
@@ -832,17 +681,15 @@ func (s *SettingsService) EnsureDefaultSettings(ctx context.Context) error {
}
case err != nil:
return fmt.Errorf("failed to check for existing setting %s: %w", defaultSetting.Key, err)
- case slices.Contains(retiredSettingValuesInternal[defaultSetting.Key], existing.Value):
- if replaceDefaultSettingErr := tx.Model(&SettingVariable{}).Where("key = ?", defaultSetting.Key).Update("value", defaultSetting.Value).Error; replaceDefaultSettingErr != nil {
+ case slices.Contains(retiredSettingValues[defaultSetting.Key], existing.Value):
+ replaceDefaultSettingErr := tx.Model(&SettingVariable{}).Where("key = ?", defaultSetting.Key).Update("value", defaultSetting.Value).Error
+ if replaceDefaultSettingErr != nil {
return fmt.Errorf("failed to replace retired default for setting %s: %w", defaultSetting.Key, replaceDefaultSettingErr)
}
}
}
return nil
- }); err != nil {
- return err
- }
- return nil
+ })
}
func (s *SettingsService) PruneUnknownSettings(ctx context.Context) error {
@@ -852,12 +699,12 @@ func (s *SettingsService) PruneUnknownSettings(ctx context.Context) error {
return err
}
- allowedKeys := allowedSettingKeys()
- if len(allowedKeys) == 0 {
- return nil
+ fields, _ := getSettingsFieldCacheInternal()
+ keys := make([]string, 0, len(fields)+1)
+ for _, field := range fields {
+ keys = append(keys, field.key)
}
-
- keys := slices.Collect(maps.Keys(allowedKeys))
+ keys = append(keys, "encryptionKey")
result := s.db.WithContext(ctx).Where("key NOT IN ?", keys).Delete(&SettingVariable{})
if result.Error != nil {
@@ -876,14 +723,38 @@ func (s *SettingsService) PersistEnvSettingsIfMissing(ctx context.Context) error
return err
}
- rt := reflect.TypeFor[Settings]()
appCfg := config.Load()
isEnvOnlyMode := appCfg.AgentMode || appCfg.UIConfigurationDisabled
+ fields, _ := getSettingsFieldCacheInternal()
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- for field := range rt.Fields() {
- if err := s.processEnvField(ctx, tx, field, isEnvOnlyMode); err != nil {
- return err
+ for _, field := range fields {
+ attrs := splitSettingAttrsInternal(field.attrs)
+ // Outside env-only mode, only persist fields explicitly marked as envOverride.
+ if slices.Contains(attrs, "internal") || (!isEnvOnlyMode && !slices.Contains(attrs, "envOverride")) {
+ continue
+ }
+ envVal, ok, _ := utils.LookupEnvOrFile(settingEnvName(field.key))
+ if !ok {
+ continue
+ }
+ envVal = kit.TrimQuotes(envVal)
+
+ var existing SettingVariable
+ err := tx.Where("key = ?", field.key).First(&existing).Error
+ switch {
+ case errors.Is(err, gorm.ErrRecordNotFound):
+ if createEnvSettingErr := tx.Create(&SettingVariable{Key: field.key, Value: envVal}).Error; createEnvSettingErr != nil {
+ return fmt.Errorf("persist env setting %s: %w", field.key, createEnvSettingErr)
+ }
+ slog.DebugContext(ctx, "Created setting from environment", "key", field.key)
+ case err != nil:
+ return fmt.Errorf("check setting %s: %w", field.key, err)
+ case existing.Value != envVal:
+ if updateEnvSettingErr := tx.Model(&existing).Update("value", envVal).Error; updateEnvSettingErr != nil {
+ return fmt.Errorf("update env setting %s: %w", field.key, updateEnvSettingErr)
+ }
+ slog.DebugContext(ctx, "Updated setting from environment", "key", field.key)
}
}
return nil
@@ -893,115 +764,26 @@ func (s *SettingsService) PersistEnvSettingsIfMissing(ctx context.Context) error
return s.LoadDatabaseSettings(context.WithoutCancel(ctx))
}
-func allowedSettingKeys() map[string]struct{} {
- allowed := make(map[string]struct{})
-
- settingsType := reflect.TypeFor[Settings]()
- for field := range settingsType.Fields() {
- key, _, _ := strings.Cut(field.Tag.Get("key"), ",")
- if key == "" {
- continue
- }
- allowed[key] = struct{}{}
- }
-
- allowed["encryptionKey"] = struct{}{}
-
- return allowed
-}
-
-func (s *SettingsService) processEnvField(ctx context.Context, tx *gorm.DB, field reflect.StructField, isEnvOnlyMode bool) error {
- tag := field.Tag.Get("key")
- key, attrs, _ := strings.Cut(tag, ",")
-
- if !s.shouldProcessField(key, attrs, isEnvOnlyMode) {
- return nil
- }
-
- envVarName := strings.ToUpper(kit.SnakeCase(key))
- envVal, ok, _ := utils.LookupEnvOrFile(envVarName)
- if !ok {
- return nil
- }
- envVal = kit.TrimQuotes(envVal)
-
- return s.upsertEnvSetting(ctx, tx, key, envVal)
-}
-
-func (s *SettingsService) shouldProcessField(key, attrs string, isEnvOnlyMode bool) bool {
- if key == "" || strings.Contains(attrs, "internal") {
- return false
- }
-
- // If not in env-only mode, only persist if it's explicitly marked as envOverride
- return isEnvOnlyMode || strings.Contains(attrs, "envOverride")
-}
-
-func (s *SettingsService) upsertEnvSetting(ctx context.Context, tx *gorm.DB, key, envVal string) error {
- var existing SettingVariable
- err := tx.Where("key = ?", key).First(&existing).Error
-
- switch {
- case errors.Is(err, gorm.ErrRecordNotFound):
- newVar := SettingVariable{Key: key, Value: envVal}
- if createEnvSettingErr := tx.Create(&newVar).Error; createEnvSettingErr != nil {
- return fmt.Errorf("persist env setting %s: %w", key, createEnvSettingErr)
- }
- slog.DebugContext(ctx, "Created setting from environment", "key", key)
- case err != nil:
- return fmt.Errorf("check setting %s: %w", key, err)
- default:
- if existing.Value != envVal {
- if updateEnvSettingErr := tx.Model(&existing).Update("value", envVal).Error; updateEnvSettingErr != nil {
- return fmt.Errorf("update env setting %s: %w", key, updateEnvSettingErr)
- }
- slog.DebugContext(ctx, "Updated setting from environment", "key", key)
- }
- }
-
- return nil
-}
-
func (s *SettingsService) ListSettings(visibility SettingVisibility) []SettingVariable {
return s.GetSettingsConfig().ToSettingVariableSlice(visibility, true)
}
// GetSettingType returns the type from the setting metadata
func (s *SettingsService) GetSettingType(key string) string {
- rt := reflect.TypeFor[Settings]()
- for field := range rt.Fields() {
- keyTag := field.Tag.Get("key")
- fieldKey, _, _ := strings.Cut(keyTag, ",")
- if fieldKey == key {
- metaTag := field.Tag.Get("meta")
- parts := strings.SplitSeq(metaTag, ";")
- for part := range parts {
- if after, ok := strings.CutPrefix(part, "type="); ok {
- return after
- }
+ _, byKey := getSettingsFieldCacheInternal()
+ if field, ok := byKey[key]; ok {
+ metaTag := reflect.TypeFor[Settings]().Field(field.index).Tag.Get("meta")
+ for part := range strings.SplitSeq(metaTag, ";") {
+ if after, found := strings.CutPrefix(part, "type="); found {
+ return after
}
- return "text" // default type
}
}
- return "text" // default if not found
-}
-
-func (s *SettingsService) setupInstanceID(ctx context.Context) error {
- instanceID := s.GetSettingsConfig().InstanceID.Value
- if instanceID != "" {
- return nil
- }
-
- err := s.UpdateSetting(ctx, "instanceId", uuid.New().String())
- if err != nil {
- return fmt.Errorf("failed to set instance ID in database: %w", err)
- }
-
- return nil
+ return "text"
}
func (s *SettingsService) settingValue[T any](ctx context.Context, key string, parse func(string) (T, error)) mo.Option[T] {
- cfg := s.getEffectiveSettingsConfigInternal(ctx)
+ cfg := s.GetSettingsOrDefaults(ctx)
value, _, _, err := cfg.FieldByKey(key)
if err != nil || value == "" {
return mo.None[T]()
@@ -1038,9 +820,8 @@ func (s *SettingsService) SetStringSetting(ctx context.Context, key, value strin
return s.UpdateSetting(ctx, key, value)
}
-// SetContainerAutoUpdateExclusionInternal adds or removes a container name from
-// the autoUpdateExcludedContainers setting. When excluded is true the container
-// is added to the list; when false it is removed.
+// SetContainerAutoUpdateExclusionInternal adds (excluded) or removes a container name
+// in the autoUpdateExcludedContainers setting.
func (s *SettingsService) SetContainerAutoUpdateExclusionInternal(ctx context.Context, containerName string, excluded bool) error {
s.writes.Lock()
defer s.writes.Unlock()
@@ -1049,23 +830,13 @@ func (s *SettingsService) SetContainerAutoUpdateExclusionInternal(ctx context.Co
}
ordered := kit.Unique(kit.TrimNonEmpty(strings.Split(s.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""), ",")))
-
if excluded {
ordered = kit.Unique(append(ordered, containerName))
} else {
- filtered := ordered[:0]
- for _, name := range ordered {
- if name != containerName {
- filtered = append(filtered, name)
- }
- }
- ordered = filtered
+ ordered = slices.DeleteFunc(ordered, func(name string) bool { return name == containerName })
}
- if err := s.updateSettingValueNoRefreshInternal(ctx, "autoUpdateExcludedContainers", strings.Join(ordered, ",")); err != nil {
- return err
- }
- return s.refreshSettingsCacheInternal(context.WithoutCancel(ctx))
+ return s.persistSettings(ctx, []SettingVariable{{Key: "autoUpdateExcludedContainers", Value: strings.Join(ordered, ",")}})
}
func (s *SettingsService) EnsureEncryptionKey(ctx context.Context) (string, error) {
@@ -1075,50 +846,14 @@ func (s *SettingsService) EnsureEncryptionKey(ctx context.Context) (string, erro
return "", err
}
- const keyName = "encryptionKey"
- var key string
-
- err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- var sv SettingVariable
- err := tx.Where("key = ?", keyName).First(&sv).Error
-
- if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
- return fmt.Errorf("failed to load encryption key: %w", err)
- }
-
- if sv.Value != "" {
- key = sv.Value
- return nil
- }
-
- notFound := errors.Is(err, gorm.ErrRecordNotFound)
+ return ensureSettingValue(ctx, s.db, "encryptionKey", "encryption key", func() (string, error) {
sum := sha256.Sum256([]byte(uuid.New().String()))
- generatedKey := base64.StdEncoding.EncodeToString(sum[:])
- key = generatedKey
-
- if notFound {
- if createErr := tx.Create(&SettingVariable{Key: keyName, Value: generatedKey}).Error; createErr != nil {
- return fmt.Errorf("failed to persist encryption key: %w", createErr)
- }
- return nil
- }
-
- if updErr := tx.Model(&SettingVariable{}).
- Where("key = ?", keyName).
- Update("value", generatedKey).Error; updErr != nil {
- return fmt.Errorf("failed to update encryption key: %w", updErr)
- }
- return nil
+ return base64.StdEncoding.EncodeToString(sum[:]), nil
})
- if err != nil {
- return "", err
- }
-
- return key, nil
}
func (s *SettingsService) EnsureJwtSigningKey(ctx context.Context) (*mldsa.PrivateKey, error) {
- seed, err := s.ensureEncryptedKeyInternal(ctx, "jwtSigningKeySeed", mldsa.PrivateKeySize)
+ seed, err := s.ensureEncryptedKey(ctx, "jwtSigningKeySeed", mldsa.PrivateKeySize)
if err != nil {
return nil, err
}
@@ -1130,66 +865,46 @@ func (s *SettingsService) EnsureJwtSigningKey(ctx context.Context) (*mldsa.Priva
}
func (s *SettingsService) EnsureBrowserSessionSigningKey(ctx context.Context) ([]byte, error) {
- return s.ensureEncryptedKeyInternal(ctx, "browserSessionSigningKey", browserSessionSigningKeySize)
+ return s.ensureEncryptedKey(ctx, "browserSessionSigningKey", browserSessionSigningKeySize)
}
-func (s *SettingsService) ensureEncryptedKeyInternal(ctx context.Context, keyName string, size int) ([]byte, error) {
+func (s *SettingsService) ensureEncryptedKey(ctx context.Context, keyName string, size int) ([]byte, error) {
s.writes.Lock()
defer s.writes.Unlock()
if err := ctx.Err(); err != nil {
return nil, err
}
- var key []byte
-
- err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- var sv SettingVariable
- err := tx.Where("key = ?", keyName).First(&sv).Error
- if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
- return fmt.Errorf("failed to load signing key: %w", err)
- }
-
- if sv.Value != "" {
- decoded, decErr := crypto.Decrypt(sv.Value)
- if decErr != nil {
- return fmt.Errorf("failed to decrypt signing key: %w", decErr)
- }
- key, decErr = base64.StdEncoding.DecodeString(decoded)
- if decErr != nil {
- return fmt.Errorf("failed to decode signing key: %w", decErr)
- }
- if len(key) != size {
- return fmt.Errorf("invalid signing key length: got %d, want %d", len(key), size)
- }
- return nil
- }
-
- key = make([]byte, size)
- if _, genErr := rand.Read(key); genErr != nil {
- return fmt.Errorf("failed to generate signing key: %w", genErr)
+ var generated []byte
+ stored, err := ensureSettingValue(ctx, s.db, keyName, "signing key", func() (string, error) {
+ generated = make([]byte, size)
+ if _, genErr := rand.Read(generated); genErr != nil {
+ return "", fmt.Errorf("failed to generate signing key: %w", genErr)
}
- encrypted, encErr := crypto.Encrypt(base64.StdEncoding.EncodeToString(key))
+ encrypted, encErr := crypto.Encrypt(base64.StdEncoding.EncodeToString(generated))
if encErr != nil {
- return fmt.Errorf("failed to encrypt signing key: %w", encErr)
- }
-
- if errors.Is(err, gorm.ErrRecordNotFound) {
- if createSigningKeyErr := tx.Create(&SettingVariable{Key: keyName, Value: encrypted}).Error; createSigningKeyErr != nil {
- return fmt.Errorf("failed to persist signing key: %w", createSigningKeyErr)
- }
- return nil
- }
- if updateSigningKeyErr := tx.Model(&SettingVariable{}).
- Where("key = ?", keyName).
- Update("value", encrypted).Error; updateSigningKeyErr != nil {
- return fmt.Errorf("failed to update signing key: %w", updateSigningKeyErr)
+ return "", fmt.Errorf("failed to encrypt signing key: %w", encErr)
}
- return nil
+ return encrypted, nil
})
if err != nil {
return nil, err
}
+ if generated != nil {
+ return generated, nil
+ }
+ decoded, err := crypto.Decrypt(stored)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decrypt signing key: %w", err)
+ }
+ key, err := base64.StdEncoding.DecodeString(decoded)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decode signing key: %w", err)
+ }
+ if len(key) != size {
+ return nil, fmt.Errorf("invalid signing key length: got %d, want %d", len(key), size)
+ }
return key, nil
}
@@ -1198,95 +913,56 @@ func (s *SettingsService) NormalizeProjectsDirectory(ctx context.Context, projec
slog.DebugContext(ctx, "PROJECTS_DIRECTORY environment variable is set, skipping normalization", "value", projectsDirEnv)
return nil
}
-
- s.writes.Lock()
- defer s.writes.Unlock()
- if err := ctx.Err(); err != nil {
- return err
- }
-
- var projectsDirSetting SettingVariable
- err := s.db.WithContext(ctx).Where("key = ?", "projectsDirectory").First(&projectsDirSetting).Error
- if errors.Is(err, gorm.ErrRecordNotFound) {
- slog.DebugContext(ctx, "No projectsDirectory setting found, skipping normalization")
- return nil
- }
- if err != nil {
- return fmt.Errorf("failed to load projectsDirectory setting: %w", err)
- }
-
- value := strings.TrimSpace(projectsDirSetting.Value)
- isMapping := strings.Contains(value, ":") && (strings.HasPrefix(value, "/") || projects.IsWindowsDrivePath(value))
- if filepath.IsAbs(value) || isMapping {
- slog.DebugContext(ctx, "Projects directory already normalized or custom, skipping", "value", projectsDirSetting.Value)
- return nil
- }
-
- cwd, _ := os.Getwd()
- absPath, absErr := filepath.Abs(value)
- if absErr != nil {
- return fmt.Errorf("failed to resolve relative path to absolute: %w", absErr)
- }
- slog.InfoContext(ctx, "Normalizing projects directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)
- if updateSettingValueNoRefreshErr := s.updateSettingValueNoRefreshInternal(ctx, "projectsDirectory", absPath); updateSettingValueNoRefreshErr != nil {
- return fmt.Errorf("failed to update projectsDirectory: %w", updateSettingValueNoRefreshErr)
- }
- if refreshSettingsCacheErr := s.refreshSettingsCacheInternal(context.WithoutCancel(ctx)); refreshSettingsCacheErr != nil {
- return refreshSettingsCacheErr
- }
- slog.InfoContext(ctx, "Successfully normalized projects directory")
- s.publishSettingsChangesInternal([]libarcane.SettingUpdate{{Key: "projectsDirectory", Value: absPath}})
- return nil
+ return s.normalizeDirectorySetting(ctx, "projectsDirectory")
}
func (s *SettingsService) NormalizeBuildsDirectory(ctx context.Context) error {
- const buildsKey = "buildsDirectory"
- envVarName := strings.ToUpper(kit.SnakeCase(buildsKey))
- if envVal, ok, _ := utils.LookupEnvOrFile(envVarName); ok && strings.TrimSpace(envVal) != "" {
+ if envVal, ok, _ := utils.LookupEnvOrFile(settingEnvName("buildsDirectory")); ok && strings.TrimSpace(envVal) != "" {
slog.DebugContext(ctx, "BUILDS_DIRECTORY environment variable is set, skipping normalization", "value", envVal)
return nil
}
+ return s.normalizeDirectorySetting(ctx, "buildsDirectory")
+}
+func (s *SettingsService) normalizeDirectorySetting(ctx context.Context, key string) error {
s.writes.Lock()
defer s.writes.Unlock()
if err := ctx.Err(); err != nil {
return err
}
- var buildsDirSetting SettingVariable
- err := s.db.WithContext(ctx).Where("key = ?", buildsKey).First(&buildsDirSetting).Error
+ var setting SettingVariable
+ err := s.db.WithContext(ctx).Where("key = ?", key).First(&setting).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
- slog.DebugContext(ctx, "No buildsDirectory setting found, skipping normalization")
+ slog.DebugContext(ctx, "No directory setting found, skipping normalization", "key", key)
return nil
}
if err != nil {
- return fmt.Errorf("failed to load buildsDirectory setting: %w", err)
+ return fmt.Errorf("failed to load %s setting: %w", key, err)
}
- value := strings.TrimSpace(buildsDirSetting.Value)
- if value == "" || filepath.IsAbs(value) {
- slog.DebugContext(ctx, "Builds directory already normalized or empty, skipping", "value", buildsDirSetting.Value)
+ value := strings.TrimSpace(setting.Value)
+ isMapping := strings.Contains(value, ":") && (strings.HasPrefix(value, "/") || projects.IsWindowsDrivePath(value))
+ if value == "" || filepath.IsAbs(value) || isMapping {
+ slog.DebugContext(ctx, "Directory setting already normalized, custom, or empty, skipping", "key", key, "value", setting.Value)
return nil
}
cwd, _ := os.Getwd()
- absPath, absErr := filepath.Abs(value)
- if absErr != nil {
- return fmt.Errorf("failed to resolve relative path to absolute: %w", absErr)
- }
- slog.InfoContext(ctx, "Normalizing builds directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)
- if updateSettingValueNoRefreshErr := s.updateSettingValueNoRefreshInternal(ctx, buildsKey, absPath); updateSettingValueNoRefreshErr != nil {
- return fmt.Errorf("failed to update buildsDirectory: %w", updateSettingValueNoRefreshErr)
+ absPath, err := filepath.Abs(value)
+ if err != nil {
+ return fmt.Errorf("failed to resolve relative path to absolute: %w", err)
}
- if refreshSettingsCacheErr := s.refreshSettingsCacheInternal(context.WithoutCancel(ctx)); refreshSettingsCacheErr != nil {
- return refreshSettingsCacheErr
+ slog.InfoContext(ctx, "Normalizing directory setting from relative to absolute path", "key", key, "from", value, "to", absPath, "base", cwd)
+ if persistSettingsErr := s.persistSettings(ctx, []SettingVariable{{Key: key, Value: absPath}}); persistSettingsErr != nil {
+ return persistSettingsErr
}
- slog.InfoContext(ctx, "Successfully normalized builds directory")
- s.publishSettingsChangesInternal([]libarcane.SettingUpdate{{Key: buildsKey, Value: absPath}})
+ slog.InfoContext(ctx, "Successfully normalized directory setting", "key", key)
+ s.publishSettingsChanges([]libarcane.SettingUpdate{{Key: key, Value: absPath}})
return nil
}
-func (s *SettingsService) enqueueEffectInternal(effect func()) error {
+func (s *SettingsService) enqueueEffect(effect func()) error {
s.effectsMu.Lock()
defer s.effectsMu.Unlock()
if s.effectsClosed {
@@ -1300,33 +976,6 @@ func (s *SettingsService) enqueueEffectInternal(effect func()) error {
return nil
}
-func (s *SettingsService) runEffectsInternal() {
- defer close(s.effectsDone)
- for {
- s.effectsMu.Lock()
- if len(s.effects) > 0 {
- effect := s.effects[0]
- s.effects[0] = nil
- s.effects = s.effects[1:]
- s.effectsMu.Unlock()
- func() { var err error; defer utils.RecoverToError(&err, "settings effect"); effect() }()
- continue
- }
- closed := s.effectsClosed
- s.effectsMu.Unlock()
- if closed {
- return
- }
- select {
- case <-s.effectsWake:
- case <-s.lifecycleCtx.Done():
- s.effectsMu.Lock()
- s.effectsClosed = true
- s.effectsMu.Unlock()
- }
- }
-}
-
// Stop drains accepted effects and joins the settings worker.
func (s *SettingsService) Stop(ctx context.Context) error {
s.writes.Lock()
diff --git a/backend/internal/settings/service_test.go b/backend/internal/settings/service_test.go
index 59ef71d37..e135751a1 100644
--- a/backend/internal/settings/service_test.go
+++ b/backend/internal/settings/service_test.go
@@ -32,7 +32,7 @@ func setupSettingsTestDB(t *testing.T) *database.DB {
return &database.DB{DB: db}
}
-func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*SettingsService, error) {
+func newSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*SettingsService, error) {
t.Helper()
svc, err := NewSettingsService(ctx, db)
if err == nil {
@@ -41,11 +41,11 @@ func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *da
return svc, err
}
-func waitForSettingsNotificationsInternal(t *testing.T, svc *SettingsService) {
+func waitForSettingsNotifications(t *testing.T, svc *SettingsService) {
t.Helper()
done := make(chan struct{})
svc.writes.Lock()
- err := svc.enqueueEffectInternal(func() { close(done) })
+ err := svc.enqueueEffect(func() { close(done) })
svc.writes.Unlock()
require.NoError(t, err)
select {
@@ -58,7 +58,7 @@ func waitForSettingsNotificationsInternal(t *testing.T, svc *SettingsService) {
func TestSettingsService_EnsureDefaultSettings_ReplacesRetiredDefaults(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, db.Create(&SettingVariable{Key: "dockerClientRefreshInterval", Value: "*/30 * * * * *"}).Error)
require.NoError(t, db.Create(&SettingVariable{Key: "autoHealInterval", Value: "*/10 * * * * *"}).Error)
@@ -75,7 +75,7 @@ func TestSettingsService_EnsureDefaultSettings_ReplacesRetiredDefaults(t *testin
func TestSettingsService_EnsureDefaultSettings_Idempotent(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -164,7 +164,7 @@ func TestSettingsService_EnsureDefaultSettings_Idempotent(t *testing.T) {
func TestSettingsService_ImageEventWatcherSettingPersists(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
require.False(t, svc.GetBoolSetting(ctx, "imageEventWatcherEnabled", true))
@@ -181,7 +181,7 @@ func TestSettingsService_ImageEventWatcherSettingPersists(t *testing.T) {
func TestSettingsService_GetSettings_UnknownKeysIgnored(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.db.WithContext(ctx).
@@ -194,7 +194,7 @@ func TestSettingsService_GetSettings_UnknownKeysIgnored(t *testing.T) {
func TestSettingsService_GetSettings_UsesCachedSnapshotWithoutDatabase(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.SetStringSetting(ctx, "baseServerUrl", "http://cached"))
@@ -210,7 +210,7 @@ func TestSettingsService_GetSettings_UsesCachedSnapshotWithoutDatabase(t *testin
func TestSettingsService_AvatarMaxUploadSizeDefaultAndUpdate(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
current, err := svc.GetSettings(ctx)
@@ -228,10 +228,10 @@ func TestSettingsService_AvatarMaxUploadSizeDefaultAndUpdate(t *testing.T) {
require.Equal(t, "8", current.AvatarMaxUploadSizeMb.Value)
}
-func TestSettingsServiceUpdateSettingsRejectsOIDCIssuerChangeWithStoredSecretInternal(t *testing.T) {
+func TestSettingsServiceUpdateSettingsRejectsOIDCIssuerChangeWithStoredSecret(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "oidcIssuerUrl", "https://issuer.example.com"))
require.NoError(t, svc.UpdateSetting(ctx, "oidcClientSecret", "old-client-secret"))
@@ -247,10 +247,10 @@ func TestSettingsServiceUpdateSettingsRejectsOIDCIssuerChangeWithStoredSecretInt
require.Equal(t, "old-client-secret", current.OidcClientSecret.Value)
}
-func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWhenClearingSecretInternal(t *testing.T) {
+func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWhenClearingSecret(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "oidcIssuerUrl", "https://issuer.example.com"))
require.NoError(t, svc.UpdateSetting(ctx, "oidcClientSecret", "old-client-secret"))
@@ -267,10 +267,10 @@ func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWhenClearingSecretIn
require.Empty(t, current.OidcClientSecret.Value)
}
-func TestSettingsServiceUpdateSettingsRejectsEnablingOIDCWithoutSecretInternal(t *testing.T) {
+func TestSettingsServiceUpdateSettingsRejectsEnablingOIDCWithoutSecret(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
_, err = svc.UpdateSettings(ctx, settings.Update{
@@ -285,10 +285,10 @@ func TestSettingsServiceUpdateSettingsRejectsEnablingOIDCWithoutSecretInternal(t
require.Equal(t, "false", current.OidcEnabled.Value)
}
-func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWithReplacementSecretInternal(t *testing.T) {
+func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWithReplacementSecret(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "oidcIssuerUrl", "https://issuer.example.com"))
require.NoError(t, svc.UpdateSetting(ctx, "oidcClientSecret", "old-client-secret"))
@@ -312,10 +312,10 @@ func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWithReplacementSecre
require.Equal(t, "Café Login", current.OidcProviderName.Value)
}
-func TestSettingsServiceUpdateSettingsRejectsTrivyServerChangeWithStoredTokenInternal(t *testing.T) {
+func TestSettingsServiceUpdateSettingsRejectsTrivyServerChangeWithStoredToken(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerToken", "old-trivy-token"))
@@ -331,10 +331,10 @@ func TestSettingsServiceUpdateSettingsRejectsTrivyServerChangeWithStoredTokenInt
require.Equal(t, "old-trivy-token", current.TrivyServerToken.Value)
}
-func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithReplacementTokenInternal(t *testing.T) {
+func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithReplacementToken(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerToken", "old-trivy-token"))
@@ -351,10 +351,10 @@ func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithReplacementToke
require.Equal(t, "new-trivy-token", current.TrivyServerToken.Value)
}
-func TestSettingsServiceUpdateSettingsAllowsClearingTrivyServerWithStoredTokenInternal(t *testing.T) {
+func TestSettingsServiceUpdateSettingsAllowsClearingTrivyServerWithStoredToken(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerToken", "old-trivy-token"))
@@ -370,10 +370,10 @@ func TestSettingsServiceUpdateSettingsAllowsClearingTrivyServerWithStoredTokenIn
require.Equal(t, "old-trivy-token", current.TrivyServerToken.Value)
}
-func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithoutStoredTokenInternal(t *testing.T) {
+func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithoutStoredToken(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))
@@ -390,7 +390,7 @@ func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithoutStoredTokenI
func TestSettingsService_PruneUnknownSettings_RemovesStaleKeys(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "projectsDirectory", "/tmp/projects"))
@@ -417,7 +417,7 @@ func TestSettingsService_GetSettings_EnvOverride_OidcMergeAccounts(t *testing.T)
db := setupSettingsTestDB(t)
t.Setenv("OIDC_MERGE_ACCOUNTS", "true")
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -431,7 +431,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyScanTimeout(t *testing.T)
db := setupSettingsTestDB(t)
t.Setenv("TRIVY_SCAN_TIMEOUT", "1800")
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -447,7 +447,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyResourceLimits(t *testing.
t.Setenv("TRIVY_CPU_LIMIT", "2.5")
t.Setenv("TRIVY_MEMORY_LIMIT_MB", "2048")
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -463,7 +463,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyNetwork(t *testing.T) {
db := setupSettingsTestDB(t)
t.Setenv("TRIVY_NETWORK", "arcane-external")
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -477,7 +477,7 @@ func TestSettingsService_GetSettings_EnvOverride_FollowProjectSymlinks(t *testin
db := setupSettingsTestDB(t)
t.Setenv("FOLLOW_PROJECT_SYMLINKS", "true")
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -492,7 +492,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyRuntimeSecurity(t *testing
t.Setenv("TRIVY_SECURITY_OPTS", "label=disable,\nlabel=type:container_runtime_t")
t.Setenv("TRIVY_PRIVILEGED", "true")
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -507,7 +507,7 @@ func TestSettingsService_GetStringSetting_EnvOverride_SwarmStackSourcesDirectory
db := setupSettingsTestDB(t)
t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", "/mnt/swarm-from-env")
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "swarmStackSourcesDirectory", "/tmp/swarm-from-db"))
@@ -515,21 +515,21 @@ func TestSettingsService_GetStringSetting_EnvOverride_SwarmStackSourcesDirectory
require.Equal(t, "/mnt/swarm-from-env", svc.GetSettingsConfig().SwarmStackSourcesDirectory.Value)
}
-func TestSettingsService_isEnvOverrideActiveInternal(t *testing.T) {
+func TestSettingsService_isEnvOverrideActive(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.False(t, svc.IsEnvOverrideActive("oidcEnabled"))
t.Setenv("OIDC_ENABLED", "false")
- svcWithOverride, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svcWithOverride, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.True(t, svcWithOverride.IsEnvOverrideActive("oidcEnabled"))
t.Setenv("AUTH_SESSION_TIMEOUT", "120")
- svcWithNonOverrideEnv, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svcWithNonOverrideEnv, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.False(t, svcWithNonOverrideEnv.IsEnvOverrideActive("authSessionTimeout"))
}
@@ -537,7 +537,7 @@ func TestSettingsService_isEnvOverrideActiveInternal(t *testing.T) {
func TestSettingsService_GetSetHelpers(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
// Defaults for missing keys
@@ -559,7 +559,7 @@ func TestSettingsService_GetSetHelpers(t *testing.T) {
func TestSettingsService_UpdateSetting(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "pruneImageMode", "all"))
@@ -572,7 +572,7 @@ func TestSettingsService_UpdateSetting(t *testing.T) {
func TestSettingsService_UpdateSettingRejectsInvalidCronBeforePersistence(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "pollingInterval", "*/5 * * * * *"))
@@ -586,7 +586,7 @@ func TestSettingsService_UpdateSettingRejectsInvalidCronBeforePersistence(t *tes
func TestSettingsService_UpdateSetting_RefreshesCachedSnapshot(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.Equal(t, "http://localhost", svc.GetSettingsConfig().BaseServerURL.Value)
@@ -602,7 +602,7 @@ func TestSettingsService_UpdateSetting_RefreshesCachedSnapshot(t *testing.T) {
func TestSettingsService_UpdateSettings_PruneModesDoNotTriggerScheduledPruneCallback(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
callbackCalls := 0
@@ -615,14 +615,14 @@ func TestSettingsService_UpdateSettings_PruneModesDoNotTriggerScheduledPruneCall
PruneContainerUntil: new("24h"),
})
require.NoError(t, err)
- waitForSettingsNotificationsInternal(t, svc)
+ waitForSettingsNotifications(t, svc)
require.Equal(t, 0, callbackCalls)
}
func TestSettingsService_UpdateSettings_ScheduledPruneScheduleTriggersCallback(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
callbackCalls := 0
@@ -634,14 +634,14 @@ func TestSettingsService_UpdateSettings_ScheduledPruneScheduleTriggersCallback(t
ScheduledPruneEnabled: new("true"),
})
require.NoError(t, err)
- waitForSettingsNotificationsInternal(t, svc)
+ waitForSettingsNotifications(t, svc)
require.Equal(t, 1, callbackCalls)
}
-func TestSettingsServiceActorSerializesConcurrentUpdatesInternal(t *testing.T) {
+func TestSettingsServiceActorSerializesConcurrentUpdates(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -666,10 +666,10 @@ func TestSettingsServiceActorSerializesConcurrentUpdatesInternal(t *testing.T) {
require.Equal(t, "/data/templates-b", current.TemplatesDirectory.Value)
}
-func TestSettingsServiceActorPublishesSnapshotBeforeAsynchronousNotificationInternal(t *testing.T) {
+func TestSettingsServiceActorPublishesSnapshotBeforeAsynchronousNotification(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -708,10 +708,10 @@ func TestSettingsServiceActorPublishesSnapshotBeforeAsynchronousNotificationInte
close(releaseCallback)
}
-func TestSettingsServiceActorNotifiesSubscriberOnceForMultipleMatchingKeysInternal(t *testing.T) {
+func TestSettingsServiceActorNotifiesSubscriberOnceForMultipleMatchingKeys(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -731,10 +731,10 @@ func TestSettingsServiceActorNotifiesSubscriberOnceForMultipleMatchingKeysIntern
require.Equal(t, int32(1), calls.Load())
}
-func TestSettingsServiceActorDoesNotPublishSensitiveValuesInternal(t *testing.T) {
+func TestSettingsServiceActorDoesNotPublishSensitiveValues(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -744,14 +744,14 @@ func TestSettingsServiceActorDoesNotPublishSensitiveValuesInternal(t *testing.T)
})
_, err = svc.UpdateSettings(ctx, settings.Update{OidcClientSecret: new("should-not-leave-settings-service")})
require.NoError(t, err)
- waitForSettingsNotificationsInternal(t, svc)
+ waitForSettingsNotifications(t, svc)
require.Zero(t, calls.Load())
}
-func TestSettingsServiceActorSerializesContainerExclusionUpdatesInternal(t *testing.T) {
+func TestSettingsServiceActorSerializesContainerExclusionUpdates(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -780,7 +780,7 @@ func BenchmarkSettingsService_GetSettings(b *testing.B) {
require.FailNowf(b, "benchmark database migration failed", "%v", autoMigrateErr)
}
settingsDB := &database.DB{DB: db}
- svc, err := newSettingsServiceForTestInternal(b, ctx, settingsDB)
+ svc, err := newSettingsServiceForTest(b, ctx, settingsDB)
if err != nil {
require.FailNowf(b, "benchmark service setup failed", "%v", err)
}
@@ -802,7 +802,7 @@ func BenchmarkSettingsService_GetSettings(b *testing.B) {
func TestSettingsService_EnsureEncryptionKey(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
k1, err := svc.EnsureEncryptionKey(ctx)
@@ -834,7 +834,7 @@ func TestSettingsService_EnsureEncryptionKey(t *testing.T) {
func TestSettingsService_LoadDatabaseSettings_ReloadsChanges(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
// Initially empty DB -> defaults (not persisted yet)
@@ -864,7 +864,7 @@ func TestSettingsService_LoadDatabaseSettings_UIConfigurationDisabled_Env(t *tes
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
// Reload explicitly (NewSettingsService already did, but explicit for clarity)
@@ -878,7 +878,7 @@ func TestSettingsService_LoadDatabaseSettings_UIConfigurationDisabled_Env(t *tes
func TestSettingsService_UpdateSettings_RefreshesCache(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -907,7 +907,7 @@ func TestSettingsService_UpdateSettings_ReturnsEnvOverriddenValues(t *testing.T)
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -929,7 +929,7 @@ func TestSettingsService_UpdateSettings_ReturnsEnvOverriddenValues(t *testing.T)
func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyScanTimeout(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -940,7 +940,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyScanTimeout(
_, err = svc.UpdateSettings(ctx, settings.Update{TrivyScanTimeout: new("1200")})
require.NoError(t, err)
- waitForSettingsNotificationsInternal(t, svc)
+ waitForSettingsNotifications(t, svc)
require.NotNil(t, callbackPayload)
require.Contains(t, callbackPayload, libarcane.SettingUpdate{Key: "trivyScanTimeout", Value: "1200"})
@@ -949,7 +949,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyScanTimeout(
func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyResourceLimits(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -964,7 +964,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyResourceLimi
TrivyMemoryLimitMb: new("3072"),
})
require.NoError(t, err)
- waitForSettingsNotificationsInternal(t, svc)
+ waitForSettingsNotifications(t, svc)
require.NotNil(t, callbackPayload)
require.Contains(t, callbackPayload, libarcane.SettingUpdate{Key: "trivyResourceLimitsEnabled", Value: "false"})
@@ -975,7 +975,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyResourceLimi
func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyConcurrentScanContainers(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -986,7 +986,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyConcurrentSc
_, err = svc.UpdateSettings(ctx, settings.Update{TrivyConcurrentScanContainers: new("4")})
require.NoError(t, err)
- waitForSettingsNotificationsInternal(t, svc)
+ waitForSettingsNotifications(t, svc)
require.NotNil(t, callbackPayload)
require.Contains(t, callbackPayload, libarcane.SettingUpdate{Key: "trivyConcurrentScanContainers", Value: "4"})
@@ -995,7 +995,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyConcurrentSc
func TestSettingsService_UpdateSettings_TrivyNetworkTriggersVulnerabilityCallback(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -1006,14 +1006,14 @@ func TestSettingsService_UpdateSettings_TrivyNetworkTriggersVulnerabilityCallbac
_, err = svc.UpdateSettings(ctx, settings.Update{TrivyNetwork: new("arcane-external")})
require.NoError(t, err)
- waitForSettingsNotificationsInternal(t, svc)
+ waitForSettingsNotifications(t, svc)
require.True(t, callbackCalled)
}
func TestSettingsService_UpdateSettings_TrivyNetworkDoesNotTriggerTimeoutCallback(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -1024,14 +1024,14 @@ func TestSettingsService_UpdateSettings_TrivyNetworkDoesNotTriggerTimeoutCallbac
_, err = svc.UpdateSettings(ctx, settings.Update{TrivyNetwork: new("arcane-external")})
require.NoError(t, err)
- waitForSettingsNotificationsInternal(t, svc)
+ waitForSettingsNotifications(t, svc)
require.Nil(t, callbackPayload)
}
func TestSettingsService_UpdateSettings_TrivyRuntimeSecurityTriggersVulnerabilityCallback(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -1045,14 +1045,14 @@ func TestSettingsService_UpdateSettings_TrivyRuntimeSecurityTriggersVulnerabilit
TrivyPrivileged: new("true"),
})
require.NoError(t, err)
- waitForSettingsNotificationsInternal(t, svc)
+ waitForSettingsNotifications(t, svc)
require.True(t, callbackCalled)
}
func TestSettingsService_UpdateSettings_TrivyRuntimeSecurityDoesNotTriggerTimeoutCallback(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -1066,7 +1066,7 @@ func TestSettingsService_UpdateSettings_TrivyRuntimeSecurityDoesNotTriggerTimeou
TrivyPrivileged: new("true"),
})
require.NoError(t, err)
- waitForSettingsNotificationsInternal(t, svc)
+ waitForSettingsNotifications(t, svc)
require.Nil(t, callbackPayload)
}
@@ -1082,7 +1082,7 @@ func TestSettingsService_LoadDatabaseSettings_InternalKeys_EnvMode(t *testing.T)
internalVal := "test-instance-id"
require.NoError(t, db.DB.Create(&SettingVariable{Key: internalKey, Value: internalVal}).Error)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
// Reload explicitly to trigger the env loading path
@@ -1096,7 +1096,7 @@ func TestSettingsService_LoadDatabaseSettings_InternalKeys_EnvMode(t *testing.T)
func TestSettingsService_NormalizeProjectsDirectory_ConvertsRelativeToAbsolute(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
// Seed with relative path
@@ -1119,7 +1119,7 @@ func TestSettingsService_NormalizeProjectsDirectory_ConvertsRelativeToAbsolute(t
func TestSettingsService_NormalizeProjectsDirectory_SkipsWhenEnvSet(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
// Seed with relative path
@@ -1138,7 +1138,7 @@ func TestSettingsService_NormalizeProjectsDirectory_SkipsWhenEnvSet(t *testing.T
func TestSettingsService_NormalizeProjectsDirectory_LeavesOtherPathsUnchanged(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
customPath := "/custom/projects/path"
@@ -1157,7 +1157,7 @@ func TestSettingsService_NormalizeProjectsDirectory_LeavesOtherPathsUnchanged(t
func TestSettingsService_NormalizeProjectsDirectory_HandlesNotFound(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
// Don't create the setting at all
@@ -1170,7 +1170,7 @@ func TestSettingsService_NormalizeProjectsDirectory_HandlesNotFound(t *testing.T
func TestSettingsService_NormalizeProjectsDirectory_UpdatesCacheAfterNormalization(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -1193,10 +1193,10 @@ func TestSettingsService_NormalizeProjectsDirectory_UpdatesCacheAfterNormalizati
require.True(t, filepath.IsAbs(cfg2.ProjectsDirectory.Value), "path should be absolute")
}
-func TestSettingsService_NormalizeProjectsDirectoryPublishesChangeInternal(t *testing.T) {
+func TestSettingsService_NormalizeProjectsDirectoryPublishesChange(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
require.NoError(t, svc.UpdateSetting(ctx, "projectsDirectory", "data/projects"))
@@ -1208,18 +1208,16 @@ func TestSettingsService_NormalizeProjectsDirectoryPublishesChangeInternal(t *te
defer unsubscribe()
require.NoError(t, svc.NormalizeProjectsDirectory(ctx, ""))
- waitForSettingsNotificationsInternal(t, svc)
+ waitForSettingsNotifications(t, svc)
updates := <-notified
require.Len(t, updates, 1)
require.Equal(t, "projectsDirectory", updates[0].Key)
require.True(t, filepath.IsAbs(updates[0].Value))
}
-// TestSettingsServiceEffectiveSnapshotMaterializedInternal verifies the
-// env-override-applied snapshot is built once per refresh: reads share one
-// pointer (no per-call clone), env overrides win over database values, and
-// an update rebuilds the snapshot while keeping the override applied.
-func TestSettingsServiceEffectiveSnapshotMaterializedInternal(t *testing.T) {
+// TestSettingsServiceEffectiveSnapshotMaterialized verifies reads share one env-override-applied
+// snapshot per refresh, and an update rebuilds it with the override still applied.
+func TestSettingsServiceEffectiveSnapshotMaterialized(t *testing.T) {
ctx := t.Context()
t.Setenv("PROJECTS_DIRECTORY", "/env/projects")
@@ -1227,7 +1225,7 @@ func TestSettingsServiceEffectiveSnapshotMaterializedInternal(t *testing.T) {
require.NoError(t, db.Create(&SettingVariable{Key: "projectsDirectory", Value: "/db/projects"}).Error)
require.NoError(t, db.Create(&SettingVariable{Key: "baseServerUrl", Value: "http://before.example"}).Error)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
// Env override beats the database value on the effective snapshot.
@@ -1257,7 +1255,7 @@ func TestSettingsService_FeatureDefaultsAndUnknownFeature(t *testing.T) {
require.NoError(t, svc.RequireFeature(t.Context(), features.VulnerabilityManagement))
require.False(t, svc.IsFeatureEnabled(t.Context(), features.ID("unknown")))
- svc, err := newSettingsServiceForTestInternal(t, t.Context(), setupSettingsTestDB(t))
+ svc, err := newSettingsServiceForTest(t, t.Context(), setupSettingsTestDB(t))
require.NoError(t, err)
require.True(t, svc.IsFeatureEnabled(t.Context(), features.VulnerabilityManagement))
require.NoError(t, svc.EnsureDefaultSettings(t.Context()))
@@ -1270,9 +1268,9 @@ func TestSettingsService_FeatureDefaultsAndUnknownFeature(t *testing.T) {
func TestSettingsService_FeaturePersistenceAndEnvironmentIsolation(t *testing.T) {
ctx := t.Context()
db := setupSettingsTestDB(t)
- svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ svc, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
- other, err := newSettingsServiceForTestInternal(t, ctx, setupSettingsTestDB(t))
+ other, err := newSettingsServiceForTest(t, ctx, setupSettingsTestDB(t))
require.NoError(t, err)
var notifications int
@@ -1286,7 +1284,7 @@ func TestSettingsService_FeaturePersistenceAndEnvironmentIsolation(t *testing.T)
disabled := "false"
_, err = svc.UpdateSettings(ctx, settings.Update{FeatureVulnerabilityManagementEnabled: &disabled})
require.NoError(t, err)
- waitForSettingsNotificationsInternal(t, svc)
+ waitForSettingsNotifications(t, svc)
require.Equal(t, 1, notifications)
require.False(t, svc.IsFeatureEnabled(ctx, features.VulnerabilityManagement))
require.True(t, other.IsFeatureEnabled(ctx, features.VulnerabilityManagement))
@@ -1298,7 +1296,7 @@ func TestSettingsService_FeaturePersistenceAndEnvironmentIsolation(t *testing.T)
require.Equal(t, http.StatusForbidden, apiErr.HTTPStatus())
require.Equal(t, common.APIErrorCodeFeatureDisabled, apiErr.Code)
- reloaded, err := newSettingsServiceForTestInternal(t, ctx, db)
+ reloaded, err := newSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.False(t, reloaded.IsFeatureEnabled(ctx, features.VulnerabilityManagement))
enabled := "true"
@@ -1313,7 +1311,7 @@ func TestSettingsService_FeaturePersistenceAndEnvironmentIsolation(t *testing.T)
func TestSettingsService_FeatureEnvironmentOverride(t *testing.T) {
t.Setenv("FEATURE_VULNERABILITY_MANAGEMENT_ENABLED", "false")
- svc, err := newSettingsServiceForTestInternal(t, t.Context(), setupSettingsTestDB(t))
+ svc, err := newSettingsServiceForTest(t, t.Context(), setupSettingsTestDB(t))
require.NoError(t, err)
enabled := "true"
_, err = svc.UpdateSettings(t.Context(), settings.Update{FeatureVulnerabilityManagementEnabled: &enabled})
@@ -1326,7 +1324,7 @@ func TestSettingsService_FeatureEnvironmentOverride(t *testing.T) {
func TestSettingsService_RejectInvalidFeatureBoolean(t *testing.T) {
for _, invalid := range []string{"", "yes", "0", "TRUE"} {
t.Run(invalid, func(t *testing.T) {
- svc, err := newSettingsServiceForTestInternal(t, t.Context(), setupSettingsTestDB(t))
+ svc, err := newSettingsServiceForTest(t, t.Context(), setupSettingsTestDB(t))
require.NoError(t, err)
_, err = svc.UpdateSettings(t.Context(), settings.Update{FeatureVulnerabilityManagementEnabled: &invalid})
require.ErrorIs(t, err, common.ErrValidation)
@@ -1343,7 +1341,7 @@ func TestSettingsService_SwarmFeatureDefaultsOffAndPersists(t *testing.T) {
var nilService *SettingsService
require.False(t, nilService.IsFeatureEnabled(t.Context(), features.Swarm))
- svc, err := newSettingsServiceForTestInternal(t, t.Context(), setupSettingsTestDB(t))
+ svc, err := newSettingsServiceForTest(t, t.Context(), setupSettingsTestDB(t))
require.NoError(t, err)
require.False(t, svc.IsFeatureEnabled(t.Context(), features.Swarm))
require.Contains(t, svc.ListSettings(SettingVisibilityPublic), SettingVariable{Key: features.SwarmSettingKey, Value: "false"})
diff --git a/backend/internal/system/children/backup/service.go b/backend/internal/system/children/backup/service.go
index 797219455..62ba31dab 100644
--- a/backend/internal/system/children/backup/service.go
+++ b/backend/internal/system/children/backup/service.go
@@ -31,6 +31,10 @@ import (
"go.getarcane.app/docker/compat"
kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/cgroup"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
+ "go.opentelemetry.io/otel/trace"
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
@@ -54,6 +58,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/backupbrowser"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
const (
@@ -139,6 +144,7 @@ type Service struct {
// backupWorkflow runs backups StartBackup accepted; policyWorkflow runs scheduled policies.
backupWorkflow *flow.Workflow
policyWorkflow *flow.Workflow
+ operations metric.Int64Counter
}
// manualBackupInput is a manual backup's workflow payload; the run row is created before it starts.
@@ -167,6 +173,12 @@ func NewService(deps Dependencies) *Service {
recoveryKeys: deps.RecoveryKeys, resolveRuntimeOptions: deps.ResolveRuntimeOptions,
jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),
}
+ var counterErr error
+ service.operations, counterErr = otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.backups",
+ metric.WithDescription("Backup operations by outcome"), metric.WithUnit("{backup}"))
+ if counterErr != nil {
+ otel.Handle(counterErr)
+ }
service.snapshots = snapshots.NewService(
deps.Engine,
deps.SQLDB,
@@ -230,6 +242,22 @@ func (s *Service) recoveryEnvironment(ctx context.Context) map[string]string {
return result
}
+// startOperation opens a backup span; the returned func ends it and counts the outcome.
+func (s *Service) startOperation(ctx context.Context, operation string, attrs ...attribute.KeyValue) (context.Context, func(err error, results ...attribute.KeyValue)) {
+ attrs = append(attrs, attribute.String("arcane.environment.id", "0"), attribute.String("arcane.backup.kind", "system"))
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "backup."+operation, trace.WithAttributes(attrs...))
+ return ctx, func(err error, results ...attribute.KeyValue) {
+ span.SetAttributes(results...)
+ tracing.End(span, err)
+ if s.operations != nil {
+ s.operations.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes(
+ attribute.String("arcane.backup.operation", operation),
+ attribute.String("arcane.backup.outcome", kit.Ternary(err == nil, "success", "failure")),
+ ))
+ }
+ }
+}
+
// acquireRun takes the shared system backup admission lease.
func (s *Service) acquireRun(ctx context.Context) (*runs.Lease, error) {
lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID)
@@ -431,6 +459,11 @@ func (s *Service) prepareBackup(ctx context.Context, trigger string, request bac
// executeBackup stages one snapshot, keeps it locally and/or replicates it to S3, then records the run's result.
func (s *Service) executeBackup(ctx context.Context, prepared *preparedSystemBackup) (_ *backuptypes.SystemBackupRun, err error) {
run, key, checkpoint := prepared.run, prepared.recoveryKey, prepared.checkpoint
+ ctx, finish := s.startOperation(ctx, "create", attribute.String("arcane.backup.id", run.ID), attribute.String("arcane.backup.trigger", run.Trigger))
+ defer func() {
+ finish(err, attribute.String("arcane.backup.target", string(run.Destination)), attribute.Int64("arcane.backup.size_bytes", run.Size),
+ attribute.String("arcane.backup.s3_destination_id", run.S3DestinationID))
+ }()
defer func() {
if err = cmp.Or(err, ctx.Err()); err != nil {
run.Status, run.Error = backuptypes.SystemBackupStatusFailed, err.Error()
@@ -560,7 +593,10 @@ func (s *Service) BrowseBackupFiles(ctx context.Context, id, recoveryKey, reques
// RestoreBackupFiles restores selected project files into the current projects
// directory after creating a safety snapshot.
-func (s *Service) RestoreBackupFiles(ctx context.Context, id string, request backuptypes.RestoreSystemBackupFilesRequest, user usertypes.Actor) error {
+func (s *Service) RestoreBackupFiles(ctx context.Context, id string, request backuptypes.RestoreSystemBackupFilesRequest, user usertypes.Actor) (opErr error) {
+ ctx, finish := s.startOperation(ctx, "restore_files", attribute.String("arcane.backup.id", id),
+ attribute.Int("arcane.result.count", len(request.Paths)), attribute.Bool("arcane.backup.select_all", request.SelectAll))
+ defer func() { finish(opErr) }()
lease, err := s.acquireRun(ctx)
if err != nil {
return err
@@ -591,7 +627,9 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, id string, request bac
return s.snapshots.RestoreFiles(ctx, dockerClient, key, *run, request.RestoreSelection, safetyBackup)
}
-func (s *Service) DeleteBackup(ctx context.Context, id, recoveryKey string) error {
+func (s *Service) DeleteBackup(ctx context.Context, id, recoveryKey string) (opErr error) {
+ ctx, finish := s.startOperation(ctx, "delete", attribute.String("arcane.backup.id", id))
+ defer func() { finish(opErr) }()
// Deletes contend with create/restore/upload on the system run lease so a
// slow operation cannot resurrect or double-forget snapshots.
lease, err := s.acquireRun(ctx)
@@ -805,7 +843,9 @@ func (s *Service) UploadBackup(ctx context.Context, id string, request backuptyp
return run, nil
}
-func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, user usertypes.Actor) error {
+func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, user usertypes.Actor) (opErr error) {
+ ctx, finish := s.startOperation(ctx, "restore", attribute.String("arcane.backup.id", id))
+ defer func() { finish(opErr) }()
run, err := s.store.Run(ctx, id)
if err != nil {
return err
diff --git a/backend/internal/telemetry/module.go b/backend/internal/telemetry/module.go
new file mode 100644
index 000000000..7b2250c10
--- /dev/null
+++ b/backend/internal/telemetry/module.go
@@ -0,0 +1,64 @@
+// Package telemetry relays browser OpenTelemetry data to the collector configured
+// through the standard OTEL_* exporter, endpoint, protocol, and header variables.
+package telemetry
+
+import (
+ "context"
+ "errors"
+ "log/slog"
+ "mime"
+ "net/http"
+ "os"
+ "time"
+
+ "github.com/labstack/echo/v5"
+ "golang.org/x/time/rate"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
+)
+
+// RegisterRoutes adds an unauthenticated relay route per enabled signal; disabled
+// signals return 404. Routes share a per-IP limit, a global limit, and a cap on concurrent requests.
+func RegisterRoutes(g *echo.Group, service *Service) {
+ if service == nil {
+ return
+ }
+ perIP := middleware.PerIPRateLimit(120, 30)
+ global := rate.NewLimiter(rate.Every(time.Minute/600), 60)
+ inFlight := make(chan struct{}, 32)
+ for signal := range service.destinations {
+ g.POST("/telemetry/"+signal, func(c *echo.Context) error {
+ req := c.Request()
+ if !global.Allow() {
+ c.Response().Header().Set("Retry-After", "60")
+ return c.NoContent(http.StatusTooManyRequests)
+ }
+ select {
+ case inFlight <- struct{}{}:
+ defer func() { <-inFlight }()
+ default:
+ return c.NoContent(http.StatusTooManyRequests)
+ }
+ // Slow uploads must not hold a relay slot indefinitely.
+ if err := http.NewResponseController(c.Response()).SetReadDeadline(time.Now().Add(10 * time.Second)); err != nil {
+ slog.DebugContext(req.Context(), "Failed to set relay read deadline", "error", err)
+ }
+ if mediaType, _, _ := mime.ParseMediaType(req.Header.Get("Content-Type")); mediaType != "application/json" {
+ return c.NoContent(http.StatusUnsupportedMediaType)
+ }
+ status, err := service.Forward(context.WithoutCancel(req.Context()), signal, req.Body)
+ switch {
+ case errors.Is(err, ErrPayloadTooLarge):
+ return c.NoContent(http.StatusRequestEntityTooLarge)
+ case errors.Is(err, ErrInvalidPayload):
+ return c.NoContent(http.StatusBadRequest)
+ case errors.Is(err, os.ErrDeadlineExceeded):
+ return c.NoContent(http.StatusRequestTimeout)
+ case err != nil:
+ slog.DebugContext(req.Context(), "Failed to relay browser telemetry", "signal", signal, "error", err)
+ return c.NoContent(http.StatusBadGateway)
+ }
+ return c.NoContent(status)
+ }, perIP)
+ }
+}
diff --git a/backend/internal/telemetry/service.go b/backend/internal/telemetry/service.go
new file mode 100644
index 000000000..ebec31eb5
--- /dev/null
+++ b/backend/internal/telemetry/service.go
@@ -0,0 +1,161 @@
+package telemetry
+
+import (
+ "bytes"
+ "cmp"
+ "context"
+ "encoding/json/v2"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "slices"
+ "strings"
+ "time"
+)
+
+const maxPayloadBytes = 512 << 10
+
+var (
+ ErrPayloadTooLarge = errors.New("telemetry payload exceeds 512 KiB")
+ ErrInvalidPayload = errors.New("telemetry payload is not OTLP/JSON from arcane-frontend")
+ // resourceKeys names each signal's top-level OTLP/JSON resource list.
+ resourceKeys = map[string]string{"traces": "resourceSpans", "metrics": "resourceMetrics", "logs": "resourceLogs"}
+)
+
+type otlpAttribute struct {
+ Key string `json:"key"`
+ Value struct {
+ StringValue string `json:"stringValue"`
+ } `json:"value"`
+}
+
+// otlpEnvelope decodes only the resource identities of an OTLP/JSON export.
+type otlpEnvelope map[string][]struct {
+ Resource struct {
+ Attributes []otlpAttribute `json:"attributes"`
+ } `json:"resource"`
+}
+
+// Service relays browser OTLP/JSON payloads to the configured collector so the
+// collector and its credentials stay server-side.
+type Service struct {
+ destinations map[string]destination
+ client *http.Client
+}
+
+type destination struct {
+ endpoint string
+ headers http.Header
+}
+
+// SDKDisabled reports whether OTEL_SDK_DISABLED turns all telemetry off.
+func SDKDisabled() bool {
+ return strings.EqualFold(strings.TrimSpace(os.Getenv("OTEL_SDK_DISABLED")), "true")
+}
+
+// Endpoint returns the OTLP/HTTP URL for signal, or "" when its exporter is
+// off, uses gRPC, or has no endpoint configured.
+func Endpoint(signal string) string {
+ upper := strings.ToUpper(signal)
+ if SDKDisabled() || strings.TrimSpace(os.Getenv("OTEL_"+upper+"_EXPORTER")) != "otlp" {
+ return ""
+ }
+ protocol := cmp.Or(os.Getenv("OTEL_EXPORTER_OTLP_"+upper+"_PROTOCOL"), os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL"), "http/protobuf")
+ if !strings.HasPrefix(protocol, "http/") {
+ return ""
+ }
+ endpoint := os.Getenv("OTEL_EXPORTER_OTLP_" + upper + "_ENDPOINT")
+ if endpoint == "" {
+ base := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+ if base == "" {
+ return ""
+ }
+ endpoint = strings.TrimRight(base, "/") + "/v1/" + signal
+ }
+ parsed, err := url.Parse(endpoint)
+ if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
+ return ""
+ }
+ return parsed.String()
+}
+
+// NewService returns nil when no signal can be relayed.
+func NewService() *Service {
+ destinations := make(map[string]destination, 3)
+ for _, signal := range []string{"traces", "metrics", "logs"} {
+ endpoint := Endpoint(signal)
+ if endpoint == "" {
+ continue
+ }
+ headers := http.Header{}
+ rawHeaders := cmp.Or(os.Getenv("OTEL_EXPORTER_OTLP_"+strings.ToUpper(signal)+"_HEADERS"), os.Getenv("OTEL_EXPORTER_OTLP_HEADERS"))
+ for pair := range strings.SplitSeq(rawHeaders, ",") {
+ key, value, ok := strings.Cut(pair, "=")
+ key = strings.TrimSpace(key)
+ if !ok || key == "" {
+ continue
+ }
+ if decoded, err := url.PathUnescape(strings.TrimSpace(value)); err == nil {
+ value = decoded
+ }
+ headers.Set(key, value)
+ }
+ destinations[signal] = destination{endpoint: endpoint, headers: headers}
+ }
+ if len(destinations) == 0 {
+ return nil
+ }
+
+ return &Service{
+ destinations: destinations,
+ client: &http.Client{
+ Timeout: 10 * time.Second,
+ CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+ },
+ }
+}
+
+// Forward posts an OTLP/JSON payload of at most 512 KiB for signal to the collector
+// and returns its status code. Every resource must identify as arcane-frontend.
+func (s *Service) Forward(ctx context.Context, signal string, body io.Reader) (int, error) {
+ dest, ok := s.destinations[signal]
+ if !ok {
+ return 0, fmt.Errorf("telemetry signal %q is not relayed", signal)
+ }
+ payload, err := io.ReadAll(io.LimitReader(body, maxPayloadBytes+1))
+ if err != nil {
+ return 0, fmt.Errorf("read %s payload: %w", signal, err)
+ }
+ if len(payload) > maxPayloadBytes {
+ return 0, ErrPayloadTooLarge
+ }
+ var envelope otlpEnvelope
+ if json.Unmarshal(payload, &envelope) != nil || len(envelope) != 1 || len(envelope[resourceKeys[signal]]) == 0 {
+ return 0, ErrInvalidPayload
+ }
+ for _, entry := range envelope[resourceKeys[signal]] {
+ if !slices.ContainsFunc(entry.Resource.Attributes, func(attr otlpAttribute) bool {
+ return attr.Key == "service.name" && attr.Value.StringValue == "arcane-frontend"
+ }) {
+ return 0, ErrInvalidPayload
+ }
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, dest.endpoint, bytes.NewReader(payload))
+ if err != nil {
+ return 0, fmt.Errorf("create %s forward request: %w", signal, err)
+ }
+ req.Header = dest.headers.Clone()
+ req.Header.Set("Content-Type", "application/json")
+
+ resp, err := s.client.Do(req)
+ if err != nil {
+ return 0, fmt.Errorf("forward %s: %w", signal, err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxPayloadBytes))
+ return resp.StatusCode, nil
+}
diff --git a/backend/internal/template/service.go b/backend/internal/template/service.go
index 0e82ebca5..169fac4a9 100644
--- a/backend/internal/template/service.go
+++ b/backend/internal/template/service.go
@@ -10,8 +10,10 @@ import (
"log/slog"
"maps"
"net/http"
+ "net/url"
"os"
"path/filepath"
+ "slices"
"strings"
"sync"
"sync/atomic"
@@ -26,6 +28,9 @@ import (
"go.getarcane.app/kit/normalization"
"go.getarcane.app/kit/pkg"
"go.getarcane.app/kit/pkg/mapping"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
"golang.org/x/sync/errgroup"
"gorm.io/gorm"
@@ -36,6 +41,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
type registryFetchMeta struct {
@@ -68,7 +74,7 @@ const (
remoteIDPrefix = "remote"
)
-var errRemoteCacheInvalidatedInternal = errors.New("remote template cache invalidated")
+var errRemoteCacheInvalidated = errors.New("remote template cache invalidated")
func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.Client, settingsService *settings.SettingsService) *TemplateService {
if httpClient == nil {
@@ -83,14 +89,13 @@ func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.C
registryFetchMeta: make(map[string]*registryFetchMeta),
registryErrors: make(map[string]string),
}
- service.safeHTTPClient = service.newSafeHTTPClientInternal(ctx)
revalidationCtx := context.WithoutCancel(ctx)
loader := func(generations []uint64) (map[uint64][]ComposeTemplate, error) {
loadCtx, cancel := context.WithTimeout(revalidationCtx, 2*time.Minute)
defer cancel()
catalogs := make(map[uint64][]ComposeTemplate, len(generations))
for _, generation := range generations {
- templates, err := service.loadRemoteTemplatesInternal(loadCtx, generation)
+ templates, err := service.loadRemoteTemplates(loadCtx, generation)
if err != nil {
return nil, err
}
@@ -105,31 +110,27 @@ func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.C
WithRevalidationErrorPolicy(hot.KeepOnError).
Build()
- if err := projects.EnsureDefaultTemplates(ctx, service.configuredTemplatesDirSettingInternal(ctx)); err != nil {
+ if err := projects.EnsureDefaultTemplates(ctx, service.configuredTemplatesDirSetting(ctx)); err != nil {
slog.WarnContext(ctx, "failed to ensure default templates", "error", err)
}
return service
}
-// configuredTemplatesDirSettingInternal returns the raw user-configured templates
-// directory string (from the settingsService), without resolving it to an absolute
-// path. Use this when calling helpers like projects.GetTemplatesDirectory that
-// perform their own resolution.
-func (s *TemplateService) configuredTemplatesDirSettingInternal(ctx context.Context) string {
+// configuredTemplatesDirSetting returns the raw templates directory setting; callers resolve it.
+func (s *TemplateService) configuredTemplatesDirSetting(ctx context.Context) string {
if s.settingsService == nil {
return ""
}
return s.settingsService.GetStringSetting(ctx, "templatesDirectory", "/app/data/templates")
}
-// getTemplatesDirectoryInternal resolves the effective templates directory by
-// applying the same resolution rules used elsewhere for the projects directory.
-func (s *TemplateService) getTemplatesDirectoryInternal(ctx context.Context) (string, error) {
- return projects.GetTemplatesDirectory(ctx, strings.TrimSpace(s.configuredTemplatesDirSettingInternal(ctx)))
+// getTemplatesDirectory resolves the effective templates directory.
+func (s *TemplateService) getTemplatesDirectory(ctx context.Context) (string, error) {
+ return projects.GetTemplatesDirectory(ctx, strings.TrimSpace(s.configuredTemplatesDirSetting(ctx)))
}
-func (s *TemplateService) remoteTemplatesInternal(ctx context.Context, refresh bool) ([]ComposeTemplate, error) {
+func (s *TemplateService) remoteTemplates(ctx context.Context, refresh bool) ([]ComposeTemplate, error) {
if s.remoteCache == nil {
return nil, errors.New("remote template cache is not initialized")
}
@@ -141,7 +142,7 @@ func (s *TemplateService) remoteTemplatesInternal(ctx context.Context, refresh b
var templates []ComposeTemplate
var err error
if refresh {
- templates, err = s.loadRemoteTemplatesInternal(ctx, generation)
+ templates, err = s.loadRemoteTemplates(ctx, generation)
s.registryMu.Lock()
if generation == s.remoteGeneration.Load() && err == nil {
s.remoteCache.Set(generation, templates)
@@ -165,11 +166,28 @@ func (s *TemplateService) remoteTemplatesInternal(ctx context.Context, refresh b
}
func (s *TemplateService) GetAllTemplates(ctx context.Context) ([]ComposeTemplate, error) {
- return s.getMergedTemplatesInternal(ctx)
+ if err := s.SyncLocalTemplatesFromFilesystem(ctx); err != nil {
+ slog.WarnContext(ctx, "failed to sync filesystem templates", "error", err)
+ }
+
+ var templates []ComposeTemplate
+ // Omit heavy content fields which are not needed for listing.
+ if err := s.db.WithContext(ctx).Omit("Content", "EnvContent").Preload("Registry").Find(&templates).Error; err != nil {
+ return nil, fmt.Errorf("failed to get local templates: %w", err)
+ }
+
+ remoteTemplates, err := s.remoteTemplates(ctx, false)
+ if err != nil {
+ slog.WarnContext(ctx, "failed to load remote templates", "error", err)
+ } else {
+ templates = append(templates, cloneRemoteTemplates(remoteTemplates)...)
+ }
+
+ return templates, nil
}
func (s *TemplateService) GetAllTemplatesPaginated(ctx context.Context, params pagination.QueryParams) ([]tmpl.Template, pagination.Response, error) {
- templates, err := s.getMergedTemplatesInternal(ctx)
+ templates, err := s.GetAllTemplates(ctx)
if err != nil {
return nil, pagination.Response{}, err
}
@@ -236,7 +254,7 @@ func (s *TemplateService) GetAllTemplatesPaginated(ctx context.Context, params p
}
func (s *TemplateService) GetTemplate(ctx context.Context, id string) (*ComposeTemplate, error) {
- if err := s.syncFilesystemTemplatesInternal(ctx); err != nil {
+ if err := s.SyncLocalTemplatesFromFilesystem(ctx); err != nil {
slog.WarnContext(ctx, "failed to sync filesystem templates", "error", err)
}
@@ -247,55 +265,33 @@ func (s *TemplateService) GetTemplate(ctx context.Context, id string) (*ComposeT
return nil, fmt.Errorf("failed to query local template: %w", err)
}
- templates, err := s.remoteTemplatesInternal(ctx, false)
+ templates, err := s.remoteTemplates(ctx, false)
if err != nil {
return nil, fmt.Errorf("template %q lookup failed: registry refresh error: %w", id, err)
}
- if found := s.lookupRemoteTemplateInternal(templates, id); found != nil {
- return found, nil
+ matchesID := func(t ComposeTemplate) bool { return t.ID == id }
+ if i := slices.IndexFunc(templates, matchesID); i >= 0 {
+ return &cloneRemoteTemplates(templates[i : i+1])[0], nil
}
- // Remote IDs (remote:registry:slug) deserve a synchronous force-refresh on miss
- // before we return "not found" — the cache may be stale or the previous refresh
- // silently returned empty.
+ // Force-refresh remote IDs (remote:registry:slug) on a miss; the cache may be stale.
if strings.HasPrefix(id, remoteIDPrefix+":") {
slog.InfoContext(ctx, "remote template not in cache, forcing registry refresh", "templateId", id, "cacheSize", len(templates))
- localTemplates, refreshErr := s.remoteTemplatesInternal(ctx, true)
+ refreshed, refreshErr := s.remoteTemplates(ctx, true)
if refreshErr != nil {
return nil, fmt.Errorf("template %q not found and registry refresh failed: %w", id, refreshErr)
}
- if found := s.lookupRemoteTemplateInternal(localTemplates, id); found != nil {
- return found, nil
+ if i := slices.IndexFunc(refreshed, matchesID); i >= 0 {
+ return &cloneRemoteTemplates(refreshed[i : i+1])[0], nil
}
- return nil, common.Classify(
- common.ErrTemplateNotFound,
- fmt.Errorf(
- "template not found: %w",
- fmt.Errorf(
- "template %q not found in any registered registry (cache size=%d after refresh)",
- id,
- len(
- localTemplates,
- ),
- ),
- ),
- )
+ notFoundErr := fmt.Errorf("template %q not found in any registered registry (cache size=%d after refresh)", id, len(refreshed))
+ return nil, common.Classify(common.ErrTemplateNotFound, fmt.Errorf("template not found: %w", notFoundErr))
}
return nil, common.Classify(common.ErrTemplateNotFound, errors.New("template not found"))
}
-func (s *TemplateService) lookupRemoteTemplateInternal(templates []ComposeTemplate, id string) *ComposeTemplate {
- for i := range templates {
- if templates[i].ID == id {
- cloned := cloneRemoteTemplates(templates[i : i+1])
- return &cloned[0]
- }
- }
- return nil
-}
-
func (s *TemplateService) CreateTemplate(ctx context.Context, template *ComposeTemplate) error {
if err := normalization.Normalize(template); err != nil {
return err
@@ -357,7 +353,7 @@ func (s *TemplateService) DeleteTemplate(ctx context.Context, id string) error {
return errors.New("cannot delete remote template directly")
}
- baseDir, err := s.getTemplatesDirectoryInternal(ctx)
+ baseDir, err := s.getTemplatesDirectory(ctx)
if err != nil {
return fmt.Errorf("failed to get templates directory: %w", err)
}
@@ -377,12 +373,9 @@ func (s *TemplateService) DeleteTemplate(ctx context.Context, id string) error {
})
}
-// readDefaultTemplateInternal reads one of the default template files from the
-// configured templates directory. These used to be read from a CWD-relative
-// "data/templates", which only resolved when the process happened to run from
-// the repository root.
-func (s *TemplateService) readDefaultTemplateInternal(ctx context.Context, fileName string) (string, error) {
- baseDir, err := s.getTemplatesDirectoryInternal(ctx)
+// readDefaultTemplate reads one of the default template files from the configured templates directory.
+func (s *TemplateService) readDefaultTemplate(ctx context.Context, fileName string) (string, error) {
+ baseDir, err := s.getTemplatesDirectory(ctx)
if err != nil {
return "", fmt.Errorf("failed to get templates directory: %w", err)
}
@@ -394,7 +387,7 @@ func (s *TemplateService) readDefaultTemplateInternal(ctx context.Context, fileN
}
func (s *TemplateService) GetComposeTemplate(ctx context.Context) string {
- content, err := s.readDefaultTemplateInternal(ctx, ".compose.template")
+ content, err := s.readDefaultTemplate(ctx, ".compose.template")
if err != nil {
slog.WarnContext(ctx, "failed to read compose template", "error", err)
return ""
@@ -403,7 +396,7 @@ func (s *TemplateService) GetComposeTemplate(ctx context.Context) string {
}
func (s *TemplateService) GetSwarmStackTemplate(ctx context.Context) string {
- content, err := s.readDefaultTemplateInternal(ctx, ".swarm-stack.template")
+ content, err := s.readDefaultTemplate(ctx, ".swarm-stack.template")
if err != nil {
slog.WarnContext(ctx, "failed to read swarm stack template", "error", err)
return projects.DefaultSwarmStackTemplate()
@@ -412,7 +405,7 @@ func (s *TemplateService) GetSwarmStackTemplate(ctx context.Context) string {
}
func (s *TemplateService) GetSwarmStackEnvTemplate(ctx context.Context) string {
- content, err := s.readDefaultTemplateInternal(ctx, ".swarm-stack.env.template")
+ content, err := s.readDefaultTemplate(ctx, ".swarm-stack.env.template")
if err != nil {
slog.WarnContext(ctx, "failed to read swarm stack env template", "error", err)
return projects.DefaultSwarmStackEnvTemplate()
@@ -421,7 +414,7 @@ func (s *TemplateService) GetSwarmStackEnvTemplate(ctx context.Context) string {
}
func (s *TemplateService) SaveComposeTemplate(ctx context.Context, content string) error {
- baseDir, err := s.getTemplatesDirectoryInternal(ctx)
+ baseDir, err := s.getTemplatesDirectory(ctx)
if err != nil {
return fmt.Errorf("failed to get templates directory: %w", err)
}
@@ -429,7 +422,7 @@ func (s *TemplateService) SaveComposeTemplate(ctx context.Context, content strin
}
func (s *TemplateService) GetEnvTemplate(ctx context.Context) string {
- content, err := s.readDefaultTemplateInternal(ctx, ".env.template")
+ content, err := s.readDefaultTemplate(ctx, ".env.template")
if err != nil {
slog.WarnContext(ctx, "failed to read env template", "error", err)
return ""
@@ -438,7 +431,7 @@ func (s *TemplateService) GetEnvTemplate(ctx context.Context) string {
}
func (s *TemplateService) SaveEnvTemplate(ctx context.Context, content string) error {
- baseDir, err := s.getTemplatesDirectoryInternal(ctx)
+ baseDir, err := s.getTemplatesDirectory(ctx)
if err != nil {
return fmt.Errorf("failed to get templates directory: %w", err)
}
@@ -474,12 +467,8 @@ func (s *TemplateService) CreateRegistry(ctx context.Context, registry *Template
return errors.New("registry URL is required")
}
if manifest, err := s.fetchRegistryManifest(ctx, registry.URL); err == nil {
- if registry.Name == "" {
- registry.Name = manifest.Name
- }
- if registry.Description == "" {
- registry.Description = manifest.Description
- }
+ registry.Name = cmp.Or(registry.Name, manifest.Name)
+ registry.Description = cmp.Or(registry.Description, manifest.Description)
} else if registry.Name == "" || registry.Description == "" {
return fmt.Errorf("failed to fetch registry manifest: %w", err)
}
@@ -502,7 +491,7 @@ func (s *TemplateService) CreateRegistry(ctx context.Context, registry *Template
return err
}
- s.invalidateRemoteCacheInternal()
+ s.invalidateRemoteCache()
return nil
}
@@ -519,8 +508,16 @@ func (s *TemplateService) UpdateRegistry(ctx context.Context, id string, updates
return fmt.Errorf("failed to find registry: %w", err)
}
- if err := s.hydrateRegistryUpdates(ctx, updates, &existing); err != nil {
- return err
+ // Hydrate blank fields from the manifest; a changed URL must yield them.
+ urlChanged := updates.URL != "" && updates.URL != existing.URL
+ manifestURL := cmp.Or(updates.URL, existing.URL)
+ if (urlChanged || updates.Name == "" || updates.Description == "") && manifestURL != "" {
+ if manifest, err := s.fetchRegistryManifest(ctx, manifestURL); err == nil {
+ updates.Name = cmp.Or(updates.Name, manifest.Name)
+ updates.Description = cmp.Or(updates.Description, manifest.Description)
+ } else if urlChanged && (updates.Name == "" || updates.Description == "") {
+ return fmt.Errorf("failed to fetch registry manifest: %w", err)
+ }
}
if err := normalization.Normalize(updates); err != nil {
@@ -537,30 +534,7 @@ func (s *TemplateService) UpdateRegistry(ctx context.Context, id string, updates
return err
}
- s.invalidateRemoteCacheInternal()
- return nil
-}
-
-func (s *TemplateService) hydrateRegistryUpdates(ctx context.Context, updates, existing *TemplateRegistry) error {
- urlChanged := updates.URL != "" && updates.URL != existing.URL
- needsHydration := updates.Name == "" || updates.Description == ""
-
- if (urlChanged || needsHydration) && (updates.URL != "" || existing.URL != "") {
- manifestURL := updates.URL
- if manifestURL == "" {
- manifestURL = existing.URL
- }
- if manifest, err := s.fetchRegistryManifest(ctx, manifestURL); err == nil {
- if updates.Name == "" {
- updates.Name = manifest.Name
- }
- if updates.Description == "" {
- updates.Description = manifest.Description
- }
- } else if urlChanged && (updates.Name == "" || updates.Description == "") {
- return fmt.Errorf("failed to fetch registry manifest: %w", err)
- }
- }
+ s.invalidateRemoteCache()
return nil
}
@@ -579,11 +553,17 @@ func (s *TemplateService) DeleteRegistry(ctx context.Context, id string) error {
return err
}
- s.invalidateRemoteCacheInternal()
+ s.invalidateRemoteCache()
return nil
}
-func (s *TemplateService) loadRemoteTemplatesInternal(ctx context.Context, generation uint64) ([]ComposeTemplate, error) {
+func (s *TemplateService) loadRemoteTemplates(ctx context.Context, generation uint64) (templates []ComposeTemplate, err error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "template.sync_remote")
+ defer func() {
+ span.SetAttributes(attribute.Int("arcane.result.count", len(templates)))
+ tracing.End(span, err)
+ }()
+
registries, err := s.GetRegistries(ctx)
if err != nil {
return nil, err
@@ -591,8 +571,8 @@ func (s *TemplateService) loadRemoteTemplatesInternal(ctx context.Context, gener
var (
mu sync.Mutex
- templates []ComposeTemplate
fetchErrors []error
+ enabledRegistries int
successfulRegistries int
)
@@ -603,11 +583,12 @@ func (s *TemplateService) loadRemoteTemplatesInternal(ctx context.Context, gener
if !reg.Enabled {
continue
}
+ enabledRegistries++
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "template worker")
- remoteTemplates, fetchRegistryTemplatesErr := s.fetchRegistryTemplatesInternal(groupCtx, &reg, generation)
+ remoteTemplates, fetchRegistryTemplatesErr := s.fetchRegistryTemplates(groupCtx, &reg, generation)
if fetchRegistryTemplatesErr != nil {
slog.WarnContext(groupCtx, "failed to fetch templates from registry", "registry", reg.Name, "url", reg.URL, "error", fetchRegistryTemplatesErr)
s.registryMu.Lock()
@@ -639,11 +620,16 @@ func (s *TemplateService) loadRemoteTemplatesInternal(ctx context.Context, gener
})
}
- if waitErr := g.Wait(); waitErr != nil {
+ waitErr := g.Wait()
+ span.SetAttributes(
+ attribute.Int("arcane.template.registry_count", enabledRegistries),
+ attribute.Int("arcane.template.failed_registry_count", len(fetchErrors)),
+ )
+ if waitErr != nil {
return nil, waitErr
}
if generation != s.remoteGeneration.Load() {
- return nil, errRemoteCacheInvalidatedInternal
+ return nil, errRemoteCacheInvalidated
}
if successfulRegistries == 0 && len(fetchErrors) > 0 {
return nil, errors.Join(fetchErrors...)
@@ -652,44 +638,52 @@ func (s *TemplateService) loadRemoteTemplatesInternal(ctx context.Context, gener
return templates, nil
}
-func (s *TemplateService) FetchRaw(ctx context.Context, url string) ([]byte, error) {
- return s.doGET(ctx, url)
-}
-
-func (s *TemplateService) doGET(ctx context.Context, url string) ([]byte, error) {
- client, req, err := s.newSafeRequestInternal(ctx, http.MethodGet, url)
+func (s *TemplateService) FetchRaw(ctx context.Context, rawURL string) ([]byte, error) {
+ client, req, err := s.newSafeRequest(ctx, http.MethodGet, rawURL)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err != nil {
- return nil, fmt.Errorf("failed to fetch %s: %w", url, err)
+ return nil, fmt.Errorf("failed to fetch %s: %w", rawURL, err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("HTTP status %d for URL %s", resp.StatusCode, url)
+ return nil, fmt.Errorf("HTTP status %d for URL %s", resp.StatusCode, rawURL)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
- return nil, fmt.Errorf("failed to read response body from %s: %w", url, err)
+ return nil, fmt.Errorf("failed to read response body from %s: %w", rawURL, err)
}
return body, nil
}
-// fetchRegistryTemplatesInternal performs a conditional GET using If-Modified-Since.
-// If the server replies 304 Not Modified, cached templates for the registry are reused.
-func (s *TemplateService) fetchRegistryTemplatesInternal(ctx context.Context, reg *TemplateRegistry, generation uint64) ([]ComposeTemplate, error) {
+// fetchRegistryTemplates performs a conditional GET; a 304 reuses the registry's cached templates.
+func (s *TemplateService) fetchRegistryTemplates(ctx context.Context, reg *TemplateRegistry, generation uint64) (templates []ComposeTemplate, err error) {
+ registryHost := ""
+ if parsedURL, parseErr := url.Parse(reg.URL); parseErr == nil {
+ registryHost = parsedURL.Host
+ }
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "template.fetch_registry", trace.WithAttributes(
+ attribute.String("arcane.template.registry_id", reg.ID),
+ attribute.String("arcane.template.registry_host", registryHost),
+ ))
+ defer func() {
+ span.SetAttributes(attribute.Int("arcane.result.count", len(templates)))
+ tracing.End(span, err)
+ }()
+
s.registryMu.RLock()
if generation != s.remoteGeneration.Load() {
s.registryMu.RUnlock()
- return nil, errRemoteCacheInvalidatedInternal
+ return nil, errRemoteCacheInvalidated
}
fetchMeta := s.registryFetchMeta[reg.ID]
s.registryMu.RUnlock()
- client, req, err := s.newSafeRequestInternal(ctx, http.MethodGet, reg.URL)
+ client, req, err := s.newSafeRequest(ctx, http.MethodGet, reg.URL)
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
@@ -703,6 +697,7 @@ func (s *TemplateService) fetchRegistryTemplatesInternal(ctx context.Context, re
}
defer func() { _ = resp.Body.Close() }()
+ span.SetAttributes(attribute.Bool("arcane.template.not_modified", resp.StatusCode == http.StatusNotModified))
if resp.StatusCode == http.StatusNotModified {
if fetchMeta != nil {
return cloneRemoteTemplates(fetchMeta.Templates), nil
@@ -723,20 +718,36 @@ func (s *TemplateService) fetchRegistryTemplatesInternal(ctx context.Context, re
return nil, fmt.Errorf("parse registry JSON: %w", unmarshalErr)
}
- templates := make([]ComposeTemplate, 0, len(regDTO.Templates))
- for _, remoteTemplate := range regDTO.Templates {
- templates = append(templates, s.convertRemoteToLocal(remoteTemplate, reg))
+ templates = make([]ComposeTemplate, 0, len(regDTO.Templates))
+ for _, remote := range regDTO.Templates {
+ templates = append(templates, ComposeTemplate{
+ ID: fmt.Sprintf("%s:%s:%s", remoteIDPrefix, reg.ID, remote.ID),
+ Name: remote.Name,
+ Description: remote.Description,
+ IsRemote: true,
+ RegistryID: mo.EmptyableToOption(strings.TrimSpace(reg.ID)).ToPointer(),
+ Registry: cloneRegistry(reg),
+ Metadata: &ComposeTemplateMetadata{
+ Version: mo.EmptyableToOption(strings.TrimSpace(remote.Version)).ToPointer(),
+ Author: mo.EmptyableToOption(strings.TrimSpace(remote.Author)).ToPointer(),
+ Tags: remote.Tags,
+ RemoteURL: mo.EmptyableToOption(strings.TrimSpace(remote.ComposeURL)).ToPointer(),
+ EnvURL: mo.EmptyableToOption(strings.TrimSpace(remote.EnvURL)).ToPointer(),
+ DocumentationURL: mo.EmptyableToOption(strings.TrimSpace(remote.DocumentationURL)).ToPointer(),
+ IconURL: mo.EmptyableToOption(strings.TrimSpace(remote.IconURL)).ToPointer(),
+ RegistryIconURL: mo.EmptyableToOption(strings.TrimSpace(remote.IconURL)).ToPointer(),
+ },
+ })
}
- lm := resp.Header.Get("Last-Modified")
newMeta := &registryFetchMeta{
- LastModified: lm,
+ LastModified: resp.Header.Get("Last-Modified"),
Templates: cloneRemoteTemplates(templates),
}
s.registryMu.Lock()
if generation != s.remoteGeneration.Load() {
s.registryMu.Unlock()
- return nil, errRemoteCacheInvalidatedInternal
+ return nil, errRemoteCacheInvalidated
}
s.registryFetchMeta[reg.ID] = newMeta
s.registryMu.Unlock()
@@ -744,8 +755,8 @@ func (s *TemplateService) fetchRegistryTemplatesInternal(ctx context.Context, re
return templates, nil
}
-func (s *TemplateService) fetchRegistryManifest(ctx context.Context, url string) (*tmpl.RemoteRegistry, error) {
- body, err := s.doGET(ctx, url)
+func (s *TemplateService) fetchRegistryManifest(ctx context.Context, rawURL string) (*tmpl.RemoteRegistry, error) {
+ body, err := s.FetchRaw(ctx, rawURL)
if err != nil {
return nil, err
}
@@ -759,95 +770,43 @@ func (s *TemplateService) fetchRegistryManifest(ctx context.Context, url string)
return &reg, nil
}
-func (s *TemplateService) convertRemoteToLocal(remote tmpl.RemoteTemplate, registry *TemplateRegistry) ComposeTemplate {
- publicID := fmt.Sprintf("%s:%s:%s", remoteIDPrefix, registry.ID, remote.ID)
-
- return ComposeTemplate{
- ID: publicID,
- Name: remote.Name,
- Description: remote.Description,
- Content: "",
- EnvContent: nil,
- IsCustom: false,
- IsRemote: true,
- RegistryID: mo.EmptyableToOption(strings.TrimSpace(registry.ID)).ToPointer(),
- Registry: cloneRegistry(registry),
- Metadata: &ComposeTemplateMetadata{
- Version: mo.EmptyableToOption(strings.TrimSpace(remote.Version)).ToPointer(),
- Author: mo.EmptyableToOption(strings.TrimSpace(remote.Author)).ToPointer(),
- Tags: remote.Tags,
- RemoteURL: mo.EmptyableToOption(strings.TrimSpace(remote.ComposeURL)).ToPointer(),
- EnvURL: mo.EmptyableToOption(strings.TrimSpace(remote.EnvURL)).ToPointer(),
- DocumentationURL: mo.EmptyableToOption(strings.TrimSpace(remote.DocumentationURL)).ToPointer(),
- IconURL: mo.EmptyableToOption(strings.TrimSpace(remote.IconURL)).ToPointer(),
- RegistryIconURL: mo.EmptyableToOption(strings.TrimSpace(remote.IconURL)).ToPointer(),
- },
- }
-}
-
-func (s *TemplateService) FetchTemplateContent(ctx context.Context, template *ComposeTemplate) (string, string, error) {
+func (s *TemplateService) FetchTemplateContent(ctx context.Context, template *ComposeTemplate) (composeContent, envContent string, err error) {
if !template.IsRemote {
- envContent := ""
- if template.EnvContent != nil {
- envContent = *template.EnvContent
- }
- return template.Content, envContent, nil
+ return template.Content, mo.PointerToOption(template.EnvContent).OrEmpty(), nil
}
if template.Metadata == nil || template.Metadata.RemoteURL == nil || strings.TrimSpace(*template.Metadata.RemoteURL) == "" {
return "", "", fmt.Errorf("remote template %q is missing compose_url in registry metadata", template.ID)
}
- return s.fetchRemoteTemplateFiles(ctx, template)
-}
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "template.fetch", trace.WithAttributes(
+ attribute.String("arcane.template.id", template.ID),
+ attribute.String("arcane.template.registry_id", mo.PointerToOption(template.RegistryID).OrEmpty()),
+ ))
+ defer func() { tracing.End(span, err) }()
-func (s *TemplateService) fetchRemoteTemplateFiles(ctx context.Context, template *ComposeTemplate) (string, string, error) {
- if template == nil || template.Metadata == nil || template.Metadata.RemoteURL == nil {
- return "", "", errors.New("not a remote template or missing remote URL")
- }
-
- composeContent, err := s.fetchURL(ctx, *template.Metadata.RemoteURL)
+ composeBody, err := s.FetchRaw(ctx, *template.Metadata.RemoteURL)
if err != nil {
return "", "", fmt.Errorf("failed to fetch compose content from %s: %w", *template.Metadata.RemoteURL, err)
}
- var envContent string
- if template.Metadata.EnvURL != nil && *template.Metadata.EnvURL != "" {
- envContent, err = s.fetchURL(ctx, *template.Metadata.EnvURL)
- if err != nil {
- slog.WarnContext(ctx, "failed to fetch env content", "url", *template.Metadata.EnvURL, "error", err)
- envContent = ""
+ if envURL := mo.PointerToOption(template.Metadata.EnvURL).OrEmpty(); envURL != "" {
+ envBody, envErr := s.FetchRaw(ctx, envURL)
+ if envErr != nil {
+ slog.WarnContext(ctx, "failed to fetch env content", "url", envURL, "error", envErr)
}
+ envContent = string(envBody)
}
- return composeContent, envContent, nil
+ return string(composeBody), envContent, nil
}
-func (s *TemplateService) fetchURL(ctx context.Context, url string) (string, error) {
- body, err := s.doGET(ctx, url)
- if err != nil {
- return "", err
- }
- return string(body), nil
-}
-
-func (s *TemplateService) newSafeHTTPClientInternal(ctx context.Context) *http.Client {
- client, err := httpx.NewSafeOutboundHTTPClient(s.httpClient, s.lookupIP)
- if err != nil {
- slog.WarnContext(ctx, "failed to configure safe HTTP client", "error", err)
- return nil
- }
- return client
-}
-
-func (s *TemplateService) newSafeRequestInternal(ctx context.Context, method, rawURL string) (*http.Client, *http.Request, error) {
+func (s *TemplateService) newSafeRequest(ctx context.Context, method, rawURL string) (*http.Client, *http.Request, error) {
parsedURL, err := httpx.ValidateSafeRemoteURL(ctx, rawURL, s.lookupIP)
if err != nil {
return nil, nil, err
}
- // Double-checked under registryMu: the lazy init ran unsynchronized, so
- // concurrent template downloads raced on s.safeHTTPClient — a data race, and
- // each loser silently built and leaked its own client.
+ // Lazily built once under registryMu (double-checked) so concurrent requests share one client.
s.registryMu.RLock()
client := s.safeHTTPClient
s.registryMu.RUnlock()
@@ -855,7 +814,11 @@ func (s *TemplateService) newSafeRequestInternal(ctx context.Context, method, ra
if client == nil {
s.registryMu.Lock()
if s.safeHTTPClient == nil {
- s.safeHTTPClient = s.newSafeHTTPClientInternal(ctx)
+ safeClient, clientErr := httpx.NewSafeOutboundHTTPClient(s.httpClient, s.lookupIP)
+ if clientErr != nil {
+ slog.WarnContext(ctx, "failed to configure safe HTTP client", "error", clientErr)
+ }
+ s.safeHTTPClient = safeClient
}
client = s.safeHTTPClient
s.registryMu.Unlock()
@@ -878,59 +841,45 @@ func (s *TemplateService) DownloadTemplate(ctx context.Context, remoteTemplate *
return nil, errors.New("template is not remote")
}
- base := s.templateBaseFromRemote(remoteTemplate)
+ base := projects.Slugify(remoteTemplate.Name)
+ if base == "" {
+ base = cmp.Or(projects.Slugify(remoteTemplate.ID[strings.LastIndex(remoteTemplate.ID, ":")+1:]), "template-"+uuid.New().String())
+ }
- dir, composePath, envPath, err := projects.EnsureTemplateDir(ctx, s.configuredTemplatesDirSettingInternal(ctx), base)
+ dir, composePath, envPath, err := projects.EnsureTemplateDir(ctx, s.configuredTemplatesDirSetting(ctx), base)
if err != nil {
return nil, err
}
srcDesc := fmt.Sprintf("Imported from %s/compose.yaml", dir)
- return s.downloadTemplateTransaction(ctx, remoteTemplate, base, composePath, envPath, srcDesc)
-}
-
-func (s *TemplateService) downloadTemplateTransaction(ctx context.Context, remoteTemplate *ComposeTemplate, base, composePath, envPath, srcDesc string) (*ComposeTemplate, error) {
var resultTemplate *ComposeTemplate
-
- err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ err = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var existing ComposeTemplate
- if err := tx.Where("is_remote = ? AND registry_id IS NULL AND (description = ? OR name = ?)", false, srcDesc, base).
- First(&existing).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
- return fmt.Errorf("failed to check existing template: %w", err)
- } else if err == nil {
- // Existing template found
- composeContent, envContent, fetchTemplateContentErr := s.FetchTemplateContent(ctx, remoteTemplate)
- if fetchTemplateContentErr != nil {
- return fmt.Errorf("failed to fetch template content for existing local template: %w", fetchTemplateContentErr)
- }
+ findErr := tx.Where("is_remote = ? AND registry_id IS NULL AND (description = ? OR name = ?)", false, srcDesc, base).First(&existing).Error
+ if findErr != nil && !errors.Is(findErr, gorm.ErrRecordNotFound) {
+ return fmt.Errorf("failed to check existing template: %w", findErr)
+ }
- envPtr, werr := projects.WriteTemplateFiles(composePath, envPath, composeContent, envContent)
- if werr != nil {
- return werr
- }
+ composeContent, envContent, fetchErr := s.FetchTemplateContent(ctx, remoteTemplate)
+ if fetchErr != nil {
+ return fmt.Errorf("failed to fetch template content for download: %w", fetchErr)
+ }
+ envPtr, writeErr := projects.WriteTemplateFiles(composePath, envPath, composeContent, envContent)
+ if writeErr != nil {
+ return writeErr
+ }
+ if findErr == nil {
existing.Content = composeContent
existing.EnvContent = envPtr
existing.Metadata = cloneTemplateMetadata(remoteTemplate.Metadata)
-
- if updateLocalTemplateErr := tx.Save(&existing).Error; updateLocalTemplateErr != nil {
- return fmt.Errorf("failed to update existing local template: %w", updateLocalTemplateErr)
+ if saveErr := tx.Save(&existing).Error; saveErr != nil {
+ return fmt.Errorf("failed to update existing local template: %w", saveErr)
}
resultTemplate = &existing
return nil
}
- // New template
- composeContent, envContent, err := s.FetchTemplateContent(ctx, remoteTemplate)
- if err != nil {
- return fmt.Errorf("failed to fetch template content for download: %w", err)
- }
-
- envPtr, werr := projects.WriteTemplateFiles(composePath, envPath, composeContent, envContent)
- if werr != nil {
- return werr
- }
-
localTemplate := &ComposeTemplate{
ID: uuid.New().String(),
Name: base,
@@ -938,14 +887,10 @@ func (s *TemplateService) downloadTemplateTransaction(ctx context.Context, remot
Content: composeContent,
EnvContent: envPtr,
IsCustom: true,
- IsRemote: false,
- RegistryID: nil,
- Registry: nil,
Metadata: cloneTemplateMetadata(remoteTemplate.Metadata),
}
-
- if createLocalTemplateErr := tx.Create(localTemplate).Error; createLocalTemplateErr != nil {
- return fmt.Errorf("failed to save local template: %w", createLocalTemplateErr)
+ if createErr := tx.Create(localTemplate).Error; createErr != nil {
+ return fmt.Errorf("failed to save local template: %w", createErr)
}
resultTemplate = localTemplate
return nil
@@ -956,21 +901,6 @@ func (s *TemplateService) downloadTemplateTransaction(ctx context.Context, remot
return resultTemplate, nil
}
-func (s *TemplateService) templateBaseFromRemote(remoteTemplate *ComposeTemplate) string {
- base := projects.Slugify(remoteTemplate.Name)
- if base != "" {
- return base
- }
- parts := strings.Split(remoteTemplate.ID, ":")
- if len(parts) > 0 {
- base = projects.Slugify(parts[len(parts)-1])
- }
- if base == "" {
- base = "template-" + uuid.New().String()
- }
- return base
-}
-
func cloneTemplateMetadata(meta *ComposeTemplateMetadata) *ComposeTemplateMetadata {
if meta == nil {
return nil
@@ -1010,7 +940,7 @@ func cloneRegistry(registry *TemplateRegistry) *TemplateRegistry {
return new(*registry)
}
-func (s *TemplateService) invalidateRemoteCacheInternal() {
+func (s *TemplateService) invalidateRemoteCache() {
s.registryMu.Lock()
defer s.registryMu.Unlock()
s.remoteGeneration.Add(1)
@@ -1021,71 +951,8 @@ func (s *TemplateService) invalidateRemoteCacheInternal() {
}
}
-func (s *TemplateService) SyncLocalTemplatesFromFilesystem(ctx context.Context) error {
- return s.syncFilesystemTemplatesInternal(ctx)
-}
-
-func (s *TemplateService) upsertFilesystemTemplate(ctx context.Context, name, desc, compose string, envPtr *string) error {
- iconURL := projects.ResolveTemplateIconURL(ctx, compose, mo.PointerToOption(envPtr).OrEmpty())
-
- return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- var existing ComposeTemplate
- // Match on description (which encodes the absolute compose path) OR name (folder name).
- // Matching on name as a fallback keeps a single row across templates-directory
- // reconfigurations or compose-filename renames within the same folder, but
- // only for templates previously imported from disk.
- q := tx.Where("is_remote = ? AND registry_id IS NULL AND (description = ? OR (name = ? AND description LIKE ?))", false, desc, name, "Imported from %").
- First(&existing)
-
- if q.Error == nil {
- existing.Name = name
- existing.Description = desc
- existing.Content = compose
- existing.EnvContent = envPtr
- existing.IsCustom = true
- existing.IsRemote = false
- setTemplateIconURL(&existing, iconURL)
- if err := tx.Save(&existing).Error; err != nil {
- return fmt.Errorf("update template %s: %w", existing.ID, err)
- }
- return nil
- }
- if !errors.Is(q.Error, gorm.ErrRecordNotFound) {
- if err := q.Error; err != nil {
- return fmt.Errorf("query existing template: %w", err)
- }
- return nil
- }
-
- tpl := &ComposeTemplate{
- ID: uuid.New().String(),
- Name: name,
- Description: desc,
- Content: compose,
- EnvContent: envPtr,
- IsCustom: true,
- IsRemote: false,
- RegistryID: nil,
- Registry: nil,
- Metadata: nil,
- }
- setTemplateIconURL(tpl, iconURL)
- if err := tx.Create(tpl).Error; err != nil {
- return fmt.Errorf("insert template %s: %w", name, err)
- }
- return nil
- })
-}
-
-func (s *TemplateService) processFolderEntry(ctx context.Context, baseDir, folder string) error {
- compose, envPtr, desc, found, err := projects.ReadFolderComposeTemplate(ctx, baseDir, folder)
- if err != nil || !found {
- return err
- }
- return s.upsertFilesystemTemplate(ctx, folder, desc, compose, envPtr)
-}
-
-func (s *TemplateService) syncFilesystemTemplatesInternal(ctx context.Context) error {
+// SyncLocalTemplatesFromFilesystem imports template folders from the templates directory, at most once per fsSyncInterval.
+func (s *TemplateService) SyncLocalTemplatesFromFilesystem(ctx context.Context) (err error) {
s.fsSyncMu.Lock()
defer s.fsSyncMu.Unlock()
@@ -1093,7 +960,14 @@ func (s *TemplateService) syncFilesystemTemplatesInternal(ctx context.Context) e
return nil
}
- dir, err := s.getTemplatesDirectoryInternal(ctx)
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "template.sync_filesystem")
+ synced, failed := 0, 0
+ defer func() {
+ span.SetAttributes(attribute.Int("arcane.result.count", synced), attribute.Int("arcane.template.failed_count", failed))
+ tracing.End(span, err)
+ }()
+
+ dir, err := s.getTemplatesDirectory(ctx)
if err != nil {
return fmt.Errorf("ensure templates dir: %w", err)
}
@@ -1111,9 +985,52 @@ func (s *TemplateService) syncFilesystemTemplatesInternal(ctx context.Context) e
if !ent.IsDirectory {
continue
}
- if processFolderEntryErr := s.processFolderEntry(ctx, dir, ent.Name); processFolderEntryErr != nil {
- slog.WarnContext(ctx, "failed to read folder template", "folder", ent.Name, "error", processFolderEntryErr)
+ compose, envPtr, desc, found, folderErr := projects.ReadFolderComposeTemplate(ctx, dir, ent.Name)
+ if folderErr == nil && !found {
+ continue
+ }
+ if folderErr == nil {
+ iconURL := projects.ResolveTemplateIconURL(ctx, compose, mo.PointerToOption(envPtr).OrEmpty())
+ folderErr = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ // Match on description (the absolute compose path) or, for disk imports, the folder name so
+ // directory reconfigurations and compose renames keep a single row.
+ var existing ComposeTemplate
+ findErr := tx.Where("is_remote = ? AND registry_id IS NULL AND (description = ? OR (name = ? AND description LIKE ?))", false, desc, ent.Name, "Imported from %").
+ First(&existing).Error
+ if findErr != nil && !errors.Is(findErr, gorm.ErrRecordNotFound) {
+ return fmt.Errorf("query existing template: %w", findErr)
+ }
+
+ tpl := &existing
+ if findErr != nil {
+ tpl = &ComposeTemplate{ID: uuid.New().String()}
+ }
+ tpl.Name = ent.Name
+ tpl.Description = desc
+ tpl.Content = compose
+ tpl.EnvContent = envPtr
+ tpl.IsCustom = true
+ tpl.IsRemote = false
+ setTemplateIconURL(tpl, iconURL)
+
+ if findErr == nil {
+ if saveErr := tx.Save(tpl).Error; saveErr != nil {
+ return fmt.Errorf("update template %s: %w", tpl.ID, saveErr)
+ }
+ return nil
+ }
+ if createErr := tx.Create(tpl).Error; createErr != nil {
+ return fmt.Errorf("insert template %s: %w", ent.Name, createErr)
+ }
+ return nil
+ })
+ }
+ if folderErr != nil {
+ failed++
+ slog.WarnContext(ctx, "failed to read folder template", "folder", ent.Name, "error", folderErr)
+ continue
}
+ synced++
}
s.lastFsSync = time.Now()
@@ -1152,17 +1069,11 @@ func (s *TemplateService) GetTemplateContentWithParsedData(ctx context.Context,
return nil, err
}
- var composeContent, envContent string
- if composeTemplate.IsRemote {
- composeContent, envContent, err = s.FetchTemplateContent(ctx, composeTemplate)
- if err != nil {
- return nil, fmt.Errorf("failed to fetch template content: %w", err)
- }
- } else {
- composeContent = composeTemplate.Content
- if composeTemplate.EnvContent != nil {
- envContent = *composeTemplate.EnvContent
- }
+ composeContent, envContent, err := s.FetchTemplateContent(ctx, composeTemplate)
+ if err != nil {
+ return nil, fmt.Errorf("failed to fetch template content: %w", err)
+ }
+ if !composeTemplate.IsRemote {
setTemplateIconURL(composeTemplate, projects.ResolveTemplateIconURL(ctx, composeContent, envContent))
}
@@ -1189,28 +1100,3 @@ func (s *TemplateService) GetTemplateContentWithParsedData(ctx context.Context,
EnvVariables: envVars,
}, nil
}
-
-func (s *TemplateService) getMergedTemplatesInternal(ctx context.Context) ([]ComposeTemplate, error) {
- if err := s.syncFilesystemTemplatesInternal(ctx); err != nil {
- slog.WarnContext(ctx, "failed to sync filesystem templates", "error", err)
- }
-
- var templates []ComposeTemplate
- // Use Omit to avoid fetching heavy content fields which are not needed for listing
- if err := s.db.WithContext(ctx).Omit("Content", "EnvContent").Preload("Registry").Find(&templates).Error; err != nil {
- return nil, fmt.Errorf("failed to get local templates: %w", err)
- }
-
- remoteTemplates, err := s.remoteTemplatesInternal(ctx, false)
- if err != nil {
- slog.WarnContext(ctx, "failed to load remote templates", "error", err)
- } else {
- copied := cloneRemoteTemplates(remoteTemplates)
-
- if len(copied) > 0 {
- templates = append(templates, copied...)
- }
- }
-
- return templates, nil
-}
diff --git a/backend/internal/template/service_test.go b/backend/internal/template/service_test.go
index 60de58db9..91e9cf45f 100644
--- a/backend/internal/template/service_test.go
+++ b/backend/internal/template/service_test.go
@@ -148,7 +148,7 @@ services:
Enabled: true,
}
- templates, err := service.fetchRegistryTemplatesInternal(t.Context(), registry, service.remoteGeneration.Load())
+ templates, err := service.fetchRegistryTemplates(t.Context(), registry, service.remoteGeneration.Load())
require.NoError(t, err)
require.Len(t, templates, 2)
require.NotNil(t, templates[0].Metadata)
@@ -156,7 +156,7 @@ services:
require.Equal(t, "https://cdn.example/good.png", *templates[0].Metadata.IconURL)
require.Nil(t, templates[1].Metadata.IconURL)
- cachedTemplates, err := service.fetchRegistryTemplatesInternal(t.Context(), registry, service.remoteGeneration.Load())
+ cachedTemplates, err := service.fetchRegistryTemplates(t.Context(), registry, service.remoteGeneration.Load())
require.NoError(t, err)
require.Len(t, cachedTemplates, 2)
require.NotNil(t, cachedTemplates[0].Metadata)
@@ -301,12 +301,12 @@ func TestGetAllTemplatesPaginated_FiltersByType(t *testing.T) {
Filters: map[string]string{"type": tt.typeFilter},
})
require.NoError(t, err)
- require.ElementsMatch(t, tt.wantIDs, templateIDsInternal(templates))
+ require.ElementsMatch(t, tt.wantIDs, templateIDs(templates))
})
}
}
-func templateIDsInternal(templates []tmpl.Template) []string {
+func templateIDs(templates []tmpl.Template) []string {
ids := make([]string, 0, len(templates))
for _, template := range templates {
ids = append(ids, template.ID)
@@ -326,7 +326,7 @@ func TestFetchRaw_BlocksUnsafeRemoteURL(t *testing.T) {
require.ErrorIs(t, err, common.ErrUnsafeRemoteURL)
}
-func TestSyncFilesystemTemplatesInternal_PopulatesIconURL(t *testing.T) {
+func TestSyncLocalTemplatesFromFilesystem_PopulatesIconURL(t *testing.T) {
tempDir := t.TempDir()
templatesRoot := filepath.Join(tempDir, "templates")
@@ -349,7 +349,7 @@ services:
registryFetchMeta: make(map[string]*registryFetchMeta),
}
- require.NoError(t, service.syncFilesystemTemplatesInternal(t.Context()))
+ require.NoError(t, service.SyncLocalTemplatesFromFilesystem(t.Context()))
var stored ComposeTemplate
require.NoError(t, service.db.WithContext(t.Context()).First(&stored, "name = ?", "example").Error)
@@ -410,7 +410,6 @@ func TestGetTemplate_ForceRefreshesRemoteCacheOnMiss(t *testing.T) {
service := NewTemplateService(t.Context(), db, client, settingsSvc)
service.lookupIP = lookupIP
- service.safeHTTPClient = service.newSafeHTTPClientInternal(t.Context())
// Cache starts empty; GetTemplate for a remote ID should force a refresh and find the template.
got, err := service.GetTemplate(t.Context(), "remote:reg-1:affine")
@@ -431,12 +430,12 @@ func minimalSettingsServiceForTest(t *testing.T) *settings.SettingsService {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&settings.SettingVariable{}))
- svc, err := newSettingsServiceForTestInternal(t, t.Context(), &database.DB{DB: db})
+ svc, err := newSettingsServiceForTest(t, t.Context(), &database.DB{DB: db})
require.NoError(t, err)
return svc
}
-func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
+func newSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
svc, err := settings.NewSettingsService(ctx, db)
if err == nil {
diff --git a/backend/internal/vulnerability/children/scan/children/trivy/service.go b/backend/internal/vulnerability/children/scan/children/trivy/service.go
index 8c169bfd2..25864f123 100644
--- a/backend/internal/vulnerability/children/scan/children/trivy/service.go
+++ b/backend/internal/vulnerability/children/scan/children/trivy/service.go
@@ -23,6 +23,9 @@ import (
"go.getarcane.app/docker"
"go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/cgroup"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
@@ -34,6 +37,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/vuln"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
const (
@@ -55,11 +59,9 @@ type Service struct {
registryAuthConfigs func(ctx context.Context) (map[string]registry.AuthConfig, error)
saveReport func(ctx context.Context, imageID, data string) error
setPhase func(imageID string, phase vulnerability.ScanPhase)
- // scanLocks allows concurrent scans of different images while preventing
- // duplicate scans of the same image.
+ // scanLocks allows concurrent scans of different images but not duplicate scans of one image.
scanLocks utils.KeyedMutex
- // trivyScanSlots limits concurrent Trivy scans (manual + scheduled) and
- // provides deterministic slot IDs for shared-cache sharding.
+ // trivyScanSlots limits concurrent manual and scheduled scans and provides stable slot IDs for cache sharding.
trivyScanSlots chan int
trivyScanMu sync.Mutex
}
@@ -84,9 +86,12 @@ func NewService(
}
}
-type trivyRuntimeSecurityInternal struct {
- securityOpts []string
- privileged bool
+// settingsConfig returns the current settings, or nil without a settings service.
+func (s *Service) settingsConfig() *settings.Settings {
+ if s.settingsService == nil {
+ return nil
+ }
+ return s.settingsService.GetSettingsConfig()
}
// A nil release without an error means the scan must wait in the queue.
@@ -111,7 +116,18 @@ func (s *Service) AcquireSlot(ctx context.Context, imageID string, wait bool) (s
}
}()
- slotCh := s.getTrivyScanSlotChannelInternal(s.ConcurrentScanContainers())
+ limit := s.ConcurrentScanContainers()
+ s.trivyScanMu.Lock()
+ // Resize only when idle so tokens never split across channels.
+ if s.trivyScanSlots == nil || (cap(s.trivyScanSlots) != limit && len(s.trivyScanSlots) == cap(s.trivyScanSlots)) {
+ s.trivyScanSlots = make(chan int, limit)
+ for i := range limit {
+ s.trivyScanSlots <- i
+ }
+ }
+ slotCh := s.trivyScanSlots
+ s.trivyScanMu.Unlock()
+
if wait {
select {
case slotID = <-slotCh:
@@ -142,72 +158,14 @@ func (s *Service) AcquireSlot(ctx context.Context, imageID string, wait bool) (s
}
func (s *Service) ConcurrentScanContainers() int {
- if s.settingsService == nil {
- return 1
- }
-
- cfg := s.settingsService.GetSettingsConfig()
- if cfg == nil {
- return 1
- }
-
- limit := cfg.TrivyConcurrentScanContainers.AsInt()
- return kit.Ternary(limit < 1, 1, limit)
-}
-
-func (s *Service) getTrivyScanSlotChannelInternal(limit int) chan int {
- if limit < 1 {
- limit = 1
- }
-
- createSlots := func(size int) chan int {
- slots := make(chan int, size)
- for i := range size {
- slots <- i
- }
- return slots
- }
-
- s.trivyScanMu.Lock()
- defer s.trivyScanMu.Unlock()
-
- if s.trivyScanSlots == nil {
- s.trivyScanSlots = createSlots(limit)
- return s.trivyScanSlots
- }
-
- if cap(s.trivyScanSlots) == limit {
- return s.trivyScanSlots
+ if cfg := s.settingsConfig(); cfg != nil {
+ return max(cfg.TrivyConcurrentScanContainers.AsInt(), 1)
}
-
- // Apply capacity changes only when idle to avoid splitting tokens across channels.
- if len(s.trivyScanSlots) == cap(s.trivyScanSlots) {
- s.trivyScanSlots = createSlots(limit)
- }
-
- return s.trivyScanSlots
-}
-
-func (s *Service) buildTrivyRegistryConfigInternal(ctx context.Context) []byte {
- if s.registryAuthConfigs == nil {
- return nil
- }
-
- authConfigs, err := s.registryAuthConfigs(ctx)
- if err != nil {
- slog.WarnContext(ctx, "failed to load registry credentials for trivy scan; continuing anonymously", "error", err)
- return nil
- }
-
- configJSON, err := vuln.BuildDockerConfigJSON(authConfigs)
- if err != nil {
- slog.WarnContext(ctx, "failed to build trivy registry config; continuing anonymously", "error", err)
- return nil
- }
-
- return configJSON
+ return 1
}
+// SetupBatchContainers starts long-running scanner containers that scheduled scans exec into,
+// avoiding a container lifecycle per image.
func (s *Service) SetupBatchContainers(ctx context.Context, count int) ([]string, map[string][]byte, func(), error) {
if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {
return nil, nil, nil, err
@@ -225,53 +183,22 @@ func (s *Service) SetupBatchContainers(ctx context.Context, count int) ([]string
serverURL, _ := s.ServerConfig()
var cacheVolume string
if serverURL == "" {
- cacheVolume, err = s.ensureTrivyCacheVolumeInternal(ctx)
+ cacheVolume, err = s.ensureTrivyCacheVolume(ctx)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to ensure trivy cache volume: %w", err)
}
}
- files := s.trivyContainerFilesInternal(ctx)
- containerIDs := make([]string, 0, max(count, 1))
- cleanup := func() {
- for _, containerID := range containerIDs {
- vuln.RemoveContainer(ctx, dockerClient, containerID, "batch trivy container")
- }
- }
-
- for range max(count, 1) {
- containerID, createErr := s.createBatchTrivyContainer(ctx, scannerImage, cacheVolume, files)
- if createErr != nil {
- cleanup()
- return nil, nil, nil, fmt.Errorf("failed to create batch trivy container: %w", createErr)
- }
- containerIDs = append(containerIDs, containerID)
- }
-
- return containerIDs, files, cleanup, nil
-}
-
-// createBatchTrivyContainer creates a long-running Trivy container that stays
-// alive so we can exec individual scan commands into it. This avoids the
-// overhead of creating/starting/removing a container for every image.
-func (s *Service) createBatchTrivyContainer(ctx context.Context, scannerImage, cacheVolume string, files map[string][]byte) (string, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
+ files := s.trivyContainerFiles(ctx)
+ runtimeOptions, err := s.resolveTrivyRuntimeOptions(ctx, dockerClient)
if err != nil {
- return "", fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- runtimeOptions, err := s.resolveTrivyRuntimeOptionsInternal(ctx, dockerClient)
- if err != nil {
- return "", err
+ return nil, nil, nil, fmt.Errorf("failed to create batch trivy container: %w", err)
}
-
env := append([]string(nil), runtimeOptions.ContainerEnv...)
if files[vuln.RegistryConfigFile] != nil {
+ // ECR tokens refresh only at batch-container creation; a multi-hour batch can outlive them.
env = append(env, "DOCKER_CONFIG="+vuln.RegistryConfigDir)
- // ECR tokens are refreshed when the batch container is created; a multi-hour
- // batch could outlive the token and fall back to anonymous pulls.
}
-
containerConfig := &container.Config{
Image: scannerImage,
Entrypoint: []string{"sh", "-c", "trap 'exit 0' TERM; while :; do sleep 1; done"},
@@ -280,40 +207,46 @@ func (s *Service) createBatchTrivyContainer(ctx context.Context, scannerImage, c
libarcane.InternalResourceLabel: "true",
},
}
-
- runtimeSecurity := s.getTrivyRuntimeSecurityInternal().OrEmpty()
- securityOpts, privileged := runtimeSecurity.securityOpts, runtimeSecurity.privileged
+ var securityOpts []string
+ privileged := false
+ if cfg := s.settingsConfig(); cfg != nil {
+ securityOpts, privileged = vuln.ParseSecurityOpts(cfg.TrivySecurityOpts.Value), cfg.TrivyPrivileged.IsTrue()
+ }
hostConfig := vuln.BuildBatchHostConfig(cacheVolume, runtimeOptions, securityOpts, privileged)
-
- resources, cpuSet, applyLimits, err := s.getTrivyContainerResourceOptionsInternal(ctx, dockerClient)
+ resources, cpuSet, applyLimits, err := s.getTrivyContainerResourceOptions(ctx, dockerClient)
if err != nil {
- return "", err
+ return nil, nil, nil, fmt.Errorf("failed to create batch trivy container: %w", err)
}
vuln.ApplyContainerResources(hostConfig, resources, cpuSet, applyLimits)
- containerID, err := s.createAndStartTrivyContainerInternal(ctx, dockerClient, "batch", containerConfig, hostConfig, files)
- if err != nil {
- return "", err
+ containerIDs := make([]string, 0, max(count, 1))
+ cleanup := func() {
+ for _, containerID := range containerIDs {
+ vuln.RemoveContainer(ctx, dockerClient, containerID, "batch trivy container")
+ }
+ }
+ for range max(count, 1) {
+ containerID, createErr := s.createAndStartTrivyContainer(ctx, dockerClient, "batch", containerConfig, hostConfig, files)
+ if createErr != nil {
+ cleanup()
+ return nil, nil, nil, fmt.Errorf("failed to create batch trivy container: %w", createErr)
+ }
+ slog.InfoContext(ctx, "batch trivy container started", "containerId", containerID)
+ containerIDs = append(containerIDs, containerID)
}
- slog.InfoContext(ctx, "batch trivy container started", "containerId", containerID)
-
- return containerID, nil
+ return containerIDs, files, cleanup, nil
}
-// ExecScan runs a trivy scan for a single image inside an
-// already-running container via docker exec.
-func (
- s *Service,
-) ExecScan(
- ctx context.Context,
- containerID, imageName, imageID, scannerVersion string,
- slotID int,
- files map[string][]byte,
-) (
- *vulnerability.ScanResult,
- error,
-) {
+// ExecScan runs a trivy scan for one image inside an already-running batch container via docker exec.
+func (s *Service) ExecScan(ctx context.Context, containerID, imageName, imageID, scannerVersion string, slotID int, files map[string][]byte) (result *vulnerability.ScanResult, err error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "trivy.scan", trace.WithAttributes(
+ attribute.String("container.image.name", imageName),
+ attribute.String("container.image.id", imageID),
+ attribute.String("arcane.trivy.mode", "exec"),
+ ))
+ defer func() { tracing.End(span, err) }()
+
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
return nil, fmt.Errorf("failed to connect to Docker: %w", err)
@@ -337,10 +270,11 @@ func (
serverURL, serverToken := s.ServerConfig()
var cacheDir string
if serverURL == "" {
- cacheDir = s.getTrivySingleScanCacheDirForSlotInternal(slotID)
+ cacheDir = fmt.Sprintf("%s/slot-%d", trivyCacheSlotRootDir, max(slotID, 0))
}
execCmd := append([]string{"trivy"}, s.buildTrivyCommandArgs(imageName, cacheDir, outputPath, files)...)
+ startTime := time.Now()
exitCode, err := docker.ExecInContainer(ctx, dockerClient, containerID, client.ExecCreateOptions{
Cmd: execCmd,
Env: vuln.ServerTokenEnv(serverToken),
@@ -351,115 +285,86 @@ func (
return nil, err
}
- if exitCode != 0 {
- errMsg := vuln.ReadTempFileExcerpt(stderrFile)
- if errMsg == "" {
- errMsg = vuln.ReadTempFileExcerpt(stdoutFile)
- }
- if errMsg == "" {
- errMsg = fmt.Sprintf("exit status %d", exitCode)
- }
- slog.WarnContext(ctx,
- "trivy scan diagnostics",
- "image", imageName,
- "imageId", imageID,
- "reason", "exec non-zero exit",
- "exitCode", int64(exitCode),
- "stdoutBytes", vuln.TempFileSize(stdoutFile),
- "stderrBytes", vuln.TempFileSize(stderrFile),
- )
- return nil, vuln.FormatExitError(int64(exitCode), errMsg, imageName)
+ run := trivyContainerResult{
+ stdoutFile: stdoutFile, stderrFile: stderrFile, duration: time.Since(startTime).Milliseconds(),
+ statusCode: int64(exitCode), containerID: containerID,
}
-
- trivyOutput, err := vuln.ReadOutputFromContainerFile(ctx, dockerClient, containerID, outputPath)
+ result, err = s.scanResult(ctx, dockerClient, run, outputPath, imageName, imageID, "exec")
if err != nil {
- slog.WarnContext(ctx,
- "failed to read trivy output file from batch container; falling back to stdout parsing",
- "containerId", containerID,
- "outputPath", outputPath,
- "error", err,
- )
- trivyOutput = nil
+ return nil, err
}
+ result.ScannerVersion = scannerVersion
+ return result, nil
+}
- var trivyReport *vulnerability.TrivyReport
- if len(trivyOutput) > 0 {
- trivyReport, err = vuln.DecodeReportFromBytes(trivyOutput)
- } else {
- trivyReport, err = vuln.DecodeReportFromFile(stdoutFile)
+// scanResult decodes a finished scan's report (output file first, then stdout), persists the raw report for
+// report-based patching, and records the result counts on the active span.
+func (s *Service) scanResult(ctx context.Context, dockerClient *client.Client, run trivyContainerResult, outputPath, imageName, imageID, mode string) (*vulnerability.ScanResult, error) {
+ diagnose := func(reason string) {
+ slog.WarnContext(ctx, "trivy scan diagnostics", "image", imageName, "imageId", imageID, "reason", mode+" "+reason, "exitCode", run.statusCode,
+ "stdoutBytes", vuln.TempFileSize(run.stdoutFile), "stderrBytes", vuln.TempFileSize(run.stderrFile))
+ }
+ if run.statusCode != 0 {
+ errMsg := cmp.Or(vuln.ReadTempFileExcerpt(run.stderrFile), vuln.ReadTempFileExcerpt(run.stdoutFile), fmt.Sprintf("exit status %d", run.statusCode))
+ diagnose("non-zero exit")
+ return nil, vuln.FormatExitError(run.statusCode, errMsg, imageName)
}
- if err != nil {
- if errors.Is(err, io.EOF) {
- errMsg := cmp.Or(vuln.ReadTempFileExcerpt(stderrFile), "trivy scan produced no output")
- slog.WarnContext(ctx,
- "trivy scan diagnostics",
- "image", imageName,
- "imageId", imageID,
- "reason", "exec empty output",
- "exitCode", int64(exitCode),
- "stdoutBytes", vuln.TempFileSize(stdoutFile),
- "stderrBytes", vuln.TempFileSize(stderrFile),
- )
- return nil, fmt.Errorf("trivy scan failed: %s", errMsg)
- }
- slog.WarnContext(ctx,
- "trivy scan diagnostics",
- "image", imageName,
- "imageId", imageID,
- "reason", "exec parse error",
- "exitCode", int64(exitCode),
- "stdoutBytes", vuln.TempFileSize(stdoutFile),
- "stderrBytes", vuln.TempFileSize(stderrFile),
- )
- return nil, fmt.Errorf("failed to parse trivy output: %w", err)
+ raw, err := vuln.ReadOutputFromContainerFile(ctx, dockerClient, run.containerID, outputPath)
+ if err != nil {
+ slog.WarnContext(ctx, "failed to read trivy output file; falling back to stdout parsing",
+ "mode", mode, "containerId", run.containerID, "outputPath", outputPath, "error", err)
+ raw = nil
}
- if trivyReport == nil {
- errMsg := cmp.Or(vuln.ReadTempFileExcerpt(stderrFile), "trivy scan produced no output")
- slog.WarnContext(ctx,
- "trivy scan diagnostics",
- "image", imageName,
- "imageId", imageID,
- "reason", "exec nil report",
- "exitCode", int64(exitCode),
- "stdoutBytes", vuln.TempFileSize(stdoutFile),
- "stderrBytes", vuln.TempFileSize(stderrFile),
- )
- return nil, fmt.Errorf("trivy scan failed: %s", errMsg)
+ var report *vulnerability.TrivyReport
+ if len(raw) > 0 {
+ report, err = vuln.DecodeReportFromBytes(raw)
+ } else {
+ report, err = vuln.DecodeReportFromFile(run.stdoutFile)
+ }
+ switch {
+ case errors.Is(err, io.EOF):
+ diagnose("empty output")
+ return nil, fmt.Errorf("trivy scan failed: %s", cmp.Or(vuln.ReadTempFileExcerpt(run.stderrFile), "trivy scan produced no output"))
+ case err != nil:
+ diagnose("parse error")
+ return nil, fmt.Errorf("failed to parse trivy output: %w", err)
+ case report == nil:
+ diagnose("nil report")
+ return nil, fmt.Errorf("trivy scan failed: %s", cmp.Or(vuln.ReadTempFileExcerpt(run.stderrFile), "trivy scan produced no output"))
}
// The name can move to another image after it was selected; never store that image's findings under imageID.
- if scanned := trivyReport.Metadata.ImageID; scanned != "" && strings.TrimPrefix(scanned, "sha256:") != strings.TrimPrefix(imageID, "sha256:") {
+ if scanned := report.Metadata.ImageID; mode == "exec" && scanned != "" && strings.TrimPrefix(scanned, "sha256:") != strings.TrimPrefix(imageID, "sha256:") {
return nil, fmt.Errorf("%s now refers to image %s, not the selected image %s", imageName, scanned, imageID)
}
- result := vulnerability.ConvertTrivyReportToScanResult(trivyReport, imageID, time.Now(), 0)
- result.ScannerVersion = scannerVersion
- result.HasReport = s.persistRawTrivyReportInternal(ctx, imageID, trivyOutput, stdoutFile)
-
- return result, nil
-}
-
-// persistRawTrivyReportInternal stores the raw Trivy JSON report in the
-// database so it can later drive report-based image patching. Reports are
-// keyed by image ID, so a re-scan replaces the previous report. When the
-// output-file read produced nothing, the stdout temp file the report was
-// decoded from is used instead.
-func (s *Service) persistRawTrivyReportInternal(ctx context.Context, imageID string, raw []byte, stdoutFallback *os.File) bool {
- if len(raw) == 0 && stdoutFallback != nil {
- if _, err := stdoutFallback.Seek(0, io.SeekStart); err == nil {
- raw, _ = io.ReadAll(stdoutFallback)
+ result := vulnerability.ConvertTrivyReportToScanResult(report, imageID, time.Now(), run.duration)
+ if len(raw) == 0 {
+ if _, seekErr := run.stdoutFile.Seek(0, io.SeekStart); seekErr == nil {
+ raw, _ = io.ReadAll(run.stdoutFile)
}
}
- if len(raw) == 0 || s.saveReport == nil {
- return false
+ if len(raw) > 0 && s.saveReport != nil {
+ if saveErr := s.saveReport(ctx, imageID, string(raw)); saveErr != nil {
+ slog.WarnContext(ctx, "failed to persist raw trivy report", "imageId", imageID, "error", saveErr)
+ } else {
+ result.HasReport = true
+ }
}
- if err := s.saveReport(ctx, imageID, string(raw)); err != nil {
- slog.WarnContext(ctx, "failed to persist raw trivy report", "imageId", imageID, "error", err)
- return false
- }
- return true
+ summary := cmp.Or(result.Summary, &vulnerability.SeveritySummary{})
+ trace.SpanFromContext(ctx).SetAttributes(
+ attribute.String("arcane.scan.status", string(result.Status)),
+ attribute.Int("vulnerability.critical", summary.Critical),
+ attribute.Int("vulnerability.high", summary.High),
+ attribute.Int("vulnerability.medium", summary.Medium),
+ attribute.Int("vulnerability.low", summary.Low),
+ attribute.Int("vulnerability.unknown", summary.Unknown),
+ attribute.Int("vulnerability.total", summary.Total),
+ )
+ slog.DebugContext(ctx, "trivy scan finished", "imageId", imageID, "imageName", imageName, "durationMs", run.duration, "status", result.Status, "vulnerabilities", summary.Total)
+ return result, nil
}
// Version returns the Trivy version from the configured scanner image.
@@ -556,72 +461,23 @@ func (s *Service) Version(ctx context.Context) string {
return vuln.ParseVersion(vuln.ReadTempFileExcerpt(stdoutFile))
}
-func (s *Service) getTrivyImageRef() string {
- toolsRegistry := ""
- if s.settingsService != nil {
- toolsRegistry = s.settingsService.GetSettingsConfig().ToolsImageRegistry.Value
+func (s *Service) resolveTrivyRuntimeOptions(ctx context.Context, dockerClient *client.Client) (vuln.RuntimeOptions, error) {
+ var configuredNetwork, configuredDockerHost string
+ if cfg := s.settingsConfig(); cfg != nil {
+ configuredNetwork, configuredDockerHost = strings.TrimSpace(cfg.TrivyNetwork.Value), strings.TrimSpace(cfg.DockerHost.Value)
}
- return volumehelper.ToolsImage(toolsRegistry)
-}
-func (s *Service) getTrivyDbRegistryInternal() string {
- if s.settingsService == nil {
- return ""
- }
- return s.settingsService.GetSettingsConfig().TrivyDbRegistry.Value
-}
-
-func (s *Service) getConfiguredTrivyNetworkModeInternal() string {
- if s.settingsService == nil {
- return ""
- }
-
- cfg := s.settingsService.GetSettingsConfig()
- if cfg == nil {
- return ""
- }
-
- return strings.TrimSpace(cfg.TrivyNetwork.Value)
-}
-
-func (s *Service) getEffectiveDockerHostInternal() string {
- if s.dockerService != nil {
- if host := strings.TrimSpace(s.dockerService.DockerHost()); host != "" {
- return host
+ options := vuln.RuntimeOptions{NetworkMode: configuredNetwork}
+ if options.NetworkMode == "" {
+ if inspect, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient); inspectErr == nil {
+ options.NetworkMode = docker.SelectAutoNetworkMode(inspect)
+ } else {
+ slog.DebugContext(ctx, "failed to auto-detect Trivy network; falling back to bridge", "error", inspectErr)
+ options.NetworkMode = vuln.DefaultNetworkMode
}
}
- if s.settingsService != nil {
- if cfg := s.settingsService.GetSettingsConfig(); cfg != nil {
- if host := strings.TrimSpace(cfg.DockerHost.Value); host != "" {
- return host
- }
- }
- }
-
- return vuln.DefaultDockerHostURI
-}
-
-func (s *Service) resolveTrivyNetworkModeInternal(ctx context.Context, dockerClient *client.Client) string {
- if configuredNetwork := s.getConfiguredTrivyNetworkModeInternal(); configuredNetwork != "" {
- return configuredNetwork
- }
-
- inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
- if err == nil {
- return docker.SelectAutoNetworkMode(inspect)
- }
-
- slog.DebugContext(ctx, "failed to auto-detect Trivy network; falling back to bridge", "error", err)
- return vuln.DefaultNetworkMode
-}
-
-func (s *Service) resolveTrivyRuntimeOptionsInternal(ctx context.Context, dockerClient *client.Client) (vuln.RuntimeOptions, error) {
- options := vuln.RuntimeOptions{
- NetworkMode: s.resolveTrivyNetworkModeInternal(ctx, dockerClient),
- }
-
- dockerHost := s.getEffectiveDockerHostInternal()
+ dockerHost := cmp.Or(strings.TrimSpace(s.dockerService.DockerHost()), configuredDockerHost, vuln.DefaultDockerHostURI)
scheme, socketPath, err := vuln.ParseDockerHost(dockerHost)
if err != nil {
return vuln.RuntimeOptions{}, fmt.Errorf("failed to resolve trivy docker host: %w", err)
@@ -659,58 +515,29 @@ func (s *Service) resolveTrivyRuntimeOptionsInternal(ctx context.Context, docker
return options, nil
}
-func (s *Service) getTrivyRuntimeSecurityInternal() mo.Option[trivyRuntimeSecurityInternal] {
- if s.settingsService == nil {
- return mo.None[trivyRuntimeSecurityInternal]()
- }
-
- cfg := s.settingsService.GetSettingsConfig()
- if cfg == nil {
- return mo.None[trivyRuntimeSecurityInternal]()
- }
-
- return mo.Some(trivyRuntimeSecurityInternal{
- securityOpts: vuln.ParseSecurityOpts(cfg.TrivySecurityOpts.Value),
- privileged: cfg.TrivyPrivileged.IsTrue(),
- })
-}
-
-func (s *Service) getTrivyScanTimeoutArg() string {
+func (s *Service) scanTimeoutArg() string {
timeoutSeconds := 0
- if s.settingsService != nil {
- timeoutSeconds = s.settingsService.GetSettingsConfig().TrivyScanTimeout.AsInt()
+ if cfg := s.settingsConfig(); cfg != nil {
+ timeoutSeconds = cfg.TrivyScanTimeout.AsInt()
}
-
return timeouts.GetDuration(timeoutSeconds, timeouts.DefaultTrivyScan).String()
}
-func (s *Service) getTrivySingleScanCacheDirForSlotInternal(slotID int) string {
- if slotID < 0 {
- slotID = 0
- }
-
- // Use a fixed cache directory per concurrency slot on the shared cache
- // volume. This avoids Trivy fs-cache lock contention while bounding cache
- // growth to the configured slot count.
- return fmt.Sprintf("%s/slot-%d", trivyCacheSlotRootDir, slotID)
-}
-
func (s *Service) EnsureImage(ctx context.Context) (string, error) {
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
return "", fmt.Errorf("failed to connect to Docker: %w", err)
}
- scannerImage := s.getTrivyImageRef()
+ toolsRegistry, pullTimeoutSeconds := "", 0
+ if cfg := s.settingsConfig(); cfg != nil {
+ toolsRegistry, pullTimeoutSeconds = cfg.ToolsImageRegistry.Value, cfg.DockerImagePullTimeout.AsInt()
+ }
+ scannerImage := volumehelper.ToolsImage(toolsRegistry)
if _, imageInspectErr := dockerClient.ImageInspect(ctx, scannerImage); imageInspectErr == nil {
return scannerImage, nil
}
- pullTimeoutSeconds := 0
- if s.settingsService != nil && s.settingsService.GetSettingsConfig() != nil {
- pullTimeoutSeconds = s.settingsService.GetSettingsConfig().DockerImagePullTimeout.AsInt()
- }
-
pullCtx, pullCancel := context.WithTimeout(ctx, timeouts.GetDuration(pullTimeoutSeconds, timeouts.DefaultDockerImagePull))
defer pullCancel()
@@ -730,7 +557,7 @@ func (s *Service) EnsureImage(ctx context.Context) (string, error) {
return scannerImage, nil
}
-func (s *Service) ensureTrivyCacheVolumeInternal(ctx context.Context) (string, error) {
+func (s *Service) ensureTrivyCacheVolume(ctx context.Context) (string, error) {
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
return "", err
@@ -753,27 +580,10 @@ func (s *Service) ensureTrivyCacheVolumeInternal(ctx context.Context) (string, e
return trivyCacheVolumeName, nil
}
-func (s *Service) getTrivyConfigFiles() (configContent, ignoreContent string) {
- if s.settingsService == nil {
- return "", ""
- }
-
- cfg := s.settingsService.GetSettingsConfig()
- if cfg == nil {
- return "", ""
- }
-
- return cfg.TrivyConfig.Value, cfg.TrivyIgnore.Value
-}
-
// ServerConfig returns the remote Trivy server URL and optional token when
// client/server mode is enabled, or empty strings for standalone (local DB) scanning.
func (s *Service) ServerConfig() (serverURL, token string) {
- if s.settingsService == nil {
- return "", ""
- }
-
- cfg := s.settingsService.GetSettingsConfig()
+ cfg := s.settingsConfig()
if cfg == nil || !cfg.TrivyServerEnabled.IsTrue() {
return "", ""
}
@@ -784,25 +594,28 @@ func (s *Service) ServerConfig() (serverURL, token string) {
// IgnoreUnfixed reports whether scans should only return vulnerabilities
// that have a known fix (--ignore-unfixed).
func (s *Service) IgnoreUnfixed() bool {
- if s.settingsService == nil {
- return false
- }
-
- cfg := s.settingsService.GetSettingsConfig()
+ cfg := s.settingsConfig()
return cfg != nil && cfg.TrivyIgnoreUnfixed.IsTrue()
}
-func (s *Service) trivyContainerFilesInternal(ctx context.Context) map[string][]byte {
+func (s *Service) trivyContainerFiles(ctx context.Context) map[string][]byte {
files := map[string][]byte{}
- if registryConfigJSON := s.buildTrivyRegistryConfigInternal(ctx); len(registryConfigJSON) > 0 {
- files[vuln.RegistryConfigFile] = registryConfigJSON
- }
- configContent, ignoreContent := s.getTrivyConfigFiles()
- if strings.TrimSpace(configContent) != "" {
- files[vuln.TrivyConfigFile] = []byte(configContent)
+ if s.registryAuthConfigs != nil {
+ if authConfigs, err := s.registryAuthConfigs(ctx); err != nil {
+ slog.WarnContext(ctx, "failed to load registry credentials for trivy scan; continuing anonymously", "error", err)
+ } else if configJSON, buildErr := vuln.BuildDockerConfigJSON(authConfigs); buildErr != nil {
+ slog.WarnContext(ctx, "failed to build trivy registry config; continuing anonymously", "error", buildErr)
+ } else if len(configJSON) > 0 {
+ files[vuln.RegistryConfigFile] = configJSON
+ }
}
- if strings.TrimSpace(ignoreContent) != "" {
- files[vuln.TrivyIgnoreFile] = []byte(ignoreContent)
+ if cfg := s.settingsConfig(); cfg != nil {
+ if strings.TrimSpace(cfg.TrivyConfig.Value) != "" {
+ files[vuln.TrivyConfigFile] = []byte(cfg.TrivyConfig.Value)
+ }
+ if strings.TrimSpace(cfg.TrivyIgnore.Value) != "" {
+ files[vuln.TrivyIgnoreFile] = []byte(cfg.TrivyIgnore.Value)
+ }
}
return files
}
@@ -810,8 +623,12 @@ func (s *Service) trivyContainerFilesInternal(ctx context.Context) map[string][]
// Flags passed here take precedence over the user-supplied trivy.yaml.
func (s *Service) buildTrivyCommandArgs(imageName, cacheDir, outputPath string, files map[string][]byte) []string {
serverURL, _ := s.ServerConfig()
- cmdArgs := []string{"image", "--format", "json", "--quiet", "--exit-code", "0", "--exit-on-eol", "0", "--timeout", s.getTrivyScanTimeoutArg()}
- cmdArgs = append(cmdArgs, vuln.ScanSourceArgs(serverURL, s.getTrivyDbRegistryInternal())...)
+ dbRegistry := ""
+ if cfg := s.settingsConfig(); cfg != nil {
+ dbRegistry = cfg.TrivyDbRegistry.Value
+ }
+ cmdArgs := []string{"image", "--format", "json", "--quiet", "--exit-code", "0", "--exit-on-eol", "0", "--timeout", s.scanTimeoutArg()}
+ cmdArgs = append(cmdArgs, vuln.ScanSourceArgs(serverURL, dbRegistry)...)
cmdArgs = append(cmdArgs, vuln.IgnoreUnfixedArgs(s.IgnoreUnfixed())...)
if files[vuln.TrivyConfigFile] != nil {
@@ -832,7 +649,7 @@ func (s *Service) buildTrivyCommandArgs(imageName, cacheDir, outputPath string,
return append(cmdArgs, imageName)
}
-func (s *Service) createAndStartTrivyContainerInternal(
+func (s *Service) createAndStartTrivyContainer(
ctx context.Context,
dockerClient *client.Client,
scope string,
@@ -859,8 +676,7 @@ func (s *Service) createAndStartTrivyContainerInternal(
vuln.RemoveContainer(ctx, dockerClient, resp.ID, "trivy container after file copy failure")
return "", fmt.Errorf("failed to copy scan files into container: %w", copyFilesToContainerErr)
}
- // DOCKER_CONFIG may still point at the target directory. When the
- // config file is absent, Trivy falls back to anonymous registry access.
+ // DOCKER_CONFIG may still point here; without the file Trivy falls back to anonymous registry access.
slog.WarnContext(ctx,
"failed to inject registry credentials into trivy container; continuing anonymously",
"scope", scope,
@@ -890,17 +706,8 @@ func (s *Service) createAndStartTrivyContainerInternal(
return resp.ID, nil
}
-func (s *Service) getTrivyContainerResourcesInternal() mo.Option[container.Resources] {
- var cfg *settings.Settings
- if s.settingsService != nil {
- cfg = s.settingsService.GetSettingsConfig()
- }
-
- return buildTrivyContainerResourcesInternal(cfg)
-}
-
-func (s *Service) getTrivyContainerResourceOptionsInternal(ctx context.Context, dockerClient *client.Client) (container.Resources, string, bool, error) {
- resources, applyLimits := s.getTrivyContainerResourcesInternal().Get()
+func (s *Service) getTrivyContainerResourceOptions(ctx context.Context, dockerClient *client.Client) (container.Resources, string, bool, error) {
+ resources, applyLimits := buildTrivyContainerResources(s.settingsConfig()).Get()
if !applyLimits || resources.NanoCPUs == 0 {
return resources, "", applyLimits, nil
}
@@ -911,13 +718,53 @@ func (s *Service) getTrivyContainerResourceOptionsInternal(ctx context.Context,
return container.Resources{}, "", false, err
}
- capabilities, err := s.getTrivyCPUCapabilitiesInternal(ctx, dockerClient)
- if err != nil {
- if ctx.Err() != nil {
+ // Capabilities, not limits, are cached per daemon so a new service instance skips the Info call.
+ key := "trivy:cpu-capabilities:v1:" + kit.SHA256Hex(dockerClient.DaemonHost()+"\x00"+dockerClient.ClientVersion())
+ var capabilities trivyCPUCapabilities
+ fresh := false
+ if s.kvService != nil {
+ raw, found, err := s.kvService.Get(ctx, key)
+ if err != nil {
+ slog.WarnContext(ctx, "failed to read cached Trivy CPU capabilities", "error", err)
+ } else if found {
+ if unmarshalErr := json.Unmarshal([]byte(raw), &capabilities); unmarshalErr != nil || capabilities.FetchedAt.IsZero() {
+ slog.WarnContext(ctx, "ignoring invalid cached Trivy CPU capabilities", "error", unmarshalErr)
+ capabilities = trivyCPUCapabilities{}
+ } else if age := time.Since(capabilities.FetchedAt); age >= 0 && age < trivyCPUCapabilitiesCacheTTL {
+ fresh = true
+ }
+ }
+ }
+
+ if !fresh {
+ infoCtx, cancel := context.WithTimeout(ctx, trivyCPUCapabilitiesLookupTimeout)
+ info, err := dockerClient.Info(infoCtx, client.InfoOptions{})
+ cancel()
+ switch {
+ case err != nil && ctx.Err() != nil:
return container.Resources{}, "", false, ctx.Err()
+ case err != nil && capabilities.FetchedAt.IsZero():
+ slog.WarnContext(ctx, "unable to determine Trivy CPU capabilities; using configured CPU limits", "error", err)
+ return resources, "", applyLimits, nil
+ case err != nil:
+ slog.WarnContext(ctx, "failed to refresh Trivy CPU capabilities; using cached capabilities", "error", err)
+ default:
+ capabilities = trivyCPUCapabilities{
+ Quota: info.Info.CPUCfsQuota && !vuln.IsSynologyDockerHost(info.Info.OperatingSystem),
+ CPUSet: info.Info.CPUSet,
+ CPUs: info.Info.NCPU,
+ FetchedAt: time.Now().UTC(),
+ }
+ if s.kvService != nil {
+ payload, marshalErr := json.Marshal(capabilities)
+ if marshalErr == nil {
+ marshalErr = s.kvService.Set(ctx, key, string(payload))
+ }
+ if marshalErr != nil {
+ slog.WarnContext(ctx, "failed to cache Trivy CPU capabilities", "error", marshalErr)
+ }
+ }
}
- slog.WarnContext(ctx, "unable to determine Trivy CPU capabilities; using configured CPU limits", "error", err)
- return resources, "", applyLimits, nil
}
if capabilities.Quota {
@@ -931,20 +778,15 @@ func (s *Service) getTrivyContainerResourceOptionsInternal(ctx context.Context,
resources.NanoCPUs = 0
if cpuSet != "" {
- slog.InfoContext(ctx,
- "docker host does not support CPU CFS quota; applying Trivy CPU limit via cpuset",
- "cpuset", cpuSet,
- )
+ slog.InfoContext(ctx, "docker host does not support CPU CFS quota; applying Trivy CPU limit via cpuset", "cpuset", cpuSet)
} else {
- slog.InfoContext(ctx,
- "docker host does not support CPU CFS quota or cpuset; skipping Trivy CPU limit",
- )
+ slog.InfoContext(ctx, "docker host does not support CPU CFS quota or cpuset; skipping Trivy CPU limit")
}
return resources, cpuSet, applyLimits, nil
}
-func buildTrivyContainerResourcesInternal(cfg *settings.Settings) mo.Option[container.Resources] {
+func buildTrivyContainerResources(cfg *settings.Settings) mo.Option[container.Resources] {
limitsEnabled := true
cpuLimitCores := defaultTrivyCPULimitCores
memoryLimitMB := defaultTrivyMemoryLimitMB
@@ -973,7 +815,7 @@ func buildTrivyContainerResourcesInternal(cfg *settings.Settings) mo.Option[cont
return mo.Some(resources)
}
-type trivyContainerResultInternal struct {
+type trivyContainerResult struct {
stdoutFile *os.File
stderrFile *os.File
duration int64
@@ -981,38 +823,80 @@ type trivyContainerResultInternal struct {
containerID string
}
-func (s *Service) runTrivyContainerInternal(
- ctx context.Context,
- dockerClient *client.Client,
- imageID string,
- containerConfig *container.Config,
- hostConfig *container.HostConfig,
- files map[string][]byte,
-) (trivyContainerResultInternal, error) {
+func (s *Service) Scan(ctx context.Context, scannerImage, imageName, imageID string, slotID int) (result *vulnerability.ScanResult, err error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "trivy.scan", trace.WithAttributes(
+ attribute.String("container.image.name", imageName),
+ attribute.String("container.image.id", imageID),
+ attribute.String("arcane.trivy.mode", "container"),
+ ))
+ defer func() { tracing.End(span, err) }()
+
+ slog.DebugContext(ctx, "running trivy scan", "imageId", imageID, "imageName", imageName, "scannerImage", scannerImage)
+
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to connect to Docker: %w", err)
+ }
+
+ serverURL, serverToken := s.ServerConfig()
+ var cacheVolume, cacheDir string
+ if serverURL == "" {
+ if cacheVolume, err = s.ensureTrivyCacheVolume(ctx); err != nil {
+ return nil, err
+ }
+ // A fixed cache dir per slot avoids fs-cache lock contention while bounding growth to the slot count.
+ cacheDir = fmt.Sprintf("%s/slot-%d", trivyCacheSlotRootDir, max(slotID, 0))
+ slog.DebugContext(ctx, "using slot-based trivy cache dir for scan", "slotId", slotID, "cacheDir", cacheDir, "concurrentScanContainers", s.ConcurrentScanContainers())
+ }
+
+ outputPath := vuln.NewOutputPath()
+ files := s.trivyContainerFiles(ctx)
+ cmdArgs := s.buildTrivyCommandArgs(imageName, cacheDir, outputPath, files)
+
+ runtimeOptions, err := s.resolveTrivyRuntimeOptions(ctx, dockerClient)
+ if err != nil {
+ return nil, err
+ }
+
+ env := append([]string(nil), runtimeOptions.ContainerEnv...)
+ if files[vuln.RegistryConfigFile] != nil {
+ env = append(env, "DOCKER_CONFIG="+vuln.RegistryConfigDir)
+ }
+ if serverToken != "" {
+ env = append(env, vuln.ServerTokenEnv(serverToken)...)
+ }
+
+ containerConfig := vuln.BuildContainerConfig(scannerImage, cmdArgs, env)
+ resources, cpuSet, applyLimits, err := s.getTrivyContainerResourceOptions(ctx, dockerClient)
+ if err != nil {
+ return nil, err
+ }
+ var securityOpts []string
+ privileged := false
+ if cfg := s.settingsConfig(); cfg != nil {
+ securityOpts, privileged = vuln.ParseSecurityOpts(cfg.TrivySecurityOpts.Value), cfg.TrivyPrivileged.IsTrue()
+ }
+ hostConfig := vuln.BuildHostConfig(cacheVolume, resources, cpuSet, applyLimits, runtimeOptions, securityOpts, privileged)
+ s.setPhase(imageID, vulnerability.ScanPhaseCreatingContainer)
+
stdoutFile, err := vuln.CreateLogTempFile("trivy-stdout-*")
if err != nil {
- return trivyContainerResultInternal{}, fmt.Errorf("failed to create trivy stdout temp file: %w", err)
+ return nil, fmt.Errorf("failed to create trivy stdout temp file: %w", err)
}
stderrFile, err := vuln.CreateLogTempFile("trivy-stderr-*")
if err != nil {
vuln.CleanupLogTempFiles(ctx, stdoutFile)
- return trivyContainerResultInternal{}, fmt.Errorf("failed to create trivy stderr temp file: %w", err)
+ return nil, fmt.Errorf("failed to create trivy stderr temp file: %w", err)
}
- success := false
- containerID := ""
- defer func() {
- if !success {
- vuln.RemoveContainer(ctx, dockerClient, containerID, "trivy scan container")
- vuln.CleanupLogTempFiles(ctx, stdoutFile, stderrFile)
- }
- }()
+ defer vuln.CleanupLogTempFiles(ctx, stdoutFile, stderrFile)
- containerID, err = s.createAndStartTrivyContainerInternal(ctx, dockerClient, "single-scan", containerConfig, hostConfig, files)
+ containerID, err := s.createAndStartTrivyContainer(ctx, dockerClient, "single-scan", containerConfig, hostConfig, files)
if err != nil {
- return trivyContainerResultInternal{}, fmt.Errorf("failed to prepare trivy scan container: %w", err)
+ return nil, fmt.Errorf("failed to prepare trivy scan container: %w", err)
}
+ defer vuln.RemoveContainer(ctx, dockerClient, containerID, "trivy scan container")
slog.DebugContext(ctx, "started trivy scan container", "containerId", containerID)
s.setPhase(imageID, vulnerability.ScanPhaseScanningImage)
@@ -1025,7 +909,7 @@ func (s *Service) runTrivyContainerInternal(
})
if err != nil {
logsCancel()
- return trivyContainerResultInternal{}, fmt.Errorf("failed to stream trivy logs: %w", err)
+ return nil, fmt.Errorf("failed to stream trivy logs: %w", err)
}
var closeLogsOnce sync.Once
@@ -1042,303 +926,63 @@ func (s *Service) runTrivyContainerInternal(
logDone := docker.StartStdCopy(logs, stdoutFile, stderrFile)
startTime := time.Now()
- waitTimeout := vuln.ContainerWaitTimeout(s.getTrivyScanTimeoutArg())
- waitCtx, waitCancel := context.WithTimeout(ctx, waitTimeout)
+ waitCtx, waitCancel := context.WithTimeout(ctx, vuln.ContainerWaitTimeout(s.scanTimeoutArg()))
defer waitCancel()
- statusCode, err := s.waitForTrivyContainer(waitCtx, dockerClient, containerID)
+ waitResult := dockerClient.ContainerWait(waitCtx, containerID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
+ statusCode, err := vuln.AwaitContainerWaitResponse(waitCtx, waitResult.Result, waitResult.Error)
if err != nil {
closeLogs()
if logErr := docker.WaitStdCopy(logDone); logErr != nil {
slog.DebugContext(ctx, "trivy log stream ended with error while handling wait failure", "error", logErr)
}
- return trivyContainerResultInternal{}, err
- }
-
- // Wait for the log copy goroutine to complete naturally. On some Docker
- // variants (e.g. Synology with Docker API 1.43), the daemon may not have
- // fully flushed container stdout before ContainerWait returns. Cancelling
- // the stream immediately after ContainerWait can truncate the JSON output
- // and cause "unexpected EOF" or empty-output decode errors.
- //
- // Strategy: wait up to logDrainDeadline for the stream to close on its own,
- // then force-close it (for Docker variants that keep the stream open after
- // container exit, e.g. older Docker builds that never send EOF).
+ switch {
+ case errors.Is(err, context.DeadlineExceeded):
+ return nil, fmt.Errorf("trivy container wait timed out: %w", err)
+ case errors.Is(err, context.Canceled):
+ return nil, fmt.Errorf("scan cancelled: %w", err)
+ default:
+ return nil, fmt.Errorf("trivy container wait failed: %w", err)
+ }
+ }
+
+ // Some daemons (e.g. Synology, API 1.43) flush stdout after ContainerWait returns, so let the stream drain
+ // before force-closing it for daemons that never send EOF; cancelling early truncates the JSON report.
const logDrainDeadline = 30 * time.Second
logDrainTimer := time.NewTimer(logDrainDeadline)
defer logDrainTimer.Stop()
select {
case logErr := <-logDone:
- closeLogs() // clean up context / connection
+ closeLogs()
if logErr != nil && !docker.IsExpectedStreamEndError(logErr) {
- return trivyContainerResultInternal{}, fmt.Errorf("failed to read trivy logs: %w", logErr)
+ return nil, fmt.Errorf("failed to read trivy logs: %w", logErr)
}
case <-logDrainTimer.C:
// Stream did not close on its own; force-close so the goroutine exits.
slog.DebugContext(ctx, "trivy log stream did not close after container exit; force-closing", "containerId", containerID)
closeLogs()
if logErr := docker.WaitStdCopy(logDone); logErr != nil {
- return trivyContainerResultInternal{}, fmt.Errorf("failed to read trivy logs: %w", logErr)
+ return nil, fmt.Errorf("failed to read trivy logs: %w", logErr)
}
}
- duration := time.Since(startTime).Milliseconds()
- success = true
- return trivyContainerResultInternal{
- stdoutFile: stdoutFile, stderrFile: stderrFile, duration: duration,
+ run := trivyContainerResult{
+ stdoutFile: stdoutFile, stderrFile: stderrFile, duration: time.Since(startTime).Milliseconds(),
statusCode: statusCode, containerID: containerID,
- }, nil
-}
-
-func (s *Service) waitForTrivyContainer(
- ctx context.Context,
- dockerClient *client.Client,
- containerID string,
-) (int64, error) {
- waitResult := dockerClient.ContainerWait(ctx, containerID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
-
- statusCode, err := vuln.AwaitContainerWaitResponse(ctx, waitResult.Result, waitResult.Error)
- if err != nil {
- if errors.Is(err, context.DeadlineExceeded) {
- return 0, fmt.Errorf("trivy container wait timed out: %w", err)
- }
- if errors.Is(err, context.Canceled) {
- return 0, fmt.Errorf("scan cancelled: %w", err)
- }
- return 0, fmt.Errorf("trivy container wait failed: %w", err)
}
- return statusCode, nil
-}
-
-func (s *Service) Scan(ctx context.Context, scannerImage, imageName, imageID string, slotID int) (*vulnerability.ScanResult, error) {
- slog.DebugContext(ctx,
- "running trivy scan",
- "imageId", imageID,
- "imageName", imageName,
- "scannerImage", scannerImage,
- )
-
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- serverURL, serverToken := s.ServerConfig()
- var cacheVolume string
- if serverURL == "" {
- cacheVolume, err = s.ensureTrivyCacheVolumeInternal(ctx)
- if err != nil {
- return nil, err
- }
- }
-
- var cacheDir string
- if serverURL == "" {
- cacheDir = s.getTrivySingleScanCacheDirForSlotInternal(slotID)
- }
- if cacheDir != "" {
- slog.DebugContext(ctx,
- "using slot-based trivy cache dir for scan",
- "slotId", slotID,
- "cacheDir", cacheDir,
- "concurrentScanContainers", s.ConcurrentScanContainers(),
- )
- }
-
- outputPath := vuln.NewOutputPath()
- files := s.trivyContainerFilesInternal(ctx)
- cmdArgs := s.buildTrivyCommandArgs(imageName, cacheDir, outputPath, files)
-
- runtimeOptions, err := s.resolveTrivyRuntimeOptionsInternal(ctx, dockerClient)
+ result, err = s.scanResult(ctx, dockerClient, run, outputPath, imageName, imageID, "container")
if err != nil {
return nil, err
}
-
- env := append([]string(nil), runtimeOptions.ContainerEnv...)
- if files[vuln.RegistryConfigFile] != nil {
- env = append(env, "DOCKER_CONFIG="+vuln.RegistryConfigDir)
- }
- if serverToken != "" {
- env = append(env, vuln.ServerTokenEnv(serverToken)...)
- }
-
- containerConfig := vuln.BuildContainerConfig(scannerImage, cmdArgs, env)
- resources, cpuSet, applyLimits, err := s.getTrivyContainerResourceOptionsInternal(ctx, dockerClient)
- if err != nil {
- return nil, err
- }
- runtimeSecurity := s.getTrivyRuntimeSecurityInternal().OrEmpty()
- securityOpts, privileged := runtimeSecurity.securityOpts, runtimeSecurity.privileged
- hostConfig := vuln.BuildHostConfig(
- cacheVolume,
- resources,
- cpuSet,
- applyLimits,
- runtimeOptions,
- securityOpts,
- privileged,
- )
- s.setPhase(imageID, vulnerability.ScanPhaseCreatingContainer)
-
- containerResult, err := s.runTrivyContainerInternal(ctx, dockerClient, imageID, containerConfig, hostConfig, files)
- if err != nil {
- return nil, err
- }
- defer vuln.CleanupLogTempFiles(ctx, containerResult.stdoutFile, containerResult.stderrFile)
- defer vuln.RemoveContainer(ctx, dockerClient, containerResult.containerID, "trivy scan container")
-
- if containerResult.statusCode != 0 {
- errMsg := vuln.ReadTempFileExcerpt(containerResult.stderrFile)
- if errMsg == "" {
- errMsg = vuln.ReadTempFileExcerpt(containerResult.stdoutFile)
- }
- if errMsg == "" {
- errMsg = fmt.Sprintf("exit status %d", containerResult.statusCode)
- }
- slog.WarnContext(ctx,
- "trivy scan diagnostics",
- "image", imageName,
- "imageId", imageID,
- "reason", "container non-zero exit",
- "exitCode", containerResult.statusCode,
- "stdoutBytes", vuln.TempFileSize(containerResult.stdoutFile),
- "stderrBytes", vuln.TempFileSize(containerResult.stderrFile),
- )
- return nil, vuln.FormatExitError(containerResult.statusCode, errMsg, imageName)
- }
-
- trivyOutput, outputErr := vuln.ReadOutputFromContainerFile(ctx, dockerClient, containerResult.containerID, outputPath)
- if outputErr != nil {
- slog.WarnContext(ctx,
- "failed to read trivy output file from single-scan container; falling back to stdout parsing",
- "containerId", containerResult.containerID,
- "outputPath", outputPath,
- "error", outputErr,
- )
- }
-
- var trivyReport *vulnerability.TrivyReport
- if len(trivyOutput) > 0 {
- trivyReport, err = vuln.DecodeReportFromBytes(trivyOutput)
- } else {
- trivyReport, err = vuln.DecodeReportFromFile(containerResult.stdoutFile)
- }
- if err != nil {
- if errors.Is(err, io.EOF) {
- errMsg := cmp.Or(vuln.ReadTempFileExcerpt(containerResult.stderrFile), "trivy scan produced no output")
- slog.WarnContext(ctx,
- "trivy scan diagnostics",
- "image", imageName,
- "imageId", imageID,
- "reason", "container empty output",
- "exitCode", containerResult.statusCode,
- "stdoutBytes", vuln.TempFileSize(containerResult.stdoutFile),
- "stderrBytes", vuln.TempFileSize(containerResult.stderrFile),
- )
- return nil, fmt.Errorf("trivy scan failed: %s", errMsg)
- }
-
- slog.WarnContext(ctx,
- "trivy scan diagnostics",
- "image", imageName,
- "imageId", imageID,
- "reason", "container parse error",
- "exitCode", containerResult.statusCode,
- "stdoutBytes", vuln.TempFileSize(containerResult.stdoutFile),
- "stderrBytes", vuln.TempFileSize(containerResult.stderrFile),
- )
- return nil, fmt.Errorf("failed to parse trivy output: %w", err)
- }
-
- if trivyReport == nil {
- errMsg := cmp.Or(vuln.ReadTempFileExcerpt(containerResult.stderrFile), "trivy scan produced no output")
- slog.WarnContext(ctx,
- "trivy scan diagnostics",
- "image", imageName,
- "imageId", imageID,
- "reason", "container nil report",
- "exitCode", containerResult.statusCode,
- "stdoutBytes", vuln.TempFileSize(containerResult.stdoutFile),
- "stderrBytes", vuln.TempFileSize(containerResult.stderrFile),
- )
- return nil, fmt.Errorf("trivy scan failed: %s", errMsg)
- }
-
- // Convert Trivy report to our ScanResult format
- result := vulnerability.ConvertTrivyReportToScanResult(trivyReport, imageID, time.Now(), containerResult.duration)
result.ScannerVersion = s.Version(ctx)
- result.HasReport = s.persistRawTrivyReportInternal(ctx, imageID, trivyOutput, containerResult.stdoutFile)
- totalVulnerabilities := 0
- if result.Summary != nil {
- totalVulnerabilities = result.Summary.Total
- }
-
- slog.DebugContext(ctx,
- "trivy scan finished",
- "imageId", imageID,
- "imageName", imageName,
- "durationMs", containerResult.duration,
- "status", result.Status,
- "vulnerabilities", totalVulnerabilities,
- )
-
return result, nil
}
-type trivyCPUCapabilitiesInternal struct {
+type trivyCPUCapabilities struct {
Quota bool `json:"quota"`
CPUSet bool `json:"cpuSet"`
CPUs int `json:"cpus"`
FetchedAt time.Time `json:"fetchedAt"`
}
-
-func (s *Service) getTrivyCPUCapabilitiesInternal(ctx context.Context, dockerClient *client.Client) (trivyCPUCapabilitiesInternal, error) {
- key := "trivy:cpu-capabilities:v1:" + kit.SHA256Hex(dockerClient.DaemonHost()+"\x00"+dockerClient.ClientVersion())
- var cached trivyCPUCapabilitiesInternal
- if s.kvService != nil {
- raw, found, err := s.kvService.Get(ctx, key)
- if err != nil {
- slog.WarnContext(ctx, "failed to read cached Trivy CPU capabilities", "error", err)
- } else if found {
- if unmarshalErr := json.Unmarshal([]byte(raw), &cached); unmarshalErr != nil || cached.FetchedAt.IsZero() {
- slog.WarnContext(ctx, "ignoring invalid cached Trivy CPU capabilities", "error", unmarshalErr)
- cached = trivyCPUCapabilitiesInternal{}
- } else if age := time.Since(cached.FetchedAt); age >= 0 && age < trivyCPUCapabilitiesCacheTTL {
- return cached, nil
- }
- }
- }
-
- infoCtx, cancel := context.WithTimeout(ctx, trivyCPUCapabilitiesLookupTimeout)
- defer cancel()
- info, err := dockerClient.Info(infoCtx, client.InfoOptions{})
- if err != nil {
- if ctx.Err() != nil {
- return trivyCPUCapabilitiesInternal{}, ctx.Err()
- }
- if !cached.FetchedAt.IsZero() {
- slog.WarnContext(ctx, "failed to refresh Trivy CPU capabilities; using cached capabilities", "error", err)
- return cached, nil
- }
- return trivyCPUCapabilitiesInternal{}, err
- }
-
- capabilities := trivyCPUCapabilitiesInternal{
- Quota: info.Info.CPUCfsQuota && !vuln.IsSynologyDockerHost(info.Info.OperatingSystem),
- CPUSet: info.Info.CPUSet,
- CPUs: info.Info.NCPU,
- FetchedAt: time.Now().UTC(),
- }
- if s.kvService != nil {
- payload, marshalErr := json.Marshal(capabilities)
- if marshalErr == nil {
- marshalErr = s.kvService.Set(ctx, key, string(payload))
- }
- if marshalErr != nil {
- slog.WarnContext(ctx, "failed to cache Trivy CPU capabilities", "error", marshalErr)
- }
- }
- return capabilities, nil
-}
diff --git a/backend/internal/vulnerability/children/scan/children/trivy/service_test.go b/backend/internal/vulnerability/children/scan/children/trivy/service_test.go
index 2ea132f40..09feb3a9f 100644
--- a/backend/internal/vulnerability/children/scan/children/trivy/service_test.go
+++ b/backend/internal/vulnerability/children/scan/children/trivy/service_test.go
@@ -24,7 +24,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/vuln"
)
-func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *database.DB) {
+func featureSettings(t *testing.T) (*settings.SettingsService, *database.DB) {
t.Helper()
gdb, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
@@ -38,8 +38,8 @@ func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *database
return svc, db
}
-func TestBuildTrivyContainerResourcesInternal_Defaults(t *testing.T) {
- resources, applyLimits := buildTrivyContainerResourcesInternal(nil).Get()
+func TestBuildTrivyContainerResources_Defaults(t *testing.T) {
+ resources, applyLimits := buildTrivyContainerResources(nil).Get()
require.True(t, applyLimits)
require.Equal(t, int64(1_000_000_000), resources.NanoCPUs)
@@ -47,14 +47,14 @@ func TestBuildTrivyContainerResourcesInternal_Defaults(t *testing.T) {
require.Equal(t, int64(0), resources.MemorySwap)
}
-func TestBuildTrivyContainerResourcesInternal_LimitsDisabled(t *testing.T) {
+func TestBuildTrivyContainerResources_LimitsDisabled(t *testing.T) {
cfg := &settings.Settings{
TrivyResourceLimitsEnabled: settings.SettingVariable{Value: "false"},
TrivyCpuLimit: settings.SettingVariable{Value: "2"},
TrivyMemoryLimitMb: settings.SettingVariable{Value: "2048"},
}
- resources, applyLimits := buildTrivyContainerResourcesInternal(cfg).Get()
+ resources, applyLimits := buildTrivyContainerResources(cfg).Get()
require.False(t, applyLimits)
require.Equal(t, int64(0), resources.NanoCPUs)
@@ -62,14 +62,14 @@ func TestBuildTrivyContainerResourcesInternal_LimitsDisabled(t *testing.T) {
require.Equal(t, int64(0), resources.MemorySwap)
}
-func TestBuildTrivyContainerResourcesInternal_CustomLimits(t *testing.T) {
+func TestBuildTrivyContainerResources_CustomLimits(t *testing.T) {
cfg := &settings.Settings{
TrivyResourceLimitsEnabled: settings.SettingVariable{Value: "true"},
TrivyCpuLimit: settings.SettingVariable{Value: "2.5"},
TrivyMemoryLimitMb: settings.SettingVariable{Value: "2048"},
}
- resources, applyLimits := buildTrivyContainerResourcesInternal(cfg).Get()
+ resources, applyLimits := buildTrivyContainerResources(cfg).Get()
require.True(t, applyLimits)
require.Equal(t, int64(2_500_000_000), resources.NanoCPUs)
@@ -100,7 +100,7 @@ func TestBuildTrivyContainerResourcesInternal_CustomLimits(t *testing.T) {
{name: "no Docker client", cpu: "0.5", noClient: true},
} {
t.Run(tc.name, func(t *testing.T) {
- settingsSvc, _ := featureSettingsInternal(t)
+ settingsSvc, _ := featureSettings(t)
require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), "trivyResourceLimitsEnabled", true))
require.NoError(t, settingsSvc.SetStringSetting(t.Context(), "trivyCpuLimit", tc.cpu))
require.NoError(t, settingsSvc.SetIntSetting(t.Context(), "trivyMemoryLimitMb", 2048))
@@ -145,7 +145,7 @@ func TestBuildTrivyContainerResourcesInternal_CustomLimits(t *testing.T) {
if tc.noClient {
dockerClient = nil
}
- actual, cpuSet, apply, err := svc.getTrivyContainerResourceOptionsInternal(ctx, dockerClient)
+ actual, cpuSet, apply, err := svc.getTrivyContainerResourceOptions(ctx, dockerClient)
if tc.expired || tc.noClient {
require.Error(t, err)
if tc.expired {
@@ -166,7 +166,7 @@ func TestBuildTrivyContainerResourcesInternal_CustomLimits(t *testing.T) {
}
for _, quotaSupported := range []bool{true, false} {
- t.Run(fmt.Sprintf("persistent capabilities quota=%t", quotaSupported), func(t *testing.T) { assertPersistentTrivyCapabilitiesInternal(t, quotaSupported) })
+ t.Run(fmt.Sprintf("persistent capabilities quota=%t", quotaSupported), func(t *testing.T) { assertPersistentTrivyCapabilities(t, quotaSupported) })
}
}
@@ -207,38 +207,29 @@ func TestBuildTrivyCommandArgs_IncludesDefaultRepositories(t *testing.T) {
require.Contains(t, cmdArgs, "ghcr.io/"+vuln.ChecksBundleRepository)
}
-func TestGetTrivySingleScanCacheDirForSlotInternal(t *testing.T) {
- svc := &Service{}
-
- require.Equal(t, "/root/.cache/arcane-scan-slots/slot-0", svc.getTrivySingleScanCacheDirForSlotInternal(0))
- require.Equal(t, "/root/.cache/arcane-scan-slots/slot-2", svc.getTrivySingleScanCacheDirForSlotInternal(2))
- // Negative slot IDs should normalize to slot-0
- require.Equal(t, "/root/.cache/arcane-scan-slots/slot-0", svc.getTrivySingleScanCacheDirForSlotInternal(-1))
-}
-
-func TestGetTrivyScanSlotChannelInternal_PrefillsSlots(t *testing.T) {
- svc := &Service{}
-
- slotCh := svc.getTrivyScanSlotChannelInternal(3)
- require.NotNil(t, slotCh)
- require.Equal(t, 3, cap(slotCh))
- require.Len(t, slotCh, 3)
+func TestAcquireSlot_PrefillsSlots(t *testing.T) {
+ settingsSvc, _ := featureSettings(t)
+ require.NoError(t, settingsSvc.SetIntSetting(t.Context(), "trivyConcurrentScanContainers", 3))
+ svc := &Service{settingsService: settingsSvc}
seen := map[int]bool{}
- for range 3 {
- slotID := <-slotCh
+ for i := range 3 {
+ slotID, release, err := svc.AcquireSlot(t.Context(), fmt.Sprintf("image-%d", i), false)
+ require.NoError(t, err)
+ require.NotNil(t, release)
+ t.Cleanup(release)
seen[slotID] = true
}
+ require.Equal(t, map[int]bool{0: true, 1: true, 2: true}, seen)
- require.True(t, seen[0])
- require.True(t, seen[1])
- require.True(t, seen[2])
- require.Empty(t, slotCh)
+ _, release, err := svc.AcquireSlot(t.Context(), "image-3", false)
+ require.NoError(t, err)
+ require.Nil(t, release, "all slots are taken")
}
-func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool) {
+func assertPersistentTrivyCapabilities(t *testing.T, quotaSupported bool) {
t.Helper()
- settingsSvc, db := featureSettingsInternal(t)
+ settingsSvc, db := featureSettings(t)
require.NoError(t, db.AutoMigrate(&kv.KVEntry{}))
store := kv.NewKVService(db)
require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), "trivyResourceLimitsEnabled", true))
@@ -262,7 +253,7 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, dockerClient.Close()) })
svc := &Service{settingsService: settingsSvc, kvService: store}
- first, firstCPUSet, apply, err := svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)
+ first, firstCPUSet, apply, err := svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)
require.NoError(t, err)
require.True(t, apply)
require.Equal(t, int32(1), requests.Load())
@@ -277,7 +268,7 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool
// A new service instance reuses persisted capabilities without calling Docker.
svc = &Service{settingsService: settingsSvc, kvService: kv.NewKVService(db)}
status.Store(http.StatusForbidden)
- cached, cpuSet, apply, err := svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)
+ cached, cpuSet, apply, err := svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)
require.NoError(t, err)
require.True(t, apply)
require.Equal(t, first, cached)
@@ -286,7 +277,7 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool
// Cache capabilities, not the configured limit.
require.NoError(t, settingsSvc.SetStringSetting(t.Context(), "trivyCpuLimit", "0.5"))
- cached, cpuSet, _, err = svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)
+ cached, cpuSet, _, err = svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)
require.NoError(t, err)
if quotaSupported {
require.Equal(t, int64(500_000_000), cached.NanoCPUs)
@@ -298,13 +289,13 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool
entries, err := store.ListByPrefix(t.Context(), "trivy:cpu-capabilities:v1:")
require.NoError(t, err)
require.Len(t, entries, 1)
- var expired trivyCPUCapabilitiesInternal
+ var expired trivyCPUCapabilities
require.NoError(t, json.Unmarshal([]byte(entries[0].Value), &expired))
expired.FetchedAt = time.Now().Add(-2 * trivyCPUCapabilitiesCacheTTL)
payload, err := json.Marshal(expired)
require.NoError(t, err)
require.NoError(t, store.Set(t.Context(), entries[0].Key, string(payload)))
- stale, staleCPUSet, apply, err := svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)
+ stale, staleCPUSet, apply, err := svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)
require.NoError(t, err)
require.True(t, apply)
require.Equal(t, cached, stale)
@@ -312,13 +303,13 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool
require.Equal(t, int32(2), requests.Load())
status.Store(http.StatusOK)
- _, _, _, err = svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)
+ _, _, _, err = svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)
require.NoError(t, err)
require.Equal(t, int32(3), requests.Load())
raw, found, err := store.Get(t.Context(), entries[0].Key)
require.NoError(t, err)
require.True(t, found)
- var refreshed trivyCPUCapabilitiesInternal
+ var refreshed trivyCPUCapabilities
require.NoError(t, json.Unmarshal([]byte(raw), &refreshed))
require.True(t, refreshed.FetchedAt.After(expired.FetchedAt))
@@ -332,7 +323,7 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool
otherClient, newErr := client.New(client.WithHost(endpoint.host), client.WithAPIVersion(endpoint.version))
require.NoError(t, newErr)
t.Cleanup(func() { require.NoError(t, otherClient.Close()) })
- fallback, fallbackCPUSet, fallbackApply, newErr := svc.getTrivyContainerResourceOptionsInternal(t.Context(), otherClient)
+ fallback, fallbackCPUSet, fallbackApply, newErr := svc.getTrivyContainerResourceOptions(t.Context(), otherClient)
require.NoError(t, newErr)
require.True(t, fallbackApply)
require.Equal(t, int64(500_000_000), fallback.NanoCPUs)
@@ -341,7 +332,7 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool
}
require.NoError(t, store.Set(t.Context(), entries[0].Key, "invalid JSON"))
- fallback, cpuSet, apply, err := svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)
+ fallback, cpuSet, apply, err := svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)
require.NoError(t, err)
require.True(t, apply)
require.Equal(t, int64(500_000_000), fallback.NanoCPUs)
@@ -352,13 +343,13 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool
// Cache storage failures must not prevent a live lookup or a safe fallback.
require.NoError(t, db.Migrator().DropTable(&kv.KVEntry{}))
status.Store(http.StatusOK)
- live, liveCPUSet, apply, err := svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)
+ live, liveCPUSet, apply, err := svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)
require.NoError(t, err)
require.True(t, apply)
require.Equal(t, cached, live)
require.Equal(t, staleCPUSet, liveCPUSet)
status.Store(http.StatusInternalServerError)
- fallback, cpuSet, apply, err = svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)
+ fallback, cpuSet, apply, err = svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)
require.NoError(t, err)
require.True(t, apply)
require.Equal(t, int64(500_000_000), fallback.NanoCPUs)
@@ -377,7 +368,7 @@ func TestVulnerabilityFeatureDisabledWhileWaitingForSlot(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- settingsSvc, _ := featureSettingsInternal(t)
+ settingsSvc, _ := featureSettings(t)
require.NoError(t, settingsSvc.SetIntSetting(t.Context(), "trivyConcurrentScanContainers", tt.slots))
var inspections atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
@@ -432,7 +423,7 @@ func TestVulnerabilityFeatureDisabledWhileWaitingForSlot(t *testing.T) {
}
func TestVulnerabilityFeatureDisabledRejectsBatchContainers(t *testing.T) {
- settingsSvc, _ := featureSettingsInternal(t)
+ settingsSvc, _ := featureSettings(t)
svc := &Service{settingsService: settingsSvc}
require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))
diff --git a/backend/internal/vulnerability/module.go b/backend/internal/vulnerability/module.go
index 72f049328..efe19184b 100644
--- a/backend/internal/vulnerability/module.go
+++ b/backend/internal/vulnerability/module.go
@@ -2,23 +2,67 @@
package vulnerability
import (
+ "context"
+ "errors"
"log/slog"
"net/http"
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/types/v2/features"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/ignores"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/query"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/scan"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
type Module struct {
service *VulnerabilityService
}
+// New wires vulnerability routes and registers the arcane.vulnerability.* gauges once.
func New(service *VulnerabilityService) *Module {
+ if service != nil && service.query != nil && service.settingsService != nil {
+ meter := otel.Meter(tracing.InstrumentationName)
+ findings, findingsErr := meter.Int64ObservableGauge("arcane.vulnerability.findings",
+ metric.WithDescription("Vulnerabilities in the latest completed scans of local images by severity"), metric.WithUnit("{vulnerability}"))
+ scanned, scannedErr := meter.Int64ObservableGauge("arcane.vulnerability.images.scanned",
+ metric.WithDescription("Local images with a completed vulnerability scan"), metric.WithUnit("{image}"))
+ actionable, actionableErr := meter.Int64ObservableGauge("arcane.vulnerability.actionable",
+ metric.WithDescription("Critical and high vulnerabilities that are not ignored"), metric.WithUnit("{vulnerability}"))
+ err := errors.Join(findingsErr, scannedErr, actionableErr)
+ if err == nil {
+ _, err = meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
+ if !service.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {
+ return nil
+ }
+ if summary, summaryErr := service.query.GetEnvironmentSummary(ctx); summaryErr != nil {
+ slog.DebugContext(ctx, "Skipping vulnerability summary metrics", "error", summaryErr)
+ } else if summary.Summary != nil {
+ o.ObserveInt64(scanned, int64(summary.ScannedImages))
+ for severity, n := range map[string]int{
+ "critical": summary.Summary.Critical, "high": summary.Summary.High, "medium": summary.Summary.Medium,
+ "low": summary.Summary.Low, "unknown": summary.Summary.Unknown,
+ } {
+ o.ObserveInt64(findings, int64(n), metric.WithAttributes(attribute.String("vulnerability.severity", severity)))
+ }
+ }
+ if count, countErr := service.ActionableCountExcludingIgnored(ctx); countErr != nil {
+ slog.DebugContext(ctx, "Skipping actionable vulnerability metric", "error", countErr)
+ } else {
+ o.ObserveInt64(actionable, int64(count))
+ }
+ return nil
+ }, findings, scanned, actionable)
+ }
+ if err != nil {
+ otel.Handle(err)
+ }
+ }
return &Module{service: service}
}
diff --git a/backend/pkg/gitutil/git.go b/backend/pkg/gitutil/git.go
index c1ca6a8ae..5dc2dbba2 100644
--- a/backend/pkg/gitutil/git.go
+++ b/backend/pkg/gitutil/git.go
@@ -15,7 +15,6 @@ import (
"path/filepath"
"regexp"
"slices"
- "sort"
"strings"
"time"
@@ -32,8 +31,13 @@ import (
"go.getarcane.app/acfs"
acfstypes "go.getarcane.app/acfs/types"
kit "go.getarcane.app/kit/pkg"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
gossh "golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
const (
@@ -54,16 +58,11 @@ const (
defaultKnownHostsPath = "/app/data/.ssh/known_hosts"
)
-// go-git's file transport execs the git binary, which doesn't exist in the
-// distroless image. Unregister it so a repository URL can never reach it.
+// go-git's file transport execs the git binary, which the distroless image lacks, so unregister it.
func init() {
client.InstallProtocol("file", nil)
- // go-git strips multi_ack/multi_ack_detailed from pack negotiation by
- // default, which Azure DevOps rejects by sending an incomplete pack
- // ("invalid reset option: object not found" on clone). Advertise them and
- // keep only thin-pack disabled, matching Flux/ArgoCD; other hosts are
- // unaffected.
+ // Azure DevOps sends an incomplete pack without multi_ack, so only thin-pack stays disabled (as in Flux/ArgoCD).
transport.UnsupportedCapabilities = []capability.Capability{
capability.ThinPack,
}
@@ -76,10 +75,9 @@ type Client struct {
var scpLikeURLPattern = regexp.MustCompile(`^[^@/]+@[^:/]+:`)
-// normalizeURLInternal coerces a repository URL into a form go-git resolves to a
-// network transport, never the file transport (which execs the git binary).
+// normalizeURL coerces a repository URL onto a network transport, never the file transport.
// Schemeless host-style URLs (e.g. github.com/org/repo.git) get https://.
-func normalizeURLInternal(raw string) (string, error) {
+func normalizeURL(raw string) (string, error) {
url := strings.TrimSpace(raw)
if url == "" {
return "", errors.New("repository URL is empty")
@@ -121,8 +119,8 @@ type AuthConfig struct {
SSHHostKeyVerification string // strict, accept_new, skip
}
-// getAuthInternal returns the appropriate transport.AuthMethod.
-func (c *Client) getAuthInternal(ctx context.Context, url string, localConfig AuthConfig) (transport.AuthMethod, error) {
+// getAuth returns the appropriate transport.AuthMethod.
+func (c *Client) getAuth(ctx context.Context, url string, localConfig AuthConfig) (transport.AuthMethod, error) {
switch localConfig.AuthType {
case "http":
if localConfig.Token != "" {
@@ -144,8 +142,7 @@ func (c *Client) getAuthInternal(ctx context.Context, url string, localConfig Au
return nil, fmt.Errorf("failed to create ssh auth: %w", err)
}
- // Configure host key verification based on mode
- hostKeyCallback, err := c.getSSHHostKeyCallbackInternal(ctx, localConfig.SSHHostKeyVerification)
+ hostKeyCallback, err := c.getSSHHostKeyCallback(ctx, localConfig.SSHHostKeyVerification)
if err != nil {
return nil, fmt.Errorf("failed to configure SSH host key verification: %w", err)
}
@@ -156,21 +153,18 @@ func (c *Client) getAuthInternal(ctx context.Context, url string, localConfig Au
return publicKeys, nil
}
return nil, errors.New("ssh key required for ssh authentication")
- case "none":
- return nil, nil
default:
return nil, nil
}
}
-// getSSHHostKeyCallbackInternal returns the SSH host key callback for the verification mode.
-func (c *Client) getSSHHostKeyCallbackInternal(ctx context.Context, mode string) (gossh.HostKeyCallback, error) {
+// getSSHHostKeyCallback returns the SSH host key callback for the verification mode.
+func (c *Client) getSSHHostKeyCallback(ctx context.Context, mode string) (gossh.HostKeyCallback, error) {
if mode == SSHHostKeyVerificationSkip {
- // Skip host key verification - intentionally insecure, user explicitly opted in via UI
return gossh.InsecureIgnoreHostKey(), nil //nolint:gosec // User explicitly chose to skip verification
}
- knownHostsPaths, pathErr := getKnownHostsPathsInternal(
+ knownHostsPaths, pathErr := getKnownHostsPaths(
c.workDir,
os.Getenv,
os.Stat,
@@ -183,16 +177,13 @@ func (c *Client) getSSHHostKeyCallbackInternal(ctx context.Context, mode string)
return knownhosts.New(knownHostsPaths...)
}
- // Other modes default to accepting and remembering new host keys.
- // Ensure the directory exists
- // os.* supports home and override paths outside acfs roots, plus append/flock.
+ // Other modes accept and remember new host keys; os.* reaches paths outside acfs roots and supports append/flock.
knownHostsPath := knownHostsPaths[0]
dir := filepath.Dir(knownHostsPath)
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, fmt.Errorf("failed to create known_hosts directory: %w", err)
}
- // Create the file if it doesn't exist
if _, err := os.Stat(knownHostsPath); os.IsNotExist(err) {
file, openFileErr := os.OpenFile(knownHostsPath, os.O_CREATE|os.O_WRONLY, 0o600)
if openFileErr != nil {
@@ -204,8 +195,7 @@ func (c *Client) getSSHHostKeyCallbackInternal(ctx context.Context, mode string)
}
return func(hostname string, remote net.Addr, key gossh.PublicKey) error {
- // Re-read known_hosts on each call to handle concurrent modifications.
- // Unreadable trust must fail closed so changed keys are never accepted.
+ // Re-read known_hosts each call for concurrent edits; unreadable trust fails closed.
existingCallback, err := knownhosts.New(knownHostsPaths...)
if err != nil {
return fmt.Errorf("failed to read known_hosts: %w", err)
@@ -213,16 +203,13 @@ func (c *Client) getSSHHostKeyCallbackInternal(ctx context.Context, mode string)
existingCallbackErr := existingCallback(hostname, remote, key)
if existingCallbackErr == nil {
- return nil // Host key matches
+ return nil
}
- // Check if it's a "key mismatch" error vs "unknown host"
if keyErr, ok := errors.AsType[*knownhosts.KeyError](existingCallbackErr); ok && len(keyErr.Want) > 0 {
- // Host is known but key doesn't match - this is a security concern
return fmt.Errorf("host key mismatch for %s (possible MITM attack): %w", hostname, existingCallbackErr)
}
- // Otherwise, host is unknown - we'll add it
- // Log persistence failures after closing the file and releasing its lock.
+ // Unknown host: save it, logging persistence failures after the file closes and the lock releases.
var saveErr error
defer func() {
if saveErr != nil {
@@ -260,20 +247,18 @@ func (c *Client) getSSHHostKeyCallbackInternal(ctx context.Context, mode string)
}, nil
}
-// getKnownHostsPathsInternal returns trusted known_hosts files; new keys are saved to the first.
-func getKnownHostsPathsInternal(
+// getKnownHostsPaths returns trusted known_hosts files; new keys are saved to the first.
+func getKnownHostsPaths(
workDir string,
getenv func(string) string,
stat func(string) (os.FileInfo, error),
userHomeDir func() (string, error),
) ([]string, error) {
- // Check environment variable first
if localPath := getenv("SSH_KNOWN_HOSTS"); localPath != "" {
return []string{localPath}, nil
}
- // Prefer Arcane's writable persistent data directory when it is available,
- // which is the case for published container images and PUID/PGID setups.
+ // Prefer Arcane's writable data directory, present in published images and PUID/PGID setups.
if info, err := stat(defaultKnownHostsDataDir); err == nil && info.IsDir() {
return []string{defaultKnownHostsPath}, nil
}
@@ -319,74 +304,78 @@ func getKnownHostsPathsInternal(
}
}
+// RepositoryAttribute names the repository on spans without credentials or query tokens.
+func RepositoryAttribute(rawURL string) attribute.KeyValue {
+ endpoint, err := transport.NewEndpoint(rawURL)
+ if err != nil {
+ return attribute.String("arcane.git.repository", "")
+ }
+ endpoint.User, endpoint.Password = "", ""
+ if i := strings.IndexAny(endpoint.Path, "?#"); i >= 0 {
+ endpoint.Path = endpoint.Path[:i]
+ }
+ return attribute.String("arcane.git.repository", endpoint.String())
+}
+
// Clone clones a repository to a temporary directory. A depth of 0 fetches the
// full history; read-only callers pass 1 to fetch only the branch tip.
-func (c *Client) Clone(ctx context.Context, url, branch string, auth AuthConfig, depth int) (string, error) {
+func (c *Client) Clone(ctx context.Context, url, branch string, auth AuthConfig, depth int) (repoPath string, err error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "gitrepo.clone", trace.WithAttributes(
+ attribute.String("arcane.git.branch", branch),
+ attribute.Int("arcane.git.depth", depth),
+ ))
+ defer func() { tracing.End(span, err) }()
+
if _, hasDeadline := ctx.Deadline(); !hasDeadline {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
}
-
- if err := ctx.Err(); err != nil {
- return "", err
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return "", ctxErr
}
- // Create a temporary directory
- workDir := c.workDir
- if workDir == "" {
- workDir = os.TempDir()
- }
- // Ensure the work directory exists
- // os.* rather than acfs: this creates the clone staging root itself (under the
- // system temp dir by default), which has to exist before acfs could open it.
- if err := os.MkdirAll(workDir, 0o755); err != nil {
- return "", fmt.Errorf("failed to create work dir: %w", err)
- }
- tmpDir, err := os.MkdirTemp(workDir, cloneScratchPrefix+"*")
+ url, err = normalizeURL(url)
if err != nil {
- return "", fmt.Errorf("failed to create temp dir: %w", err)
- }
-
- url, err = normalizeURLInternal(url)
- if err != nil {
- _ = os.RemoveAll(tmpDir)
return "", err
}
-
- authMethod, err := c.getAuthInternal(ctx, url, auth)
+ span.SetAttributes(RepositoryAttribute(url))
+ authMethod, err := c.getAuth(ctx, url, auth)
if err != nil {
- _ = os.RemoveAll(tmpDir)
return "", err
}
- cloneOptions := &git.CloneOptions{
- URL: url,
- Progress: nil,
- Depth: depth,
+ // os.* rather than acfs: the clone staging root must exist before acfs could open it.
+ workDir := cmp.Or(c.workDir, os.TempDir())
+ if err = os.MkdirAll(workDir, 0o755); err != nil {
+ return "", fmt.Errorf("failed to create work dir: %w", err)
}
-
- if authMethod != nil {
- cloneOptions.Auth = authMethod
+ tmpDir, err := os.MkdirTemp(workDir, cloneScratchPrefix+"*")
+ if err != nil {
+ return "", fmt.Errorf("failed to create temp dir: %w", err)
}
+ cloneOptions := &git.CloneOptions{URL: url, Auth: authMethod, Depth: depth}
if branch != "" {
cloneOptions.ReferenceName = plumbing.NewBranchReferenceName(branch)
cloneOptions.SingleBranch = true
}
- _, err = git.PlainCloneContext(ctx, tmpDir, false, cloneOptions)
+ repo, err := git.PlainCloneContext(ctx, tmpDir, false, cloneOptions)
if err != nil && depth > 0 && strings.Contains(err.Error(), "shallow") {
// Servers without shallow support, go-git's own included, get a full clone.
cloneOptions.Depth = 0
if err = os.RemoveAll(tmpDir); err == nil {
- _, err = git.PlainCloneContext(ctx, tmpDir, false, cloneOptions)
+ repo, err = git.PlainCloneContext(ctx, tmpDir, false, cloneOptions)
}
}
if err != nil {
_ = os.RemoveAll(tmpDir)
return "", fmt.Errorf("failed to clone repository: %w", err)
}
+ if head, headErr := repo.Head(); headErr == nil {
+ span.SetAttributes(attribute.String("arcane.git.commit", head.Hash().String()))
+ }
return tmpDir, nil
}
@@ -415,111 +404,79 @@ type BranchInfo struct {
IsDefault bool
}
-// ListBranches lists all branches in a remote repository
+// ListBranches lists a remote repository's branches, default branch first.
func (c *Client) ListBranches(ctx context.Context, url string, auth AuthConfig) ([]BranchInfo, error) {
- if err := ctx.Err(); err != nil {
- return nil, err
- }
- refs, err := c.listRemoteReferencesInternal(ctx, url, auth)
+ refs, err := c.listRemoteReferences(ctx, url, auth)
if err != nil {
return nil, err
}
- var branches []BranchInfo
+ // HEAD is a symbolic reference to the default branch.
var defaultBranch string
-
- // Find the default branch (HEAD points to it)
- for _, ref := range refs {
- if ref.Name().String() == "HEAD" {
- // HEAD is a symbolic reference that points to the default branch
- if ref.Target().IsBranch() {
- defaultBranch = ref.Target().Short()
- }
- break
- }
+ if i := slices.IndexFunc(refs, func(ref *plumbing.Reference) bool { return ref.Name().String() == "HEAD" }); i >= 0 && refs[i].Target().IsBranch() {
+ defaultBranch = refs[i].Target().Short()
}
- // Collect all branches
+ var branches []BranchInfo
seen := make(map[string]bool)
for _, ref := range refs {
- if ref.Name().IsBranch() {
- branchName := ref.Name().Short()
- if seen[branchName] {
- continue
- }
- seen[branchName] = true
-
- branches = append(branches, BranchInfo{
- Name: branchName,
- IsDefault: branchName == defaultBranch,
- })
+ branchName := ref.Name().Short()
+ if !ref.Name().IsBranch() || seen[branchName] {
+ continue
}
+ seen[branchName] = true
+ branches = append(branches, BranchInfo{Name: branchName, IsDefault: branchName == defaultBranch})
}
- // Sort branches with default first
- sort.Slice(branches, func(i, j int) bool {
- if branches[i].IsDefault {
- return true
- }
- if branches[j].IsDefault {
- return false
+ slices.SortFunc(branches, func(a, b BranchInfo) int {
+ if a.IsDefault != b.IsDefault {
+ return kit.Ternary(a.IsDefault, -1, 1)
}
- return branches[i].Name < branches[j].Name
+ return strings.Compare(a.Name, b.Name)
})
-
return branches, nil
}
// ProbeRemote verifies that a remote repository is reachable without cloning it.
func (c *Client) ProbeRemote(ctx context.Context, url string, auth AuthConfig) error {
- if err := ctx.Err(); err != nil {
- return err
- }
-
- _, err := c.listRemoteReferencesInternal(ctx, url, auth)
+ _, err := c.listRemoteReferences(ctx, url, auth)
return err
}
-func (c *Client) listRemoteReferencesInternal(ctx context.Context, url string, auth AuthConfig) ([]*plumbing.Reference, error) {
- url, err := normalizeURLInternal(url)
+func (c *Client) listRemoteReferences(ctx context.Context, url string, auth AuthConfig) (refs []*plumbing.Reference, err error) {
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "gitrepo.list_remote")
+ defer func() { tracing.End(span, err) }()
+
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return nil, ctxErr
+ }
+ url, err = normalizeURL(url)
if err != nil {
return nil, err
}
-
- authMethod, err := c.getAuthInternal(ctx, url, auth)
+ span.SetAttributes(RepositoryAttribute(url))
+ authMethod, err := c.getAuth(ctx, url, auth)
if err != nil {
return nil, err
}
- // Create a remote without cloning
- rem := git.NewRemote(nil, &config.RemoteConfig{
- Name: "origin",
- URLs: []string{url},
- })
-
- listOptions := &git.ListOptions{}
- if authMethod != nil {
- listOptions.Auth = authMethod
- }
-
listCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
- refs, err := rem.ListContext(listCtx, listOptions)
+ rem := git.NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{url}})
+ refs, err = rem.ListContext(listCtx, &git.ListOptions{Auth: authMethod})
if err != nil {
return nil, fmt.Errorf("failed to list remote references: %w", err)
}
-
+ span.SetAttributes(attribute.Int("arcane.result.count", len(refs)))
return refs, nil
}
// ValidatePath ensures the path is safe and doesn't escape the repo
func ValidatePath(repoPath, requestedPath string) error {
- // Clean the paths
cleanRepoPath := filepath.Clean(repoPath)
cleanRequestedPath := filepath.Clean(filepath.Join(repoPath, requestedPath))
- // Check if the requested path is within the repo using relative path validation
rel, err := filepath.Rel(cleanRepoPath, cleanRequestedPath)
if err != nil {
return fmt.Errorf("invalid path: %w", err)
@@ -531,9 +488,8 @@ func ValidatePath(repoPath, requestedPath string) error {
return nil
}
-// BrowseTree returns the file tree at the specified path. The clone directory
-// is the confinement root: paths that escape it, and symbolic links pointing
-// outside it, are rejected rather than followed.
+// BrowseTree returns the file tree at targetPath, confined to the clone directory: escaping paths and
+// symlinks pointing outside it are rejected rather than followed.
func (c *Client) BrowseTree(ctx context.Context, repoPath, targetPath string) ([]gitops.FileTreeNode, error) {
if err := ctx.Err(); err != nil {
return nil, err
@@ -558,7 +514,6 @@ func (c *Client) BrowseTree(ctx context.Context, repoPath, targetPath string) ([
if errErr := ctx.Err(); errErr != nil {
return nil, errErr
}
- // Skip .git directory
if child.Name == ".git" {
continue
}
@@ -576,9 +531,7 @@ func (c *Client) BrowseTree(ctx context.Context, repoPath, targetPath string) ([
return nodes, nil
}
-// Cleanup removes a temporary repository directory
-// Stays on os.RemoveAll: acfs refuses to remove its own root, and here the whole
-// clone directory itself is what gets deleted.
+// Cleanup removes a temporary repository directory with os.RemoveAll, since acfs refuses to remove its own root.
func (c *Client) Cleanup(repoPath string) error {
return os.RemoveAll(repoPath)
}
@@ -593,10 +546,7 @@ func (c *Client) Discard(ctx context.Context, repoPath string) {
// PurgeScratchDirs removes clone scratch dirs ("gitops-*") under the work dir
// whose mtime is older than maxAge. maxAge <= 0 removes all (boot sweep).
func (c *Client) PurgeScratchDirs(ctx context.Context, maxAge time.Duration) (int, error) {
- root := c.workDir
- if root == "" {
- root = os.TempDir()
- }
+ root := cmp.Or(c.workDir, os.TempDir())
entries, err := os.ReadDir(root)
if err != nil {
@@ -634,27 +584,13 @@ func (c *Client) PurgeScratchDirs(ctx context.Context, maxAge time.Duration) (in
return removed, nil
}
-// CommitInfo holds information about a git commit
-type CommitInfo struct {
- Hash string
- Author string
- Message string
- Date time.Time
-}
-
// TestConnection tests if the repository can be accessed with the given credentials
func (c *Client) TestConnection(ctx context.Context, url, branch string, auth AuthConfig) error {
- if err := ctx.Err(); err != nil {
- return err
- }
-
tmpDir, err := c.Clone(ctx, url, branch, auth, 1)
if err != nil {
return err
}
- defer func() {
- _ = c.Cleanup(tmpDir)
- }()
+ c.Discard(ctx, tmpDir)
return nil
}
@@ -685,10 +621,7 @@ type SyncFileInfo struct {
Content []byte
Size int64
IsBinary bool
- // Executable mirrors git's +x bit so callers can preserve it on the
- // destination. Required for lifecycle hooks: a script committed as
- // 100755 must arrive in the project workspace runnable, otherwise the
- // lifecycle runner fails to exec it.
+ // Executable mirrors git's +x bit so lifecycle hook scripts arrive runnable.
Executable bool
}
@@ -700,10 +633,8 @@ type DirectoryWalkResult struct {
SkippedBinaries int
}
-// WalkDirectory walks the directory containing the compose file and returns all files.
-// It enforces limits on file count, total size, and skips large binary files.
-// The composePath is the path to the compose file within the repo - the directory
-// containing this file will be walked.
+// WalkDirectory returns every file in the compose file's directory, enforcing file count and
+// total size limits and skipping large binary files.
func (c *Client) WalkDirectory(ctx context.Context, repoPath, composePath string,
maxFiles int, maxTotalSize, maxBinarySize int64,
) (*DirectoryWalkResult, error) {
@@ -711,12 +642,10 @@ func (c *Client) WalkDirectory(ctx context.Context, repoPath, composePath string
return nil, err
}
- // Validate compose path
if err := ValidatePath(repoPath, composePath); err != nil {
return nil, fmt.Errorf("invalid compose path: %w", err)
}
- // Get the directory containing the compose file
syncDir := filepath.Dir(filepath.Join(repoPath, composePath))
result := &DirectoryWalkResult{
@@ -803,18 +732,13 @@ func IsBinaryContent(content []byte) bool {
return false
}
- // Check first 512 bytes (or less if file is smaller)
- checkSize := min(len(content), 512)
-
- // Use net/http's content type detection
+ checkSize := min(len(content), binarySniffBytes)
contentType := nethttp.DetectContentType(content[:checkSize])
- // Text types are not binary
if strings.HasPrefix(contentType, "text/") {
return false
}
- // Common text-based application types
textAppTypes := []string{
"application/json",
"application/xml",
@@ -830,13 +754,11 @@ func IsBinaryContent(content []byte) bool {
}
}
- // For application/octet-stream, do additional null-byte check
- // Text files rarely have null bytes, so their presence indicates binary
+ // Text files rarely contain null bytes, so they mark octet-stream content as binary.
if contentType == "application/octet-stream" {
return slices.Contains(content[:checkSize], 0)
}
- // Everything else is considered binary
return strings.HasPrefix(contentType, "application/") ||
strings.HasPrefix(contentType, "image/") ||
strings.HasPrefix(contentType, "video/") ||
diff --git a/backend/pkg/gitutil/git_test.go b/backend/pkg/gitutil/git_test.go
index cb95984e7..545a16902 100644
--- a/backend/pkg/gitutil/git_test.go
+++ b/backend/pkg/gitutil/git_test.go
@@ -109,7 +109,7 @@ func TestGetKnownHostsPath(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- result, err := getKnownHostsPathsInternal(
+ result, err := getKnownHostsPaths(
tt.workDir,
func(key string) string {
require.Equal(t, "SSH_KNOWN_HOSTS", key)
@@ -165,7 +165,7 @@ func TestGetSSHHostKeyCallback(t *testing.T) {
t.Run("skip mode returns InsecureIgnoreHostKey", func(t *testing.T) {
t.Setenv("SSH_KNOWN_HOSTS", filepath.Join(t.TempDir(), "missing", "known_hosts"))
- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationSkip)
+ callback, err := client.getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationSkip)
require.NoError(t, err,
"unexpected error: %v", err)
@@ -185,7 +185,7 @@ func TestGetSSHHostKeyCallback(t *testing.T) {
knownHostsPath := filepath.Join(tmpDir, "known_hosts")
t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)
- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), "")
+ callback, err := client.getSSHHostKeyCallback(t.Context(), "")
require.NoError(t, err,
"unexpected error: %v", err)
@@ -199,7 +199,7 @@ func TestGetSSHHostKeyCallback(t *testing.T) {
knownHostsPath := filepath.Join(tmpDir, "known_hosts")
t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)
- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)
+ callback, err := client.getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)
require.NoError(t, err,
"unexpected error: %v", err)
@@ -218,7 +218,7 @@ func TestAddHostKey(t *testing.T) {
key := generateTestPublicKey(t)
t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)
- callback, err := NewClient("").getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)
+ callback, err := NewClient("").getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)
require.NoError(t, err)
err = callback("example.com:22", &net.TCPAddr{}, key)
@@ -241,7 +241,7 @@ func TestAddHostKey(t *testing.T) {
t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)
key := generateTestPublicKey(t)
- callback, callbackErr := NewClient("").getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)
+ callback, callbackErr := NewClient("").getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)
require.NoError(t, callbackErr)
var wg sync.WaitGroup
errChan := make(chan error, 10)
@@ -283,7 +283,7 @@ func TestCreateAcceptNewHostKeyCallback(t *testing.T) {
t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)
client := NewClient("")
- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)
+ callback, err := client.getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)
require.NoError(t, err,
"unexpected error: %v", err)
@@ -305,7 +305,7 @@ func TestCreateAcceptNewHostKeyCallback(t *testing.T) {
t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)
client := NewClient("")
- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)
+ callback, err := client.getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)
require.NoError(t, err,
"unexpected error: %v", err)
@@ -334,7 +334,7 @@ func TestCreateAcceptNewHostKeyCallback(t *testing.T) {
t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)
client := NewClient("")
- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)
+ callback, err := client.getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)
require.NoError(t, err,
"unexpected error: %v", err)
@@ -349,7 +349,7 @@ func TestCreateAcceptNewHostKeyCallback(t *testing.T) {
"first callback returned error: %v", err)
// A new client must reuse the persisted trust file.
- callback, err = NewClient("").getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationStrict)
+ callback, err = NewClient("").getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationStrict)
require.NoError(t, err)
err = callback("192.168.1.1:22", addr, key)
@@ -363,7 +363,7 @@ func TestCreateAcceptNewHostKeyCallback(t *testing.T) {
t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)
client := NewClient("")
- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)
+ callback, err := client.getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)
require.NoError(t, err,
"unexpected error: %v", err)
@@ -431,7 +431,7 @@ func TestNewClient(t *testing.T) {
})
}
-func writeFileInternal(t *testing.T, dir, name string, content []byte) {
+func writeFile(t *testing.T, dir, name string, content []byte) {
t.Helper()
targetPath := filepath.Join(dir, name)
{
@@ -453,9 +453,9 @@ func minimalCompose() []byte {
func TestWalkDirectory_BasicWalk(t *testing.T) {
tmpDir := t.TempDir()
- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())
- writeFileInternal(t, tmpDir, "file1.txt", []byte("hello world"))
- writeFileInternal(t, tmpDir, "file2.txt", []byte("another file"))
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "file1.txt", []byte("hello world"))
+ writeFile(t, tmpDir, "file2.txt", []byte("another file"))
client := NewClient("")
result, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 0, 0)
@@ -472,9 +472,9 @@ func TestWalkDirectory_BasicWalk(t *testing.T) {
func TestWalkDirectory_PreservesExecutableBit(t *testing.T) {
tmpDir := t.TempDir()
- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())
- writeFileInternal(t, tmpDir, "scripts/hook.sh", []byte("#!/bin/sh\necho hi\n"))
- writeFileInternal(t, tmpDir, "README.md", []byte("readme"))
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "scripts/hook.sh", []byte("#!/bin/sh\necho hi\n"))
+ writeFile(t, tmpDir, "README.md", []byte("readme"))
{
err := os.Chmod(filepath.Join(tmpDir, "scripts", "hook.sh"), 0o755)
require.NoError(t, err,
@@ -509,11 +509,11 @@ func TestWalkDirectory_PreservesExecutableBit(t *testing.T) {
func TestWalkDirectory_MaxFilesLimit(t *testing.T) {
tmpDir := t.TempDir()
- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())
- writeFileInternal(t, tmpDir, "a.txt", []byte("a"))
- writeFileInternal(t, tmpDir, "b.txt", []byte("b"))
- writeFileInternal(t, tmpDir, "c.txt", []byte("c"))
- writeFileInternal(t, tmpDir, "d.txt", []byte("d"))
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "a.txt", []byte("a"))
+ writeFile(t, tmpDir, "b.txt", []byte("b"))
+ writeFile(t, tmpDir, "c.txt", []byte("c"))
+ writeFile(t, tmpDir, "d.txt", []byte("d"))
client := NewClient("")
_, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 3, 0, 0)
@@ -527,11 +527,11 @@ func TestWalkDirectory_MaxFilesLimit(t *testing.T) {
func TestWalkDirectory_MaxFilesUnlimited(t *testing.T) {
tmpDir := t.TempDir()
- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())
- writeFileInternal(t, tmpDir, "a.txt", []byte("a"))
- writeFileInternal(t, tmpDir, "b.txt", []byte("b"))
- writeFileInternal(t, tmpDir, "c.txt", []byte("c"))
- writeFileInternal(t, tmpDir, "d.txt", []byte("d"))
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "a.txt", []byte("a"))
+ writeFile(t, tmpDir, "b.txt", []byte("b"))
+ writeFile(t, tmpDir, "c.txt", []byte("c"))
+ writeFile(t, tmpDir, "d.txt", []byte("d"))
client := NewClient("")
result, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 0, 0)
@@ -545,9 +545,9 @@ func TestWalkDirectory_MaxFilesUnlimited(t *testing.T) {
func TestWalkDirectory_MaxTotalSizeLimit(t *testing.T) {
tmpDir := t.TempDir()
- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())
- writeFileInternal(t, tmpDir, "big1.txt", []byte(strings.Repeat("x", 40)))
- writeFileInternal(t, tmpDir, "big2.txt", []byte(strings.Repeat("y", 40)))
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "big1.txt", []byte(strings.Repeat("x", 40)))
+ writeFile(t, tmpDir, "big2.txt", []byte(strings.Repeat("y", 40)))
client := NewClient("")
_, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 50, 0)
@@ -561,9 +561,9 @@ func TestWalkDirectory_MaxTotalSizeLimit(t *testing.T) {
func TestWalkDirectory_MaxTotalSizeUnlimited(t *testing.T) {
tmpDir := t.TempDir()
- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())
- writeFileInternal(t, tmpDir, "big1.txt", []byte(strings.Repeat("x", 500)))
- writeFileInternal(t, tmpDir, "big2.txt", []byte(strings.Repeat("y", 500)))
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "big1.txt", []byte(strings.Repeat("x", 500)))
+ writeFile(t, tmpDir, "big2.txt", []byte(strings.Repeat("y", 500)))
client := NewClient("")
result, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 0, 0)
@@ -577,10 +577,10 @@ func TestWalkDirectory_MaxTotalSizeUnlimited(t *testing.T) {
func TestWalkDirectory_MaxBinarySizeSkips(t *testing.T) {
tmpDir := t.TempDir()
- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
// Binary content: null bytes cause IsBinaryContent to return true
binaryContent := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13}
- writeFileInternal(t, tmpDir, "data.bin", binaryContent)
+ writeFile(t, tmpDir, "data.bin", binaryContent)
client := NewClient("")
result, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 0, 5)
@@ -594,9 +594,9 @@ func TestWalkDirectory_MaxBinarySizeSkips(t *testing.T) {
func TestWalkDirectory_MaxBinarySizeUnlimited(t *testing.T) {
tmpDir := t.TempDir()
- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
binaryContent := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13}
- writeFileInternal(t, tmpDir, "data.bin", binaryContent)
+ writeFile(t, tmpDir, "data.bin", binaryContent)
client := NewClient("")
result, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 0, 0)
@@ -613,8 +613,8 @@ func TestWalkDirectory_MaxBinarySizeUnlimited(t *testing.T) {
func TestWalkDirectory_LargeTextFileNotSkippedByBinaryLimit(t *testing.T) {
tmpDir := t.TempDir()
- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())
- writeFileInternal(t, tmpDir, "notes.txt", []byte(strings.Repeat("plain text\n", 32)))
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "notes.txt", []byte(strings.Repeat("plain text\n", 32)))
client := NewClient("")
result, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 0, 16)
@@ -631,8 +631,8 @@ func TestWalkDirectory_LargeTextFileNotSkippedByBinaryLimit(t *testing.T) {
func TestWalkDirectory_ComposeInSubdirectory(t *testing.T) {
tmpDir := t.TempDir()
- writeFileInternal(t, tmpDir, "subdir/docker-compose.yml", minimalCompose())
- writeFileInternal(t, tmpDir, "subdir/dynamic_config.yml", []byte("http:\n routers: {}\n"))
+ writeFile(t, tmpDir, "subdir/docker-compose.yml", minimalCompose())
+ writeFile(t, tmpDir, "subdir/dynamic_config.yml", []byte("http:\n routers: {}\n"))
client := NewClient("")
result, err := client.WalkDirectory(t.Context(), tmpDir, "subdir/docker-compose.yml", 0, 0, 0)
@@ -661,8 +661,8 @@ func TestWalkDirectory_ComposeInSubdirectory(t *testing.T) {
func TestWalkDirectory_NestedSiblingFile(t *testing.T) {
tmpDir := t.TempDir()
- writeFileInternal(t, tmpDir, "subdir/docker-compose.yml", minimalCompose())
- writeFileInternal(t, tmpDir, "subdir/config/dynamic_config.yml", []byte("tls:\n certificates: []\n"))
+ writeFile(t, tmpDir, "subdir/docker-compose.yml", minimalCompose())
+ writeFile(t, tmpDir, "subdir/config/dynamic_config.yml", []byte("tls:\n certificates: []\n"))
client := NewClient("")
result, err := client.WalkDirectory(t.Context(), tmpDir, "subdir/docker-compose.yml", 0, 0, 0)
@@ -691,8 +691,8 @@ func TestWalkDirectory_NestedSiblingFile(t *testing.T) {
func TestWalkDirectory_SpecialCharsInPath(t *testing.T) {
tmpDir := t.TempDir()
- writeFileInternal(t, tmpDir, "traefik (nl10)/docker-compose.yml", minimalCompose())
- writeFileInternal(t, tmpDir, "traefik (nl10)/config/dynamic_config.yml", []byte("http:\n middlewares: {}\n"))
+ writeFile(t, tmpDir, "traefik (nl10)/docker-compose.yml", minimalCompose())
+ writeFile(t, tmpDir, "traefik (nl10)/config/dynamic_config.yml", []byte("http:\n middlewares: {}\n"))
client := NewClient("")
result, err := client.WalkDirectory(t.Context(), tmpDir, "traefik (nl10)/docker-compose.yml", 0, 0, 0)
@@ -836,7 +836,7 @@ func TestNormalizeURL(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
- got, err := normalizeURLInternal(tc.in)
+ got, err := normalizeURL(tc.in)
if tc.wantErr {
require.Error(t, err,
diff --git a/backend/pkg/gitutil/git_write.go b/backend/pkg/gitutil/git_write.go
index c05d5859d..49c9e4872 100644
--- a/backend/pkg/gitutil/git_write.go
+++ b/backend/pkg/gitutil/git_write.go
@@ -210,7 +210,7 @@ func (c *Client) CheckoutForWrite(ctx context.Context, url, branch string, auth
if strings.TrimSpace(branch) == "" {
return nil, errors.New("branch is required")
}
- normalized, err := normalizeURLInternal(url)
+ normalized, err := normalizeURL(url)
if err != nil {
return nil, err
}
@@ -361,7 +361,7 @@ func stageCommitFilesInternal(ctx context.Context, checkout *WriteCheckout, work
// pushBranchInternal pushes the checkout branch without force and verifies the
// remote now points at commit.
func (c *Client) pushBranchInternal(ctx context.Context, checkout *WriteCheckout, commit string, auth AuthConfig) error {
- authMethod, err := c.getAuthInternal(ctx, checkout.url, auth)
+ authMethod, err := c.getAuth(ctx, checkout.url, auth)
if err != nil {
return err
}
@@ -400,7 +400,7 @@ func isPushRejectedInternal(err error) bool {
// RemoteBranchHead resolves the remote head of branch without cloning.
func (c *Client) RemoteBranchHead(ctx context.Context, url, branch string, auth AuthConfig) (string, bool, error) {
- refs, err := c.listRemoteReferencesInternal(ctx, url, auth)
+ refs, err := c.listRemoteReferences(ctx, url, auth)
if err != nil {
return "", false, kit.Ternary(errors.Is(err, transport.ErrEmptyRemoteRepository), nil, err)
}
diff --git a/backend/pkg/libarcane/edge/client.go b/backend/pkg/libarcane/edge/client.go
index 2c50f2795..e601e7727 100644
--- a/backend/pkg/libarcane/edge/client.go
+++ b/backend/pkg/libarcane/edge/client.go
@@ -2,8 +2,8 @@ package edge
import (
"bytes"
+ "cmp"
"context"
- "crypto/tls"
"errors"
"fmt"
"io"
@@ -18,7 +18,6 @@ import (
"github.com/cenkalti/backoff/v5"
"github.com/coder/websocket"
- "github.com/samber/mo"
kit "go.getarcane.app/kit/pkg"
wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"
@@ -46,36 +45,47 @@ const (
// DefaultRequestTimeout is the timeout for executing local requests
DefaultRequestTimeout = 5 * time.Minute
// DefaultGRPCRegistrationTimeout bounds how long the agent waits for the
- // manager to acknowledge tunnel registration on any transport before
- // treating it as a failed transport attempt.
+ // manager to acknowledge tunnel registration on any transport.
DefaultGRPCRegistrationTimeout = 10 * time.Second
// DefaultWebSocketPreferenceTTL keeps websocket as the preferred transport
// for a short period after a successful auto-mode fallback.
DefaultWebSocketPreferenceTTL = 2 * time.Minute
- // maxWebSocketPreferenceTTL caps the exponential backoff applied to the
- // websocket preference window while gRPC keeps failing, so a persistently
- // broken gRPC path is retried at most this often but never disabled.
+ // maxWebSocketPreferenceTTL caps the backoff of the websocket preference window
+ // while gRPC keeps failing, so a broken gRPC path is retried but never disabled.
maxWebSocketPreferenceTTL = 30 * time.Minute
defaultCommandChunkSize = 256 * 1024
-
- // errTunnelRegistrationTimeout marks a registration attempt where the manager
- // accepted the connection but never answered the register message.
-
- // errEstablishedTunnelSessionEnded marks a session that the manager accepted
- // and later dropped, as opposed to a transport that never connected.
-
)
var (
+ // errTunnelRegistrationTimeout marks a manager that accepted the connection but never answered registration.
errTunnelRegistrationTimeout = errors.New("timed out waiting for tunnel registration response")
-
+ // errEstablishedTunnelSessionEnded marks an accepted session the manager later dropped.
errEstablishedTunnelSessionEnded = errors.New("established edge tunnel session ended")
)
-func (t *commandRequestTransfer) stopInternal() {
- if t == nil {
- return
- }
+// localDialSkipHeaders lists headers not forwarded when the agent dials its own server for a
+// proxied WebSocket: handshake headers and browser headers that fail the local origin check.
+var localDialSkipHeaders = map[string]bool{
+ "Sec-Websocket-Key": true,
+ "Sec-Websocket-Version": true,
+ "Sec-Websocket-Extensions": true,
+ "Upgrade": true,
+ "Connection": true,
+ "Host": true,
+ "Origin": true,
+ "Cookie": true,
+ "Authorization": true,
+ "Referer": true,
+ "Sec-Fetch-Dest": true,
+ "Sec-Fetch-Mode": true,
+ "Sec-Fetch-Site": true,
+ "Sec-Fetch-User": true,
+ "Sec-Ch-Ua": true,
+ "Sec-Ch-Ua-Mobile": true,
+ "Sec-Ch-Ua-Platform": true,
+}
+
+func (t *commandRequestTransfer) stop() {
t.timerMu.Lock()
defer t.timerMu.Unlock()
if t.timer != nil {
@@ -90,19 +100,6 @@ func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {
reconnectInterval = 5 * time.Second
}
- managerURL := ""
- if managerBaseURL := strings.TrimRight(httpx.ManagerBaseURL(cfg.ManagerApiUrl), "/"); managerBaseURL != "" {
- // Convert HTTP to WebSocket URL
- managerURL = HTTPToWebSocketURL(managerBaseURL) + "/api/tunnel/connect"
- }
- managerGRPCAddr := httpx.ManagerGRPCAddr(cfg.ManagerApiUrl)
-
- // Get local port for WebSocket dialing
- localPort := cfg.Port
- if localPort == "" {
- localPort = "3552" // Default port
- }
-
return &TunnelClient{
cfg: cfg,
handler: handler,
@@ -111,9 +108,8 @@ func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {
registrationTimeout: DefaultGRPCRegistrationTimeout,
websocketPreferenceTTL: DefaultWebSocketPreferenceTTL,
healthySessionDuration: healthyTunnelSessionDuration,
- managerURL: managerURL,
- managerGRPCAddr: managerGRPCAddr,
- localPort: localPort,
+ managerGRPCAddr: httpx.ManagerGRPCAddr(cfg.ManagerApiUrl),
+ localPort: cmp.Or(cfg.Port, "3552"),
requestTimeout: DefaultRequestTimeout,
agentInstanceID: uuid.New().String(),
}
@@ -125,50 +121,43 @@ func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error)
defer close(errCh)
}
slog.InfoContext(ctx, "Starting edge agent session client", StartupLogAttrs(c.cfg)...)
- // A fixed reconnect interval meant a permanently broken or unauthorized agent
- // hammered the manager's auth and DB path forever. Back off exponentially
- // (with jitter) up to maxReconnectInterval, and reset only once a session has
- // stayed up long enough to count as healthy.
+ // Back off exponentially (with jitter) so a broken or unauthorized agent does not hammer the
+ // manager, resetting only once a session has stayed up long enough to count as healthy.
reconnectBackoff := backoff.NewExponentialBackOff()
reconnectBackoff.InitialInterval = c.reconnectInterval
reconnectBackoff.MaxInterval = maxReconnectInterval
+ serve := kit.Ternary(UsePollEdgeTransport(c.cfg), c.connectAndServePoll, c.connectAndServeManagedTunnel)
for {
- select {
- case <-ctx.Done():
+ if ctx.Err() != nil {
slog.InfoContext(ctx, "Edge tunnel client shutting down")
return
- default:
- sessionStart := time.Now()
- if err := c.connectAndServe(ctx); err != nil {
- slog.WarnContext(ctx, "Edge tunnel disconnected", "error", err)
- if errCh != nil {
- select {
- case errCh <- err:
- default:
- }
+ }
+ sessionStart := time.Now()
+ if err := serve(ctx); err != nil {
+ slog.WarnContext(ctx, "Edge tunnel disconnected", "error", err)
+ if errCh != nil {
+ select {
+ case errCh <- err:
+ default:
}
}
+ }
+ if time.Since(sessionStart) >= c.healthySessionDuration {
+ reconnectBackoff.Reset()
+ }
- if time.Since(sessionStart) >= healthyTunnelSessionDuration {
- reconnectBackoff.Reset()
- }
-
- // Wait before reconnecting
- delay := reconnectBackoff.NextBackOff()
- reconnectTimer := time.NewTimer(delay)
- select {
- case <-ctx.Done():
- reconnectTimer.Stop()
- return
- case <-reconnectTimer.C:
- reconnectTimer.Stop()
- slog.InfoContext(ctx, "Attempting to reconnect edge tunnel", "delay", delay)
- }
+ delay := reconnectBackoff.NextBackOff()
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(delay):
+ slog.InfoContext(ctx, "Attempting to reconnect edge tunnel", "delay", delay)
}
}
}
+// StartupLogAttrs returns the slog attributes describing the agent's edge transport configuration.
func StartupLogAttrs(cfg *Config) []any {
if cfg == nil {
return []any{
@@ -179,53 +168,47 @@ func StartupLogAttrs(cfg *Config) []any {
}
}
- controlPlane := kit.Ternary(UsePollEdgeTransport(cfg), EdgeTransportPoll, "managed")
-
+ poll := UsePollEdgeTransport(cfg)
+ managerGRPCAddr := strings.TrimSpace(httpx.ManagerGRPCAddr(cfg.ManagerApiUrl))
+ managerBaseURL := strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl))
managedSessionTransports := make([]string, 0, 2)
- if UseGRPCEdgeTransport(cfg) || (UsePollEdgeTransport(cfg) && strings.TrimSpace(httpx.ManagerGRPCAddr(cfg.ManagerApiUrl)) != "") {
+ if UseGRPCEdgeTransport(cfg) || (poll && managerGRPCAddr != "") {
managedSessionTransports = append(managedSessionTransports, EdgeTransportGRPC)
}
- if UseWebSocketEdgeTransport(cfg) || (UsePollEdgeTransport(cfg) && strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)) != "") {
+ if UseWebSocketEdgeTransport(cfg) || (poll && managerBaseURL != "") {
managedSessionTransports = append(managedSessionTransports, EdgeTransportWebSocket)
}
attrs := []any{
- "control_plane", controlPlane,
+ "control_plane", kit.Ternary(poll, EdgeTransportPoll, "managed"),
"managed_session_transports", managedSessionTransports,
"security_mode", NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode),
}
-
if managerAPIURL := strings.TrimSpace(cfg.ManagerApiUrl); managerAPIURL != "" {
attrs = append(attrs, "manager_api_url", managerAPIURL)
}
- if managerGRPCAddr := strings.TrimSpace(httpx.ManagerGRPCAddr(cfg.ManagerApiUrl)); managerGRPCAddr != "" {
+ if managerGRPCAddr != "" {
attrs = append(attrs, "manager_grpc_addr", managerGRPCAddr)
}
- if managerBaseURL := strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)); managerBaseURL != "" {
+ if managerBaseURL != "" {
attrs = append(attrs, "manager_base_url", managerBaseURL)
}
-
return attrs
}
-// connectAndServe establishes a connection and handles messages.
-func (c *TunnelClient) connectAndServe(ctx context.Context) error {
- if UsePollEdgeTransport(c.cfg) {
- return c.connectAndServePoll(ctx)
- }
- return c.connectAndServeManagedTunnelInternal(ctx)
-}
-
-func (c *TunnelClient) connectAndServeManagedTunnelInternal(ctx context.Context) error {
- transports := c.managedTunnelTransportsInternal()
+func (c *TunnelClient) connectAndServeManagedTunnel(ctx context.Context) error {
+ transports := c.managedTunnelTransports()
if transports.grpc {
- if preferredUntil, ok := c.preferredWebSocketUntilInternal(time.Now()).Get(); ok {
+ c.transportPreferenceMu.RLock()
+ preferredUntil, grpcFailureStreak := c.preferWebSocketUntil, c.grpcFailureStreak
+ c.transportPreferenceMu.RUnlock()
+ if transports.websocket && time.Now().Before(preferredUntil) {
slog.InfoContext(ctx, "Temporarily preferring websocket edge tunnel transport after recent websocket success",
"preferredUntil", preferredUntil,
- "grpcFailureStreak", c.grpcFailureStreakInternal(),
- "managerWsUrl", c.managerWebSocketURLInternal(),
+ "grpcFailureStreak", grpcFailureStreak,
+ "managerWsUrl", c.managerWebSocketURL(),
)
- return c.connectAndServeWebSocketInternal(ctx)
+ return c.connectAndServeWebSocket(ctx)
}
sessionStart := time.Now()
@@ -233,21 +216,21 @@ func (c *TunnelClient) connectAndServeManagedTunnelInternal(ctx context.Context)
if ctx.Err() != nil {
return ctx.Err()
}
- // A healthy gRPC session that the manager dropped (restart, redeploy)
- // is not a broken gRPC path: let the outer loop back off and retry
- // gRPC first instead of racing a websocket dial against the restart.
- if errors.Is(err, errEstablishedTunnelSessionEnded) && time.Since(sessionStart) >= c.healthySessionDurationInternal() {
+ // A healthy session the manager dropped (restart, redeploy) is not a broken gRPC path:
+ // let the outer loop back off and retry gRPC instead of racing a websocket dial.
+ if errors.Is(err, errEstablishedTunnelSessionEnded) && time.Since(sessionStart) >= c.healthySessionDuration {
return err
}
- c.noteGRPCTunnelFailureInternal()
+ c.transportPreferenceMu.Lock()
+ c.grpcFailureStreak++
+ c.transportPreferenceMu.Unlock()
if transports.websocket {
- managerWSURL := c.managerWebSocketURLInternal()
slog.WarnContext(ctx, "gRPC edge tunnel connection failed, falling back to websocket transport",
"error", err,
"managerGrpcAddr", c.managerGRPCAddr,
- "managerWsUrl", managerWSURL,
+ "managerWsUrl", c.managerWebSocketURL(),
)
- if wsErr := c.connectAndServeWebSocketInternal(ctx); wsErr != nil {
+ if wsErr := c.connectAndServeWebSocket(ctx); wsErr != nil {
// Keep both transport failures in the chain for errors.Is/errors.As traversal.
return fmt.Errorf("gRPC edge tunnel failed: %w; websocket fallback failed: %w", err, wsErr)
}
@@ -258,48 +241,31 @@ func (c *TunnelClient) connectAndServeManagedTunnelInternal(ctx context.Context)
return nil
}
if transports.websocket {
- return c.connectAndServeWebSocketInternal(ctx)
+ return c.connectAndServeWebSocket(ctx)
}
return errors.New("no edge tunnel transport is available")
}
-func (c *TunnelClient) managedTunnelTransportsInternal() managedTunnelTransportsInternal {
+func (c *TunnelClient) managedTunnelTransports() managedTunnelTransportsInternal {
if c == nil || c.cfg == nil {
return managedTunnelTransportsInternal{}
}
managerGRPCAvailable := strings.TrimSpace(c.managerGRPCAddr) != ""
- managerWebSocketAvailable := c.managerWebSocketURLInternal() != ""
- transport := NormalizeEdgeTransport(c.cfg.EdgeTransport)
-
- switch transport {
- case EdgeTransportAuto:
- return managedTunnelTransportsInternal{
- grpc: managerGRPCAvailable,
- websocket: managerWebSocketAvailable,
- }
- case EdgeTransportPoll:
- return managedTunnelTransportsInternal{
- grpc: managerGRPCAvailable,
- websocket: managerWebSocketAvailable,
- }
+ managerWebSocketAvailable := c.managerWebSocketURL() != ""
+ switch NormalizeEdgeTransport(c.cfg.EdgeTransport) {
+ case EdgeTransportAuto, EdgeTransportPoll:
+ return managedTunnelTransportsInternal{grpc: managerGRPCAvailable, websocket: managerWebSocketAvailable}
case EdgeTransportWebSocket:
- return managedTunnelTransportsInternal{
- websocket: managerWebSocketAvailable,
- }
+ return managedTunnelTransportsInternal{websocket: managerWebSocketAvailable}
case EdgeTransportGRPC:
- return managedTunnelTransportsInternal{
- grpc: managerGRPCAvailable,
- }
+ return managedTunnelTransportsInternal{grpc: managerGRPCAvailable}
default:
return managedTunnelTransportsInternal{}
}
}
-func (c *TunnelClient) managerWebSocketURLInternal() string {
- if c == nil {
- return ""
- }
+func (c *TunnelClient) managerWebSocketURL() string {
if managerURL := strings.TrimSpace(c.managerURL); managerURL != "" {
return managerURL
}
@@ -313,129 +279,19 @@ func (c *TunnelClient) managerWebSocketURLInternal() string {
return HTTPToWebSocketURL(managerBaseURL) + "/api/tunnel/connect"
}
-func (c *TunnelClient) registrationTimeoutInternal() time.Duration {
- if c == nil || c.registrationTimeout <= 0 {
- return DefaultGRPCRegistrationTimeout
- }
- return c.registrationTimeout
-}
-
-func (c *TunnelClient) requestTimeoutInternal() time.Duration {
- if c == nil || c.requestTimeout <= 0 {
- return DefaultRequestTimeout
- }
- return c.requestTimeout
-}
-
-func (c *TunnelClient) healthySessionDurationInternal() time.Duration {
- if c == nil || c.healthySessionDuration <= 0 {
- return healthyTunnelSessionDuration
- }
- return c.healthySessionDuration
-}
-
-func (c *TunnelClient) websocketPreferenceTTLInternal() time.Duration {
- if c == nil || c.websocketPreferenceTTL <= 0 {
- return DefaultWebSocketPreferenceTTL
- }
- return c.websocketPreferenceTTL
-}
-
-func (c *TunnelClient) preferredWebSocketUntilInternal(now time.Time) mo.Option[time.Time] {
- // A nil client yields a zero transports struct, so this also guards nil.
- transports := c.managedTunnelTransportsInternal()
- if !transports.grpc || !transports.websocket {
- return mo.None[time.Time]()
- }
-
- c.transportPreferenceMu.RLock()
- defer c.transportPreferenceMu.RUnlock()
-
- if c.preferWebSocketUntil.IsZero() || !now.Before(c.preferWebSocketUntil) {
- return mo.None[time.Time]()
- }
-
- return mo.Some(c.preferWebSocketUntil)
-}
-
-func (c *TunnelClient) markTransportConnectedInternal(transport string) {
- if c == nil {
- return
- }
-
- c.transportPreferenceMu.Lock()
- defer c.transportPreferenceMu.Unlock()
-
- switch transport {
- case EdgeTransportGRPC:
- c.preferWebSocketUntil = time.Time{}
- c.grpcFailureStreak = 0
- case EdgeTransportWebSocket:
- if transports := c.managedTunnelTransportsInternal(); transports.grpc && transports.websocket {
- // Back the preference window off exponentially while gRPC keeps
- // failing so each reconnect does not re-pay the registration
- // timeout against a persistently broken gRPC path.
- ttl := c.websocketPreferenceTTLInternal()
- if streak := min(c.grpcFailureStreak, 4); streak > 0 {
- ttl = min(ttl<<streak, maxWebSocketPreferenceTTL)
- }
- c.preferWebSocketUntil = time.Now().Add(ttl)
- }
- }
-}
-
-func (c *TunnelClient) noteGRPCTunnelFailureInternal() {
- if c == nil {
- return
- }
- c.transportPreferenceMu.Lock()
- defer c.transportPreferenceMu.Unlock()
- c.grpcFailureStreak++
-}
-
-func (c *TunnelClient) grpcFailureStreakInternal() int {
- if c == nil {
- return 0
- }
- c.transportPreferenceMu.RLock()
- defer c.transportPreferenceMu.RUnlock()
- return c.grpcFailureStreak
-}
-
-func (c *TunnelClient) registerMessageInternal() *TunnelMessage {
- capabilities := AdvertisedEdgeCommands()
- capabilities = append(capabilities, tunnelCapabilityChunkedRequest, tunnelCapabilityProtoParity, tunnelCapabilityCommandCredit)
- registration, _ := c.registration.Load()
- return &TunnelMessage{
- Type: MessageTypeRegister,
- AgentToken: c.cfg.AgentToken,
- AgentInstance: c.agentInstanceID,
- Capabilities: capabilities,
- ResumeSession: registration.sessionID,
- }
-}
-
-func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context, conn TunnelConnection) (*TunnelMessage, error) {
- if c == nil {
- return nil, errors.New("edge tunnel connection is not initialized")
- }
- if conn == nil {
- return nil, errors.New("edge tunnel connection is not initialized")
- }
-
+func (c *TunnelClient) awaitRegistration(ctx context.Context, conn TunnelConnection) (*TunnelMessage, error) {
type registrationResult struct {
msg *TunnelMessage
err error
}
- timeout := c.registrationTimeoutInternal()
recvCh := make(chan registrationResult, 1)
go func() {
msg, err := conn.Receive()
recvCh <- registrationResult{msg: msg, err: err}
}()
- timer := time.NewTimer(timeout)
+ timer := time.NewTimer(c.registrationTimeout)
defer timer.Stop()
select {
@@ -444,7 +300,7 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context, conn Tunne
return nil, ctx.Err()
case <-timer.C:
_ = conn.Close()
- return nil, fmt.Errorf("after %s: %w", timeout, errTunnelRegistrationTimeout)
+ return nil, fmt.Errorf("after %s: %w", c.registrationTimeout, errTunnelRegistrationTimeout)
case result := <-recvCh:
if result.err != nil {
return nil, fmt.Errorf("failed to receive tunnel registration response: %w", result.err)
@@ -466,9 +322,9 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context, conn Tunne
}
}
-// serveTunnelSessionInternal runs the shared register/heartbeat/message
-// lifecycle on an established tunnel connection, for every transport.
-func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn TunnelConnection, managerAddr string) error {
+// serveTunnelSession runs the shared register/heartbeat/message lifecycle on an established
+// tunnel connection, for every transport.
+func (c *TunnelClient) serveTunnelSession(ctx context.Context, conn TunnelConnection, managerAddr string) error {
box := &connBox{conn: conn}
c.conn.Store(box)
setActiveAgentTunnelConn(conn)
@@ -477,8 +333,24 @@ func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn Tunn
stopCloseWatch := context.AfterFunc(connCtx, func() { _ = conn.Close() })
defer func() {
connCancel()
- c.closeAllStreams()
- c.clearCommandRequestTransfersInternal(conn)
+ // Tear down streams and pending transfers so a reconnect cannot leak goroutines or sockets.
+ c.activeStreams.Range(func(key, value any) bool {
+ streamID, idOK := key.(string)
+ stream, streamOK := value.(*activeWSStream)
+ if idOK && streamOK {
+ c.closeWebSocketStream(streamID, stream)
+ }
+ return true
+ })
+ c.requestTransfers.Range(func(transferID, value any) bool {
+ switch transfer, ok := value.(*commandRequestTransfer); {
+ case !ok:
+ c.requestTransfers.Delete(transferID)
+ case transfer.conn == conn && c.requestTransfers.CompareAndDelete(transferID, transfer):
+ transfer.stop()
+ }
+ return true
+ })
_ = conn.Close()
stopCloseWatch()
drainCtx, cancelDrain := context.WithTimeout(context.WithoutCancel(ctx), tunnelSessionWorkerDrainTimeout)
@@ -490,14 +362,22 @@ func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn Tunn
clearActiveAgentTunnelConn(conn)
}()
- if err := conn.Send(c.registerMessageInternal()); err != nil {
+ registration, _ := c.registration.Load()
+ registerMsg := &TunnelMessage{
+ Type: MessageTypeRegister,
+ AgentToken: c.cfg.AgentToken,
+ AgentInstance: c.agentInstanceID,
+ Capabilities: append(AdvertisedEdgeCommands(), tunnelCapabilityChunkedRequest, tunnelCapabilityProtoParity, tunnelCapabilityCommandCredit),
+ ResumeSession: registration.sessionID,
+ }
+ if err := conn.Send(registerMsg); err != nil {
// A rejected gRPC stream can report EOF from Send; Recv carries the RPC status.
if conn.Transport() != EdgeTransportGRPC || !errors.Is(err, io.EOF) {
return fmt.Errorf("failed to send %s tunnel register message: %w", conn.Transport(), err)
}
}
- registerMsg, err := c.awaitRegistrationInternal(ctx, conn)
+ registerResp, err := c.awaitRegistration(ctx, conn)
if err != nil {
return err
}
@@ -505,17 +385,25 @@ func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn Tunn
slog.InfoContext(ctx, "Edge tunnel connected to manager",
"transport", conn.Transport(),
"managerAddr", managerAddr,
- "environmentId", registerMsg.EnvironmentID,
- "sessionId", registerMsg.SessionID,
+ "environmentId", registerResp.EnvironmentID,
+ "sessionId", registerResp.SessionID,
)
- c.markTransportConnectedInternal(conn.Transport())
+ c.transportPreferenceMu.Lock()
+ switch conn.Transport() {
+ case EdgeTransportGRPC:
+ c.preferWebSocketUntil, c.grpcFailureStreak = time.Time{}, 0
+ case EdgeTransportWebSocket:
+ if transports := c.managedTunnelTransports(); transports.grpc && transports.websocket {
+ // Back off the preference window while gRPC keeps failing so reconnects do not re-pay its timeout.
+ ttl := min(c.websocketPreferenceTTL<<min(c.grpcFailureStreak, 4), maxWebSocketPreferenceTTL)
+ c.preferWebSocketUntil = time.Now().Add(ttl)
+ }
+ }
+ c.transportPreferenceMu.Unlock()
workers.Go(func() { c.heartbeatLoop(connCtx, conn) })
- if messageLoopErr := c.messageLoop(connCtx, conn, &workers); messageLoopErr != nil {
- return fmt.Errorf("%w: %w", errEstablishedTunnelSessionEnded, messageLoopErr)
- }
- return nil
+ return fmt.Errorf("%w: %w", errEstablishedTunnelSessionEnded, c.messageLoop(connCtx, conn, &workers))
}
// heartbeatLoop sends periodic heartbeats
@@ -532,12 +420,7 @@ func (c *TunnelClient) heartbeatLoop(ctx context.Context, conn TunnelConnection)
return
}
- msg := &TunnelMessage{
- ID: uuid.New().String(),
- Type: MessageTypeHeartbeat,
- }
-
- if err := conn.Send(msg); err != nil {
+ if err := conn.Send(&TunnelMessage{ID: uuid.New().String(), Type: MessageTypeHeartbeat}); err != nil {
slog.WarnContext(ctx, "Failed to send heartbeat", "error", err)
// Force reconnect so the manager does not keep stale state without heartbeats.
if closeErr := conn.Close(); closeErr != nil {
@@ -553,96 +436,92 @@ func (c *TunnelClient) heartbeatLoop(ctx context.Context, conn TunnelConnection)
// messageLoop processes incoming messages from the manager
func (c *TunnelClient) messageLoop(ctx context.Context, conn TunnelConnection, workers *sync.WaitGroup) error {
for {
- select {
- case <-ctx.Done():
- return ctx.Err()
- default:
- msg, err := conn.Receive()
- if err != nil {
- return fmt.Errorf("failed to receive message: %w", err)
- }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ msg, err := conn.Receive()
+ if err != nil {
+ return fmt.Errorf("failed to receive message: %w", err)
+ }
- switch msg.Type {
- case MessageTypeRequest:
- workers.Go(func() { c.handleRequest(ctx, conn, msg) })
- case MessageTypeCommandRequest:
- if transferID := strings.TrimSpace(msg.Metadata[bodyTransferMetadataKey]); transferID != "" {
- c.beginCommandRequestTransferInternal(ctx, conn, transferID, msg)
- } else {
- workers.Go(func() { c.handleCommandRequest(ctx, conn, msg) })
+ switch msg.Type {
+ case MessageTypeRequest:
+ workers.Go(func() { c.handleRequest(ctx, conn, msg) })
+ case MessageTypeCommandRequest:
+ if transferID := strings.TrimSpace(msg.Metadata[bodyTransferMetadataKey]); transferID != "" {
+ c.beginCommandRequestTransfer(ctx, conn, transferID, msg)
+ } else {
+ workers.Go(func() { c.handleCommandRequest(ctx, conn, msg) })
+ }
+ case MessageTypeFileChunk:
+ c.handleCommandRequestChunk(ctx, msg, workers)
+ case MessageTypeWebSocketStart, MessageTypeStreamOpen:
+ c.handleWebSocketStart(ctx, conn, msg, workers)
+ case MessageTypeWebSocketData, MessageTypeStreamData:
+ c.handleStreamData(ctx, msg)
+ case MessageTypeWebSocketClose, MessageTypeStreamClose:
+ if value, ok := c.activeStreams.Load(msg.ID); ok {
+ if stream, isStream := value.(*activeWSStream); isStream {
+ c.closeWebSocketStream(msg.ID, stream)
+ slog.DebugContext(ctx, "Closed WebSocket stream", "streamId", msg.ID)
}
- case MessageTypeFileChunk:
- c.handleCommandRequestChunkInternal(ctx, msg, workers)
- case MessageTypeWebSocketStart:
- c.handleWebSocketStart(ctx, conn, msg, workers)
- case MessageTypeStreamOpen:
- c.handleStreamOpen(ctx, conn, msg, workers)
- case MessageTypeWebSocketData:
- c.handleWebSocketData(ctx, msg)
- case MessageTypeStreamData:
- c.handleStreamData(ctx, msg)
- case MessageTypeWebSocketClose:
- c.handleWebSocketClose(ctx, msg)
- case MessageTypeStreamClose:
- c.handleStreamClose(ctx, msg)
- case MessageTypeCancelRequest:
- slog.DebugContext(ctx, "Ignoring edge cancel request on agent", "id", msg.ID)
- case MessageTypeCommandCredit:
- if value, ok := c.commandRecorders.Load(msg.ID); ok {
- if recorder, isRecorder := value.(*commandResponseRecorder); isRecorder {
- recorder.inFlight.Add(-msg.Credit)
- select {
- case recorder.credited <- struct{}{}:
- default:
- }
+ }
+ case MessageTypeCancelRequest:
+ slog.DebugContext(ctx, "Ignoring edge cancel request on agent", "id", msg.ID)
+ case MessageTypeCommandCredit:
+ if value, ok := c.commandRecorders.Load(msg.ID); ok {
+ if recorder, isRecorder := value.(*commandResponseRecorder); isRecorder {
+ recorder.inFlight.Add(-msg.Credit)
+ select {
+ case recorder.credited <- struct{}{}:
+ default:
}
}
- case MessageTypeResponse, MessageTypeHeartbeat, MessageTypeStreamEnd, MessageTypeEvent, MessageTypeCommandAck, MessageTypeCommandOutput, MessageTypeCommandComplete:
- slog.DebugContext(ctx, "Ignoring message type on agent", "type", msg.Type)
- case MessageTypeHeartbeatAck:
- slog.DebugContext(ctx, "Received heartbeat ack")
- case MessageTypeRegisterResponse:
- if !msg.Accepted {
- return fmt.Errorf("manager rejected tunnel registration: %s", msg.Error)
- }
- slog.InfoContext(ctx, "Edge tunnel re-registered",
- "transport", conn.Transport(),
- "environmentId", msg.EnvironmentID,
- )
- case MessageTypeRegister:
- slog.DebugContext(ctx, "Ignoring register message on agent")
- default:
- slog.WarnContext(ctx, "Unknown message type", "type", msg.Type)
}
+ case MessageTypeResponse, MessageTypeHeartbeat, MessageTypeStreamEnd, MessageTypeEvent, MessageTypeCommandAck, MessageTypeCommandOutput, MessageTypeCommandComplete:
+ slog.DebugContext(ctx, "Ignoring message type on agent", "type", msg.Type)
+ case MessageTypeHeartbeatAck:
+ slog.DebugContext(ctx, "Received heartbeat ack")
+ case MessageTypeRegisterResponse:
+ if !msg.Accepted {
+ return fmt.Errorf("manager rejected tunnel registration: %s", msg.Error)
+ }
+ slog.InfoContext(ctx, "Edge tunnel re-registered",
+ "transport", conn.Transport(),
+ "environmentId", msg.EnvironmentID,
+ )
+ case MessageTypeRegister:
+ slog.DebugContext(ctx, "Ignoring register message on agent")
+ default:
+ slog.WarnContext(ctx, "Unknown message type", "type", msg.Type)
}
}
}
-func (c *TunnelClient) beginCommandRequestTransferInternal(ctx context.Context, conn TunnelConnection, transferID string, msg *TunnelMessage) {
+func (c *TunnelClient) beginCommandRequestTransfer(ctx context.Context, conn TunnelConnection, transferID string, msg *TunnelMessage) {
if c == nil || conn == nil || msg == nil {
return
}
- timeout := c.requestTimeoutInternal()
transfer := &commandRequestTransfer{request: msg, conn: conn}
transfer.timerMu.Lock()
previous, loaded := c.requestTransfers.Swap(transferID, transfer)
- transfer.timer = time.AfterFunc(timeout, func() {
+ transfer.timer = time.AfterFunc(c.requestTimeout, func() {
if c.requestTransfers.CompareAndDelete(transferID, transfer) {
slog.WarnContext(ctx, "Command body transfer expired", "transferId", transferID, "commandId", msg.ID)
- c.sendCommandCompleteInternal(conn, msg.ID, http.StatusRequestTimeout, "command body transfer timed out")
+ c.sendCommandComplete(conn, msg.ID, http.StatusRequestTimeout, "command body transfer timed out")
}
})
transfer.timerMu.Unlock()
if loaded {
if previous, ok := previous.(*commandRequestTransfer); ok {
- previous.stopInternal()
- c.sendCommandCompleteInternal(previous.conn, previous.request.ID, http.StatusBadRequest, "duplicate command body transfer ID")
+ previous.stop()
+ c.sendCommandComplete(previous.conn, previous.request.ID, http.StatusBadRequest, "duplicate command body transfer ID")
}
}
}
-func (c *TunnelClient) handleCommandRequestChunkInternal(ctx context.Context, msg *TunnelMessage, workers *sync.WaitGroup) {
+func (c *TunnelClient) handleCommandRequestChunk(ctx context.Context, msg *TunnelMessage, workers *sync.WaitGroup) {
if c == nil || msg == nil {
return
}
@@ -660,8 +539,8 @@ func (c *TunnelClient) handleCommandRequestChunkInternal(ctx context.Context, ms
}
if msg.Sequence != transfer.nextSequence {
if c.requestTransfers.CompareAndDelete(msg.ID, transfer) {
- transfer.stopInternal()
- c.sendCommandCompleteInternal(transfer.conn, transfer.request.ID, http.StatusBadRequest, "command body chunks arrived out of order")
+ transfer.stop()
+ c.sendCommandComplete(transfer.conn, transfer.request.ID, http.StatusBadRequest, "command body chunks arrived out of order")
}
return
}
@@ -673,36 +552,15 @@ func (c *TunnelClient) handleCommandRequestChunkInternal(ctx context.Context, ms
}
if c.requestTransfers.CompareAndDelete(msg.ID, transfer) {
- transfer.stopInternal()
+ transfer.stop()
transfer.request.Body = append([]byte(nil), transfer.body.Bytes()...)
workers.Go(func() { c.handleCommandRequest(ctx, transfer.conn, transfer.request) })
}
}
-func (c *TunnelClient) clearCommandRequestTransfersInternal(conn TunnelConnection) {
- if c == nil {
- return
- }
-
- c.requestTransfers.Range(func(transferID, value any) bool {
- transfer, ok := value.(*commandRequestTransfer)
- if !ok {
- c.requestTransfers.Delete(transferID)
- return true
- }
- if transfer.conn != conn {
- return true
- }
- if c.requestTransfers.CompareAndDelete(transferID, transfer) {
- transfer.stopInternal()
- }
- return true
- })
-}
-
func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {
if !ValidateEdgeCommand(msg.Command, msg.Method, msg.Path, false) {
- c.sendCommandCompleteInternal(conn, msg.ID, http.StatusBadRequest, "unsupported edge command")
+ c.sendCommandComplete(conn, msg.ID, http.StatusBadRequest, "unsupported edge command")
return
}
if conn == nil {
@@ -713,12 +571,12 @@ func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConn
return
}
- reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeoutInternal())
+ reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeout)
defer cancel()
req, err := c.buildLocalHTTPRequest(reqCtx, msg)
if err != nil {
- c.sendCommandCompleteInternal(conn, msg.ID, http.StatusInternalServerError, fmt.Sprintf("failed to create request: %v", err))
+ c.sendCommandComplete(conn, msg.ID, http.StatusInternalServerError, fmt.Sprintf("failed to create request: %v", err))
return
}
@@ -744,30 +602,17 @@ func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConn
func (c *TunnelClient) buildLocalHTTPRequest(ctx context.Context, msg *TunnelMessage) (*http.Request, error) {
var body io.Reader
- var bodyBytes []byte
if len(msg.Body) > 0 {
- bodyBytes = append([]byte(nil), msg.Body...)
- body = bytes.NewReader(bodyBytes)
- }
-
- path := msg.Path
- if msg.Query != "" {
- path = path + "?" + msg.Query
+ body = bytes.NewReader(append([]byte(nil), msg.Body...))
}
+ path := msg.Path + kit.Ternary(msg.Query != "", "?"+msg.Query, "")
req, err := http.NewRequestWithContext(ctx, msg.Method, path, body)
if err != nil {
return nil, err
}
- if bodyBytes != nil {
- req.GetBody = func() (io.ReadCloser, error) {
- return io.NopCloser(bytes.NewReader(bodyBytes)), nil
- }
- }
-
- // Set headers. Note: Go's net/http does not populate req.Host from
- // Header.Set("Host", ...) — it must be set explicitly on the field.
+ // net/http ignores Header.Set("Host", ...); the field must be set explicitly.
for k, v := range msg.Headers {
if http.CanonicalHeaderKey(k) == "Host" {
req.Host = v
@@ -779,9 +624,9 @@ func (c *TunnelClient) buildLocalHTTPRequest(ctx context.Context, msg *TunnelMes
return req, nil
}
-// agentAuthCredentialsInternal returns the canonical header/metadata credential
+// agentAuthCredentials returns the canonical header/metadata credential
// set every agent transport presents, so the three transports cannot drift.
-func agentAuthCredentialsInternal(token string) map[string]string {
+func agentAuthCredentials(token string) map[string]string {
return map[string]string{
HeaderAgentToken: token,
HeaderAPIKey: token,
@@ -789,50 +634,44 @@ func agentAuthCredentialsInternal(token string) map[string]string {
}
}
-// handleRequest processes an incoming request and sends back a response
+// handleRequest serves a tunneled request locally, streaming the response on gRPC connections.
func (c *TunnelClient) handleRequest(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {
- if isGRPCConnection(conn) {
- c.handleRequestStreaming(ctx, conn, msg)
- return
- }
-
- reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeoutInternal())
+ reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeout)
defer cancel()
reqCtx = context.WithValue(reqCtx, internalTunnelRequestContextKey{}, true)
+ streaming := conn.Transport() == EdgeTransportGRPC
- slog.DebugContext(reqCtx, "Processing tunneled request", "id", msg.ID, "method", msg.Method, "path", msg.Path, "bodyLength", len(msg.Body))
+ slog.DebugContext(reqCtx, "Processing tunneled request", "id", msg.ID, "method", msg.Method, "path", msg.Path, "bodyLength", len(msg.Body), "streaming", streaming)
req, err := c.buildLocalHTTPRequest(reqCtx, msg)
if err != nil {
- c.sendErrorResponse(conn, msg.ID, http.StatusInternalServerError, fmt.Sprintf("failed to create request: %v", err))
+ _ = conn.Send(&TunnelMessage{
+ ID: msg.ID,
+ Type: MessageTypeResponse,
+ Status: http.StatusInternalServerError,
+ Body: fmt.Appendf(nil, "failed to create request: %v", err),
+ })
return
}
- // Use a response recorder to capture the response
- rw := &responseRecorder{
- headers: make(http.Header),
- statusCode: http.StatusOK,
- }
-
- // Execute the request through the local handler
- c.handler.ServeHTTP(rw, req)
-
- // Build response message
- respHeaders := make(map[string]string)
- for k, v := range rw.headers {
- if len(v) > 0 {
- respHeaders[k] = v[0]
+ if streaming {
+ recorder := &streamingResponseRecorder{requestID: msg.ID, conn: conn, headers: make(http.Header), statusCode: http.StatusOK}
+ c.handler.ServeHTTP(recorder, req)
+ if closeErr := recorder.Close(); closeErr != nil {
+ slog.WarnContext(reqCtx, "Failed to finalize streamed response", "id", msg.ID, "error", closeErr)
}
+ return
}
+ rw := &responseRecorder{headers: make(http.Header), statusCode: http.StatusOK}
+ c.handler.ServeHTTP(rw, req)
resp := &TunnelMessage{
ID: msg.ID,
Type: MessageTypeResponse,
Status: rw.statusCode,
- Headers: respHeaders,
+ Headers: flattenResponseHeaders(rw.headers),
Body: rw.body.Bytes(),
}
-
if sendErr := conn.Send(resp); sendErr != nil {
slog.ErrorContext(reqCtx, "Failed to send response", "id", msg.ID, "error", sendErr)
} else {
@@ -840,48 +679,19 @@ func (c *TunnelClient) handleRequest(ctx context.Context, conn TunnelConnection,
}
}
-func isGRPCConnection(conn TunnelConnection) bool {
- if conn == nil {
- return false
- }
- return conn.Transport() == EdgeTransportGRPC
-}
-
-func (c *TunnelClient) handleRequestStreaming(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {
- reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeoutInternal())
- defer cancel()
- reqCtx = context.WithValue(reqCtx, internalTunnelRequestContextKey{}, true)
-
- slog.DebugContext(reqCtx, "Processing tunneled request (streaming)", "id", msg.ID, "method", msg.Method, "path", msg.Path, "bodyLength", len(msg.Body))
-
- req, err := c.buildLocalHTTPRequest(reqCtx, msg)
- if err != nil {
- c.sendErrorResponse(conn, msg.ID, http.StatusInternalServerError, fmt.Sprintf("failed to create request: %v", err))
- return
- }
-
- recorder := newStreamingResponseRecorder(msg.ID, conn)
- c.handler.ServeHTTP(recorder, req)
-
- if closeErr := recorder.Close(); closeErr != nil {
- slog.WarnContext(reqCtx, "Failed to finalize streamed response", "id", msg.ID, "error", closeErr)
- }
-}
-
-// handleWebSocketStart handles a WebSocket stream start request from the manager.
+// handleWebSocketStart proxies a manager WebSocket or stream-open request to the local server.
func (c *TunnelClient) handleWebSocketStart(ctx context.Context, conn TunnelConnection, msg *TunnelMessage, workers *sync.WaitGroup) {
- if msg.Command == "" {
- if commandName, ok := ResolveEdgeCommandName(http.MethodGet, msg.Path, true).Get(); ok {
- msg.Command = commandName
- }
- }
streamID := msg.ID
+ if msg.Type == MessageTypeStreamOpen && !ValidateEdgeCommand(msg.Command, http.MethodGet, msg.Path, true) {
+ c.sendStreamCloseMessage(conn, streamID, "unsupported edge stream")
+ return
+ }
slog.DebugContext(ctx, "Starting WebSocket stream", "streamId", streamID, "path", msg.Path)
- localURL := c.buildLocalWebSocketURLInternal(msg)
- headers := c.buildLocalWebSocketHeadersInternal(msg)
-
- ws, resp, err := c.dialLocalWebSocket(ctx, localURL, headers)
+ localURL := c.buildLocalWebSocketURL(msg)
+ dialCtx, cancelDial := context.WithTimeout(ctx, 30*time.Second)
+ defer cancelDial()
+ ws, resp, err := websocket.Dial(dialCtx, localURL, &websocket.DialOptions{HTTPHeader: c.buildLocalWebSocketHeaders(msg)})
// coder/websocket leaves resp.Body nil on a successful handshake.
if resp != nil && resp.Body != nil {
defer func() { _ = resp.Body.Close() }()
@@ -892,111 +702,58 @@ func (c *TunnelClient) handleWebSocketStart(ctx context.Context, conn TunnelConn
attrs = append(attrs, "status", resp.StatusCode)
if resp.Body != nil {
buf := make([]byte, 512)
- n, _ := resp.Body.Read(buf)
- if n > 0 {
+ if n, _ := resp.Body.Read(buf); n > 0 {
attrs = append(attrs, "response_body", string(buf[:n]))
}
}
}
slog.ErrorContext(ctx, "Failed to dial local WebSocket", attrs...)
- c.sendWebSocketClose(conn, streamID)
+ c.sendStreamCloseMessage(conn, streamID, "")
return
}
+ // Local frames are forwarded into tunnel messages, so allow that size instead of the 32KB default.
+ ws.SetReadLimit(maxGRPCTunnelMessageSize)
streamCtx, cancel := context.WithCancel(ctx)
- stream := c.registerStream(conn, streamID, ws, cancel)
+ stream := &activeWSStream{ws: ws, conn: conn, cancel: cancel, dataCh: make(chan wsPayload, 100)}
+ c.activeStreams.Store(streamID, stream)
- workers.Go(func() { c.startLocalWebSocketReadLoop(ctx, streamCtx, streamID, ws, stream) })
+ workers.Go(func() {
+ defer c.closeWebSocketStream(streamID, stream)
+ for {
+ msgType, data, readErr := ws.Read(streamCtx)
+ if readErr != nil {
+ if !wshub.IsExpectedClose(readErr) {
+ slog.DebugContext(ctx, "Local WebSocket read error", "error", readErr)
+ }
+ c.sendStreamCloseMessage(conn, streamID, "")
+ return
+ }
+ if sendErr := conn.Send(&TunnelMessage{ID: streamID, Type: MessageTypeWebSocketData, Body: data, WSMessageType: int(msgType)}); sendErr != nil {
+ slog.DebugContext(ctx, "Failed to send WebSocket data to manager", "error", sendErr)
+ return
+ }
+ }
+ })
workers.Go(func() { c.startLocalWebSocketWriteLoop(ctx, streamCtx, ws, stream, cancel) })
}
-func (c *TunnelClient) handleStreamOpen(ctx context.Context, conn TunnelConnection, msg *TunnelMessage, workers *sync.WaitGroup) {
- if !ValidateEdgeCommand(msg.Command, http.MethodGet, msg.Path, true) {
- c.sendStreamCloseMessage(conn, msg.ID, "unsupported edge stream")
- return
- }
-
- c.handleWebSocketStart(ctx, conn, &TunnelMessage{
- ID: msg.ID,
- Type: MessageTypeWebSocketStart,
- Command: msg.Command,
- Path: msg.Path,
- Query: msg.Query,
- Headers: msg.Headers,
- }, workers)
-}
-
-func (c *TunnelClient) buildLocalWebSocketURLInternal(msg *TunnelMessage) string {
- path := msg.Path
- if msg.Query != "" {
- path = path + "?" + msg.Query
- }
- host := c.localWebSocketHostInternal()
- return "ws://" + net.JoinHostPort(host, c.localPort) + path
-}
-
-func (c *TunnelClient) localWebSocketHostInternal() string {
- listenHost := strings.TrimSpace(c.cfg.Listen)
- if listenHost == "" {
- return "localhost"
- }
-
- // LISTEN may be just a host ("0.0.0.0"), host:port ("0.0.0.0:3552"),
- // IPv6 ("::"), or bracketed IPv6 with port ("[::]:3552").
- if strings.HasPrefix(listenHost, ":") {
- return "localhost"
+func (c *TunnelClient) buildLocalWebSocketURL(msg *TunnelMessage) string {
+ // LISTEN may be a host, host:port, IPv6, or bracketed IPv6 with port.
+ host := strings.TrimSpace(c.cfg.Listen)
+ if splitHost, _, err := net.SplitHostPort(host); err == nil {
+ host = splitHost
}
- if host, _, err := net.SplitHostPort(listenHost); err == nil {
- listenHost = host
+ if host = strings.Trim(host, "[]"); slices.Contains([]string{"", "0.0.0.0", "::"}, host) {
+ host = "localhost"
}
-
- trimmed := strings.Trim(listenHost, "[]")
-
- switch trimmed {
- case "", "0.0.0.0", "::":
- return "localhost"
- default:
- return trimmed
- }
-}
-
-// localDialSkipHeaders lists headers that must not be forwarded when the
-// agent dials its own local HTTP server for a proxied WebSocket stream.
-// This includes:
-// - Standard WebSocket handshake headers (coder/websocket sets its own)
-// - Browser-specific headers that were forwarded through the tunnel from
-// the manager. These cause handshake failures because the agent's
-// WebSocket upgrader validates the Origin against localhost, not the
-// browser's remote origin.
-var localDialSkipHeaders = map[string]bool{
- // WebSocket handshake (coder/websocket adds its own)
- "Sec-Websocket-Key": true,
- "Sec-Websocket-Version": true,
- "Sec-Websocket-Extensions": true,
- "Upgrade": true,
- "Connection": true,
- "Host": true,
-
- // Browser headers forwarded through the tunnel that are invalid
- // for a server-to-server local dial.
- "Origin": true,
- "Cookie": true,
- "Authorization": true,
- "Referer": true,
- "Sec-Fetch-Dest": true,
- "Sec-Fetch-Mode": true,
- "Sec-Fetch-Site": true,
- "Sec-Fetch-User": true,
- "Sec-Ch-Ua": true,
- "Sec-Ch-Ua-Mobile": true,
- "Sec-Ch-Ua-Platform": true,
+ return "ws://" + net.JoinHostPort(host, c.localPort) + msg.Path + kit.Ternary(msg.Query != "", "?"+msg.Query, "")
}
-func (c *TunnelClient) buildLocalWebSocketHeadersInternal(msg *TunnelMessage) http.Header {
+func (c *TunnelClient) buildLocalWebSocketHeaders(msg *TunnelMessage) http.Header {
headers := http.Header{}
for k, v := range msg.Headers {
- canonicalKey := http.CanonicalHeaderKey(k)
- if !localDialSkipHeaders[canonicalKey] {
+ if canonicalKey := http.CanonicalHeaderKey(k); !localDialSkipHeaders[canonicalKey] {
headers.Set(canonicalKey, v)
}
}
@@ -1009,32 +766,6 @@ func (c *TunnelClient) buildLocalWebSocketHeadersInternal(msg *TunnelMessage) ht
return headers
}
-func (c *TunnelClient) dialLocalWebSocket(ctx context.Context, localURL string, headers http.Header) (*websocket.Conn, *http.Response, error) {
- dialCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
- defer cancel()
-
- ws, resp, err := websocket.Dial(dialCtx, localURL, &websocket.DialOptions{HTTPHeader: headers})
- if err != nil {
- return nil, resp, err
- }
- // Local stream frames (exec output, log lines) are forwarded into tunnel
- // messages capped at maxGRPCTunnelMessageSize; allow the same size here
- // instead of coder/websocket's 32KB default.
- ws.SetReadLimit(maxGRPCTunnelMessageSize)
- return ws, resp, nil
-}
-
-func (c *TunnelClient) registerStream(conn TunnelConnection, streamID string, ws *websocket.Conn, cancel context.CancelFunc) *activeWSStream {
- stream := &activeWSStream{
- ws: ws,
- conn: conn,
- cancel: cancel,
- dataCh: make(chan wsPayload, 100),
- }
- c.activeStreams.Store(streamID, stream)
- return stream
-}
-
func (c *TunnelClient) closeWebSocketStream(streamID string, stream *activeWSStream) {
stream.mu.Lock()
if stream.closed {
@@ -1050,45 +781,6 @@ func (c *TunnelClient) closeWebSocketStream(streamID string, stream *activeWSStr
c.activeStreams.Delete(streamID)
}
-// closeAllStreams tears down every active WebSocket stream. It is called when
-// the message loop exits so a reconnect cannot leak stream goroutines, local
-// sockets, or activeStreams entries. closeWebSocketStream is idempotent, so it
-// is safe to race a concurrent manager-driven stream close.
-func (c *TunnelClient) closeAllStreams() {
- c.activeStreams.Range(func(key, value any) bool {
- streamID, ok := key.(string)
- if !ok {
- return true
- }
- if stream, localOk := value.(*activeWSStream); localOk {
- c.closeWebSocketStream(streamID, stream)
- }
- return true
- })
-}
-
-func (c *TunnelClient) startLocalWebSocketReadLoop(ctx, streamCtx context.Context, streamID string, ws *websocket.Conn, stream *activeWSStream) {
- defer func() {
- c.closeWebSocketStream(streamID, stream)
- }()
-
- for {
- msgType, data, err := ws.Read(streamCtx)
- if err != nil {
- if !wshub.IsExpectedClose(err) {
- slog.DebugContext(ctx, "Local WebSocket read error", "error", err)
- }
- c.sendWebSocketClose(stream.conn, streamID)
- return
- }
-
- if sendWebSocketDataErr := c.sendWebSocketData(stream.conn, streamID, int(msgType), data); sendWebSocketDataErr != nil {
- slog.DebugContext(ctx, "Failed to send WebSocket data to manager", "error", sendWebSocketDataErr)
- return
- }
- }
-}
-
func (c *TunnelClient) startLocalWebSocketWriteLoop(ctx, streamCtx context.Context, ws *websocket.Conn, stream *activeWSStream, cancel context.CancelFunc) {
for {
select {
@@ -1098,12 +790,11 @@ func (c *TunnelClient) startLocalWebSocketWriteLoop(ctx, streamCtx context.Conte
if !ok {
return
}
- msgType := websocket.MessageType(payload.messageType)
- if msgType != websocket.MessageText && msgType != websocket.MessageBinary {
+ if !isForwardableWSMessage(payload.messageType) {
slog.WarnContext(ctx, "Dropping WebSocket message with unsupported type", "messageType", payload.messageType)
continue
}
- if err := ws.Write(streamCtx, msgType, payload.data); err != nil {
+ if err := ws.Write(streamCtx, websocket.MessageType(payload.messageType), payload.data); err != nil {
slog.DebugContext(ctx, "Failed to write to local WebSocket", "error", err)
cancel()
return
@@ -1112,38 +803,11 @@ func (c *TunnelClient) startLocalWebSocketWriteLoop(ctx, streamCtx context.Conte
}
}
-func (c *TunnelClient) sendWebSocketData(conn TunnelConnection, streamID string, msgType int, data []byte) error {
- if conn == nil {
- return ErrNoActiveAgentTunnel
- }
- wsDataMsg := &TunnelMessage{
- ID: streamID,
- Type: MessageTypeWebSocketData,
- Body: data,
- WSMessageType: msgType,
- }
- return conn.Send(wsDataMsg)
-}
-
-func (c *TunnelClient) sendWebSocketClose(conn TunnelConnection, streamID string) {
- c.sendStreamCloseMessage(conn, streamID, "")
-}
-
func (c *TunnelClient) sendStreamCloseMessage(conn TunnelConnection, streamID, message string) {
if conn == nil {
return
}
- closeMsg := &TunnelMessage{
- ID: streamID,
- Type: MessageTypeStreamClose,
- Error: message,
- }
- _ = conn.Send(closeMsg)
-}
-
-// handleWebSocketData handles incoming WebSocket data from the manager.
-func (c *TunnelClient) handleWebSocketData(ctx context.Context, msg *TunnelMessage) {
- c.handleStreamData(ctx, msg)
+ _ = conn.Send(&TunnelMessage{ID: streamID, Type: MessageTypeStreamClose, Error: message})
}
func (c *TunnelClient) handleStreamData(ctx context.Context, msg *TunnelMessage) {
@@ -1171,39 +835,7 @@ func (c *TunnelClient) handleStreamData(ctx context.Context, msg *TunnelMessage)
}
}
-// handleWebSocketClose handles WebSocket close from the manager.
-func (c *TunnelClient) handleWebSocketClose(ctx context.Context, msg *TunnelMessage) {
- c.handleStreamClose(ctx, msg)
-}
-
-func (c *TunnelClient) handleStreamClose(ctx context.Context, msg *TunnelMessage) {
- streamRaw, ok := c.activeStreams.Load(msg.ID)
- if !ok {
- return
- }
- stream, ok := streamRaw.(*activeWSStream)
- if !ok {
- return
- }
- c.closeWebSocketStream(msg.ID, stream)
- slog.DebugContext(ctx, "Closed WebSocket stream", "streamId", msg.ID)
-}
-
-// sendErrorResponse sends an error response
-func (c *TunnelClient) sendErrorResponse(conn TunnelConnection, requestID string, status int, message string) {
- if conn == nil {
- return
- }
- resp := &TunnelMessage{
- ID: requestID,
- Type: MessageTypeResponse,
- Status: status,
- Body: []byte(message),
- }
- _ = conn.Send(resp)
-}
-
-func (c *TunnelClient) sendCommandCompleteInternal(conn TunnelConnection, commandID string, status int, message string) {
+func (c *TunnelClient) sendCommandComplete(conn TunnelConnection, commandID string, status int, message string) {
if conn == nil {
return
}
@@ -1222,7 +854,6 @@ func (r *commandResponseRecorder) Header() http.Header {
func (r *commandResponseRecorder) Write(b []byte) (int, error) {
r.mu.Lock()
defer r.mu.Unlock()
- originalLen := len(b)
if r.flushErr != nil {
return 0, r.flushErr
@@ -1239,19 +870,12 @@ func (r *commandResponseRecorder) Write(b []byte) (int, error) {
if err := r.flushBufferLocked(); err != nil {
return 0, err
}
-
- for len(b) > 0 {
- chunk := b
- if len(chunk) > defaultCommandChunkSize {
- chunk = chunk[:defaultCommandChunkSize]
- }
+ for chunk := range slices.Chunk(b, defaultCommandChunkSize) {
if err := r.sendOutputLocked(chunk); err != nil {
return 0, err
}
- b = b[len(chunk):]
}
-
- return originalLen, nil
+ return len(b), nil
}
func (r *commandResponseRecorder) WriteHeader(statusCode int) {
@@ -1286,37 +910,24 @@ func (r *commandResponseRecorder) Close() error {
return r.flushErr
}
- if !r.streaming {
- err := r.conn.Send(&TunnelMessage{
- ID: r.commandID,
- Type: MessageTypeCommandComplete,
- Status: r.statusCode,
- Headers: flattenResponseHeadersInternal(r.headers),
- Body: append([]byte(nil), r.buffer.Bytes()...),
- Streaming: false,
- Command: r.commandName,
- })
- if err != nil {
- return err
- }
- r.closed = true
- return nil
- }
-
- if err := r.flushBufferLocked(); err != nil {
- return err
- }
- if err := r.conn.Send(&TunnelMessage{
+ complete := &TunnelMessage{
ID: r.commandID,
Type: MessageTypeCommandComplete,
Status: r.statusCode,
- Headers: flattenResponseHeadersInternal(r.headers),
- Streaming: true,
+ Headers: flattenResponseHeaders(r.headers),
+ Streaming: r.streaming,
Command: r.commandName,
- }); err != nil {
+ }
+ if r.streaming {
+ if err := r.flushBufferLocked(); err != nil {
+ return err
+ }
+ } else {
+ complete.Body = append([]byte(nil), r.buffer.Bytes()...)
+ }
+ if err := r.conn.Send(complete); err != nil {
return err
}
-
r.closed = true
return nil
}
@@ -1353,7 +964,7 @@ func (r *commandResponseRecorder) sendOutputLocked(chunk []byte) error {
}
if r.sequence == 0 {
msg.Status = r.statusCode
- msg.Headers = flattenResponseHeadersInternal(r.headers)
+ msg.Headers = flattenResponseHeaders(r.headers)
}
if err := r.conn.Send(msg); err != nil {
return err
@@ -1362,10 +973,7 @@ func (r *commandResponseRecorder) sendOutputLocked(chunk []byte) error {
return nil
}
-func flattenResponseHeadersInternal(headers http.Header) map[string]string {
- if len(headers) == 0 {
- return nil
- }
+func flattenResponseHeaders(headers http.Header) map[string]string {
out := make(map[string]string, len(headers))
for k, vs := range headers {
if len(vs) > 0 {
@@ -1387,15 +995,6 @@ func (r *responseRecorder) WriteHeader(statusCode int) {
r.statusCode = statusCode
}
-func newStreamingResponseRecorder(requestID string, conn TunnelConnection) *streamingResponseRecorder {
- return &streamingResponseRecorder{
- requestID: requestID,
- conn: conn,
- headers: make(http.Header),
- statusCode: http.StatusOK,
- }
-}
-
func (r *streamingResponseRecorder) Header() http.Header {
return r.headers
}
@@ -1404,22 +1003,10 @@ func (r *streamingResponseRecorder) Write(b []byte) (int, error) {
r.mu.Lock()
defer r.mu.Unlock()
- if !r.wroteHeader {
- if err := r.writeHeaderLocked(r.statusCode); err != nil {
- return 0, err
- }
- }
-
- if len(b) == 0 {
- return 0, nil
+ if err := r.writeHeaderLocked(); err != nil {
+ return 0, err
}
-
- originalLen := len(b)
- for len(b) > 0 {
- chunk := b
- if len(chunk) > defaultCommandChunkSize {
- chunk = chunk[:defaultCommandChunkSize]
- }
+ for chunk := range slices.Chunk(b, defaultCommandChunkSize) {
if err := r.conn.Send(&TunnelMessage{
ID: r.requestID,
Type: MessageTypeStreamData,
@@ -1427,31 +1014,21 @@ func (r *streamingResponseRecorder) Write(b []byte) (int, error) {
}); err != nil {
return 0, err
}
- b = b[len(chunk):]
}
- return originalLen, nil
+ return len(b), nil
}
func (r *streamingResponseRecorder) WriteHeader(statusCode int) {
r.mu.Lock()
defer r.mu.Unlock()
-
r.statusCode = statusCode
- if r.wroteHeader {
- return
- }
- if err := r.writeHeaderLocked(statusCode); err != nil {
- return
- }
+ _ = r.writeHeaderLocked()
}
func (r *streamingResponseRecorder) Flush() {
r.mu.Lock()
defer r.mu.Unlock()
-
- if !r.wroteHeader {
- _ = r.writeHeaderLocked(r.statusCode)
- }
+ _ = r.writeHeaderLocked()
}
func (r *streamingResponseRecorder) Close() error {
@@ -1461,37 +1038,27 @@ func (r *streamingResponseRecorder) Close() error {
if r.closed {
return nil
}
-
- if !r.wroteHeader {
- if err := r.writeHeaderLocked(r.statusCode); err != nil {
- return err
- }
+ if err := r.writeHeaderLocked(); err != nil {
+ return err
}
-
- if err := r.conn.Send(&TunnelMessage{
- ID: r.requestID,
- Type: MessageTypeStreamEnd,
- }); err != nil {
+ if err := r.conn.Send(&TunnelMessage{ID: r.requestID, Type: MessageTypeStreamEnd}); err != nil {
return err
}
-
r.closed = true
return nil
}
-func (r *streamingResponseRecorder) writeHeaderLocked(statusCode int) error {
- respHeaders := make(map[string]string)
- for k, v := range r.headers {
- if len(v) > 0 {
- respHeaders[k] = v[0]
- }
+// writeHeaderLocked sends the response status and headers once, marked as a streamed response.
+func (r *streamingResponseRecorder) writeHeaderLocked() error {
+ if r.wroteHeader {
+ return nil
}
+ respHeaders := flattenResponseHeaders(r.headers)
respHeaders[tunnelStreamHeader] = "1"
-
if err := r.conn.Send(&TunnelMessage{
ID: r.requestID,
Type: MessageTypeResponse,
- Status: statusCode,
+ Status: r.statusCode,
Headers: respHeaders,
}); err != nil {
return err
@@ -1505,21 +1072,15 @@ func StartTunnelClient(ctx context.Context, cfg *Config, handler http.Handler) (
if !cfg.EdgeAgent {
return nil, errors.New("edge tunnel disabled")
}
-
- if UseGRPCEdgeTransport(cfg) {
- if httpx.ManagerGRPCAddr(cfg.ManagerApiUrl) == "" {
- return nil, errors.New("MANAGER_API_URL with a valid host is required for gRPC transport")
- }
+ if UseGRPCEdgeTransport(cfg) && httpx.ManagerGRPCAddr(cfg.ManagerApiUrl) == "" {
+ return nil, errors.New("MANAGER_API_URL with a valid host is required for gRPC transport")
}
-
if UseWebSocketEdgeTransport(cfg) && strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)) == "" {
return nil, errors.New("MANAGER_API_URL is required for websocket transport")
}
-
if UsePollEdgeTransport(cfg) && strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)) == "" {
return nil, errors.New("MANAGER_API_URL is required for poll transport")
}
-
if cfg.AgentToken == "" {
return nil, errors.New("AGENT_TOKEN is required")
}
@@ -1538,25 +1099,18 @@ func StartTunnelClient(ctx context.Context, cfg *Config, handler http.Handler) (
})
}
-func (c *TunnelClient) connectAndServeWebSocketInternal(ctx context.Context) error {
- managerWSURL := c.managerWebSocketURLInternal()
+func (c *TunnelClient) connectAndServeWebSocket(ctx context.Context) error {
+ managerWSURL := c.managerWebSocketURL()
if managerWSURL == "" {
return errors.New("manager WebSocket URL is empty")
}
- transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
- if strings.HasPrefix(strings.ToLower(managerWSURL), "wss://") {
- tlsConfig, err := buildManagerClientTLSConfig(c.cfg)
- if err != nil {
- return fmt.Errorf("failed to configure edge websocket TLS: %w", err)
- }
- if tlsConfig == nil {
- tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12}
- }
- transport.TLSClientConfig = tlsConfig
+ httpClient, err := NewManagerHTTPClient(c.cfg, 0)
+ if err != nil {
+ return fmt.Errorf("failed to configure edge websocket TLS: %w", err)
}
headers := http.Header{}
- for header, value := range agentAuthCredentialsInternal(c.cfg.AgentToken) {
+ for header, value := range agentAuthCredentials(c.cfg.AgentToken) {
headers.Set(header, value)
}
@@ -1566,7 +1120,7 @@ func (c *TunnelClient) connectAndServeWebSocketInternal(ctx context.Context) err
dialCtx, dialCancel := context.WithTimeout(ctx, 30*time.Second)
defer dialCancel()
conn, resp, err := websocket.Dial(dialCtx, managerWSURL, &websocket.DialOptions{
- HTTPClient: &http.Client{Transport: transport},
+ HTTPClient: httpClient,
HTTPHeader: headers,
})
if err != nil {
@@ -1578,5 +1132,5 @@ func (c *TunnelClient) connectAndServeWebSocketInternal(ctx context.Context) err
return fmt.Errorf("failed to connect to manager websocket endpoint: %w", err)
}
- return c.serveTunnelSessionInternal(ctx, NewTunnelConn(conn), managerWSURL)
+ return c.serveTunnelSession(ctx, NewTunnelConn(conn), managerWSURL)
}
diff --git a/backend/pkg/libarcane/edge/client_test.go b/backend/pkg/libarcane/edge/client_test.go
index 436b6039a..52e6a2978 100644
--- a/backend/pkg/libarcane/edge/client_test.go
+++ b/backend/pkg/libarcane/edge/client_test.go
@@ -193,10 +193,8 @@ func TestTunnelClient_WebSocketProxy(t *testing.T) {
time.Sleep(100 * time.Millisecond)
}
-// TestTunnelClient_WebSocket_ReconnectClosesStreams verifies that a reconnect
-// reclaims the goroutines, local sockets, and activeStreams entries opened on
-// the previous connection instead of leaking them. Run under -race it also
-// exercises concurrent c.conn access across the reassignment on reconnect.
+// TestTunnelClient_WebSocket_ReconnectClosesStreams verifies a reconnect reclaims the previous
+// connection's goroutines, sockets, and activeStreams entries, and races c.conn under -race.
func TestTunnelClient_WebSocket_ReconnectClosesStreams(t *testing.T) {
// Local WS echo server the agent proxies to.
localServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -370,50 +368,6 @@ func TestTunnelClient_HandleRequest_Errors(t *testing.T) {
time.Sleep(100 * time.Millisecond)
}
-func TestTunnelClient_InternalHelpers(t *testing.T) {
- // Mock connection
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- conn, err := websocket.Accept(w, r, nil)
- if err != nil {
- return
- }
- defer func() { _ = conn.CloseNow() }()
-
- for {
- _, _, readErr := conn.Read(r.Context())
- if readErr != nil {
- return
- }
- }
- }))
- defer server.Close()
-
- cfg := &Config{
- ManagerApiUrl: server.URL,
- AgentToken: "test-token",
- }
- client := NewTunnelClient(cfg, nil)
-
- // Manually connect
- serverURL := "ws" + strings.TrimPrefix(server.URL, "http")
- conn, _, err := websocket.Dial(t.Context(), serverURL, nil)
- require.NoError(t, err)
- defer func() { _ = conn.CloseNow() }()
-
- tunnelConn := NewTunnelConn(conn)
- client.conn.Store(&connBox{conn: tunnelConn})
-
- // Test sendWebSocketData
- err = client.sendWebSocketData(tunnelConn, "stream-1", int(websocket.MessageText), []byte("data"))
- require.NoError(t, err)
-
- // Test sendWebSocketClose
- client.sendWebSocketClose(tunnelConn, "stream-1")
-
- // Test sendErrorResponse
- client.sendErrorResponse(tunnelConn, "req-1", 500, "error")
-}
-
func TestTunnelClient_BuildLocalWebSocketURL(t *testing.T) {
tests := []struct {
name string
@@ -492,22 +446,22 @@ func TestTunnelClient_BuildLocalWebSocketURL(t *testing.T) {
Path: testCase.path,
Query: testCase.query,
}
- assert.Equal(t, testCase.expected, client.buildLocalWebSocketURLInternal(msg))
+ assert.Equal(t, testCase.expected, client.buildLocalWebSocketURL(msg))
})
}
}
-func TestTunnelClient_GRPCConnectMethodInternal(t *testing.T) {
+func TestTunnelClient_GRPCConnectMethod(t *testing.T) {
client := NewTunnelClient(&Config{}, http.NotFoundHandler())
assert.Equal(t, "/api/tunnel/connect", client.grpcConnectMethodInternal())
}
-func TestTunnelClient_buildLocalWebSocketHeadersInternal(t *testing.T) {
+func TestTunnelClient_buildLocalWebSocketHeaders(t *testing.T) {
client := NewTunnelClient(&Config{
AgentToken: "agent-token",
}, http.NotFoundHandler())
- headers := client.buildLocalWebSocketHeadersInternal(&TunnelMessage{
+ headers := client.buildLocalWebSocketHeaders(&TunnelMessage{
Headers: map[string]string{
"sec-websocket-key": "abc",
"sec-websocket-version": "13",
@@ -525,12 +479,12 @@ func TestTunnelClient_buildLocalWebSocketHeadersInternal(t *testing.T) {
assert.Equal(t, "agent-token", headers.Get("X-Arcane-Agent-Token"))
}
-func TestTunnelClient_buildLocalWebSocketHeadersInternal_FiltersBrowserHeaders(t *testing.T) {
+func TestTunnelClient_buildLocalWebSocketHeaders_FiltersBrowserHeaders(t *testing.T) {
client := NewTunnelClient(&Config{
AgentToken: "agent-token",
}, http.NotFoundHandler())
- headers := client.buildLocalWebSocketHeadersInternal(&TunnelMessage{
+ headers := client.buildLocalWebSocketHeaders(&TunnelMessage{
Headers: map[string]string{
// Browser headers that should be stripped
"Origin": "https://docker.example.com",
@@ -630,13 +584,13 @@ func TestTunnelClient_DialLocalWebSocket_StripsForwardedBrowserHeaders(t *testin
},
}
- headers := client.buildLocalWebSocketHeadersInternal(msg)
+ headers := client.buildLocalWebSocketHeaders(msg)
assert.Empty(t, headers.Get("Host"))
assert.Empty(t, headers.Get("Origin"))
assert.Empty(t, headers.Get("Cookie"))
assert.Empty(t, headers.Get("Authorization"))
- ws, _, err := client.dialLocalWebSocket(t.Context(), client.buildLocalWebSocketURLInternal(msg), headers)
+ ws, _, err := websocket.Dial(t.Context(), client.buildLocalWebSocketURL(msg), &websocket.DialOptions{HTTPHeader: headers})
require.NoError(t, err)
defer func() { _ = ws.CloseNow() }()
@@ -646,20 +600,6 @@ func TestTunnelClient_DialLocalWebSocket_StripsForwardedBrowserHeaders(t *testin
assert.Equal(t, "ok", string(body))
}
-func TestTunnelClient_IsGRPCConnectionInternal(t *testing.T) {
- t.Run("nil connection", func(t *testing.T) {
- assert.False(t, isGRPCConnection(nil))
- })
-
- t.Run("grpc connection", func(t *testing.T) {
- assert.True(t, isGRPCConnection(NewGRPCAgentTunnelConn(nil)))
- })
-
- t.Run("non-grpc connection", func(t *testing.T) {
- assert.False(t, isGRPCConnection(&fakeTunnelConnForTransportCheck{}))
- })
-}
-
func TestTunnelClient_HandleRequest_GRPCConfigWithWebSocketConnUsesNonStreamingResponse(t *testing.T) {
localHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !assert.Equal(t, "/local/api", r.URL.Path) {
@@ -701,30 +641,6 @@ func TestTunnelClient_HeartbeatLoop_ClosesConnectionOnSendFailure(t *testing.T)
assert.True(t, conn.closeCalled)
}
-type fakeTunnelConnForTransportCheck struct{}
-
-func (f *fakeTunnelConnForTransportCheck) Send(_ *TunnelMessage) error {
- return nil
-}
-
-func (f *fakeTunnelConnForTransportCheck) Receive() (*TunnelMessage, error) {
- return nil, nil
-}
-
-func (f *fakeTunnelConnForTransportCheck) IsExpectedReceiveError(error) bool {
- return false
-}
-
-func (f *fakeTunnelConnForTransportCheck) Close() error {
- return nil
-}
-
-func (f *fakeTunnelConnForTransportCheck) IsClosed() bool {
- return false
-}
-
-func (f *fakeTunnelConnForTransportCheck) Transport() string { return EdgeTransportWebSocket }
-
type capturingTunnelConnForHandleRequest struct {
sent []*TunnelMessage
}
@@ -799,7 +715,7 @@ type blockingRegistrationConn struct {
receiveStartOnce sync.Once
}
-func newBlockingRegistrationConnInternal() *blockingRegistrationConn {
+func newBlockingRegistrationConn() *blockingRegistrationConn {
return &blockingRegistrationConn{
closedCh: make(chan struct{}),
receiveStarted: make(chan struct{}),
@@ -867,7 +783,7 @@ func TestTunnelClient_GRPC_EndToEnd(t *testing.T) {
tunnelServer.WaitForCleanupDone()
}()
- managerURL, stopManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)
+ managerURL, stopManager := startTestGRPCTunnelServerOnAPIPath(t, ctx, tunnelServer)
defer stopManager()
localHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -935,7 +851,7 @@ func TestTunnelClient_GRPC_ChunkedRequestBodyEndToEnd(t *testing.T) {
tunnelServer.WaitForCleanupDone()
}()
- managerURL, stopManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)
+ managerURL, stopManager := startTestGRPCTunnelServerOnAPIPath(t, ctx, tunnelServer)
defer stopManager()
wantBody := bytes.Repeat([]byte("arcane-edge-chunk"), 320*1024)
@@ -1078,7 +994,7 @@ func TestTunnelClient_connectAndServeGRPC_RegistrationRejected(t *testing.T) {
tunnelServer.WaitForCleanupDone()
}()
- managerURL, stopManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)
+ managerURL, stopManager := startTestGRPCTunnelServerOnAPIPath(t, ctx, tunnelServer)
defer stopManager()
client := NewTunnelClient(&Config{
@@ -1100,7 +1016,7 @@ func TestTunnelClient_connectAndServeGRPC_TimesOutWithoutRegisterResponse(t *tes
defer cancel()
service := &stallingTunnelService{}
- managerURL, stopManager := startTestTunnelServiceOnAPIPathInternal(t, ctx, service)
+ managerURL, stopManager := startTestTunnelServiceOnAPIPath(t, ctx, service)
defer stopManager()
client := NewTunnelClient(&Config{
@@ -1120,29 +1036,29 @@ func TestTunnelClient_connectAndServeGRPC_TimesOutWithoutRegisterResponse(t *tes
assert.True(t, active == nil || active.conn.IsClosed())
}
-func TestTunnelClient_awaitRegistrationInternal_ClosesConnOnContextDone(t *testing.T) {
+func TestTunnelClient_awaitRegistration_ClosesConnOnContextDone(t *testing.T) {
ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
defer cancel()
- conn := newBlockingRegistrationConnInternal()
+ conn := newBlockingRegistrationConn()
client := &TunnelClient{
registrationTimeout: time.Second,
}
client.conn.Store(&connBox{conn: conn})
- msg, err := client.awaitRegistrationInternal(ctx, conn)
+ msg, err := client.awaitRegistration(ctx, conn)
require.Nil(t, msg)
require.ErrorIs(t, err, context.DeadlineExceeded)
assert.True(t, conn.IsClosed())
assert.EqualValues(t, 1, conn.closeCount.Load())
}
-func TestTunnelClient_awaitRegistrationInternal_ClosesAttemptConnWhenClientConnChanges(t *testing.T) {
+func TestTunnelClient_awaitRegistration_ClosesAttemptConnWhenClientConnChanges(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
- firstConn := newBlockingRegistrationConnInternal()
- secondConn := newBlockingRegistrationConnInternal()
+ firstConn := newBlockingRegistrationConn()
+ secondConn := newBlockingRegistrationConn()
t.Cleanup(func() {
_ = firstConn.Close()
_ = secondConn.Close()
@@ -1159,7 +1075,7 @@ func TestTunnelClient_awaitRegistrationInternal_ClosesAttemptConnWhenClientConnC
}
resultCh := make(chan registrationResult, 1)
go func() {
- msg, err := client.awaitRegistrationInternal(ctx, firstConn)
+ msg, err := client.awaitRegistration(ctx, firstConn)
resultCh <- registrationResult{msg: msg, err: err}
}()
@@ -1207,7 +1123,7 @@ func TestTunnelClient_GRPC_WebSocketProxyEndToEnd(t *testing.T) {
tunnelServer.WaitForCleanupDone()
}()
- managerURL, stopManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)
+ managerURL, stopManager := startTestGRPCTunnelServerOnAPIPath(t, ctx, tunnelServer)
defer stopManager()
headerTokenCh := make(chan string, 1)
@@ -1342,7 +1258,7 @@ func TestTunnelClient_GRPC_WebSocketProxyEndToEnd(t *testing.T) {
}
}
-func TestTunnelClient_connectAndServe_WebSocketConfigFallsBackToWebSocket(t *testing.T) {
+func TestTunnelClient_connectAndServeManagedTunnel_WebSocketConfigFallsBackToWebSocket(t *testing.T) {
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
@@ -1375,7 +1291,7 @@ func TestTunnelClient_connectAndServe_WebSocketConfigFallsBackToWebSocket(t *tes
}
client := NewTunnelClient(cfg, http.NotFoundHandler())
- err := client.connectAndServe(ctx)
+ err := client.connectAndServeManagedTunnel(ctx)
require.Error(t, err)
select {
@@ -1392,16 +1308,16 @@ func TestTunnelClient_managedTunnelTransports_AutoEnablesGRPCAndWebSocket(t *tes
AgentToken: "valid-token",
}, http.NotFoundHandler())
- transports := client.managedTunnelTransportsInternal()
+ transports := client.managedTunnelTransports()
assert.True(t, transports.grpc)
assert.True(t, transports.websocket)
}
-func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCUnavailable(t *testing.T) {
+func TestTunnelClient_connectAndServeManagedTunnel_AutoFallsBackToWebSocketWhenGRPCUnavailable(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
- managerURL, wsConnectedCh, stopManager := startTestWebSocketTunnelManagerInternal(t, ctx)
+ managerURL, wsConnectedCh, stopManager := startTestWebSocketTunnelManager(t, ctx)
defer stopManager()
client := NewTunnelClient(&Config{
@@ -1409,14 +1325,14 @@ func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCUnavailabl
ManagerApiUrl: managerURL,
AgentToken: "valid-token",
}, http.NotFoundHandler())
- grpcAddr, releaseGRPCAddr := reserveTCPAddressInternal(t)
+ grpcAddr, releaseGRPCAddr := reserveTCPAddress(t)
defer releaseGRPCAddr()
client.managerGRPCAddr = grpcAddr
client.registrationTimeout = 100 * time.Millisecond
errCh := make(chan error, 1)
go func() {
- errCh <- client.connectAndServe(ctx)
+ errCh <- client.connectAndServeManagedTunnel(ctx)
}()
select {
@@ -1434,14 +1350,14 @@ func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCUnavailabl
}
}
-func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCSetupHangs(t *testing.T) {
+func TestTunnelClient_connectAndServeManagedTunnel_AutoFallsBackToWebSocketWhenGRPCSetupHangs(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
- grpcAddr, stopGRPC := startHangingTCPServerInternal(t, ctx)
+ grpcAddr, stopGRPC := startHangingTCPServer(t, ctx)
defer stopGRPC()
- managerURL, wsConnectedCh, stopManager := startTestWebSocketTunnelManagerInternal(t, ctx)
+ managerURL, wsConnectedCh, stopManager := startTestWebSocketTunnelManager(t, ctx)
defer stopManager()
client := NewTunnelClient(&Config{
@@ -1454,7 +1370,7 @@ func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCSetupHangs
errCh := make(chan error, 1)
go func() {
- errCh <- client.connectAndServe(ctx)
+ errCh <- client.connectAndServeManagedTunnel(ctx)
}()
select {
@@ -1472,11 +1388,11 @@ func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCSetupHangs
}
}
-func TestTunnelClient_connectAndServe_GRPCDoesNotFallbackToWebSocket(t *testing.T) {
+func TestTunnelClient_connectAndServeManagedTunnel_GRPCDoesNotFallbackToWebSocket(t *testing.T) {
ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
defer cancel()
- managerURL, wsConnectedCh, stopManager := startTestWebSocketTunnelManagerInternal(t, ctx)
+ managerURL, wsConnectedCh, stopManager := startTestWebSocketTunnelManager(t, ctx)
defer stopManager()
client := NewTunnelClient(&Config{
@@ -1484,12 +1400,12 @@ func TestTunnelClient_connectAndServe_GRPCDoesNotFallbackToWebSocket(t *testing.
ManagerApiUrl: managerURL,
AgentToken: "valid-token",
}, http.NotFoundHandler())
- grpcAddr, releaseGRPCAddr := reserveTCPAddressInternal(t)
+ grpcAddr, releaseGRPCAddr := reserveTCPAddress(t)
defer releaseGRPCAddr()
client.managerGRPCAddr = grpcAddr
client.registrationTimeout = 100 * time.Millisecond
- err := client.connectAndServe(ctx)
+ err := client.connectAndServeManagedTunnel(ctx)
require.Error(t, err)
select {
@@ -1499,7 +1415,7 @@ func TestTunnelClient_connectAndServe_GRPCDoesNotFallbackToWebSocket(t *testing.
}
}
-func TestTunnelClient_connectAndServe_OpensGRPCWhenAvailable(t *testing.T) {
+func TestTunnelClient_connectAndServeManagedTunnel_OpensGRPCWhenAvailable(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
@@ -1518,7 +1434,7 @@ func TestTunnelClient_connectAndServe_OpensGRPCWhenAvailable(t *testing.T) {
go tunnelServer.StartCleanupLoop(ctx)
defer tunnelServer.WaitForCleanupDone()
- managerURL, stopManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)
+ managerURL, stopManager := startTestGRPCTunnelServerOnAPIPath(t, ctx, tunnelServer)
defer stopManager()
client := NewTunnelClient(&Config{
@@ -1529,7 +1445,7 @@ func TestTunnelClient_connectAndServe_OpensGRPCWhenAvailable(t *testing.T) {
errCh := make(chan error, 1)
go func() {
- errCh <- client.connectAndServe(ctx)
+ errCh <- client.connectAndServeManagedTunnel(ctx)
}()
require.Eventually(t, func() bool {
@@ -1551,7 +1467,7 @@ func TestTunnelClient_connectAndServe_OpensGRPCWhenAvailable(t *testing.T) {
}
}
-func startTestWebSocketTunnelManagerInternal(t *testing.T, ctx context.Context) (string, <-chan struct{}, func()) {
+func startTestWebSocketTunnelManager(t *testing.T, ctx context.Context) (string, <-chan struct{}, func()) {
t.Helper()
wsConnectedCh := make(chan struct{}, 1)
@@ -1595,7 +1511,7 @@ func startTestWebSocketTunnelManagerInternal(t *testing.T, ctx context.Context)
return managerServer.URL, wsConnectedCh, managerServer.Close
}
-func reserveTCPAddressInternal(t *testing.T) (string, func()) {
+func reserveTCPAddress(t *testing.T) (string, func()) {
t.Helper()
lis, err := net.Listen("tcp", "127.0.0.1:0")
@@ -1605,7 +1521,7 @@ func reserveTCPAddressInternal(t *testing.T) (string, func()) {
}
}
-func startHangingTCPServerInternal(t *testing.T, ctx context.Context) (string, func()) {
+func startHangingTCPServer(t *testing.T, ctx context.Context) (string, func()) {
t.Helper()
lis, err := net.Listen("tcp", "127.0.0.1:0")
@@ -1638,12 +1554,12 @@ func startHangingTCPServerInternal(t *testing.T, ctx context.Context) (string, f
return lis.Addr().String(), stop
}
-func startTestGRPCTunnelServerOnAPIPathInternal(t *testing.T, ctx context.Context, tunnelServer *TunnelServer) (string, func()) {
+func startTestGRPCTunnelServerOnAPIPath(t *testing.T, ctx context.Context, tunnelServer *TunnelServer) (string, func()) {
t.Helper()
- return startTestTunnelServiceOnAPIPathInternal(t, ctx, tunnelServer)
+ return startTestTunnelServiceOnAPIPath(t, ctx, tunnelServer)
}
-func startTestTunnelServiceOnAPIPathInternal(t *testing.T, ctx context.Context, service tunnelpb.TunnelServiceServer) (string, func()) {
+func startTestTunnelServiceOnAPIPath(t *testing.T, ctx context.Context, service tunnelpb.TunnelServiceServer) (string, func()) {
t.Helper()
var serverOptions []grpc.ServerOption
@@ -1702,7 +1618,7 @@ func startTestTunnelServiceOnAPIPathInternal(t *testing.T, ctx context.Context,
return "http://" + lis.Addr().String(), cleanup
}
-func TestTunnelClient_connectAndServe_AutoDoesNotFallbackWhenEstablishedGRPCSessionDrops(t *testing.T) {
+func TestTunnelClient_connectAndServeManagedTunnel_AutoDoesNotFallbackWhenEstablishedGRPCSessionDrops(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
@@ -1721,10 +1637,10 @@ func TestTunnelClient_connectAndServe_AutoDoesNotFallbackWhenEstablishedGRPCSess
go tunnelServer.StartCleanupLoop(ctx)
defer tunnelServer.WaitForCleanupDone()
- grpcManagerURL, stopGRPCManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)
+ grpcManagerURL, stopGRPCManager := startTestGRPCTunnelServerOnAPIPath(t, ctx, tunnelServer)
defer stopGRPCManager()
- wsManagerURL, wsConnectedCh, stopWSManager := startTestWebSocketTunnelManagerInternal(t, ctx)
+ wsManagerURL, wsConnectedCh, stopWSManager := startTestWebSocketTunnelManager(t, ctx)
defer stopWSManager()
client := NewTunnelClient(&Config{
@@ -1739,7 +1655,7 @@ func TestTunnelClient_connectAndServe_AutoDoesNotFallbackWhenEstablishedGRPCSess
errCh := make(chan error, 1)
go func() {
- errCh <- client.connectAndServe(ctx)
+ errCh <- client.connectAndServeManagedTunnel(ctx)
}()
var tunnel *AgentTunnel
@@ -1768,7 +1684,7 @@ func TestTunnelClient_connectAndServe_AutoDoesNotFallbackWhenEstablishedGRPCSess
require.FailNow(t, "dropped gRPC session must not fall back to websocket")
default:
}
- assert.Zero(t, client.grpcFailureStreakInternal())
+ assert.Zero(t, client.grpcFailureStreak)
cancel()
}
@@ -1794,7 +1710,7 @@ func TestTunnelClient_connectAndServePoll_OpensGRPCWhenRequired(t *testing.T) {
tunnelServer.WaitForCleanupDone()
}()
- managerURL, stopManager := startTestPollAndGRPCManagerInternal(t, ctx, tunnelServer, TunnelPollResponse{
+ managerURL, stopManager := startTestPollAndGRPCManager(t, ctx, tunnelServer, TunnelPollResponse{
Status: TunnelStatusRequired,
PollIntervalSeconds: 1,
})
@@ -1808,7 +1724,7 @@ func TestTunnelClient_connectAndServePoll_OpensGRPCWhenRequired(t *testing.T) {
errCh := make(chan error, 1)
go func() {
- errCh <- client.connectAndServe(ctx)
+ errCh <- client.connectAndServePoll(ctx)
}()
require.Eventually(t, func() bool {
@@ -1889,7 +1805,7 @@ func TestTunnelClient_connectAndServePoll_OpensWebSocketWhenRequired(t *testing.
}, http.NotFoundHandler())
client.registrationTimeout = 100 * time.Millisecond
- err := client.connectAndServe(ctx)
+ err := client.connectAndServePoll(ctx)
require.Error(t, err)
require.ErrorIs(t, err, context.DeadlineExceeded)
@@ -1902,7 +1818,7 @@ func TestTunnelClient_connectAndServePoll_OpensWebSocketWhenRequired(t *testing.
assert.GreaterOrEqual(t, pollCount.Load(), int32(1))
}
-func TestTunnelClient_pollTunnelControlInternal_UsesConfiguredHTTPClient(t *testing.T) {
+func TestTunnelClient_pollTunnelControl_UsesConfiguredHTTPClient(t *testing.T) {
t.Parallel()
managerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -1974,7 +1890,7 @@ func TestTunnelClient_connectAndServePoll_DoesNotOpenWebSocketWhenIdle(t *testin
AgentToken: "valid-token",
}, http.NotFoundHandler())
- err := client.connectAndServe(ctx)
+ err := client.connectAndServePoll(ctx)
require.Error(t, err)
require.ErrorIs(t, err, context.DeadlineExceeded)
@@ -2050,7 +1966,7 @@ func TestTunnelClient_connectAndServePoll_RetriesAfterTransientPollError(t *test
}, http.NotFoundHandler())
client.registrationTimeout = 100 * time.Millisecond
- err := client.connectAndServe(ctx)
+ err := client.connectAndServePoll(ctx)
require.Error(t, err)
require.ErrorIs(t, err, context.DeadlineExceeded)
@@ -2063,7 +1979,7 @@ func TestTunnelClient_connectAndServePoll_RetriesAfterTransientPollError(t *test
assert.GreaterOrEqual(t, pollCount.Load(), int32(2))
}
-func TestTunnelClient_stopPollManagedSessionInternal_DeadlineExceededReturnsTimeoutMessage(t *testing.T) {
+func TestTunnelClient_stopPollManagedSession_DeadlineExceededReturnsTimeoutMessage(t *testing.T) {
t.Parallel()
session := &pollManagedTunnelSession{
@@ -2079,7 +1995,7 @@ func TestTunnelClient_stopPollManagedSessionInternal_DeadlineExceededReturnsTime
assert.EqualError(t, err, "timed out waiting for poll-managed websocket session to stop")
}
-func TestTunnelClient_syncPollManagedSessionInternal_IdleUsesBoundedStopTimeout(t *testing.T) {
+func TestTunnelClient_syncPollManagedSession_IdleUsesBoundedStopTimeout(t *testing.T) {
previousTimeout := defaultPollManagedSessionStopTimeout
defaultPollManagedSessionStopTimeout = 20 * time.Millisecond
defer func() {
@@ -2107,7 +2023,7 @@ func TestTunnelClient_syncPollManagedSessionInternal_IdleUsesBoundedStopTimeout(
require.Less(t, time.Since(started), defaultPollManagedSessionStopTimeout)
}
-func startTestPollAndGRPCManagerInternal(t *testing.T, ctx context.Context, service tunnelpb.TunnelServiceServer, pollResp TunnelPollResponse) (string, func()) {
+func startTestPollAndGRPCManager(t *testing.T, ctx context.Context, service tunnelpb.TunnelServiceServer, pollResp TunnelPollResponse) (string, func()) {
t.Helper()
var serverOptions []grpc.ServerOption
@@ -2210,10 +2126,10 @@ func TestTunnelClient_InternalRequestSkipsSlogEcho(t *testing.T) {
{
name: "streaming response",
run: func(t *testing.T, client *TunnelClient) {
- conn := &fakeTunnelConn{}
+ conn := &fakeTunnelConn{transport: EdgeTransportGRPC}
client.conn.Store(&connBox{conn: conn})
- client.handleRequestStreaming(t.Context(), conn, &TunnelMessage{
+ client.handleRequest(t.Context(), conn, &TunnelMessage{
ID: "req-stream",
Type: MessageTypeRequest,
Method: http.MethodGet,
@@ -2314,7 +2230,7 @@ func (f *fakeTunnelConn) Transport() string {
return f.transport
}
-func TestTunnelClient_serveTunnelSessionInternal_GRPCSendEOFSurfacesRegistrationError(t *testing.T) {
+func TestTunnelClient_serveTunnelSession_GRPCSendEOFSurfacesRegistrationError(t *testing.T) {
conn := &fakeTunnelConn{
sendErr: io.EOF,
receiveErr: status.Error(codes.Unauthenticated, "invalid agent token"),
@@ -2322,7 +2238,7 @@ func TestTunnelClient_serveTunnelSessionInternal_GRPCSendEOFSurfacesRegistration
}
client := NewTunnelClient(&Config{}, http.NotFoundHandler())
- err := client.serveTunnelSessionInternal(t.Context(), conn, "manager.test")
+ err := client.serveTunnelSession(t.Context(), conn, "manager.test")
require.Error(t, err)
assert.Equal(t, codes.Unauthenticated, status.Code(err))
assert.Contains(t, err.Error(), "invalid agent token")
@@ -2330,7 +2246,7 @@ func TestTunnelClient_serveTunnelSessionInternal_GRPCSendEOFSurfacesRegistration
func TestStreamingResponseRecorder_Sequence(t *testing.T) {
conn := &fakeTunnelConn{}
- r := newStreamingResponseRecorder("req-1", conn)
+ r := &streamingResponseRecorder{requestID: "req-1", conn: conn, headers: make(http.Header), statusCode: http.StatusOK}
r.Header().Set("Content-Type", "text/plain")
@@ -2361,7 +2277,7 @@ func TestStreamingResponseRecorder_Sequence(t *testing.T) {
func TestStreamingResponseRecorder_WriteHeaderAndClose(t *testing.T) {
conn := &fakeTunnelConn{}
- r := newStreamingResponseRecorder("req-2", conn)
+ r := &streamingResponseRecorder{requestID: "req-2", conn: conn, headers: make(http.Header), statusCode: http.StatusOK}
r.WriteHeader(http.StatusCreated)
require.NoError(t, r.Close())
@@ -2411,7 +2327,7 @@ func TestConnectAndServeWebSocket_CancelUnblocksMessageLoop(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
done := make(chan error, 1)
- go func() { done <- client.connectAndServeWebSocketInternal(ctx) }()
+ go func() { done <- client.connectAndServeWebSocket(ctx) }()
select {
case <-registered:
diff --git a/backend/pkg/libarcane/edge/client_transport_grpc.go b/backend/pkg/libarcane/edge/client_transport_grpc.go
index 5f8f7ea85..660ce54a0 100644
--- a/backend/pkg/libarcane/edge/client_transport_grpc.go
+++ b/backend/pkg/libarcane/edge/client_transport_grpc.go
@@ -76,7 +76,7 @@ func (c *TunnelClient) connectAndServeGRPC(ctx context.Context) error {
// metadata.New lowercases the keys itself.
streamCtx, streamCancel := context.WithCancel(metadata.NewOutgoingContext(ctx,
- metadata.New(agentAuthCredentialsInternal(c.cfg.AgentToken))))
+ metadata.New(agentAuthCredentials(c.cfg.AgentToken))))
defer streamCancel()
method := c.grpcConnectMethodInternal()
@@ -85,7 +85,7 @@ func (c *TunnelClient) connectAndServeGRPC(ctx context.Context) error {
return fmt.Errorf("failed to open tunnel stream: %w", err)
}
- if serveTunnelSessionErr := c.serveTunnelSessionInternal(ctx, NewGRPCAgentTunnelConn(stream, streamCancel), managerAddr); serveTunnelSessionErr != nil {
+ if serveTunnelSessionErr := c.serveTunnelSession(ctx, NewGRPCAgentTunnelConn(stream, streamCancel), managerAddr); serveTunnelSessionErr != nil {
if errors.Is(serveTunnelSessionErr, errTunnelRegistrationTimeout) {
// The channel already reached Ready, so TCP/TLS works but gRPC
// framing was never answered end to end.
@@ -104,7 +104,7 @@ func (c *TunnelClient) waitForGRPCReadyInternal(ctx context.Context, conn *grpc.
return errors.New("manager gRPC connection is not initialized")
}
- timeout := c.registrationTimeoutInternal()
+ timeout := c.registrationTimeout
readyCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
diff --git a/backend/pkg/libarcane/edge/client_transport_poll.go b/backend/pkg/libarcane/edge/client_transport_poll.go
index 76d5e71cf..c552d2dd5 100644
--- a/backend/pkg/libarcane/edge/client_transport_poll.go
+++ b/backend/pkg/libarcane/edge/client_transport_poll.go
@@ -137,7 +137,7 @@ func (c *TunnelClient) syncPollManagedSessionInternal(ctx context.Context, sessi
slog.InfoContext(ctx, "Poll control plane requested edge tunnel",
"status", status,
"managerGrpcAddr", c.managerGRPCAddr,
- "managerWsUrl", c.managerWebSocketURLInternal(),
+ "managerWsUrl", c.managerWebSocketURL(),
)
return c.startPollManagedSessionInternal(ctx), nil
case TunnelStatusIdle:
@@ -204,7 +204,7 @@ func (c *TunnelClient) pollTunnelControlInternal(ctx context.Context, httpClient
return nil, fmt.Errorf("failed to create poll request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
- for header, value := range agentAuthCredentialsInternal(c.cfg.AgentToken) {
+ for header, value := range agentAuthCredentials(c.cfg.AgentToken) {
req.Header.Set(header, value)
}
@@ -231,7 +231,7 @@ func (c *TunnelClient) startPollManagedSessionInternal(ctx context.Context) *pol
done := make(chan error, 1)
go func() {
- err := c.connectAndServeManagedTunnelInternal(sessionCtx)
+ err := c.connectAndServeManagedTunnel(sessionCtx)
if sessionCtx.Err() != nil {
err = nil
}
diff --git a/backend/pkg/libarcane/edge/command_client.go b/backend/pkg/libarcane/edge/command_client.go
index 3b1ff9e12..b491b88b6 100644
--- a/backend/pkg/libarcane/edge/command_client.go
+++ b/backend/pkg/libarcane/edge/command_client.go
@@ -1,6 +1,7 @@
package edge
import (
+ "cmp"
"context"
"errors"
"fmt"
@@ -8,53 +9,90 @@ import (
"math"
"net/http"
"slices"
+ "sync"
"time"
"uuid"
+
+ "go.getarcane.app/kit/pkg"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
+ "go.opentelemetry.io/otel/propagation"
+ "go.opentelemetry.io/otel/trace"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
const (
tunnelCapabilityChunkedRequest = "chunked-request"
- // tunnelCapabilityProtoParity signals the peer can decode the full
- // TunnelMessage vocabulary natively over gRPC (stream_data/stream_end
- // manager->agent, cancel_request agent->manager) instead of the legacy
- // re-encoded forms.
+ // tunnelCapabilityProtoParity signals the peer decodes the full TunnelMessage vocabulary natively over gRPC.
tunnelCapabilityProtoParity = "proto-parity-v1"
- // Credit-based flow control for command output keeps at most commandCreditWindow bytes
- // unconsumed by the manager. Agents offer it; only the manager sends the grant, which older
- // managers that echo agent capabilities cannot produce.
+ // Command output credit keeps at most commandCreditWindow bytes unconsumed by the manager.
+ // Agents offer it; only the manager sends the grant, which older echoing managers cannot produce.
tunnelCapabilityCommandCredit = "command-credit-v1"
tunnelCapabilityCommandCreditGrant = "command-credit-grant-v1"
commandCreditWindow = 8 << 20
bodyTransferMetadataKey = "body_transfer_id"
)
+// commandDuration is created once and shared by every CommandClient.
+var commandDuration = sync.OnceValue(func() metric.Float64Histogram {
+ histogram, err := otel.Meter(tracing.InstrumentationName).Float64Histogram("arcane.edge.command.duration",
+ metric.WithDescription("Edge tunnel command round-trip duration"),
+ metric.WithUnit("s"),
+ metric.WithExplicitBucketBoundaries(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300),
+ )
+ if err != nil {
+ otel.Handle(err)
+ }
+ return histogram
+})
+
func NewCommandClient() *CommandClient {
return &CommandClient{}
}
-func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *CommandRequest) (*CommandResult, error) {
- if ctx == nil {
- return nil, errors.New("context is required")
- }
- if err := validateConnectedTunnelInternal(tunnel); err != nil {
+func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *CommandRequest) (result *CommandResult, err error) {
+ commandName, err := resolveCommand(ctx, tunnel, req, false)
+ if err != nil {
return nil, err
}
- if req == nil {
- return nil, errors.New("command request is required")
- }
+ requestID := cmp.Or(req.ID, uuid.New().String())
- commandName := req.Command
- if commandName == "" {
- resolved, ok := ResolveEdgeCommandName(req.Method, req.Path, false).Get()
- if !ok {
- return nil, fmt.Errorf("unsupported edge command for %s %s", req.Method, req.Path)
+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "edge.command "+commandName,
+ trace.WithSpanKind(trace.SpanKindClient),
+ trace.WithAttributes(attribute.String("arcane.edge.command", commandName), attribute.String("http.request.method", req.Method)),
+ )
+ started := time.Now()
+ defer func() {
+ tracing.End(span, err)
+ var outcome string
+ switch {
+ case err == nil:
+ outcome = "success"
+ case errors.Is(err, context.Canceled):
+ outcome = "canceled"
+ case errors.Is(err, context.DeadlineExceeded):
+ outcome = "timeout"
+ default:
+ outcome = "error"
}
- commandName = resolved
- }
+ attrs := metric.WithAttributes(attribute.String("arcane.edge.command", commandName), attribute.String("arcane.edge.outcome", outcome))
+ commandDuration().Record(context.WithoutCancel(ctx), time.Since(started).Seconds(), attrs)
+ }()
- requestID := req.ID
- if requestID == "" {
- requestID = uuid.New().String()
+ // Replace any inbound trace headers so the agent's in-process request
+ // continues this span.
+ headers := req.Headers
+ if span.SpanContext().IsValid() {
+ carrier := make(propagation.MapCarrier, len(req.Headers)+2)
+ for k, v := range req.Headers {
+ if key := http.CanonicalHeaderKey(k); key != "Traceparent" && key != "Tracestate" {
+ carrier[k] = v
+ }
+ }
+ otel.GetTextMapPropagator().Inject(ctx, carrier)
+ headers = carrier
}
timeoutMillis := req.TimeoutMillis
@@ -69,7 +107,7 @@ func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *C
Method: req.Method,
Path: req.Path,
Query: req.Query,
- Headers: req.Headers,
+ Headers: headers,
Body: req.Body,
TimeoutMillis: timeoutMillis,
SessionID: tunnel.SessionID,
@@ -82,9 +120,9 @@ func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *C
}
defer tunnel.Pending.Delete(requestID)
- chunkRequestBody := len(req.Body) > defaultCommandChunkSize && slices.Contains(tunnel.Capabilities, tunnelCapabilityChunkedRequest)
- if chunkRequestBody {
- transferID := uuid.New().String()
+ var transferID string
+ if len(req.Body) > defaultCommandChunkSize && slices.Contains(tunnel.Capabilities, tunnelCapabilityChunkedRequest) {
+ transferID = uuid.New().String()
msg.Body = nil
msg.Metadata = map[string]string{bodyTransferMetadataKey: transferID}
}
@@ -92,18 +130,17 @@ func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *C
if sendErr := tunnel.Conn.Send(msg); sendErr != nil {
return nil, fmt.Errorf("tunnel request failed: %w", sendErr)
}
- if chunkRequestBody {
- transferID := msg.Metadata[bodyTransferMetadataKey]
+ if transferID != "" {
for sequence, offset := int64(0), 0; offset < len(req.Body); sequence++ {
end := min(offset+defaultCommandChunkSize, len(req.Body))
- if sendErr2 := tunnel.Conn.Send(&TunnelMessage{
+ if sendErr := tunnel.Conn.Send(&TunnelMessage{
ID: transferID,
Type: MessageTypeFileChunk,
Body: req.Body[offset:end],
Sequence: sequence,
EOF: end == len(req.Body),
- }); sendErr2 != nil {
- return nil, fmt.Errorf("tunnel request body transfer failed: %w", sendErr2)
+ }); sendErr != nil {
+ return nil, fmt.Errorf("tunnel request body transfer failed: %w", sendErr)
}
offset = end
}
@@ -134,27 +171,10 @@ func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *C
}
func (c *CommandClient) OpenStream(ctx context.Context, tunnel *AgentTunnel, req *CommandRequest) error {
- if ctx == nil {
- return errors.New("context is required")
- }
- if err := validateConnectedTunnelInternal(tunnel); err != nil {
+ commandName, err := resolveCommand(ctx, tunnel, req, true)
+ if err != nil {
return err
}
- if req == nil {
- return errors.New("command request is required")
- }
- if req.ID == "" {
- return errors.New("stream ID is required")
- }
-
- commandName := req.Command
- if commandName == "" {
- resolved, ok := ResolveEdgeCommandName(http.MethodGet, req.Path, true).Get()
- if !ok {
- return fmt.Errorf("unsupported edge stream target %q", req.Path)
- }
- commandName = resolved
- }
msg := &TunnelMessage{
ID: req.ID,
@@ -177,7 +197,7 @@ func (c *CommandClient) OpenStream(ctx context.Context, tunnel *AgentTunnel, req
var DefaultCommandClient = NewCommandClient()
// grantCommandCredit queues credit for a command without blocking. A per-tunnel sender coalesces
-// queued credits and sends them, so a stalled connection never holds up the response reader.
+// queued credits, so a stalled connection never holds up the response reader.
func (t *AgentTunnel) grantCommandCredit(ctx context.Context, commandID string, consumed int64) {
t.creditOnce.Do(func() {
t.credits = make(map[string]int64)
@@ -212,9 +232,25 @@ func (t *AgentTunnel) grantCommandCredit(ctx context.Context, commandID string,
}
}
-func validateConnectedTunnelInternal(tunnel *AgentTunnel) error {
- if tunnel == nil || tunnel.Conn == nil || tunnel.Conn.IsClosed() {
- return errors.New("edge tunnel is not connected")
+// resolveCommand validates a command request and resolves its edge command name.
+func resolveCommand(ctx context.Context, tunnel *AgentTunnel, req *CommandRequest, stream bool) (string, error) {
+ switch {
+ case ctx == nil:
+ return "", errors.New("context is required")
+ case !tunnel.connected():
+ return "", errors.New("edge tunnel is not connected")
+ case req == nil:
+ return "", errors.New("command request is required")
+ case stream && req.ID == "":
+ return "", errors.New("stream ID is required")
+ case req.Command != "":
+ return req.Command, nil
+ }
+ if name, ok := ResolveEdgeCommandName(kit.Ternary(stream, http.MethodGet, req.Method), req.Path, stream).Get(); ok {
+ return name, nil
+ }
+ if stream {
+ return "", fmt.Errorf("unsupported edge stream target %q", req.Path)
}
- return nil
+ return "", fmt.Errorf("unsupported edge command for %s %s", req.Method, req.Path)
}
diff --git a/backend/pkg/libarcane/edge/poll_control.go b/backend/pkg/libarcane/edge/poll_control.go
index 79e38de68..e731852b6 100644
--- a/backend/pkg/libarcane/edge/poll_control.go
+++ b/backend/pkg/libarcane/edge/poll_control.go
@@ -19,14 +19,10 @@ import (
const (
// DefaultTunnelPollInterval is how often poll-mode agents should check in.
DefaultTunnelPollInterval = 2 * time.Second
- // DefaultPollRuntimeTTL is how long a poll check-in is considered fresh for
- // runtime status reporting when no live tunnel is currently open.
+ // DefaultPollRuntimeTTL is the minimum time a poll check-in stays fresh for runtime status.
DefaultPollRuntimeTTL = 6 * time.Second
- // DefaultTunnelDemandTTL is how long the manager should keep an edge tunnel
- // marked as required after a user/API request touches the environment. It is
- // deliberately longer than the default environment health interval (2m) so
- // health checks refreshing the demand keep idle poll-mode tunnels open
- // instead of letting them flap between checks.
+ // DefaultTunnelDemandTTL keeps a touched environment's tunnel required; it exceeds the 2m health
+ // interval so health checks keep idle poll-mode tunnels open instead of flapping.
DefaultTunnelDemandTTL = 5 * time.Minute
// TunnelStatusIdle indicates that no reverse tunnel is currently needed.
@@ -37,15 +33,9 @@ const (
TunnelStatusActive = "ACTIVE"
)
-func pollRuntimeTTLInternal(state PollRuntimeState) time.Duration {
- ttl := DefaultPollRuntimeTTL
- if state.PollIntervalSeconds > 0 {
- intervalTTL := time.Duration(state.PollIntervalSeconds) * time.Second * 3
- if intervalTTL > ttl {
- ttl = intervalTTL
- }
- }
- return ttl
+// pollRuntimeTTL is how long a check-in stays fresh: three poll intervals, never under DefaultPollRuntimeTTL.
+func pollRuntimeTTL(state PollRuntimeState) time.Duration {
+ return max(DefaultPollRuntimeTTL, time.Duration(state.PollIntervalSeconds)*time.Second*3)
}
// NewTunnelDemandRegistry creates a new tunnel demand registry.
@@ -164,7 +154,7 @@ func (r *PollRuntimeRegistry) Get(envID string, now time.Time) mo.Option[PollRun
return mo.None[PollRuntimeState]()
}
- ttl := pollRuntimeTTLInternal(state)
+ ttl := pollRuntimeTTL(state)
if now.Sub(*state.LastPollAt) > ttl {
r.mu.Lock()
@@ -173,7 +163,7 @@ func (r *PollRuntimeRegistry) Get(envID string, now time.Time) mo.Option[PollRun
r.mu.Unlock()
return mo.None[PollRuntimeState]()
}
- ttl = pollRuntimeTTLInternal(state)
+ ttl = pollRuntimeTTL(state)
if now.Sub(*state.LastPollAt) > ttl {
delete(r.states, envID)
r.mu.Unlock()
@@ -185,58 +175,21 @@ func (r *PollRuntimeRegistry) Get(envID string, now time.Time) mo.Option[PollRun
return mo.Some(state)
}
-func decodeTunnelPollRequestInternal(c *echo.Context) (*TunnelPollRequest, error) {
- if c == nil || c.Request() == nil || c.Request().Body == nil {
- return &TunnelPollRequest{}, nil
- }
-
- req := c.Request()
- defer func() { _ = req.Body.Close() }()
-
- var pollReq TunnelPollRequest
- if err := json.UnmarshalRead(req.Body, &pollReq); err != nil {
- if errors.Is(err, http.ErrBodyReadAfterClose) {
- return &TunnelPollRequest{}, nil
- }
- if errors.Is(err, io.EOF) {
- return &TunnelPollRequest{}, nil
- }
- return nil, err
- }
-
- return &pollReq, nil
-}
-
-func (s *TunnelServer) pollStatusInternal(envID string) TunnelPollResponse {
- hasActiveTunnel := false
- activeTransport := ""
-
- if tunnel, ok := s.registry.Get(envID).Get(); ok && tunnel != nil && tunnel.Conn != nil && !tunnel.Conn.IsClosed() {
- hasActiveTunnel = true
- activeTransport = tunnel.Conn.Transport()
- }
-
- status := GetDemandRegistry().DesiredStatus(envID, hasActiveTunnel, time.Now())
- return TunnelPollResponse{
- Status: status,
- PollIntervalSeconds: int(DefaultTunnelPollInterval / time.Second),
- ActiveTransport: activeTransport,
- Connected: hasActiveTunnel,
- }
-}
-
// HandlePoll is the HTTP control-plane endpoint used by poll-mode agents.
func (s *TunnelServer) HandlePoll(c *echo.Context) error {
req := c.Request()
ctx := req.Context()
- if _, err := decodeTunnelPollRequestInternal(c); err != nil {
- return c.JSON(http.StatusBadRequest, map[string]any{"error": "invalid poll payload"})
+ if req.Body != nil {
+ var pollReq TunnelPollRequest
+ err := json.UnmarshalRead(req.Body, &pollReq)
+ _ = req.Body.Close()
+ if err != nil && !errors.Is(err, http.ErrBodyReadAfterClose) && !errors.Is(err, io.EOF) {
+ return c.JSON(http.StatusBadRequest, map[string]any{"error": "invalid poll payload"})
+ }
}
- // In proxy-terminated mTLS deployments, the client certificate is consumed
- // by the TLS terminator before this request reaches Arcane. The token is
- // still needed as the poll protocol's environment lookup claim.
+ // Proxy-terminated mTLS consumes the client certificate, so the token remains the environment claim.
token, source := agentToken(req.Header.Values)
if token == "" {
slog.WarnContext(ctx, "Edge poll request without token")
@@ -264,10 +217,18 @@ func (s *TunnelServer) HandlePoll(c *echo.Context) error {
return c.JSON(http.StatusUnauthorized, map[string]any{"error": identityErr.Error()})
}
- pollInterval := DefaultTunnelPollInterval
- GetPollRuntimeRegistry().Update(envID, pollInterval, time.Now())
+ now := time.Now()
+ GetPollRuntimeRegistry().Update(envID, DefaultTunnelPollInterval, now)
- resp := s.pollStatusInternal(envID)
- resp.PollIntervalSeconds = int(pollInterval / time.Second)
+ tunnel, _ := s.registry.Get(envID).Get()
+ connected := tunnel.connected()
+ resp := TunnelPollResponse{
+ Status: GetDemandRegistry().DesiredStatus(envID, connected, now),
+ PollIntervalSeconds: int(DefaultTunnelPollInterval / time.Second),
+ Connected: connected,
+ }
+ if connected {
+ resp.ActiveTransport = tunnel.Conn.Transport()
+ }
return c.JSON(http.StatusOK, resp)
}
diff --git a/backend/pkg/libarcane/edge/registry.go b/backend/pkg/libarcane/edge/registry.go
index 7fd9e75b2..0b4345ae6 100644
--- a/backend/pkg/libarcane/edge/registry.go
+++ b/backend/pkg/libarcane/edge/registry.go
@@ -8,6 +8,8 @@ import (
"time"
"github.com/samber/mo"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
)
@@ -32,6 +34,11 @@ func (t *AgentTunnel) UpdateHeartbeat() {
t.mu.Unlock()
}
+// connected reports whether the tunnel has a live, open connection.
+func (t *AgentTunnel) connected() bool {
+ return t != nil && t.Conn != nil && !t.Conn.IsClosed()
+}
+
// GetLastHeartbeat returns the last heartbeat time
func (t *AgentTunnel) GetLastHeartbeat() time.Time {
t.mu.RLock()
@@ -115,20 +122,14 @@ func (r *TunnelRegistry) Register(envID string, tunnel *AgentTunnel) {
slog.ErrorContext(ctx, "Failed to register edge agent tunnel", "environmentId", envID, "error", "tunnel is required")
return
}
- previous,
- err := r.tunnels.ApplyTyped(ctx,
- "register edge tunnel",
- func(tunnels map[string]*AgentTunnel) (*AgentTunnel,
- bool,
- error,
- ) {
- if r.stopped.Load() {
- return nil, false, errors.New("edge tunnel registry stopped")
- }
- previous := tunnels[envID]
- tunnels[envID] = tunnel
- return previous, true, nil
- })
+ previous, err := r.tunnels.ApplyTyped(ctx, "register edge tunnel", func(tunnels map[string]*AgentTunnel) (*AgentTunnel, bool, error) {
+ if r.stopped.Load() {
+ return nil, false, errors.New("edge tunnel registry stopped")
+ }
+ previous := tunnels[envID]
+ tunnels[envID] = tunnel
+ return previous, true, nil
+ })
if err != nil {
slog.ErrorContext(ctx, "Failed to register edge agent tunnel", "environmentId", envID, "error", err)
return
@@ -140,7 +141,7 @@ func (r *TunnelRegistry) Register(envID string, tunnel *AgentTunnel) {
slog.InfoContext(ctx, "Edge agent tunnel registered")
}
-type registerSessionResultInternal struct {
+type registerSessionResult struct {
accepted bool
drainPrevious bool
reason string
@@ -158,8 +159,8 @@ func (r *TunnelRegistry) RegisterSession(ctx context.Context, tunnel *AgentTunne
return false, false, "environment ID is required", nil
}
- result, err := r.tunnels.ApplyTyped(ctx, "register edge tunnel session", func(tunnels map[string]*AgentTunnel) (registerSessionResultInternal, bool, error) {
- var result registerSessionResultInternal
+ result, err := r.tunnels.ApplyTyped(ctx, "register edge tunnel session", func(tunnels map[string]*AgentTunnel) (registerSessionResult, bool, error) {
+ var result registerSessionResult
if r.stopped.Load() {
return result, false, errors.New("edge tunnel registry stopped")
}
@@ -173,7 +174,7 @@ func (r *TunnelRegistry) RegisterSession(ctx context.Context, tunnel *AgentTunne
tunnelMetadata := tunnel.MetadataSnapshot()
existingStale := staleAfter > 0 && time.Since(existingMetadata.LastHeartbeat) > staleAfter
sameAgentInstance := existingMetadata.AgentInstance != "" && tunnelMetadata.AgentInstance != "" && existingMetadata.AgentInstance == tunnelMetadata.AgentInstance
- if existing.Conn != nil && !existing.Conn.IsClosed() && !existingStale && !sameAgentInstance {
+ if existing.connected() && !existingStale && !sameAgentInstance {
result.reason = "another edge agent session is already active"
return result, false, nil
}
@@ -218,15 +219,15 @@ func (r *TunnelRegistry) UnregisterCurrent(ctx context.Context, envID string, cu
return false, false
}
- type resultInternal struct {
+ type unregisterResult struct {
removed bool
activeReplacement bool
}
- result, err := r.tunnels.ApplyTyped(ctx, "unregister current edge tunnel", func(tunnels map[string]*AgentTunnel) (resultInternal, bool, error) {
- var result resultInternal
+ result, err := r.tunnels.ApplyTyped(ctx, "unregister current edge tunnel", func(tunnels map[string]*AgentTunnel) (unregisterResult, bool, error) {
+ var result unregisterResult
existing := tunnels[envID]
if existing != current {
- result.activeReplacement = existing != nil && existing.Conn != nil && !existing.Conn.IsClosed()
+ result.activeReplacement = existing.connected()
return result, false, nil
}
delete(tunnels, envID)
@@ -284,6 +285,40 @@ func (r *TunnelRegistry) Stop(ctx context.Context) error {
return err
}
+// ObserveAgents reports connected edge agents by transport through the arcane.edge.agents gauge.
+// Poll-mode agents count as "poll" while their check-ins are fresh and no tunnel is open.
+func (r *TunnelRegistry) ObserveAgents(meter metric.Meter) (metric.Registration, error) {
+ gauge, err := meter.Int64ObservableGauge("arcane.edge.agents",
+ metric.WithDescription("Connected edge agents by transport"),
+ metric.WithUnit("{agent}"),
+ )
+ if err != nil {
+ return nil, err
+ }
+ return meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
+ counts := map[string]int64{EdgeTransportWebSocket: 0, EdgeTransportGRPC: 0, EdgeTransportPoll: 0}
+ tunneled := make(map[string]bool)
+ for _, tunnel := range r.tunnels.Values() {
+ if tunnel.connected() {
+ counts[tunnel.Conn.Transport()]++
+ tunneled[tunnel.EnvironmentID] = true
+ }
+ }
+ polls, now := GetPollRuntimeRegistry(), time.Now()
+ polls.mu.RLock()
+ for envID, state := range polls.states {
+ if !tunneled[envID] && state.LastPollAt != nil && now.Sub(*state.LastPollAt) <= pollRuntimeTTL(state) {
+ counts[EdgeTransportPoll]++
+ }
+ }
+ polls.mu.RUnlock()
+ for transport, count := range counts {
+ o.ObserveInt64(gauge, count, metric.WithAttributes(attribute.String("arcane.edge.transport", transport)))
+ }
+ return nil
+ }, gauge)
+}
+
var (
defaultRegistryMu sync.RWMutex
defaultRegistry = NewTunnelRegistry()
diff --git a/backend/pkg/libarcane/edge/server.go b/backend/pkg/libarcane/edge/server.go
index f8f9469a4..07d5bcbcc 100644
--- a/backend/pkg/libarcane/edge/server.go
+++ b/backend/pkg/libarcane/edge/server.go
@@ -14,6 +14,7 @@ import (
"path/filepath"
"slices"
"strings"
+ "sync"
"time"
"uuid"
@@ -21,6 +22,9 @@ import (
"github.com/labstack/echo/v5"
"go.getarcane.app/acfs/atomic"
kit "go.getarcane.app/kit/pkg"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
@@ -32,6 +36,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
)
@@ -47,6 +52,9 @@ const (
maxPendingBacklogMessages = 4096
)
+// observeAgentsOnce registers the connected-agents gauge for the first (manager) tunnel server.
+var observeAgentsOnce sync.Once
+
// EnvironmentResolver resolves an agent token to an environment ID.
type EnvironmentResolver func(ctx context.Context, token string) (environmentID string, err error)
@@ -69,6 +77,11 @@ func NewTunnelServerWithRegistry(registry *TunnelRegistry, resolver EnvironmentR
if registry == nil {
registry = NewTunnelRegistry()
}
+ observeAgentsOnce.Do(func() {
+ if _, err := registry.ObserveAgents(otel.Meter(tracing.InstrumentationName)); err != nil {
+ otel.Handle(err)
+ }
+ })
return &TunnelServer{
registry: registry,
@@ -362,16 +375,31 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, envID string,
tunnel.Capabilities = registerMsg.Capabilities
tunnel.Transport = conn.Transport()
+ // The span covers the connection lifetime; ctx stays unparented so per-message work doesn't nest under it.
+ _, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "edge.tunnel "+tunnel.Transport,
+ trace.WithSpanKind(trace.SpanKindServer),
+ trace.WithAttributes(
+ attribute.String("arcane.edge.transport", tunnel.Transport),
+ attribute.String("arcane.environment.id", envID),
+ attribute.String("arcane.edge.session_id", tunnel.SessionID),
+ attribute.String("arcane.edge.security_mode", securityMode),
+ attribute.String("arcane.edge.agent_instance", tunnel.AgentInstance),
+ ))
+ var spanErr error
+ defer func() { tracing.End(span, spanErr) }()
+
accepted, drainPrevious, rejectReason, err := s.registry.RegisterSession(callbackCtx, tunnel, TunnelStaleTimeout)
if err != nil {
slog.ErrorContext(ctx, "Failed to register edge agent session", "environmentId", envID, "agentInstanceId", tunnel.AgentInstance, "error", err)
_ = tunnel.CloseWithReason("edge agent session registration unavailable")
+ spanErr = err
return
}
if !accepted {
slog.WarnContext(ctx, "Rejected duplicate edge agent session", "environmentId", envID, "agentInstanceId", tunnel.AgentInstance, "reason", rejectReason)
_ = conn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, EnvironmentID: envID, Error: rejectReason})
_ = tunnel.CloseWithReason(rejectReason)
+ spanErr = errors.New(rejectReason)
return
}
@@ -401,6 +429,7 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, envID string,
}); sendErr != nil {
slog.WarnContext(ctx, "Failed to send register response", "environmentId", envID, "error", sendErr)
_ = tunnel.CloseWithReason("")
+ spanErr = sendErr
return
}
s.updateConnectionStatus(callbackCtx, tunnel, true)
@@ -419,6 +448,7 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, envID string,
if receiveErr != nil {
if !conn.IsExpectedReceiveError(receiveErr) {
slog.WarnContext(ctx, "Error receiving from edge tunnel", "environmentId", envID, "error", receiveErr)
+ spanErr = receiveErr
}
return
}
@@ -458,10 +488,8 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, envID string,
}
}
-// enqueue hands msg to the consumer without blocking the tunnel. Behind a full channel, messages
-// wait in order for up to pendingDeliveryTimeout each. A consumer that stalls or falls past the
-// backlog limits is failed once, and a stream also tells the agent to stop. Queued messages
-// outlive a tunnel close so the collector can still drain a fully received response.
+// enqueue hands msg to the consumer without blocking the tunnel; queued messages wait in order and outlive
+// a tunnel close. A consumer that stalls past pendingDeliveryTimeout or the backlog limits is failed once.
func (p *PendingRequest) enqueue(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage, closeStream bool) {
retained := func(m *TunnelMessage) int {
size := len(m.Body) + len(m.Error)
@@ -582,7 +610,7 @@ func (s *TunnelServer) updateConnectionStatus(ctx context.Context, tunnel *Agent
s.statusMu.Lock()
defer s.statusMu.Unlock()
if !connected {
- if active, ok := s.registry.Get(tunnel.EnvironmentID).Get(); ok && active != tunnel && active.Conn != nil && !active.Conn.IsClosed() {
+ if active, _ := s.registry.Get(tunnel.EnvironmentID).Get(); active != tunnel && active.connected() {
return
}
}
diff --git a/backend/pkg/libarcane/edge/tls.go b/backend/pkg/libarcane/edge/tls.go
index a6f386e43..a194ba057 100644
--- a/backend/pkg/libarcane/edge/tls.go
+++ b/backend/pkg/libarcane/edge/tls.go
@@ -33,6 +33,7 @@ import (
"go.getarcane.app/acfs/atomic"
kit "go.getarcane.app/kit/pkg"
libcrypto "go.getarcane.app/sys/crypto"
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
@@ -101,12 +102,7 @@ func NewManagerHTTPClient(cfg *Config, timeout time.Duration) (*http.Client, err
if tlsConfig != nil {
transport.TLSClientConfig = tlsConfig
}
-
- client := &http.Client{Transport: transport}
- if timeout > 0 {
- client.Timeout = timeout
- }
- return client, nil
+ return &http.Client{Transport: otelhttp.NewTransport(transport), Timeout: max(timeout, 0)}, nil
}
// PrepareManagerMTLSAssetsWithContext generates the manager CA when edge mTLS is
@@ -345,7 +341,7 @@ func enrollAgentMTLSAssets(ctx context.Context, cfg *Config, assetsDir, certPath
if reqErr != nil {
return fmt.Errorf("failed to create edge mTLS enrollment request: %w", reqErr)
}
- for header, value := range agentAuthCredentialsInternal(cfg.AgentToken) {
+ for header, value := range agentAuthCredentials(cfg.AgentToken) {
req.Header.Set(header, value)
}
diff --git a/backend/pkg/libarcane/edge/transport.go b/backend/pkg/libarcane/edge/transport.go
index 14cf9ba9c..100523f97 100644
--- a/backend/pkg/libarcane/edge/transport.go
+++ b/backend/pkg/libarcane/edge/transport.go
@@ -83,11 +83,10 @@ func UsePollEdgeTransport(cfg *Config) bool {
// GetActiveTunnelTransport returns the currently active tunnel transport for an environment.
func GetActiveTunnelTransport(envID string) mo.Option[string] {
- tunnel, ok := GetRegistry().Get(envID).Get()
- if !ok || tunnel == nil || tunnel.Conn == nil || tunnel.Conn.IsClosed() {
+ tunnel, ok := GetActiveTunnel(envID).Get()
+ if !ok {
return mo.None[string]()
}
-
if transport := tunnel.Conn.Transport(); transport != "" {
return mo.Some(transport)
}
@@ -96,13 +95,11 @@ func GetActiveTunnelTransport(envID string) mo.Option[string] {
// GetTunnelRuntimeState returns live metadata for an active tunnel.
func GetTunnelRuntimeState(envID string) mo.Option[*TunnelRuntimeState] {
- tunnel, ok := GetRegistry().Get(envID).Get()
- if !ok || tunnel == nil || tunnel.Conn == nil || tunnel.Conn.IsClosed() {
+ tunnel, ok := GetActiveTunnel(envID).Get()
+ if !ok {
return mo.None[*TunnelRuntimeState]()
}
-
- // One snapshot under a single read lock, so the reported fields all describe
- // the same moment and none of them race the tunnel's writers.
+ // One snapshot under a single read lock keeps the fields consistent and race-free.
meta := tunnel.MetadataSnapshot()
return mo.Some(&TunnelRuntimeState{
diff --git a/backend/pkg/libarcane/ws/metrics.go b/backend/pkg/libarcane/ws/metrics.go
index fbc5826ef..504ef7f2e 100644
--- a/backend/pkg/libarcane/ws/metrics.go
+++ b/backend/pkg/libarcane/ws/metrics.go
@@ -1,31 +1,39 @@
package ws
import (
+ "context"
+ "maps"
+ "slices"
"strconv"
"sync"
"sync/atomic"
"time"
systemtypes "github.com/getarcaneapp/arcane/types/v2/system"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
)
// WebSocketMetrics tracks active WebSocket connections and their counts.
type WebSocketMetrics struct {
- projectLogsActive atomic.Int64
- containerLogsActive atomic.Int64
- containerStats atomic.Int64
- containerExec atomic.Int64
- systemStats atomic.Int64
- serviceLogsActive atomic.Int64
- diagnosticsActive atomic.Int64
- seq atomic.Uint64
- mu sync.RWMutex
- connections map[string]systemtypes.WebSocketConnectionInfo
+ counters map[string]*atomic.Int64
+ seq atomic.Uint64
+ mu sync.RWMutex
+ connections map[string]systemtypes.WebSocketConnectionInfo
}
// NewWebSocketMetrics creates a new WebSocketMetrics instance.
func NewWebSocketMetrics() *WebSocketMetrics {
return &WebSocketMetrics{
+ counters: map[string]*atomic.Int64{
+ systemtypes.WSKindProjectLogs: {},
+ systemtypes.WSKindContainerLogs: {},
+ systemtypes.WSKindContainerStats: {},
+ systemtypes.WSKindContainerExec: {},
+ systemtypes.WSKindSystemStats: {},
+ systemtypes.WSKindServiceLogs: {},
+ systemtypes.WSKindDiagnostics: {},
+ },
connections: make(map[string]systemtypes.WebSocketConnectionInfo),
}
}
@@ -33,29 +41,43 @@ func NewWebSocketMetrics() *WebSocketMetrics {
// Snapshot returns a point-in-time copy of the active connection counts.
func (m *WebSocketMetrics) Snapshot() systemtypes.WebSocketMetricsSnapshot {
return systemtypes.WebSocketMetricsSnapshot{
- ProjectLogsActive: m.projectLogsActive.Load(),
- ContainerLogsActive: m.containerLogsActive.Load(),
- ContainerStats: m.containerStats.Load(),
- ContainerExec: m.containerExec.Load(),
- SystemStats: m.systemStats.Load(),
- ServiceLogsActive: m.serviceLogsActive.Load(),
- DiagnosticsActive: m.diagnosticsActive.Load(),
+ ProjectLogsActive: m.counters[systemtypes.WSKindProjectLogs].Load(),
+ ContainerLogsActive: m.counters[systemtypes.WSKindContainerLogs].Load(),
+ ContainerStats: m.counters[systemtypes.WSKindContainerStats].Load(),
+ ContainerExec: m.counters[systemtypes.WSKindContainerExec].Load(),
+ SystemStats: m.counters[systemtypes.WSKindSystemStats].Load(),
+ ServiceLogsActive: m.counters[systemtypes.WSKindServiceLogs].Load(),
+ DiagnosticsActive: m.counters[systemtypes.WSKindDiagnostics].Load(),
}
}
+// ObserveConnections reports active connection counts by kind through the arcane.websocket.connections gauge.
+func (m *WebSocketMetrics) ObserveConnections(meter metric.Meter) error {
+ gauge, err := meter.Int64ObservableGauge("arcane.websocket.connections",
+ metric.WithDescription("Active WebSocket connections by kind"),
+ metric.WithUnit("{connection}"),
+ )
+ if err != nil {
+ return err
+ }
+ _, err = meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
+ for kind, counter := range m.counters {
+ o.ObserveInt64(gauge, counter.Load(), metric.WithAttributes(attribute.String("kind", kind)))
+ }
+ return nil
+ }, gauge)
+ return err
+}
+
// Connections returns a snapshot of all tracked WebSocket connections.
func (m *WebSocketMetrics) Connections() []systemtypes.WebSocketConnectionInfo {
m.mu.RLock()
defer m.mu.RUnlock()
- result := make([]systemtypes.WebSocketConnectionInfo, 0, len(m.connections))
- for _, info := range m.connections {
- result = append(result, info)
- }
- return result
+ return slices.AppendSeq(make([]systemtypes.WebSocketConnectionInfo, 0, len(m.connections)), maps.Values(m.connections))
}
-// RegisterConnection adds a connection to the tracker and increments the
-// appropriate kind counter. Returns the assigned connection ID.
+// RegisterConnection adds a connection to the tracker and increments its kind counter.
+// Returns the assigned connection ID.
func (m *WebSocketMetrics) RegisterConnection(info systemtypes.WebSocketConnectionInfo) string {
if info.ID == "" {
info.ID = "ws-" + strconv.FormatUint(m.seq.Add(1), 10)
@@ -66,43 +88,19 @@ func (m *WebSocketMetrics) RegisterConnection(info systemtypes.WebSocketConnecti
m.mu.Lock()
m.connections[info.ID] = info
m.mu.Unlock()
- m.applyDelta(info.Kind, 1)
+ if counter, ok := m.counters[info.Kind]; ok {
+ counter.Add(1)
+ }
return info.ID
}
-// UnregisterConnection removes a connection from the tracker and decrements
-// the appropriate kind counter.
+// UnregisterConnection removes a connection from the tracker and decrements its kind counter.
func (m *WebSocketMetrics) UnregisterConnection(id string) {
- if id == "" {
- return
- }
- var info systemtypes.WebSocketConnectionInfo
m.mu.Lock()
- if existing, ok := m.connections[id]; ok {
- info = existing
- delete(m.connections, id)
- }
+ info := m.connections[id]
+ delete(m.connections, id)
m.mu.Unlock()
- if info.Kind != "" {
- m.applyDelta(info.Kind, -1)
- }
-}
-
-func (m *WebSocketMetrics) applyDelta(kind string, delta int64) {
- switch kind {
- case systemtypes.WSKindProjectLogs:
- m.projectLogsActive.Add(delta)
- case systemtypes.WSKindContainerLogs:
- m.containerLogsActive.Add(delta)
- case systemtypes.WSKindContainerStats:
- m.containerStats.Add(delta)
- case systemtypes.WSKindContainerExec:
- m.containerExec.Add(delta)
- case systemtypes.WSKindSystemStats:
- m.systemStats.Add(delta)
- case systemtypes.WSKindServiceLogs:
- m.serviceLogsActive.Add(delta)
- case systemtypes.WSKindDiagnostics:
- m.diagnosticsActive.Add(delta)
+ if counter, ok := m.counters[info.Kind]; ok {
+ counter.Add(-1)
}
}
diff --git a/backend/pkg/libarcane/ws/proxy.go b/backend/pkg/libarcane/ws/proxy.go
index bc14da766..29b9c77e1 100644
--- a/backend/pkg/libarcane/ws/proxy.go
+++ b/backend/pkg/libarcane/ws/proxy.go
@@ -10,12 +10,17 @@ import (
"github.com/coder/websocket"
httpxtypes "github.com/getarcaneapp/arcane/types/v2/httpx"
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
)
-// dialClient is shared by every upstream dial instead of building a transport per connection.
-var dialClient = httpx.NewHTTPClient(httpxtypes.ClientOptions{TLSHandshakeTimeout: 10 * time.Second})
+// dialClient is shared by every upstream dial; its span covers the whole proxied session.
+var dialClient = func() *http.Client {
+ client := httpx.NewHTTPClient(httpxtypes.ClientOptions{TLSHandshakeTimeout: 10 * time.Second})
+ client.Transport = otelhttp.NewTransport(client.Transport)
+ return client
+}()
// ProxyHTTP upgrades the incoming client connection and bridges it to remoteWS.
//
@@ -50,12 +55,11 @@ func ProxyHTTP(w http.ResponseWriter, r *http.Request, remoteWS string, header h
_ = resp.Body.Close()
}
if err != nil {
- slog.ErrorContext(dialCtx, "failed to dial remote websocket", "remoteWs", remoteWS, "err", err, "respStatus", func() int {
- if resp != nil {
- return resp.StatusCode
- }
- return 0
- }())
+ respStatus := 0
+ if resp != nil {
+ respStatus = resp.StatusCode
+ }
+ slog.ErrorContext(dialCtx, "failed to dial remote websocket", "remoteWs", remoteWS, "err", err, "respStatus", respStatus)
_ = clientConn.Close(websocket.StatusBadGateway, "")
return err
}
@@ -70,53 +74,31 @@ func ProxyHTTP(w http.ResponseWriter, r *http.Request, remoteWS string, header h
defer pumpCancel()
errc := make(chan struct{}, 2)
-
- // client -> remote
- go func() {
- defer func() { errc <- struct{}{} }()
- for {
- mt, msg, readErr := clientConn.Read(pumpCtx)
- if readErr != nil {
- closeRelay(r.Context(), remoteConn, readErr)
- return
- }
- if writeErr := remoteConn.Write(pumpCtx, mt, msg); writeErr != nil {
- return
- }
- }
- }()
-
- // remote -> client
- go func() {
+ pump := func(src, dst *websocket.Conn) {
defer func() { errc <- struct{}{} }()
for {
- mt, msg, readErr := remoteConn.Read(pumpCtx)
+ mt, msg, readErr := src.Read(pumpCtx)
if readErr != nil {
- closeRelay(r.Context(), clientConn, readErr)
+ // Forward the peer's close status so the terminating reason survives the proxy hop.
+ if ce, ok := errors.AsType[websocket.CloseError](readErr); ok {
+ code := ce.Code
+ if code == websocket.StatusNoStatusRcvd {
+ code = websocket.StatusNormalClosure
+ }
+ if closeErr := dst.Close(code, ce.Reason); closeErr != nil {
+ slog.DebugContext(r.Context(), "failed to relay websocket close", "status", int(code), "err", closeErr)
+ }
+ }
return
}
- if writeErr := clientConn.Write(pumpCtx, mt, msg); writeErr != nil {
+ if writeErr := dst.Write(pumpCtx, mt, msg); writeErr != nil {
return
}
}
- }()
+ }
+ go pump(clientConn, remoteConn)
+ go pump(remoteConn, clientConn)
<-errc
return nil
}
-
-// closeRelay forwards a peer's close status to the other side of the
-// bridge so the terminating reason survives the proxy hop.
-func closeRelay(ctx context.Context, conn *websocket.Conn, err error) {
- var ce websocket.CloseError
- if !errors.As(err, &ce) {
- return
- }
- code := ce.Code
- if code == websocket.StatusNoStatusRcvd {
- code = websocket.StatusNormalClosure
- }
- if closeErr := conn.Close(code, ce.Reason); closeErr != nil {
- slog.DebugContext(ctx, "failed to relay websocket close", "status", int(code), "err", closeErr)
- }
-}
diff --git a/backend/pkg/remenv/client.go b/backend/pkg/remenv/client.go
index c996e9d1c..c3c0fe059 100644
--- a/backend/pkg/remenv/client.go
+++ b/backend/pkg/remenv/client.go
@@ -11,6 +11,8 @@ import (
"net/http"
"strings"
"time"
+
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)
const (
@@ -22,9 +24,8 @@ const (
var ErrEnvironmentUnavailable = errors.New("edge agent is not connected")
-// ExtractBearerToken returns the token portion of an "Authorization: Bearer <token>"
-// header value. It is case-insensitive on the scheme and trims surrounding whitespace.
-// Returns an empty string if the header is empty or does not use the Bearer scheme.
+// ExtractBearerToken returns the trimmed token from an "Authorization: Bearer <token>" value,
+// or "" when the scheme (case-insensitive) is not Bearer.
func ExtractBearerToken(headerValue string) string {
headerValue = strings.TrimSpace(headerValue)
if headerValue == "" {
@@ -36,9 +37,7 @@ func ExtractBearerToken(headerValue string) string {
return strings.TrimSpace(headerValue[len(bearerScheme):])
}
-// RedactedTokenFingerprint returns a short, redacted identifier for a token
-// suitable for debug logs. It is intentionally not reversible and never
-// reveals the full secret.
+// RedactedTokenFingerprint returns a short, non-reversible token identifier for debug logs.
func RedactedTokenFingerprint(token string) string {
token = strings.TrimSpace(token)
if len(token) <= 10 {
@@ -93,42 +92,38 @@ type Client struct {
}
func NewClient(httpClient *http.Client, tunnel TunnelTransport) *Client {
- if httpClient == nil {
- httpClient = &http.Client{Timeout: 30 * time.Second}
+ client := &http.Client{Timeout: 30 * time.Second}
+ if httpClient != nil {
+ copied := *httpClient
+ client = &copied
}
+ client.Transport = otelhttp.NewTransport(client.Transport)
return &Client{
- httpClient: httpClient,
+ httpClient: client,
tunnel: tunnel,
}
}
func (c *Client) Do(ctx context.Context, req Request) (*Response, error) {
if req.IsEdge {
- return c.doViaTunnelInternal(ctx, req)
- }
-
- return c.doDirectHTTPInternal(ctx, req)
-}
-
-func (c *Client) doViaTunnelInternal(ctx context.Context, req Request) (*Response, error) {
- if c.tunnel == nil {
- return nil, &TransportError{Err: errors.New("edge transport unavailable")}
- }
-
- if err := c.tunnel.EnsureAvailable(ctx, req.EnvironmentID); err != nil {
- return nil, &TransportError{Err: err}
- }
-
- resp, err := c.tunnel.Do(ctx, req.EnvironmentID, req.Method, req.Path, cloneHeaders(req.Headers), req.Body)
- if err != nil {
- return nil, &TransportError{Err: err}
+ if c.tunnel == nil {
+ return nil, &TransportError{Err: errors.New("edge transport unavailable")}
+ }
+ if err := c.tunnel.EnsureAvailable(ctx, req.EnvironmentID); err != nil {
+ return nil, &TransportError{Err: err}
+ }
+ headers := maps.Clone(req.Headers)
+ if headers == nil {
+ headers = map[string]string{}
+ }
+ resp, err := c.tunnel.Do(ctx, req.EnvironmentID, req.Method, req.Path, headers, req.Body)
+ if err != nil {
+ return nil, &TransportError{Err: err}
+ }
+ return resp, nil
}
- return resp, nil
-}
-
-func (c *Client) doDirectHTTPInternal(ctx context.Context, req Request) (*Response, error) {
var bodyReader io.Reader
if len(req.Body) > 0 {
bodyReader = bytes.NewReader(req.Body)
@@ -154,26 +149,23 @@ func (c *Client) doDirectHTTPInternal(ctx context.Context, req Request) (*Respon
return nil, &TransportError{Err: fmt.Errorf("failed to read response body: %w", err)}
}
- return &Response{
- StatusCode: resp.StatusCode,
- Body: respBody,
- Headers: flattenHeaders(resp.Header),
- }, nil
+ headers := make(map[string]string, len(resp.Header))
+ for key, values := range resp.Header {
+ if len(values) > 0 {
+ headers[key] = values[0]
+ }
+ }
+ return &Response{StatusCode: resp.StatusCode, Body: respBody, Headers: headers}, nil
}
func (r *Response) RequireSuccess() error {
- if r != nil && r.StatusCode >= http.StatusOK && r.StatusCode < http.StatusMultipleChoices {
- return nil
- }
-
if r == nil {
return &StatusError{}
}
-
- return &StatusError{
- StatusCode: r.StatusCode,
- Body: r.Body,
+ if r.StatusCode >= http.StatusOK && r.StatusCode < http.StatusMultipleChoices {
+ return nil
}
+ return &StatusError{StatusCode: r.StatusCode, Body: r.Body}
}
type TransportError struct {
@@ -189,16 +181,15 @@ func (e *TransportError) Unavailable() bool {
if errors.Is(e.Err, context.DeadlineExceeded) || errors.Is(e.Err, ErrEnvironmentUnavailable) {
return true
}
- var networkErr net.Error
- if errors.As(e.Err, &networkErr) && networkErr.Timeout() {
+ if networkErr, ok := errors.AsType[net.Error](e.Err); ok && networkErr.Timeout() {
return true
}
- var operationErr *net.OpError
- if !errors.As(e.Err, &operationErr) || operationErr.Op != "dial" {
+ operationErr, ok := errors.AsType[*net.OpError](e.Err)
+ if !ok || operationErr.Op != "dial" {
return false
}
- var addressErr *net.AddrError
- return !errors.As(operationErr, &addressErr)
+ _, isAddressErr := errors.AsType[*net.AddrError](operationErr)
+ return !isAddressErr
}
func (e *TransportError) Error() string {
@@ -248,14 +239,8 @@ func (e *DecodeError) Unwrap() error {
return e.Err
}
-// ApplyAgentTokenHeaders sets the agent token in both X- header forms on the
-// given http.Header. It deliberately does NOT set the Authorization header:
-// these helpers are used by the manager when proxying user requests through
-// to remote environments, and the Authorization header in that case carries
-// the calling user's bearer or session token. Direct agent->manager call
-// sites (poll, websocket dial, grpc dial, mTLS enroll) set
-// `Authorization: Bearer <token>` themselves so the token survives reverse
-// proxies that strip non-standard X- headers.
+// ApplyAgentTokenHeaders sets the agent token in both X- header forms. It never sets
+// Authorization, which carries the proxied user's own token.
func ApplyAgentTokenHeaders(headers http.Header, accessToken *string) {
if headers == nil || accessToken == nil || strings.TrimSpace(*accessToken) == "" {
return
@@ -265,8 +250,7 @@ func ApplyAgentTokenHeaders(headers http.Header, accessToken *string) {
headers.Set(HeaderAPIKey, *accessToken)
}
-// ApplyAgentTokenHeaderMap mirrors ApplyAgentTokenHeaders for map-based
-// header carriers. Same rationale: do not write Authorization here.
+// ApplyAgentTokenHeaderMap mirrors ApplyAgentTokenHeaders for map-based header carriers.
func ApplyAgentTokenHeaderMap(headers map[string]string, accessToken *string) {
if headers == nil || accessToken == nil || strings.TrimSpace(*accessToken) == "" {
return
@@ -275,27 +259,3 @@ func ApplyAgentTokenHeaderMap(headers map[string]string, accessToken *string) {
headers[HeaderAgentToken] = *accessToken
headers[HeaderAPIKey] = *accessToken
}
-
-func flattenHeaders(headers http.Header) map[string]string {
- if len(headers) == 0 {
- return map[string]string{}
- }
-
- out := make(map[string]string, len(headers))
- for key, values := range headers {
- if len(values) > 0 {
- out[key] = values[0]
- }
- }
- return out
-}
-
-func cloneHeaders(headers map[string]string) map[string]string {
- if len(headers) == 0 {
- return map[string]string{}
- }
-
- out := make(map[string]string, len(headers))
- maps.Copy(out, headers)
- return out
-}
diff --git a/backend/pkg/scheduler/analytics_job.go b/backend/pkg/scheduler/analytics_job.go
index 46916f152..70deba030 100644
--- a/backend/pkg/scheduler/analytics_job.go
+++ b/backend/pkg/scheduler/analytics_job.go
@@ -16,6 +16,7 @@ import (
"github.com/cenkalti/backoff/v5"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
kit "go.getarcane.app/kit/pkg"
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
@@ -54,6 +55,10 @@ func NewAnalyticsJob(
if httpClient == nil {
httpClient = &http.Client{Timeout: 30 * time.Second}
}
+ // Trace a copy so the injected shared client keeps its raw transport.
+ traced := *httpClient
+ traced.Transport = otelhttp.NewTransport(httpClient.Transport)
+ httpClient = &traced
heartbeatURL := kit.Ternary(!cfg.Environment.IsProdEnvironment(), devHeartbeatEndpoint, defaultHeartbeatEndpoint)
return &AnalyticsJob{
settingsService: settingsService,
diff --git a/backend/pkg/utils/httpx/safe_remote.go b/backend/pkg/utils/httpx/safe_remote.go
index ca282c40a..465136c70 100644
--- a/backend/pkg/utils/httpx/safe_remote.go
+++ b/backend/pkg/utils/httpx/safe_remote.go
@@ -8,148 +8,116 @@ import (
"net/http"
"net/netip"
"net/url"
+ "slices"
"strings"
"time"
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
)
type LookupIPFunc func(ctx context.Context, host string) ([]net.IP, error)
-// blockedRemotePrefixes contains special-use ranges that should never be treated
-// as valid public registry destinations. We intentionally keep this list explicit
-// because Go exposes helpers for some classes (for example private, loopback, and
-// link-local) but does not provide one "publicly routable on the internet" check
-// that covers the full SSRF threat model. These prefixes complement the net.IP
-// helper checks in isBlockedIPInternal and make the policy reviewable in one place.
-var blockedRemotePrefixes = mustParsePrefixesInternal(
- "0.0.0.0/8",
- "100.64.0.0/10",
- "127.0.0.0/8",
- "169.254.0.0/16",
- "192.0.0.0/24",
- "192.0.2.0/24",
- "198.18.0.0/15",
- "198.51.100.0/24",
- "203.0.113.0/24",
- "224.0.0.0/4",
- "240.0.0.0/4",
- "::/128",
- "::1/128",
- "2001:db8::/32",
- "fc00::/7",
- "fe80::/10",
- "ff00::/8",
-)
+// blockedRemotePrefixes lists special-use ranges the netip.Addr helpers in resolveAllowedIPs do not cover,
+// keeping the SSRF policy reviewable in one place.
+var blockedRemotePrefixes = []netip.Prefix{
+ netip.MustParsePrefix("0.0.0.0/8"),
+ netip.MustParsePrefix("100.64.0.0/10"),
+ netip.MustParsePrefix("127.0.0.0/8"),
+ netip.MustParsePrefix("169.254.0.0/16"),
+ netip.MustParsePrefix("192.0.0.0/24"),
+ netip.MustParsePrefix("192.0.2.0/24"),
+ netip.MustParsePrefix("198.18.0.0/15"),
+ netip.MustParsePrefix("198.51.100.0/24"),
+ netip.MustParsePrefix("203.0.113.0/24"),
+ netip.MustParsePrefix("224.0.0.0/4"),
+ netip.MustParsePrefix("240.0.0.0/4"),
+ netip.MustParsePrefix("::/128"),
+ netip.MustParsePrefix("::1/128"),
+ netip.MustParsePrefix("2001:db8::/32"),
+ netip.MustParsePrefix("fc00::/7"),
+ netip.MustParsePrefix("fe80::/10"),
+ netip.MustParsePrefix("ff00::/8"),
+}
func DefaultLookupIP(ctx context.Context, host string) ([]net.IP, error) {
return net.DefaultResolver.LookupIP(ctx, "ip", host)
}
func ValidateSafeRemoteURL(ctx context.Context, rawURL string, lookupIP LookupIPFunc) (*url.URL, error) {
- if lookupIP == nil {
- lookupIP = DefaultLookupIP
- }
-
- parsed, err := url.Parse(rawURL)
+ parsed, err := ValidateOutboundHTTPURL(rawURL)
if err != nil {
- return nil, common.Classify(common.ErrUnsafeRemoteURL, fmt.Errorf("Remote URL is not allowed: %w", err)) //nolint:staticcheck // Preserve the existing error message.
- }
-
- scheme := strings.ToLower(parsed.Scheme)
- if scheme != "http" && scheme != "https" {
- //nolint:staticcheck // Preserve the existing error message.
- return nil,
- common.Classify(common.ErrUnsafeRemoteURL,
- fmt.Errorf("Remote URL is not allowed: %w",
- fmt.Errorf("unsupported URL scheme %q",
- scheme)))
+ return nil, unsafeRemoteURLError(err)
}
- if parsed.User != nil {
- //nolint:staticcheck // Preserve the existing error message.
- return nil,
- common.Classify(common.ErrUnsafeRemoteURL,
- fmt.Errorf("Remote URL is not allowed: %w",
- errors.New("URL credentials are not allowed")))
+ host := strings.ToLower(parsed.Hostname())
+ if host == "localhost" || strings.HasSuffix(host, ".localhost") {
+ return nil, unsafeRemoteURLError(errors.New("missing or blocked hostname"))
}
- host := parsed.Hostname()
- if host == "" || isBlockedHostnameInternal(host) {
- //nolint:staticcheck // Preserve the existing error message.
- return nil,
- common.Classify(common.ErrUnsafeRemoteURL,
- fmt.Errorf("Remote URL is not allowed: %w",
- errors.New("missing or blocked hostname")))
- }
-
- ips, err := resolveAllowedIPsInternal(ctx, host, lookupIP)
- if err != nil || len(ips) == 0 {
- if err == nil {
- err = errors.New("host did not resolve to an allowed IP")
- }
- return nil, common.Classify(common.ErrUnsafeRemoteURL, fmt.Errorf("Remote URL is not allowed: %w", err)) //nolint:staticcheck // Preserve the existing error message.
+ if _, err = resolveAllowedIPs(ctx, parsed.Hostname(), lookupIP); err != nil {
+ return nil, err
}
-
return parsed, nil
}
func NewSafeOutboundHTTPClient(base *http.Client, lookupIP LookupIPFunc) (*http.Client, error) {
- if lookupIP == nil {
- lookupIP = DefaultLookupIP
- }
if base == nil {
base = http.DefaultClient
}
- transport, err := cloneHTTPTransportInternal(base.Transport)
- if err != nil {
- return nil, err
+ var transport *http.Transport
+ switch t := base.Transport.(type) {
+ case nil:
+ defaultTransport, ok := http.DefaultTransport.(*http.Transport)
+ if !ok {
+ return nil, errors.New("http.DefaultTransport is not *http.Transport")
+ }
+ transport = defaultTransport.Clone()
+ case *http.Transport:
+ transport = t.Clone()
+ default:
+ return nil, fmt.Errorf("unsupported HTTP transport type %T", base.Transport)
}
baseDial := transport.DialContext
if baseDial == nil {
- dialer := &net.Dialer{
- Timeout: 30 * time.Second,
- KeepAlive: 30 * time.Second,
- }
+ dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}
baseDial = dialer.DialContext
}
transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
- host, port, splitHostPortErr := net.SplitHostPort(address)
- if splitHostPortErr != nil {
- return nil, common.Classify(common.ErrUnsafeRemoteURL, fmt.Errorf("Remote URL is not allowed: %w", splitHostPortErr)) //nolint:staticcheck // Preserve the existing error message.
+ host, port, err := net.SplitHostPort(address)
+ if err != nil {
+ return nil, unsafeRemoteURLError(err)
}
- ips, splitHostPortErr := resolveAllowedIPsInternal(ctx, host, lookupIP)
- if splitHostPortErr != nil {
- return nil, splitHostPortErr
+ ips, err := resolveAllowedIPs(ctx, host, lookupIP)
+ if err != nil {
+ return nil, err
}
- var lastErr error
+ var dialErr error
for _, ip := range ips {
- conn, dialErr := baseDial(ctx, network, net.JoinHostPort(ip.String(), port))
- if dialErr == nil {
+ conn, connErr := baseDial(ctx, network, net.JoinHostPort(ip.String(), port))
+ if connErr == nil {
return conn, nil
}
- lastErr = dialErr
- }
-
- if lastErr != nil {
- return nil, lastErr
+ dialErr = connErr
}
- return nil, errors.New("failed to resolve remote host")
+ return nil, dialErr
}
client := *base
- client.Transport = transport
+ // Instrumented after the SSRF dialer is installed; the wrapper never dials itself.
+ client.Transport = otelhttp.NewTransport(transport)
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
- if _, validateSafeRemoteURLErr := ValidateSafeRemoteURL(req.Context(), req.URL.String(), lookupIP); validateSafeRemoteURLErr != nil {
- return validateSafeRemoteURLErr
+ if _, err := ValidateSafeRemoteURL(req.Context(), req.URL.String(), lookupIP); err != nil {
+ return err
}
if base.CheckRedirect != nil {
return base.CheckRedirect(req, via)
@@ -160,37 +128,18 @@ func NewSafeOutboundHTTPClient(base *http.Client, lookupIP LookupIPFunc) (*http.
return &client, nil
}
-func cloneHTTPTransportInternal(base http.RoundTripper) (*http.Transport, error) {
- switch t := base.(type) {
- case nil:
- defaultTransport, ok := http.DefaultTransport.(*http.Transport)
- if !ok {
- return nil, errors.New("http.DefaultTransport is not *http.Transport")
+// resolveAllowedIPs resolves host (or parses an IP literal) and rejects it if any address is blocked.
+func resolveAllowedIPs(ctx context.Context, host string, lookupIP LookupIPFunc) ([]net.IP, error) {
+ literal, _, _ := strings.Cut(host, "%")
+ ips := []net.IP{net.ParseIP(literal)}
+ if ips[0] == nil {
+ if lookupIP == nil {
+ lookupIP = DefaultLookupIP
}
- return defaultTransport.Clone(), nil
- case *http.Transport:
- return t.Clone(), nil
- default:
- return nil, fmt.Errorf("unsupported HTTP transport type %T", base)
- }
-}
-
-func resolveAllowedIPsInternal(ctx context.Context, host string, lookupIP LookupIPFunc) ([]net.IP, error) {
- if parsedIP := parseIPLiteralInternal(host); parsedIP != nil {
- if isBlockedIPInternal(parsedIP) {
- //nolint:staticcheck // Preserve the existing error message.
- return nil,
- common.Classify(common.ErrUnsafeRemoteURL,
- fmt.Errorf("Remote URL is not allowed: %w",
- fmt.Errorf("blocked IP address %s",
- parsedIP)))
+ var err error
+ if ips, err = lookupIP(ctx, host); err != nil {
+ return nil, unsafeRemoteURLError(err)
}
- return []net.IP{parsedIP}, nil
- }
-
- ips, err := lookupIP(ctx, host)
- if err != nil {
- return nil, err
}
allowed := make([]net.IP, 0, len(ips))
@@ -198,66 +147,22 @@ func resolveAllowedIPsInternal(ctx context.Context, host string, lookupIP Lookup
if ip == nil {
continue
}
- if isBlockedIPInternal(ip) {
- //nolint:staticcheck // Preserve the existing error message.
- return nil,
- common.Classify(common.ErrUnsafeRemoteURL,
- fmt.Errorf("Remote URL is not allowed: %w",
- fmt.Errorf("blocked IP address %s",
- ip)))
+ addr, ok := netip.AddrFromSlice(ip)
+ addr = addr.Unmap()
+ blocked := !ok || addr.IsLoopback() || addr.IsPrivate() || addr.IsMulticast() || addr.IsUnspecified() ||
+ addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() || slices.ContainsFunc(blockedRemotePrefixes, func(p netip.Prefix) bool { return p.Contains(addr) })
+ if blocked {
+ return nil, unsafeRemoteURLError(fmt.Errorf("blocked IP address %s", ip))
}
allowed = append(allowed, ip)
}
if len(allowed) == 0 {
- //nolint:staticcheck // Preserve the existing error message.
- return nil,
- common.Classify(common.ErrUnsafeRemoteURL,
- fmt.Errorf("Remote URL is not allowed: %w",
- errors.New("host did not resolve to an allowed IP")))
+ return nil, unsafeRemoteURLError(errors.New("host did not resolve to an allowed IP"))
}
-
return allowed, nil
}
-func parseIPLiteralInternal(host string) net.IP {
- host = strings.Trim(strings.TrimSpace(host), "[]")
- if zoneIdx := strings.Index(host, "%"); zoneIdx != -1 {
- host = host[:zoneIdx]
- }
- return net.ParseIP(host)
-}
-
-func isBlockedHostnameInternal(host string) bool {
- host = strings.ToLower(strings.TrimSpace(host))
- return host == "localhost" || strings.HasSuffix(host, ".localhost")
-}
-
-func isBlockedIPInternal(ip net.IP) bool {
- if ip.IsLoopback() || ip.IsPrivate() || ip.IsMulticast() || ip.IsUnspecified() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
- return true
- }
-
- // net.IP helper methods do not cover every special-use range we want to keep
- // off-limits for remote registry fetching, so we also check the explicit prefix list above.
- addr, ok := netip.AddrFromSlice(ip)
- if !ok {
- return true
- }
-
- for _, prefix := range blockedRemotePrefixes {
- if prefix.Contains(addr.Unmap()) {
- return true
- }
- }
-
- return false
-}
-
-func mustParsePrefixesInternal(raw ...string) []netip.Prefix {
- prefixes := make([]netip.Prefix, 0, len(raw))
- for _, cidr := range raw {
- prefixes = append(prefixes, netip.MustParsePrefix(cidr))
- }
- return prefixes
+func unsafeRemoteURLError(err error) error {
+ return common.Classify(common.ErrUnsafeRemoteURL, fmt.Errorf("Remote URL is not allowed: %w", err)) //nolint:staticcheck // Preserve the existing error message.
}
diff --git a/backend/pkg/utils/httpx/safe_remote_test.go b/backend/pkg/utils/httpx/safe_remote_test.go
index d6446cedd..764c9d66d 100644
--- a/backend/pkg/utils/httpx/safe_remote_test.go
+++ b/backend/pkg/utils/httpx/safe_remote_test.go
@@ -65,7 +65,12 @@ func TestNewSafeOutboundHTTPClient_BlocksUnsafeRedirectTarget(t *testing.T) {
}))
defer server.Close()
- baseClient := newRedirectTestClient(server.Listener.Addr().String())
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
+ dialer := &net.Dialer{}
+ return dialer.DialContext(ctx, network, server.Listener.Addr().String())
+ }
+ baseClient := &http.Client{Transport: transport, Timeout: 5 * time.Second}
lookupIP := func(ctx context.Context, host string) ([]net.IP, error) {
return []net.IP{net.ParseIP("93.184.216.34")}, nil
}
@@ -77,16 +82,3 @@ func TestNewSafeOutboundHTTPClient_BlocksUnsafeRedirectTarget(t *testing.T) {
require.Error(t, err)
require.ErrorIs(t, err, common.ErrUnsafeRemoteURL)
}
-
-func newRedirectTestClient(listenerAddr string) *http.Client {
- transport := http.DefaultTransport.(*http.Transport).Clone()
- transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
- dialer := &net.Dialer{}
- return dialer.DialContext(ctx, network, listenerAddr)
- }
-
- return &http.Client{
- Transport: transport,
- Timeout: 5 * time.Second,
- }
-}
diff --git a/backend/pkg/utils/notifications/deliver.go b/backend/pkg/utils/notifications/deliver.go
index 520ff029f..f32fb8046 100644
--- a/backend/pkg/utils/notifications/deliver.go
+++ b/backend/pkg/utils/notifications/deliver.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"net/mail"
+ "regexp"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
)
@@ -64,9 +65,26 @@ func Deliver(ctx context.Context, provider NotificationProvider, config database
if !ok {
return false, nil
}
- return true, deliver(ctx, config, c)
+ if deliverErr := deliver(ctx, config, c); deliverErr != nil {
+ return true, deliveryError{message: webhookURL.ReplaceAllString(deliverErr.Error(), "${1}${2}"), err: deliverErr}
+ }
+ return true, nil
+}
+
+// webhookURL matches http(s) URLs in provider errors. Webhook tokens often sit in the path or userinfo,
+// so only the scheme and host are kept.
+var webhookURL = regexp.MustCompile(`(https?://)(?:[^@/?#\s"']*@)?([^/?#\s"']+)[^\s"']*`)
+
+// deliveryError reports a provider failure without its webhook URL while keeping the cause for errors.Is.
+type deliveryError struct {
+ message string
+ err error
}
+func (e deliveryError) Error() string { return e.message }
+
+func (e deliveryError) Unwrap() error { return e.err }
+
func deliverDiscord(ctx context.Context, config database.JSON, c Content) error {
discordConfig, err := DecodeConfig[DiscordConfig](config, "Discord")
if err != nil {
diff --git a/backend/pkg/utils/notifications/generic_sender.go b/backend/pkg/utils/notifications/generic_sender.go
index 59175fb0f..01ba63c64 100644
--- a/backend/pkg/utils/notifications/generic_sender.go
+++ b/backend/pkg/utils/notifications/generic_sender.go
@@ -16,25 +16,18 @@ import (
"time"
"github.com/nicholas-fedor/shoutrrr"
- shoutrrrTypes "github.com/nicholas-fedor/shoutrrr/pkg/types"
- kit "go.getarcane.app/kit/pkg"
+ "github.com/nicholas-fedor/shoutrrr/pkg/types"
+ "go.getarcane.app/kit/pkg"
)
-// genericPayloadTemplateID is the Shoutrrr template ID under which a user's
-// payload template is registered on the generic service instance.
+// genericPayloadTemplateID names the Shoutrrr template a user's payload template is registered under.
const genericPayloadTemplateID = "arcane"
-// genericHTTPClient bounds every generic webhook request. Shoutrrr's generic
-// service falls back to a client with no timeout when none is injected, and
-// the payload-template path sends through the located service directly,
-// bypassing the router's own per-send timeout.
+// genericHTTPClient bounds generic webhook sends; Shoutrrr's default client has no timeout.
var genericHTTPClient = &http.Client{Timeout: 15 * time.Second}
-// resolveWebhookURLInternal parses and normalises the configured webhook URL,
-// adding a default scheme when the user omitted one. It is the single source
-// of truth for scheme normalisation and host validation used by both
-// BuildGenericURL and sendGenericDirectInternal.
-func resolveWebhookURLInternal(config GenericConfig) (*url.URL, error) {
+// resolveWebhookURL parses the configured webhook URL, adding a default scheme when omitted.
+func resolveWebhookURL(config GenericConfig) (*url.URL, error) {
if config.WebhookURL == "" {
return nil, errors.New("webhook URL is empty")
}
@@ -67,48 +60,24 @@ func resolveWebhookURLInternal(config GenericConfig) (*url.URL, error) {
return parsed, nil
}
-// BuildGenericURL converts GenericConfig to Shoutrrr URL format for generic webhooks
+// BuildGenericURL converts GenericConfig to Shoutrrr URL format for generic webhooks.
func BuildGenericURL(config GenericConfig) (string, error) {
- webhookURL, err := resolveWebhookURLInternal(config)
+ webhookURL, err := resolveWebhookURL(config)
if err != nil {
return "", err
}
- // Start from the user's existing query parameters. Shoutrrr's generic
- // service preserves any query keys it does not recognise, so provider
- // tokens embedded in the webhook URL (e.g. PushPlus's `?token=...`) flow
- // straight through to the outbound HTTP request untouched.
- //
- // For Shoutrrr config keys (template, contenttype, method, titlekey,
- // messagekey, disabletls) we only fill in defaults / configured values
- // when the user has not already set the same key inline in the URL.
- // That way an explicit `?template=custom` or `?disabletls=yes` from the
- // user is always respected and never silently overwritten by the
- // provider settings or the URL-scheme-derived TLS flag.
+ // Unknown query keys (e.g. PushPlus `?token=`) pass through; Shoutrrr config keys set inline win over settings.
query := webhookURL.Query()
-
setDefault := func(key, value string) {
- if value == "" {
- return
- }
- if query.Get(key) != "" {
- return
+ if value != "" && query.Get(key) == "" {
+ query.Set(key, value)
}
- query.Set(key, value)
}
- // Default to the JSON template — Shoutrrr's JSON template marshals the
- // notification params as a flat JSON object at the root level, which is
- // the format most providers (PushPlus, custom APIs, Home Assistant, etc.)
- // expect. When the user configured a payload template, point Shoutrrr at
- // the named template registered at send time instead, and default the
- // content type to JSON since that is what payload templates target.
+ // Payload templates use the named template registered at send time and default to JSON content.
hasPayloadTemplate := strings.TrimSpace(config.PayloadTemplate) != ""
- if hasPayloadTemplate {
- setDefault("template", genericPayloadTemplateID)
- } else {
- setDefault("template", "json")
- }
+ setDefault("template", kit.Ternary(hasPayloadTemplate, genericPayloadTemplateID, "json"))
setDefault("contenttype", config.ContentType)
if hasPayloadTemplate {
setDefault("contenttype", "application/json")
@@ -117,8 +86,6 @@ func BuildGenericURL(config GenericConfig) (string, error) {
setDefault("titlekey", config.TitleKey)
setDefault("messagekey", config.MessageKey)
- // Determine TLS setting from the webhook URL scheme (http/https) when the
- // user has not already passed `disabletls` explicitly.
switch strings.ToLower(webhookURL.Scheme) {
case "http":
setDefault("disabletls", "yes")
@@ -126,12 +93,9 @@ func BuildGenericURL(config GenericConfig) (string, error) {
setDefault("disabletls", "no")
}
- // Add custom headers as query parameters with @ prefix
- if len(config.CustomHeaders) > 0 {
- for key, value := range config.CustomHeaders {
- // Shoutrrr uses @ prefix for headers
- query.Set("@"+key, value)
- }
+ // Shoutrrr reads headers from @-prefixed query keys.
+ for key, value := range config.CustomHeaders {
+ query.Set("@"+key, value)
}
shoutrrrURL := &url.URL{
@@ -144,10 +108,8 @@ func BuildGenericURL(config GenericConfig) (string, error) {
return shoutrrrURL.String(), nil
}
-// EventVars builds the per-event variables exposed to a generic webhook
-// payload template. The names must stay clear of Shoutrrr generic config keys
-// (title, template, contenttype, method, messagekey, titlekey, disabletls) —
-// a colliding param would mutate the per-send service config.
+// EventVars builds the per-event payload template variables.
+// Names must not collide with Shoutrrr generic config keys, which would mutate the per-send config.
func EventVars(environmentName, environmentID string, event NotificationEventType) map[string]string {
return map[string]string{
"environment": environmentName,
@@ -157,11 +119,7 @@ func EventVars(environmentName, environmentID string, event NotificationEventTyp
}
}
-// jsonEscapeString escapes value for embedding inside a JSON string literal:
-// json.Marshal yields a quoted string, and the surrounding quotes are trimmed
-// so the template author keeps control of the quoting. Applied only on the
-// payload-template path — the default template=json path is escaped by
-// Shoutrrr's own json.Marshal.
+// jsonEscapeString escapes value for embedding inside a JSON string literal, without the surrounding quotes.
func jsonEscapeString(value string) string {
encoded, err := json.Marshal(value)
if err != nil {
@@ -170,56 +128,29 @@ func jsonEscapeString(value string) string {
return strings.TrimSuffix(strings.TrimPrefix(string(encoded), `"`), `"`)
}
-// genericTemplateDataInternal builds the variable map a payload template is
-// rendered with on the direct-HTTP path: the JSON-escaped title and message
-// under the configured titlekey/messagekey — mirroring how Shoutrrr's
-// createSendParams keys the send params — plus the escaped per-event vars.
-func genericTemplateDataInternal(config GenericConfig, title, message string, vars map[string]string) map[string]string {
- titleKey := cmp.Or(config.TitleKey, "title")
- messageKey := cmp.Or(config.MessageKey, "message")
-
- data := make(map[string]string, len(vars)+2)
- for key, value := range vars {
- data[key] = jsonEscapeString(value)
- }
- data[titleKey] = jsonEscapeString(title)
- data[messageKey] = jsonEscapeString(message)
- return data
-}
-
-// RenderGenericPayloadTemplate renders the configured payload template with
-// the same variables the Shoutrrr send path exposes. Used by the direct-HTTP
-// path (SuccessBodyContains) and by save-time validation.
+// RenderGenericPayloadTemplate renders the payload template with the JSON-escaped title, message, and event vars.
func RenderGenericPayloadTemplate(config GenericConfig, title, message string, vars map[string]string) (string, error) {
tmpl, err := template.New(genericPayloadTemplateID).Parse(config.PayloadTemplate)
if err != nil {
return "", fmt.Errorf("invalid webhook payload template: %w", err)
}
+ // Keyed by titlekey/messagekey to mirror Shoutrrr's send params.
+ data := make(map[string]string, len(vars)+2)
+ for key, value := range vars {
+ data[key] = jsonEscapeString(value)
+ }
+ data[cmp.Or(config.TitleKey, "title")] = jsonEscapeString(title)
+ data[cmp.Or(config.MessageKey, "message")] = jsonEscapeString(message)
+
var rendered bytes.Buffer
- if executeErr := tmpl.Execute(&rendered, genericTemplateDataInternal(config, title, message, vars)); executeErr != nil {
+ if executeErr := tmpl.Execute(&rendered, data); executeErr != nil {
return "", fmt.Errorf("failed to render webhook payload template: %w", executeErr)
}
return rendered.String(), nil
}
-// isJSONContentType reports whether the configured content type denotes JSON.
-// An empty content type counts as JSON because that is the default applied to
-// outgoing generic webhook requests.
-func isJSONContentType(contentType string) bool {
- if contentType == "" {
- return true
- }
- // Ignore any parameters such as "; charset=utf-8".
- mediaType := strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0]))
- return mediaType == "application/json" || strings.HasSuffix(mediaType, "+json")
-}
-
-// ValidateGenericPayloadTemplate checks a configured payload template at save
-// time: it must parse, execute against sample variables, and — when the
-// content type is JSON (the default) — render valid JSON. Shoutrrr renders
-// the template internally at send time, so validating here surfaces mistakes
-// against the user's own configuration instead of as an opaque send failure.
+// ValidateGenericPayloadTemplate checks that a payload template renders, and renders valid JSON for JSON content types.
func ValidateGenericPayloadTemplate(config GenericConfig) error {
if strings.TrimSpace(config.PayloadTemplate) == "" {
return nil
@@ -231,27 +162,25 @@ func ValidateGenericPayloadTemplate(config GenericConfig) error {
return err
}
- if isJSONContentType(config.ContentType) && !jsontext.Value(rendered).IsValid() {
+ // An empty content type defaults to JSON.
+ mediaType, _, _ := strings.Cut(config.ContentType, ";")
+ mediaType = strings.ToLower(strings.TrimSpace(mediaType))
+ isJSON := config.ContentType == "" || mediaType == "application/json" || strings.HasSuffix(mediaType, "+json")
+ if isJSON && !jsontext.Value(rendered).IsValid() {
return errors.New("webhook payload template did not render valid JSON")
}
return nil
}
-// SendGenericWithTitle sends a message with title via Shoutrrr Generic webhook.
-// When config.SuccessBodyContains is set the response body is also inspected —
-// this is necessary for providers (e.g. PushPlus) that always return HTTP 200
-// but embed a success/failure indicator inside the JSON body.
+// SendGenericWithTitle sends a message with title via the Shoutrrr generic webhook.
+// SuccessBodyContains also checks the response body, for providers that always return HTTP 200.
func SendGenericWithTitle(ctx context.Context, config GenericConfig, title, message string, vars map[string]string) error {
if config.WebhookURL == "" {
return errors.New("webhook URL is empty")
}
- // When the caller needs response-body validation we make the HTTP request
- // ourselves so that we can inspect the body. Otherwise we delegate to
- // shoutrrr, which preserves the existing behaviour for everyone who does
- // not set SuccessBodyContains.
if config.SuccessBodyContains != "" {
- return sendGenericDirectInternal(ctx, config, title, message, vars)
+ return sendGenericDirect(ctx, config, title, message, vars)
}
shoutrrrURL, err := BuildGenericURL(config)
@@ -259,16 +188,37 @@ func SendGenericWithTitle(ctx context.Context, config GenericConfig, title, mess
return fmt.Errorf("failed to build shoutrrr Generic URL: %w", err)
}
- senderOptions := shoutrrrTypes.SenderOptions{HTTPClient: genericHTTPClient}
+ senderOptions := types.SenderOptions{HTTPClient: genericHTTPClient}
- // A user-supplied inline `?template=<id>` in the webhook URL wins over the
- // default "arcane" ID, so the payload template must be registered under the
- // effective ID for Shoutrrr to resolve it. An inline `template=json`
- // selects Shoutrrr's built-in JSON template instead, which does its own
- // escaping — send through the plain path so params are not double-escaped.
+ // Register the payload template under the URL's effective template ID; an inline json template escapes on its own.
if strings.TrimSpace(config.PayloadTemplate) != "" {
- if templateID := effectiveGenericTemplateIDInternal(shoutrrrURL); templateID != "json" && templateID != "JSON" {
- return sendGenericTemplatedInternal(config, shoutrrrURL, templateID, senderOptions, title, message, vars)
+ templateID := genericPayloadTemplateID
+ if parsed, parseErr := url.Parse(shoutrrrURL); parseErr == nil {
+ templateID = cmp.Or(parsed.Query().Get("template"), genericPayloadTemplateID)
+ }
+ if templateID != "json" && templateID != "JSON" {
+ // Templates are per service instance, which router.Send does not expose, so send through the located service.
+ sender, createErr := shoutrrr.CreateSenderWithOptions(senderOptions)
+ if createErr != nil {
+ return fmt.Errorf("failed to create shoutrrr Generic sender: %w", createErr)
+ }
+ service, locateErr := sender.Locate(shoutrrrURL)
+ if locateErr != nil {
+ return fmt.Errorf("failed to initialize shoutrrr Generic service: %w", locateErr)
+ }
+ if setTemplateStringErr := service.SetTemplateString(templateID, config.PayloadTemplate); setTemplateStringErr != nil {
+ return fmt.Errorf("invalid webhook payload template: %w", setTemplateStringErr)
+ }
+
+ // text/template does no escaping, so every value is JSON-escaped to keep the payload valid.
+ params := types.Params{"title": jsonEscapeString(title)}
+ for key, value := range vars {
+ params[key] = jsonEscapeString(value)
+ }
+ if sendErr := service.Send(jsonEscapeString(message), &params); sendErr != nil {
+ return fmt.Errorf("failed to send Generic webhook message via shoutrrr: %w", sendErr)
+ }
+ return nil
}
}
@@ -277,13 +227,9 @@ func SendGenericWithTitle(ctx context.Context, config GenericConfig, title, mess
return fmt.Errorf("failed to create shoutrrr Generic sender: %w", err)
}
- // Build params with title. Always use "title" as the param key — Shoutrrr's
- // generic service maps it to the configured titlekey in the JSON payload.
- params := shoutrrrTypes.Params{}
- params["title"] = cmp.Or(title, params["title"])
-
- errs := sender.Send(message, &params)
- for _, err := range errs {
+ // Shoutrrr maps the "title" param to the configured titlekey.
+ params := types.Params{"title": title}
+ for _, err := range sender.Send(message, &params) {
if err != nil {
return fmt.Errorf("failed to send Generic webhook message with title via shoutrrr: %w", err)
}
@@ -291,64 +237,10 @@ func SendGenericWithTitle(ctx context.Context, config GenericConfig, title, mess
return nil
}
-// effectiveGenericTemplateIDInternal returns the template ID the built
-// Shoutrrr URL selects for payload rendering. BuildGenericURL always sets a
-// template query key, so an empty result only happens on an unparseable URL;
-// fall back to the default ID in that case.
-func effectiveGenericTemplateIDInternal(shoutrrrURL string) string {
- parsed, err := url.Parse(shoutrrrURL)
- if err != nil {
- return genericPayloadTemplateID
- }
- if id := parsed.Query().Get("template"); id != "" {
- return id
- }
- return genericPayloadTemplateID
-}
-
-// sendGenericTemplatedInternal sends through Shoutrrr's generic service with
-// the user's payload template registered on the service instance under
-// templateID — the ID the URL's template key selects. Templates are resolved
-// per service instance, which router.Send does not expose, so the service is
-// obtained via Locate — the full initService path (scheme extraction, config
-// parse, HTTP client injection) — and sent on directly.
-func sendGenericTemplatedInternal(config GenericConfig, shoutrrrURL, templateID string, opts shoutrrrTypes.SenderOptions, title, message string, vars map[string]string) error {
- sender, err := shoutrrr.CreateSenderWithOptions(opts)
- if err != nil {
- return fmt.Errorf("failed to create shoutrrr Generic sender: %w", err)
- }
-
- service, err := sender.Locate(shoutrrrURL)
- if err != nil {
- return fmt.Errorf("failed to initialize shoutrrr Generic service: %w", err)
- }
-
- if setTemplateStringErr := service.SetTemplateString(templateID, config.PayloadTemplate); setTemplateStringErr != nil {
- return fmt.Errorf("invalid webhook payload template: %w", setTemplateStringErr)
- }
-
- // Shoutrrr renders the registered template over the send params, remapping
- // the "title" param to the configured titlekey and adding the message under
- // the configured messagekey. Every value is JSON-escaped here, so
- // {"text": "{{.message}}"} stays valid JSON for hostile message content —
- // text/template itself does no escaping.
- params := shoutrrrTypes.Params{"title": jsonEscapeString(title)}
- for key, value := range vars {
- params[key] = jsonEscapeString(value)
- }
-
- if sendErr := service.Send(jsonEscapeString(message), &params); sendErr != nil {
- return fmt.Errorf("failed to send Generic webhook message via shoutrrr: %w", sendErr)
- }
- return nil
-}
-
-// sendGenericDirectInternal makes the webhook HTTP call directly, giving access
-// to the response body so that provider-level success/failure can be detected
-// even when the HTTP status is always 200. It renders the payload template when
-// one is configured so the two features compose.
-func sendGenericDirectInternal(ctx context.Context, config GenericConfig, title, message string, vars map[string]string) error {
- webhookURL, err := resolveWebhookURLInternal(config)
+// sendGenericDirect calls the webhook directly so the response body can be checked for SuccessBodyContains.
+// Kept separate from SendGenericWithTitle to stay under the gocognit limit.
+func sendGenericDirect(ctx context.Context, config GenericConfig, title, message string, vars map[string]string) error {
+ webhookURL, err := resolveWebhookURL(config)
if err != nil {
return err
}
@@ -361,11 +253,8 @@ func sendGenericDirectInternal(ctx context.Context, config GenericConfig, title,
}
body = []byte(rendered)
} else {
- // Build JSON payload using the configured message/title keys.
- msgKey := cmp.Or(config.MessageKey, "message")
titleKey := cmp.Or(config.TitleKey, "title")
-
- payload := map[string]string{msgKey: message}
+ payload := map[string]string{cmp.Or(config.MessageKey, "message"): message}
payload[titleKey] = cmp.Or(title, payload[titleKey])
body, err = json.Marshal(payload)
@@ -375,15 +264,11 @@ func sendGenericDirectInternal(ctx context.Context, config GenericConfig, title,
}
method := cmp.Or(strings.ToUpper(config.Method), http.MethodPost)
-
req, err := http.NewRequestWithContext(ctx, method, webhookURL.String(), bytes.NewReader(body))
if err != nil {
return fmt.Errorf("failed to create webhook request: %w", err)
}
-
- contentType := cmp.Or(config.ContentType, "application/json")
- req.Header.Set("Content-Type", contentType)
-
+ req.Header.Set("Content-Type", cmp.Or(config.ContentType, "application/json"))
for k, v := range config.CustomHeaders {
req.Header.Set(k, v)
}
diff --git a/backend/pkg/utils/tracing/tracing.go b/backend/pkg/utils/tracing/tracing.go
new file mode 100644
index 000000000..78df00458
--- /dev/null
+++ b/backend/pkg/utils/tracing/tracing.go
@@ -0,0 +1,23 @@
+// Package tracing holds shared helpers for Arcane's own OpenTelemetry spans.
+package tracing
+
+import (
+ "context"
+ "errors"
+
+ "go.opentelemetry.io/otel/codes"
+ "go.opentelemetry.io/otel/trace"
+)
+
+// InstrumentationName identifies spans, metrics, and logs emitted by Arcane code.
+const InstrumentationName = "github.com/getarcaneapp/arcane/backend"
+
+// End records err on span, marks the span failed, and ends it. Context
+// cancellation is treated as a normal outcome.
+func End(span trace.Span, err error) {
+ if err != nil && !errors.Is(err, context.Canceled) {
+ span.RecordError(err)
+ span.SetStatus(codes.Error, err.Error())
+ }
+ span.End()
+}
diff --git a/docker/examples/compose.otel.yaml b/docker/examples/compose.otel.yaml
new file mode 100644
index 000000000..10f2e94ae
--- /dev/null
+++ b/docker/examples/compose.otel.yaml
@@ -0,0 +1,33 @@
+services:
+ arcane:
+ image: ghcr.io/getarcaneapp/manager:latest
+ container_name: arcane
+ ports:
+ - '3552:3552'
+ volumes:
+ - /var/run/docker.sock:/var/run/docker.sock
+ - arcane-data:/app/data
+ environment:
+ - ENCRYPTION_KEY=replace_me_with_a_random_32_char_value
+ - TZ=UTC
+ - OTEL_TRACES_EXPORTER=otlp
+ - OTEL_METRICS_EXPORTER=otlp
+ - OTEL_LOGS_EXPORTER=otlp
+ - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+ # Keep 25% of new traces; traces started by a sampled parent are always kept.
+ # - OTEL_TRACES_SAMPLER=parentbased_traceidratio
+ # - OTEL_TRACES_SAMPLER_ARG=0.25
+ depends_on:
+ - otel-collector
+ restart: unless-stopped
+
+ otel-collector:
+ image: otel/opentelemetry-collector-contrib:latest
+ container_name: otel-collector
+ command: ['--config=/etc/otelcol/config.yaml']
+ volumes:
+ - ./otel/collector.yaml:/etc/otelcol/config.yaml:ro
+ restart: unless-stopped
+
+volumes:
+ arcane-data:
diff --git a/docker/examples/otel/collector.yaml b/docker/examples/otel/collector.yaml
new file mode 100644
index 000000000..c11d34bc0
--- /dev/null
+++ b/docker/examples/otel/collector.yaml
@@ -0,0 +1,38 @@
+receivers:
+ otlp:
+ protocols:
+ http:
+ endpoint: 0.0.0.0:4318
+ grpc:
+ endpoint: 0.0.0.0:4317
+
+processors:
+ memory_limiter:
+ check_interval: 1s
+ limit_mib: 256
+ spike_limit_mib: 64
+ batch: {}
+
+exporters:
+ debug:
+ verbosity: basic
+ # Send to your backend instead of (or alongside) debug output.
+ # otlphttp:
+ # endpoint: https://otlp.example.com
+ # headers:
+ # authorization: Bearer ${env:OTLP_TOKEN}
+
+service:
+ pipelines:
+ traces:
+ receivers: [otlp]
+ processors: [memory_limiter, batch]
+ exporters: [debug]
+ metrics:
+ receivers: [otlp]
+ processors: [memory_limiter, batch]
+ exporters: [debug]
+ logs:
+ receivers: [otlp]
+ processors: [memory_limiter, batch]
+ exporters: [debug]
diff --git a/frontend/package.json b/frontend/package.json
index 3aaf80eb4..235b57f64 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -33,6 +33,20 @@
"@fontsource-variable/montserrat": "^5.3.0",
"@internationalized/date": "^3.12.4",
"@lezer/highlight": "^1.2.5",
+ "@opentelemetry/api": "^1.9.1",
+ "@opentelemetry/api-logs": "^0.223.0",
+ "@opentelemetry/exporter-logs-otlp-http": "^0.223.0",
+ "@opentelemetry/exporter-metrics-otlp-http": "^0.223.0",
+ "@opentelemetry/exporter-trace-otlp-http": "^0.223.0",
+ "@opentelemetry/instrumentation": "^0.223.0",
+ "@opentelemetry/instrumentation-document-load": "^0.68.0",
+ "@opentelemetry/instrumentation-fetch": "^0.223.0",
+ "@opentelemetry/opentelemetry-browser-detector": "^0.223.0",
+ "@opentelemetry/resources": "^2.12.0",
+ "@opentelemetry/sdk-logs": "^0.223.0",
+ "@opentelemetry/sdk-metrics": "^2.12.0",
+ "@opentelemetry/sdk-trace-web": "^2.12.0",
+ "@opentelemetry/semantic-conventions": "^1.43.0",
"@simplewebauthn/browser": "^14.0.0",
"@sveltejs/kit": "3.0.0",
"@tanstack/svelte-query": "^6.3.1",
@@ -61,6 +75,7 @@
"tailwind-variants": "^3.3.1",
"temporal-polyfill": "^1.0.5",
"vaul-svelte": "1.0.0-next.7",
+ "web-vitals": "^6.2.3",
"yaml": "^2.9.1",
"zod": "^4.6.5"
},
diff --git a/frontend/src/hooks.client.ts b/frontend/src/hooks.client.ts
index 2696ac99b..73779f2c5 100644
--- a/frontend/src/hooks.client.ts
+++ b/frontend/src/hooks.client.ts
@@ -1,12 +1,14 @@
import type { HandleClientError } from '@sveltejs/kit/hooks';
import { extractApiErrorMessage } from '#lib/utils/api.js';
+import { recordError } from '#lib/utils/telemetry.js';
export const handleError: HandleClientError = async ({ kind, error }) => {
if (kind !== 'unknown') {
return;
}
+ recordError(error, 'sveltekit.handleError');
if (error && typeof error === 'object' && 'response' in error) {
const responseStatus = (error as { response?: { status?: number } }).response?.status;
const apiErrorMessage = extractApiErrorMessage(error);
diff --git a/frontend/src/lib/services/api-service.ts b/frontend/src/lib/services/api-service.ts
index 07920c0aa..4d0d6cea6 100644
--- a/frontend/src/lib/services/api-service.ts
+++ b/frontend/src/lib/services/api-service.ts
@@ -1,4 +1,4 @@
-import ky, { HTTPError as KyHTTPError, NetworkError, TimeoutError, type Options as KyOptions, type SearchParamsOption } from 'ky';
+import ky, { HTTPError as KyHTTPError, TimeoutError, type Options as KyOptions, type SearchParamsOption } from 'ky';
import { toast } from 'svelte-sonner';
import { m } from '#lib/paraglide/messages.js';
@@ -39,18 +39,13 @@ export class SessionCancelledError extends Error {
// Browsers may surface a logout abort as a plain AbortError instead of the abort reason.
export function isSessionCancelledError(error: unknown): boolean {
if (error instanceof SessionCancelledError) return true;
- return error instanceof Error && error.name === 'AbortError' && sessionControllerInternal.signal.aborted;
+ return error instanceof Error && error.name === 'AbortError' && sessionController.signal.aborted;
}
export class APIError extends Error {
config: InternalRequestConfig & { method?: string; url?: string };
request?: { url: string };
- response?: {
- data: any;
- headers: Headers;
- raw: Response;
- status: number;
- };
+ response?: APIResponse;
status?: number;
constructor(
@@ -63,99 +58,68 @@ export class APIError extends Error {
cause?: unknown;
}
) {
- super(message);
+ super(message, { cause: options.cause });
this.name = options.name ?? 'APIError';
- if (options.cause !== undefined) {
- (this as Error & { cause?: unknown }).cause = options.cause;
- }
this.config = options.config;
this.request = options.requestUrl ? { url: options.requestUrl } : undefined;
- this.response = options.response
- ? {
- data: options.response.data,
- headers: options.response.headers,
- raw: options.response.raw,
- status: options.response.status
- }
- : undefined;
+ this.response = options.response;
this.status = options.response?.status;
}
}
-const problemMessageFactoriesInternal: Record<string, () => string> = {
+const problemMessageFactories: Record<string, () => string> = {
'urn:arcane:problem:password-policy:basic': m.security_password_policy_basic_tooltip,
'urn:arcane:problem:password-policy:standard': m.security_password_policy_standard_tooltip,
'urn:arcane:problem:password-policy:strong': m.security_password_policy_strong_tooltip
};
-function asRecordInternal(value: unknown): Record<string, unknown> | undefined {
+function asRecord(value: unknown): Record<string, unknown> | undefined {
return value !== null && typeof value === 'object' ? (value as Record<string, unknown>) : undefined;
}
-function nonEmptyStringInternal(value: unknown): string | undefined {
+function nonEmptyString(value: unknown): string | undefined {
return typeof value === 'string' && value.trim() ? value : undefined;
}
export function extractServerMessage(data: unknown, includeErrors = false): string | undefined {
- const outer = asRecordInternal(data);
- const innerValue = outer?.['data'] ?? data;
- const stringValue = nonEmptyStringInternal(innerValue);
+ const innerValue = asRecord(data)?.['data'] ?? data;
+ const stringValue = nonEmptyString(innerValue);
if (stringValue) return stringValue;
- const inner = asRecordInternal(innerValue);
+ const inner = asRecord(innerValue);
if (!inner) return undefined;
if (inner['code'] === 'feature_disabled' && inner['feature'] === 'vulnerabilityManagement') {
return m.features_vulnerability_disabled();
}
- const problemType = nonEmptyStringInternal(inner['type']);
- const localizedMessage = problemType ? problemMessageFactoriesInternal[problemType] : undefined;
+ const problemType = nonEmptyString(inner['type']);
+ const localizedMessage = problemType ? problemMessageFactories[problemType] : undefined;
if (localizedMessage) return localizedMessage();
for (const key of ['error', 'message', 'detail', 'error_description']) {
- const message = nonEmptyStringInternal(inner[key]);
+ const message = nonEmptyString(inner[key]);
if (message) return message;
}
- if (includeErrors && Array.isArray(inner['errors']) && inner['errors'].length > 0) {
- const first = inner['errors'][0];
- const firstString = nonEmptyStringInternal(first);
- if (firstString) return firstString;
-
- const firstError = asRecordInternal(first);
- return nonEmptyStringInternal(firstError?.['message']) ?? nonEmptyStringInternal(firstError?.['error']);
- }
-
- return undefined;
+ if (!includeErrors || !Array.isArray(inner['errors'])) return undefined;
+ const first: unknown = inner['errors'][0];
+ const firstError = asRecord(first);
+ return nonEmptyString(first) ?? nonEmptyString(firstError?.['message']) ?? nonEmptyString(firstError?.['error']);
}
-// Batch id applied to mutating requests STARTED synchronously inside
-// runWithActivityBatchId, so the backend groups the resulting activities under
-// one Activity Center row. The scope never spans an await: fn must kick off
-// its requests synchronously and return their promises, so an unrelated action
-// started from any other task can never inherit the batch id.
-let activeActivityBatchIdInternal: string | null = null;
+// Batch id for mutating requests started synchronously inside runWithActivityBatchId; it never spans an await,
+// so fn must start its requests synchronously and return their promises.
+let activeActivityBatchId: string | null = null;
export function runWithActivityBatchId<T>(batchId: string, fn: () => T): T {
- activeActivityBatchIdInternal = batchId;
+ activeActivityBatchId = batchId;
try {
return fn();
} finally {
- activeActivityBatchIdInternal = null;
+ activeActivityBatchId = null;
}
}
-function isBodyInit(value: unknown): value is BodyInit {
- return (
- value instanceof FormData ||
- value instanceof URLSearchParams ||
- value instanceof Blob ||
- value instanceof ArrayBuffer ||
- ArrayBuffer.isView(value) ||
- typeof value === 'string'
- );
-}
-
async function parseResponseBody(response: Response, responseType: APIRequestConfig['responseType'] = 'json'): Promise<any> {
if (responseType === 'blob') {
return response.blob();
@@ -182,187 +146,92 @@ async function parseResponseBody(response: Response, responseType: APIRequestCon
}
}
-async function parseErrorResponseBody(error: KyHTTPError): Promise<any> {
- if (error.data !== undefined) {
- return error.data;
- }
-
- if (error.response.bodyUsed) {
- return undefined;
- }
-
- return parseResponseBody(error.response.clone());
-}
-
-function normalizeUrl(baseURL: string, url: string): string {
- if (/^[a-z]+:\/\//i.test(url)) {
- return url;
- }
-
- const trimmedBase = baseURL.replace(/\/+$/, '');
- const trimmedUrl = url.replace(/^\/+/, '');
-
- if (/^[a-z]+:\/\//i.test(trimmedBase)) {
- return new URL(trimmedUrl, `${trimmedBase}/`).toString();
- }
-
- return `${trimmedBase}/${trimmedUrl}`;
-}
-
-function getRequestPath(url: string, baseURL: string): string {
- let reqUrl = url;
- try {
- if (/^https?:\/\//i.test(reqUrl)) {
- reqUrl = new URL(reqUrl).pathname;
- } else if (baseURL && /^https?:\/\//i.test(baseURL)) {
- reqUrl = new URL(reqUrl.replace(/^\/+/, ''), `${baseURL.replace(/\/+$/, '')}/`).pathname;
- }
- } catch {
- // ignore URL parse errors and fall back to the raw request URL
- }
-
- if (reqUrl.startsWith('/api')) {
- reqUrl = reqUrl.slice(4) || '/';
- }
-
- return reqUrl;
-}
-
let tokenRefreshHandler: (() => Promise<string | null>) | null = null;
-// Set true while a manager self-update / fleet "Update All" is running. During that
-// window the backend briefly restarts and returns version-mismatch 401s; we must not
-// bounce the user to /login — the session is recoverable once the backend is back.
-let upgradeInProgressInternal = false;
+// True while a manager self-update or fleet "Update All" runs; its version-mismatch 401s
+// are recoverable and must not bounce the user to /login.
+let upgradeInProgress = false;
// A confirmed manager restart still needs document recovery after polling ends.
-let upgradeReloadPendingInternal = false;
-let upgradeReloadStartedInternal = false;
-let unauthorizedRedirectStartedInternal = false;
-const skipAuthPathsInternal = [
+let upgradeReloadPending = false;
+let upgradeReloadStarted = false;
+let unauthorizedRedirectStarted = false;
+const skipAuthPaths = [
'/auth/login',
'/auth/logout',
'/auth/refresh',
'/auth/oidc',
- '/auth/oidc/login',
- '/auth/oidc/callback',
'/auth/passkey',
'/auth/mfa',
'/auth/auto-login',
- '/auth/auto-login-config',
'/settings/public'
];
// Each session owns one controller; logout aborts it and keeps it aborted until
// the next sign-in, so protected requests fail fast in between.
-let sessionControllerInternal = new AbortController();
-
-// Requests needed to sign in again, which must work after logout.
-function isSessionFreePathInternal(path: string): boolean {
- if (path.startsWith('/auth/refresh')) return false;
- return (
- skipAuthPathsInternal.some((prefix) => path.startsWith(prefix)) ||
- ['/oidc/url', '/oidc/callback', '/oidc/status'].includes(path) ||
- path.startsWith('/app-version') ||
- path.endsWith('/settings/public')
- );
-}
+let sessionController = new AbortController();
/** Combines a caller signal with the current session, so logout cancels the request. */
export function withSessionSignal(signal?: AbortSignal | null): AbortSignal {
- const session = sessionControllerInternal.signal;
+ const session = sessionController.signal;
return signal ? AbortSignal.any([signal, session]) : session;
}
export function currentSessionSignal(): AbortSignal {
- return sessionControllerInternal.signal;
+ return sessionController.signal;
}
-type UnauthorizedActionInternal = 'none' | 'redirect' | 'reload' | 'retry';
-
-function isAuthPagePathInternal(pathname: string): boolean {
+export function isAuthPagePath(pathname: string): boolean {
return ['/login', '/logout', '/oidc', '/auth/oidc', '/mobile/passkey'].some((prefix) => pathname.startsWith(prefix));
}
+type UnauthorizedAction = 'none' | 'redirect' | 'reload' | 'retry';
+
export async function handleUnauthorizedResponseInternal(
requestPath: string,
retry = false,
serverMsg?: string | null
-): Promise<UnauthorizedActionInternal> {
- const session = sessionControllerInternal.signal;
+): Promise<UnauthorizedAction> {
+ const session = sessionController.signal;
if (typeof window === 'undefined' || retry || session.aborted) {
return 'none';
}
const isVersionMismatch = serverMsg?.toLowerCase().includes('application has been updated') ?? false;
- const isAuthApi = skipAuthPathsInternal.some((path) => requestPath.startsWith(path));
+ const isAuthApi = skipAuthPaths.some((path) => requestPath.startsWith(path));
const pathname = window.location.pathname || '/';
- const isOnAuthPage = isAuthPagePathInternal(pathname);
- if (isAuthApi || isOnAuthPage) {
+ if (isAuthApi || isAuthPagePath(pathname)) {
return 'none';
}
- if (unauthorizedRedirectStartedInternal) return 'redirect';
+ if (unauthorizedRedirectStarted) return 'redirect';
- // During a server self-update the backend briefly returns version-mismatch 401s.
- // Refresh first so the new backend rotates both tokens, then replace the stale
- // frontend document exactly once. Only transport failures are recoverable restart
- // noise; a rejected or missing refresh token is a terminal authentication failure.
- const recoverable = isVersionMismatch || upgradeInProgressInternal;
+ // During a self-update, refresh so the new backend rotates tokens, then reload the stale document once.
+ // Only transport failures are recoverable; a rejected refresh is terminal.
+ const recoverable = isVersionMismatch || upgradeInProgress;
if (!tokenRefreshHandler) return 'none';
const operationResult = await tryCatch(
(async () => {
await tokenRefreshHandler();
- if (!isVersionMismatch || (!upgradeInProgressInternal && !upgradeReloadPendingInternal)) return 'retry';
- if (!upgradeReloadStartedInternal) {
- upgradeReloadStartedInternal = true;
+ if (!isVersionMismatch || (!upgradeInProgress && !upgradeReloadPending)) return 'retry';
+ if (!upgradeReloadStarted) {
+ upgradeReloadStarted = true;
window.location.reload();
}
return 'reload';
})()
);
if (session.aborted) return 'none';
- if (operationResult.error !== null) {
- const error = operationResult.error;
- const isTransientRefreshFailure =
- error instanceof APIError && (error.name === 'NetworkError' || error.name === 'TimeoutError');
- if (recoverable && isTransientRefreshFailure) return 'none';
- if (!unauthorizedRedirectStartedInternal) {
- unauthorizedRedirectStartedInternal = true;
- window.location.replace(`/login?redirect=${encodeURIComponent(pathname)}`);
- }
- return 'redirect';
- } else {
- return operationResult.data;
- }
-}
+ if (operationResult.error === null) return operationResult.data;
-function buildRequestOptionsInternal(method: string, data: unknown, config: InternalRequestConfig): KyOptions {
- const headers = new Headers(config.headers);
- const isMutation = !['GET', 'HEAD'].includes(method.toUpperCase());
- if (activeActivityBatchIdInternal && isMutation && !headers.has('X-Arcane-Batch-Id')) {
- headers.set('X-Arcane-Batch-Id', activeActivityBatchIdInternal);
+ const error = operationResult.error;
+ const isTransientRefreshFailure = error instanceof APIError && (error.name === 'NetworkError' || error.name === 'TimeoutError');
+ if (recoverable && isTransientRefreshFailure) return 'none';
+ if (!unauthorizedRedirectStarted) {
+ unauthorizedRedirectStarted = true;
+ window.location.replace(`/login?redirect=${encodeURIComponent(pathname)}`);
}
- const options: KyOptions = {
- cache: config.cache,
- method,
- headers,
- retry: config.retry ?? 0,
- searchParams: config.params,
- signal: config.signal,
- timeout: config.timeout ?? false
- };
-
- const bodyData = data !== undefined ? data : config.data;
- if (bodyData !== undefined && bodyData !== null) {
- if (isBodyInit(bodyData)) {
- options.body = bodyData;
- } else {
- options.json = bodyData;
- }
- }
-
- return options;
+ return 'redirect';
}
class APIClient {
@@ -389,23 +258,58 @@ class APIClient {
config: InternalRequestConfig = {}
): Promise<APIResponse<T>> {
const baseURL = config.baseURL ?? this.defaults.baseURL;
- const requestUrl = normalizeUrl(baseURL, url);
- const requestConfig = {
- ...config,
- baseURL,
- method,
- url
- };
- const session = isSessionFreePathInternal(getRequestPath(url, baseURL)) ? undefined : sessionControllerInternal.signal;
+ const requestUrl = /^[a-z]+:\/\//i.test(url) ? url : `${baseURL.replace(/\/+$/, '')}/${url.replace(/^\/+/, '')}`;
+ const requestConfig = { ...config, baseURL, method, url };
+ let requestPath = requestUrl;
+ try {
+ requestPath = new URL(requestUrl, 'http://localhost').pathname;
+ } catch {
+ // Fall back to the raw request URL.
+ }
+ if (requestPath.startsWith('/api')) requestPath = requestPath.slice(4) || '/';
+
+ // Requests needed to sign in again must keep working after logout.
+ const isSessionFree =
+ !requestPath.startsWith('/auth/refresh') &&
+ (skipAuthPaths.some((prefix) => requestPath.startsWith(prefix)) ||
+ ['/oidc/url', '/oidc/callback', '/oidc/status'].includes(requestPath) ||
+ requestPath.startsWith('/app-version') ||
+ requestPath.endsWith('/settings/public'));
+ const session = isSessionFree ? undefined : sessionController.signal;
const signal = session ? withSessionSignal(config.signal) : config.signal;
+ const upperMethod = method.toUpperCase();
const operationResult = await tryCatch(
(async () => {
- const options = buildRequestOptionsInternal(method, data, { ...config, signal });
+ const headers = new Headers(config.headers);
+ if (activeActivityBatchId && !['GET', 'HEAD'].includes(upperMethod) && !headers.has('X-Arcane-Batch-Id')) {
+ headers.set('X-Arcane-Batch-Id', activeActivityBatchId);
+ }
+ const options: KyOptions = {
+ cache: config.cache,
+ method,
+ headers,
+ retry: config.retry ?? 0,
+ searchParams: config.params,
+ signal,
+ timeout: config.timeout ?? false
+ };
+ const body = data ?? config.data;
+ if (
+ body instanceof FormData ||
+ body instanceof URLSearchParams ||
+ body instanceof Blob ||
+ body instanceof ArrayBuffer ||
+ ArrayBuffer.isView(body) ||
+ typeof body === 'string'
+ ) {
+ options.body = body as BodyInit;
+ } else if (body !== undefined && body !== null) {
+ options.json = body;
+ }
const response = await this.client(requestUrl, options);
- const parsed =
- method.toUpperCase() === 'HEAD' ? undefined : await parseResponseBody(response.clone(), config.responseType);
+ const parsed = upperMethod === 'HEAD' ? undefined : await parseResponseBody(response.clone(), config.responseType);
if (session?.aborted) throw new SessionCancelledError();
return {
data: parsed as T,
@@ -415,84 +319,32 @@ class APIClient {
};
})()
);
- if (operationResult.error !== null) {
- const error = operationResult.error;
- if (session?.aborted) {
- throw new SessionCancelledError();
- }
- if (error instanceof KyHTTPError) {
- return this.handleHttpError<T>(error, method, url, data, config, requestConfig, requestUrl, session);
- }
-
- if (error instanceof TimeoutError) {
- throw new APIError('Request timed out', {
- cause: error,
- config: requestConfig,
- name: 'TimeoutError',
- requestUrl
- });
- }
-
- if (error instanceof NetworkError) {
- throw new APIError(error.message, {
- cause: error,
- config: requestConfig,
- name: 'NetworkError',
- requestUrl
- });
- }
-
- if (error instanceof Error) {
- throw new APIError(error.message, {
- cause: error,
- config: requestConfig,
- name: error.name || 'APIError',
- requestUrl
- });
- }
+ if (operationResult.error === null) return operationResult.data;
- throw new APIError('Unknown error', {
+ const error = operationResult.error;
+ if (session?.aborted) throw new SessionCancelledError();
+ if (!(error instanceof KyHTTPError)) {
+ let message = 'Unknown error';
+ if (error instanceof TimeoutError) message = 'Request timed out';
+ else if (error instanceof Error) message = error.message;
+ throw new APIError(message, {
cause: error,
config: requestConfig,
+ name: (error instanceof Error && error.name) || undefined,
requestUrl
});
- } else {
- return operationResult.data;
}
- }
- private async handleHttpError<T>(
- error: KyHTTPError,
- method: string,
- url: string,
- data: unknown,
- config: InternalRequestConfig,
- requestConfig: InternalRequestConfig & { baseURL: string; method: string; url: string },
- requestUrl: string,
- session: AbortSignal | undefined
- ): Promise<APIResponse<T>> {
const errorResponse = error.response;
- const parsed = await parseErrorResponseBody(error);
+ const parsed =
+ error.data !== undefined || errorResponse.bodyUsed ? error.data : await parseResponseBody(errorResponse.clone());
if (session?.aborted) throw new SessionCancelledError();
- const response: APIResponse = {
- data: parsed,
- headers: errorResponse.headers,
- raw: errorResponse,
- status: errorResponse.status
- };
-
- if (errorResponse.status === 401 && typeof window !== 'undefined' && !config._retry) {
- const action = await handleUnauthorizedResponseInternal(
- getRequestPath(url, requestConfig.baseURL),
- !!config._retry,
- extractServerMessage(parsed)
- );
+
+ if (errorResponse.status === 401) {
+ const action = await handleUnauthorizedResponseInternal(requestPath, config._retry, extractServerMessage(parsed));
if (session?.aborted) throw new SessionCancelledError();
if (action === 'retry') {
- return this.performRequest<T>(method, url, data, {
- ...config,
- _retry: true
- });
+ return this.performRequest<T>(method, url, data, { ...config, _retry: true });
}
if (action === 'redirect' || action === 'reload') {
return new Promise(() => {});
@@ -509,7 +361,7 @@ class APIClient {
config: requestConfig,
name: 'HTTPError',
requestUrl,
- response
+ response: { data: parsed, headers: errorResponse.headers, raw: errorResponse, status: errorResponse.status }
});
}
@@ -551,35 +403,31 @@ abstract class BaseAPIService {
// Toggled by the update flows (Update All / local update center) so an in-progress
// self-update restart is treated as a recoverable reconnect, not a logout.
static setUpgradeInProgress(value: boolean) {
- if (!upgradeReloadPendingInternal && (!value || !upgradeInProgressInternal)) {
- upgradeReloadStartedInternal = false;
+ if (!upgradeReloadPending && (!value || !upgradeInProgress)) {
+ upgradeReloadStarted = false;
}
- upgradeInProgressInternal = value;
+ upgradeInProgress = value;
}
/** Cancels the current session's work and blocks protected requests until beginSession. */
static endSession() {
- sessionControllerInternal.abort(new SessionCancelledError());
+ sessionController.abort(new SessionCancelledError());
}
static beginSession() {
- if (sessionControllerInternal.signal.aborted) {
- sessionControllerInternal = new AbortController();
+ if (sessionController.signal.aborted) {
+ sessionController = new AbortController();
}
}
static confirmUpgradeRestart() {
- upgradeReloadPendingInternal = true;
+ upgradeReloadPending = true;
}
protected async handleResponse<T>(promise: Promise<APIResponse>): Promise<T> {
- const response = await promise;
- const payload = response.data;
- const extracted =
- payload && typeof payload === 'object' && 'data' in payload && (payload as any).data !== undefined
- ? (payload as any).data
- : payload;
- return extracted as T;
+ const payload = (await promise).data;
+ const inner = asRecord(payload)?.['data'];
+ return (inner !== undefined ? inner : payload) as T;
}
}
diff --git a/frontend/src/lib/types/settings.ts b/frontend/src/lib/types/settings.ts
index 04c5d0d03..8b003b285 100644
--- a/frontend/src/lib/types/settings.ts
+++ b/frontend/src/lib/types/settings.ts
@@ -80,6 +80,9 @@ export type Settings = {
autoInjectEnv: boolean;
backupVolumeName?: string;
edgeMTLSManagerCAAvailable?: boolean;
+ frontendTracingEnabled?: boolean;
+ frontendMetricsEnabled?: boolean;
+ frontendLogsEnabled?: boolean;
authLocalEnabled: boolean;
authSessionTimeout: number;
diff --git a/frontend/src/lib/utils/settings.svelte.ts b/frontend/src/lib/utils/settings.svelte.ts
index 8f8f25499..cf93c966d 100644
--- a/frontend/src/lib/utils/settings.svelte.ts
+++ b/frontend/src/lib/utils/settings.svelte.ts
@@ -12,7 +12,6 @@ const LOCAL_SETTING_KEYS = new Set([
'oidcAutoRedirectToProvider',
'oidcMergeAccounts',
'oidcSkipTlsVerify',
- 'oidcAutoRedirectToProvider',
'oidcClientId',
'oidcClientSecret',
'oidcIssuerUrl',
@@ -21,6 +20,9 @@ const LOCAL_SETTING_KEYS = new Set([
'oidcProviderName',
'oidcProviderLogoUrl',
'edgeMTLSManagerCAAvailable',
+ 'frontendTracingEnabled',
+ 'frontendMetricsEnabled',
+ 'frontendLogsEnabled',
'experimentalFeaturesEnabled',
'developmentBrandingEnabled',
'apnsEnabled'
@@ -46,33 +48,39 @@ export function preventDefault<T extends Event>(fn: (event: T) => unknown) {
}
export function createForm<T extends z.ZodType<Record<string, unknown>>>(schema: T, initialValues: z.infer<T>) {
- const inputs = $state<FormInputs<z.infer<T>>>(initializeInputs(initialValues));
+ const schemaKeys = Object.keys(schema instanceof z.ZodObject ? schema.shape : {});
+ const inputs = $state(
+ Object.fromEntries(
+ schemaKeys
+ .filter((key) => Object.hasOwn(initialValues, key))
+ .map((key) => [key, { value: initialValues[key as keyof z.infer<T>], error: null }])
+ ) as FormInputs<z.infer<T>>
+ );
let errors = $state.raw<z.ZodError<z.infer<T>>>();
- function initializeInputs(values: z.infer<T>): FormInputs<z.infer<T>> {
- const fields = {} as FormInputs<z.infer<T>>;
- const schemaShape = schema instanceof z.ZodObject ? schema.shape : {};
- for (const key of Object.keys(schemaShape) as (keyof z.infer<T>)[]) {
- if (Object.prototype.hasOwnProperty.call(values, key)) {
- fields[key] = { value: values[key], error: null };
- }
- }
- return fields;
+ function parse() {
+ const values = Object.fromEntries(
+ Object.entries(inputs).map(([key, input]) => {
+ const value: unknown = input.value;
+ if (typeof value === 'string') return [key, value.trim()];
+ if (Array.isArray(value)) return [key, value.map((item: unknown) => (typeof item === 'string' ? item.trim() : item))];
+ return [key, value];
+ })
+ );
+ return { values, result: schema.safeParse(values) };
}
function validate() {
- const values = Object.fromEntries(Object.entries(inputs).map(([key, input]) => [key, input.value]));
- const result = schema.safeParse(values);
+ const { result } = parse();
errors = result.error;
for (const key of Object.keys(inputs) as (keyof z.infer<T>)[]) {
inputs[key].error = result.error?.issues.find((issue) => issue.path[0] === key)?.message ?? null;
}
- return result.success ? data() : null;
+ return result.success ? (result.data as z.infer<T>) : null;
}
function data() {
- const values = Object.fromEntries(Object.entries(inputs).map(([key, input]) => [key, trimValue(input.value)]));
- const result = schema.safeParse(values);
+ const { values, result } = parse();
return (result.success ? result.data : values) as z.infer<T>;
}
@@ -86,12 +94,6 @@ export function createForm<T extends z.ZodType<Record<string, unknown>>>(schema:
inputs[key].value = value;
}
- function trimValue(value: unknown): unknown {
- if (typeof value === 'string') return value.trim();
- if (Array.isArray(value)) return value.map((item: unknown) => (typeof item === 'string' ? item.trim() : item));
- return value;
- }
-
return {
schema,
inputs,
diff --git a/frontend/src/lib/utils/telemetry.ts b/frontend/src/lib/utils/telemetry.ts
new file mode 100644
index 000000000..fab0a28a6
--- /dev/null
+++ b/frontend/src/lib/utils/telemetry.ts
@@ -0,0 +1,237 @@
+import type { NavigationType } from '$app/navigation';
+import type { Context, Counter, Histogram, Span } from '@opentelemetry/api';
+import type { LogRecord } from '@opentelemetry/api-logs';
+import type { Resource } from '@opentelemetry/resources';
+import type { Metric } from 'web-vitals';
+
+type Signal = 'traces' | 'metrics' | 'logs';
+
+const SERVICE_NAME = 'arcane-frontend';
+// Keeps each export well under the relay's 512 KiB request limit.
+const MAX_EXPORT_BATCH_SIZE = 32;
+// Aborted fetches and logout cancellations are expected, not failures.
+const CANCELLED_ERRORS = ['AbortError', 'SessionCancelledError'];
+
+// Query strings and fragments can carry OIDC codes and tokens, so they never leave the browser.
+function stripUrlQuery(value: string): string {
+ return value.replace(/(https?:\/\/[^\s?#"']*)[?#][^\s"']*/g, '$1').replace(/^(\/[^\s?#]*)[?#].*$/, '$1');
+}
+
+let otel: typeof import('@opentelemetry/api') | undefined;
+let resource: Promise<Resource> | undefined;
+const initializations: Partial<Record<Signal, Promise<void>>> = {};
+const providers: { forceFlush(): Promise<void> }[] = [];
+let route = '/';
+let navigationStart = 0;
+let navigated = false;
+let pageSpan: Span | undefined;
+let pageContext: Context | undefined;
+let navigationMetrics: { count: Counter; duration: Histogram } | undefined;
+let emitErrorLog: ((record: LogRecord) => void) | undefined;
+
+const initializers: Record<Signal, (resource: Resource) => Promise<void>> = {
+ traces: async (resource) => {
+ const [api, sdk, exporter, instrumentation, documentLoad, fetch] = await Promise.all([
+ import('@opentelemetry/api'),
+ import('@opentelemetry/sdk-trace-web'),
+ import('@opentelemetry/exporter-trace-otlp-http'),
+ import('@opentelemetry/instrumentation'),
+ import('@opentelemetry/instrumentation-document-load'),
+ import('@opentelemetry/instrumentation-fetch')
+ ]);
+ // Spans started outside any active span (fetches, logs) belong to the current pageview.
+ class PageContextManager extends sdk.StackContextManager {
+ override active(): Context {
+ const active = super.active();
+ return pageContext && !api.trace.getSpan(active) ? pageContext : active;
+ }
+ }
+ const provider = new sdk.WebTracerProvider({
+ resource,
+ spanProcessors: [
+ {
+ onStart: () => {},
+ onEnd: (span) => {
+ for (const attributes of [span.attributes, ...span.events.map((event) => event.attributes ?? {})]) {
+ for (const [key, value] of Object.entries(attributes)) {
+ if (key === 'url.query') delete attributes[key];
+ else if (typeof value === 'string') attributes[key] = stripUrlQuery(value);
+ }
+ }
+ },
+ forceFlush: () => Promise.resolve(),
+ shutdown: () => Promise.resolve()
+ },
+ new sdk.BatchSpanProcessor(new exporter.OTLPTraceExporter({ url: '/api/telemetry/traces' }), {
+ maxExportBatchSize: MAX_EXPORT_BATCH_SIZE
+ })
+ ]
+ });
+ provider.register({ contextManager: new PageContextManager() });
+ providers.push(provider);
+ instrumentation.registerInstrumentations({
+ tracerProvider: provider,
+ instrumentations: [
+ new documentLoad.DocumentLoadInstrumentation(),
+ new fetch.FetchInstrumentation({
+ // The SSE stream never ends, and tracing it would keep the connection open after the app cancels it.
+ ignoreUrls: [/\/api\/telemetry\//, /\/api\/stream(\?|$)/],
+ applyCustomAttributesOnSpan: (span, _request, result) => {
+ if (result instanceof Response || (result instanceof Error && CANCELLED_ERRORS.includes(result.name))) return;
+ span.setAttribute('error.type', result instanceof Error ? result.name : 'Error');
+ span.setStatus({ code: api.SpanStatusCode.ERROR, message: stripUrlQuery(result.message) });
+ }
+ })
+ ]
+ });
+ otel = api;
+ // Tracing can finish loading after the first navigation; start the pageview it missed.
+ if (navigated) startPageView(route, navigationStart);
+ },
+ metrics: async (resource) => {
+ const [sdk, exporter, vitals] = await Promise.all([
+ import('@opentelemetry/sdk-metrics'),
+ import('@opentelemetry/exporter-metrics-otlp-http'),
+ import('web-vitals')
+ ]);
+ const provider = new sdk.MeterProvider({
+ resource,
+ readers: [
+ new sdk.PeriodicExportingMetricReader({
+ exporter: new exporter.OTLPMetricExporter({ url: '/api/telemetry/metrics' }),
+ exportIntervalMillis: 60_000
+ })
+ ]
+ });
+ providers.push(provider);
+ const meter = provider.getMeter(SERVICE_NAME);
+ navigationMetrics = {
+ count: meter.createCounter('browser.navigation.count', { description: 'Client-side navigations' }),
+ duration: meter.createHistogram('browser.navigation.duration', {
+ unit: 'ms',
+ description: 'Time until a navigation completes'
+ })
+ };
+ const webVitals: Record<Metric['name'], Histogram> = {
+ CLS: meter.createHistogram('browser.web_vital.cls', {
+ advice: { explicitBucketBoundaries: [0, 0.01, 0.05, 0.1, 0.15, 0.25, 0.5, 1] }
+ }),
+ FCP: meter.createHistogram('browser.web_vital.fcp', { unit: 'ms' }),
+ INP: meter.createHistogram('browser.web_vital.inp', { unit: 'ms' }),
+ LCP: meter.createHistogram('browser.web_vital.lcp', { unit: 'ms' }),
+ TTFB: meter.createHistogram('browser.web_vital.ttfb', { unit: 'ms' })
+ };
+ const report = (metric: Metric) =>
+ webVitals[metric.name].record(metric.value, { 'http.route': route, 'web_vital.rating': metric.rating });
+ vitals.onCLS(report);
+ vitals.onFCP(report);
+ vitals.onINP(report);
+ vitals.onLCP(report);
+ vitals.onTTFB(report);
+ },
+ logs: async (resource) => {
+ const [api, sdk, exporter] = await Promise.all([
+ import('@opentelemetry/api-logs'),
+ import('@opentelemetry/sdk-logs'),
+ import('@opentelemetry/exporter-logs-otlp-http')
+ ]);
+ const provider = new sdk.LoggerProvider({
+ resource,
+ processors: [
+ new sdk.BatchLogRecordProcessor({
+ exporter: new exporter.OTLPLogExporter({ url: '/api/telemetry/logs' }),
+ maxExportBatchSize: MAX_EXPORT_BATCH_SIZE
+ })
+ ]
+ });
+ providers.push(provider);
+ const logger = provider.getLogger(SERVICE_NAME);
+ emitErrorLog = (record) => logger.emit({ ...record, severityNumber: api.SeverityNumber.ERROR, severityText: 'ERROR' });
+ window.addEventListener('error', (event) => recordError(event.error ?? event.message, 'window.error'));
+ window.addEventListener('unhandledrejection', (event) => recordError(event.reason, 'window.unhandledrejection'));
+ }
+};
+
+// Lazily loads the SDK for each enabled signal once; nothing loads while every signal is off.
+export function setTelemetry(enabled: Record<Signal, boolean>, version: string): Promise<void> {
+ if (typeof window === 'undefined') return Promise.resolve();
+ const signals = (Object.keys(initializers) as Signal[]).filter((signal) => enabled[signal]);
+ if (signals.length === 0) return Promise.resolve();
+
+ const loaded = (resource ??= Promise.all([
+ import('@opentelemetry/resources'),
+ import('@opentelemetry/semantic-conventions'),
+ import('@opentelemetry/opentelemetry-browser-detector'),
+ import('#lib/utils/streaming.js')
+ ]).then(([resources, semconv, detector, streaming]) => {
+ const flush = (event: Event) => {
+ if (event.type === 'pagehide') pageSpan?.end();
+ else if (document.visibilityState !== 'hidden') return;
+ for (const provider of providers) void provider.forceFlush().catch(() => {});
+ };
+ window.addEventListener('visibilitychange', flush);
+ window.addEventListener('pagehide', flush);
+ return resources
+ .resourceFromAttributes({
+ [semconv.ATTR_SERVICE_NAME]: SERVICE_NAME,
+ [semconv.ATTR_SERVICE_VERSION]: version,
+ // Each page session owns its cumulative metric series, so tabs never overwrite each other.
+ [semconv.ATTR_SERVICE_INSTANCE_ID]: streaming.streamCacheBuster()
+ })
+ .merge(resources.detectResources({ detectors: [detector.browserDetector] }));
+ }));
+ // A failed chunk download is forgotten so the next navigation retries instead of staying off.
+ loaded.catch(() => {
+ if (resource === loaded) resource = undefined;
+ });
+ return Promise.all(
+ signals.map(
+ (signal) =>
+ (initializations[signal] ??= loaded.then(initializers[signal]).catch(() => {
+ delete initializations[signal];
+ }))
+ )
+ ).then(() => {});
+}
+
+// Starts the pageview span that parents this navigation's requests.
+export function startPageView(routeId: string | null | undefined, startTime = performance.now()): void {
+ route = (routeId ?? '').replace(/\/\([^)]*\)/g, '') || '/';
+ navigationStart = startTime;
+ navigated = true;
+ if (!otel) return;
+ pageSpan?.end();
+ pageSpan = otel.trace
+ .getTracer(SERVICE_NAME)
+ .startSpan(`pageview ${route}`, { root: true, startTime, attributes: { 'http.route': route } });
+ pageContext = otel.trace.setSpan(otel.ROOT_CONTEXT, pageSpan);
+}
+
+// The initial 'enter' navigation starts its pageview here and is measured from the document's time origin.
+export function finishNavigation(routeId: string | null | undefined, type: NavigationType): void {
+ if (type === 'enter') startPageView(routeId, 0);
+ const attributes = { 'http.route': route, 'navigation.type': type };
+ navigationMetrics?.count.add(1, attributes);
+ navigationMetrics?.duration.record(performance.now() - navigationStart, attributes);
+}
+
+// Emits a sanitized error log with the current route and trace context.
+export function recordError(error: unknown, source: string): void {
+ if (!emitErrorLog || (error instanceof Error && CANCELLED_ERRORS.includes(error.name))) return;
+ try {
+ const message = stripUrlQuery(error instanceof Error ? error.message : String(error)).slice(0, 1_000);
+ emitErrorLog({
+ eventName: 'exception',
+ body: message,
+ attributes: {
+ 'exception.type': error instanceof Error ? error.name : typeof error,
+ 'exception.message': message,
+ 'exception.stacktrace': error instanceof Error && error.stack ? stripUrlQuery(error.stack).slice(0, 4_000) : undefined,
+ 'http.route': route,
+ 'error.source': source
+ }
+ });
+ } catch {
+ // Telemetry must never affect the app.
+ }
+}
diff --git a/frontend/src/routes/+layout.svelte b/frontend/src/routes/+layout.svelte
index c937507e0..b206cb592 100644
--- a/frontend/src/routes/+layout.svelte
+++ b/frontend/src/routes/+layout.svelte
@@ -1,6 +1,6 @@
<script lang="ts">
- import { browser, dev } from '$app/env';
- import { refreshAll } from '$app/navigation';
+ import { dev } from '$app/env';
+ import { afterNavigate, beforeNavigate, refreshAll } from '$app/navigation';
import { navigating, page } from '$app/state';
import { QueryClientProvider } from '@tanstack/svelte-query';
import { SvelteQueryDevtools } from '@tanstack/svelte-query-devtools';
@@ -17,9 +17,10 @@
import { IsMobile } from '#lib/hooks/is-mobile.svelte.js';
import { IsTablet } from '#lib/hooks/is-tablet.svelte.js';
import { m } from '#lib/paraglide/messages.js';
+ import { isAuthPagePath } from '#lib/services/api-service.js';
import settingsStore from '#lib/stores/config-store.svelte.js';
import { environmentStore } from '#lib/stores/environment.store.svelte.js';
- import { cn } from '#lib/utils.js';
+ import { finishNavigation, startPageView } from '#lib/utils/telemetry.js';
import type { LayoutProps } from './$types';
@@ -28,40 +29,31 @@
let { data, children }: LayoutProps = $props();
onMount(() => {
- if (!dev && browser && 'serviceWorker' in navigator) {
+ if (!dev && 'serviceWorker' in navigator) {
navigator.serviceWorker.register('/service-worker.js', { type: 'module' });
}
});
- const settings = $derived(data.settings);
+ beforeNavigate((navigation) => {
+ if (!navigation.willUnload) startPageView(navigation.to?.route.id);
+ });
+
+ afterNavigate((navigation) => finishNavigation(navigation.to?.route.id, navigation.type));
const isMobile = new IsMobile();
const isTablet = new IsTablet();
- // Mirror the effective layout onto <html> so CSS (sidebar visibility, etc.) can follow the
- // per-device layout preference instead of raw viewport breakpoints. `$effect.pre` runs
- // before the first DOM commit, so the attribute is present at first paint.
+ // Mirror the per-device layout onto <html> for CSS; `$effect.pre` sets it before first paint.
$effect.pre(() => {
document.documentElement.setAttribute('data-layout', isMobile.current ? 'mobile' : 'desktop');
});
- const isNavigating = $derived(navigating.type !== null);
- const isAuthPage = $derived(
- String(page.url.pathname).startsWith('/login') ||
- String(page.url.pathname).startsWith('/logout') ||
- String(page.url.pathname).startsWith('/oidc') ||
- String(page.url.pathname).startsWith('/mobile/passkey')
- );
+ const isAuthPage = $derived(isAuthPagePath(page.url.pathname));
- const autoLoginEnabled = $derived(settingsStore.autoLoginEnabled.current);
const showPasswordChangeDialog = $derived(
- !!(data.user && data.user.requiresPasswordChange && !isAuthPage && !autoLoginEnabled)
+ !!data.user?.requiresPasswordChange && !isAuthPage && !settingsStore.autoLoginEnabled.current
);
- function handlePasswordChangeSuccess() {
- refreshAll();
- }
-
const resourceName = $derived.by((): string | undefined => {
switch (page.route.id) {
case '/(app)/projects/[projectId]':
@@ -94,13 +86,13 @@
</svelte:head>
<QueryClientProvider client={data.queryClient}>
- <div class={cn('flex min-h-dvh flex-col', 'bg-transparent')}>
- {#if !settings && data.user && page.route.id !== '/(app)/environments/[id]'}
+ <div class="flex min-h-dvh flex-col bg-transparent">
+ {#if !data.settings && data.user && page.route.id !== '/(app)/environments/[id]'}
<Error message={m.error_occurred()} showButton={true} />
{:else}
<Tooltip.Provider>
{@render children()}
- <FirstLoginPasswordDialog open={showPasswordChangeDialog} onSuccess={handlePasswordChangeSuccess} />
+ <FirstLoginPasswordDialog open={showPasswordChangeDialog} onSuccess={() => refreshAll()} />
{#if dev}
<SvelteQueryDevtools />
{/if}
@@ -123,5 +115,5 @@
}}
/>
<ConfirmDialog />
- <LoadingIndicator active={isNavigating} thickness="h-1.5" />
+ <LoadingIndicator active={navigating.type !== null} thickness="h-1.5" />
</QueryClientProvider>
diff --git a/frontend/src/routes/+layout.ts b/frontend/src/routes/+layout.ts
index fcc5ec44c..4bd92ace5 100644
--- a/frontend/src/routes/+layout.ts
+++ b/frontend/src/routes/+layout.ts
@@ -17,10 +17,10 @@ import { featureStore } from '#lib/stores/features.store.svelte.js';
import userStore from '#lib/stores/user-store.svelte.js';
import { versionStore } from '#lib/stores/version.store.svelte.js';
import type { PermissionsManifest, User } from '#lib/types/auth.js';
-import type { SearchPaginationSortRequest } from '#lib/types/shared.js';
import { isAuthRejectionError } from '#lib/utils/api.js';
import { getAuthRedirectPath, userHasPermission } from '#lib/utils/auth.js';
import { getEffectiveLandingPage } from '#lib/utils/navigation.js';
+import { setTelemetry } from '#lib/utils/telemetry.js';
import { tryCatch } from '#lib/utils/try-catch.js';
import type { LayoutLoad } from './$types';
@@ -44,14 +44,8 @@ const queryClient = new QueryClient({
let authenticatedUserId: string | null | undefined;
-// The layout load re-runs on every navigation and re-checks the session. A
-// transient failure of that check (network blip, 5xx, 429) must not sign the
-// user out of the SPA: that clears every cache, bounces through /login and
-// lands on the landing page. Only an explicit rejection means the session is
-// gone; anything else keeps the user we already know about.
-function resolveUserAfterLoadFailureInternal(error: unknown): User | null {
- if (isAuthRejectionError(error) || isSessionCancelledError(error)) return null;
- return userStore.current;
+function dataOrNull<T>(promise: Promise<T>): Promise<T | null> {
+ return tryCatch(promise).then((result) => (result.error ? null : result.data));
}
export const load: LayoutLoad = async ({ url }) => {
@@ -76,9 +70,11 @@ export const load: LayoutLoad = async ({ url }) => {
})
: Promise.resolve(null);
let [user, autoLoginConfig] = await Promise.all([
- tryCatch(userService.getCurrentUser()).then((result) =>
- result.error ? resolveUserAfterLoadFailureInternal(result.error) : result.data
- ),
+ // Only an explicit rejection signs the user out; transient check failures keep the known user.
+ tryCatch(userService.getCurrentUser()).then((result): User | null => {
+ if (!result.error) return result.data;
+ return isAuthRejectionError(result.error) || isSessionCancelledError(result.error) ? null : userStore.current;
+ }),
autoLoginConfigRequest
]);
@@ -115,17 +111,8 @@ export const load: LayoutLoad = async ({ url }) => {
let permissionsManifestLoadFailed = false;
if (user) {
// Initialize environment store (required for settings service)
- const environmentRequestOptions: SearchPaginationSortRequest = {
- pagination: {
- page: 1,
- limit: -1
- }
- };
-
- const environmentsRequest = tryCatch(environmentManagementService.getEnvironments(environmentRequestOptions));
- const permissionsManifestRequest = tryCatch(roleService.getPermissionsManifest()).then((result) =>
- result.error ? null : result.data
- );
+ const environmentsRequest = tryCatch(environmentManagementService.getEnvironments({ pagination: { page: 1, limit: -1 } }));
+ const permissionsManifestRequest = dataOrNull(roleService.getPermissionsManifest());
const environments = await environmentsRequest;
if (!environments.error) {
await environmentStore.initialize(environments.data.data);
@@ -135,23 +122,15 @@ export const load: LayoutLoad = async ({ url }) => {
await environmentStore.initialize([]);
}
- const settingsRequest = userHasPermission(user, 'settings:read')
- ? tryCatch(settingsService.getSettings()).then(async (result) => {
- if (!result.error) return result.data;
- const publicSettings = await tryCatch(settingsService.getPublicSettings(environmentStore.selected?.id ?? '0'));
- return publicSettings.error ? null : publicSettings.data;
- })
- : tryCatch(settingsService.getPublicSettings(environmentStore.selected?.id ?? '0')).then((result) =>
- result.error ? null : result.data
- );
+ const settingsRequest = (
+ userHasPermission(user, 'settings:read') ? dataOrNull(settingsService.getSettings()) : Promise.resolve(null)
+ ).then((loaded) => loaded ?? dataOrNull(settingsService.getPublicSettings(environmentStore.selected?.id ?? '0')));
featureStore.connect(queryClient);
const currentEnvironmentId = await environmentStore.getCurrentEnvironmentId();
const featuresRequest = featureStore.refresh(currentEnvironmentId);
// Optional discovery: skip it without swarm:read and never toast a denial.
const swarmStatusRequest = userHasPermission(user, 'swarm:read', currentEnvironmentId)
- ? tryCatch(swarmService.getSwarmStatus(currentEnvironmentId, { suppressAccessDeniedToast: true })).then((result) =>
- result.error ? null : result.data
- )
+ ? dataOrNull(swarmService.getSwarmStatus(currentEnvironmentId, { suppressAccessDeniedToast: true }))
: Promise.resolve(null);
const [loadedSettings, loadedSwarmStatus, loadedPermissionsManifest] = await Promise.all([
settingsRequest,
@@ -159,9 +138,7 @@ export const load: LayoutLoad = async ({ url }) => {
permissionsManifestRequest
]);
// Keep recovery/settings pages reachable while a selected agent is offline.
- settings =
- loadedSettings ??
- (await tryCatch(settingsService.getPublicSettings()).then((result) => (result.error ? null : result.data)));
+ settings = loadedSettings ?? (await dataOrNull(settingsService.getPublicSettings()));
await featuresRequest;
swarmEnabled = loadedSwarmStatus?.enabled;
permissionsManifest = loadedPermissionsManifest;
@@ -171,7 +148,7 @@ export const load: LayoutLoad = async ({ url }) => {
await environmentStore.initialize([]);
// Try to fetch public settings for login page configuration
- settings = await tryCatch(settingsService.getPublicSettings()).then((result) => (result.error ? null : result.data));
+ settings = await dataOrNull(settingsService.getPublicSettings());
}
if (settings) {
@@ -180,6 +157,18 @@ export const load: LayoutLoad = async ({ url }) => {
const versionInformation = await versionInformationRequest;
versionStore.seed(versionInformation);
+ // Waiting lets the first page view be traced, but a stalled SDK download must never hold up the app.
+ await Promise.race([
+ setTelemetry(
+ {
+ traces: settings?.frontendTracingEnabled ?? false,
+ metrics: settings?.frontendMetricsEnabled ?? false,
+ logs: settings?.frontendLogsEnabled ?? false
+ },
+ versionInformation.currentVersion
+ ),
+ new Promise((resolve) => setTimeout(resolve, 2_000))
+ ]);
const redirectPath = getAuthRedirectPath(
url.pathname,
diff --git a/go.work.sum b/go.work.sum
index a2c6e1b08..240b6b557 100644
--- a/go.work.sum
+++ b/go.work.sum
@@ -2327,8 +2327,6 @@ github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxza
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ=
github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI=
-github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos=
-github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
@@ -2874,10 +2872,6 @@ go.mongodb.org/mongo-driver/v2 v2.8.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzyb
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
-go.opentelemetry.io/contrib/bridges/otelslog v0.20.1 h1:5sHc4ToTFjfSZCtGAAM6jPunICAmJX73htv372T4ipc=
-go.opentelemetry.io/contrib/bridges/otelslog v0.20.1/go.mod h1:oa6kgvyz/3GYW04dohd0++xJIH4xdQY8PAbpeCMaM8M=
-go.opentelemetry.io/contrib/bridges/prometheus v0.71.0 h1:9qgxsFLskbDMXl8WMqThoF6w8yGJgCumn9qRc67OmnI=
-go.opentelemetry.io/contrib/bridges/prometheus v0.71.0/go.mod h1:2rCjF4F2siiTeLCzJsaGZ3CK0XIoimCSKXEBPdv+Je0=
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs=
go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts=
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE=
@@ -2888,8 +2882,6 @@ go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmf
go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs=
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw=
go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s=
-go.opentelemetry.io/contrib/exporters/autoexport v0.71.0 h1:VCsJbp0YLyPtx2tu5Vgv2a2/qLoaMCj8hT2uZ34+Mx0=
-go.opentelemetry.io/contrib/exporters/autoexport v0.71.0/go.mod h1:qxZqn7e10f6ajmMCkg/47rMS7qQYfaOl2nj/4aytHUQ=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc=
@@ -2903,6 +2895,12 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
go.opentelemetry.io/contrib/propagators/envcar v0.71.0 h1:r7TFdi1B3O5ai64tONEuBTyxeQaLKVYf75vLiV7ZsDc=
go.opentelemetry.io/contrib/propagators/envcar v0.71.0/go.mod h1:uAnx0mCFm63a47qI4f/hcoI72gVYrGdxEBL+6c500Ns=
+go.opentelemetry.io/contrib/propagators/jaeger v1.19.0 h1:mGrx7XEAE+7ybCLM0T6iRl/jUTuHg6qKUJAtsAlknec=
+go.opentelemetry.io/contrib/propagators/jaeger v1.19.0/go.mod h1:cHWVPhYWMZOanEf1qexqMIRhr4TKVjZWBKwZTL/tdR4=
+go.opentelemetry.io/contrib/propagators/opencensus v0.44.0 h1:foCvUO/MnWlwcS2GFCR/27k25UwolVwp3uYMGBguEvo=
+go.opentelemetry.io/contrib/propagators/opencensus v0.44.0/go.mod h1:IUCrK+YXh4EO4dbh/l9NbWUHValpE3odollsVTjfpc4=
+go.opentelemetry.io/contrib/propagators/ot v1.19.0 h1:vODRLMlKN4ApM8ri0UDk8nnEeISuwxpf67sE7PmOHhE=
+go.opentelemetry.io/contrib/propagators/ot v1.19.0/go.mod h1:S2Uc7th2ZmLiHu0lrCmDCgTQ/y5Nbbis+TNjR1jjm4Q=
go.opentelemetry.io/contrib/zpages v0.62.0 h1:9fUYTLmrK0x/lweM2uM+BOx069jLx8PxVqWhegGJ9Bo=
go.opentelemetry.io/contrib/zpages v0.62.0/go.mod h1:C8kXoiC1Ytvereztus2R+kqdSa6W/MZ8FfS8Zwj+LiM=
go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
@@ -2916,12 +2914,12 @@ go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOX
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
+go.opentelemetry.io/otel/bridge/opencensus v0.41.0 h1:VBpeaTbrvLFHvRtsyCJXjsTaicBNrAFdmctiN1k6WNI=
+go.opentelemetry.io/otel/bridge/opencensus v0.41.0/go.mod h1:yCQB5IKRhgjlbTLc91+ixcZc2/8BncGGJ+CS3dZJwtY=
go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4=
go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI=
-go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 h1:Bu39F5tzJct+f2IZbB8989fwyTps3c8e7EsUQsz+vs8=
-go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0/go.mod h1:dJUwod88EsFgYCqrDHaSPzhiY9pBUpt0d85/qSfua7k=
-go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 h1:lYk7RmxdLK865qLwibroNGldHa1U7SWKYYvNjlK7PIo=
-go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0/go.mod h1:6GvlND0H0xdUJanOtIAn0xfwLkauh1tmsYEEVSMDdqY=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A=
@@ -2941,14 +2939,8 @@ go.opentelemetry.io/otel/exporters/prometheus v0.66.0 h1:vkrK8PAznv2NKt2r+kdu252
go.opentelemetry.io/otel/exporters/prometheus v0.66.0/go.mod h1:V/UB6D3vMF/UBOL5igAsAYnk1nG/bzYYTzvsB16cy7o=
go.opentelemetry.io/otel/exporters/prometheus v0.67.0 h1:7IefDa35e6V3NoiqIeLDMDxMFyZDk5qcoC0Ax4cC16E=
go.opentelemetry.io/otel/exporters/prometheus v0.67.0/go.mod h1:nsPI1awTg5Vmg1YrommL2mVarVGlqc4yXOoKAkPRD0c=
-go.opentelemetry.io/otel/exporters/prometheus v0.68.0 h1:QOf2IftqQwITVRJpnn0M7M9ZCbgWfxz4P7i9C9yc2N4=
-go.opentelemetry.io/otel/exporters/prometheus v0.68.0/go.mod h1:bgSvqu2TWGXiz7yr5UTMfObH8oqxJWHTnubQ3ef9BO4=
-go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0 h1:kvMAiLEudKmk+CSG+iYbU8vTUGNNDaf/V09OO5lrTwI=
-go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0/go.mod h1:L9Dlksri+MdT1cb2gIiA1cJJYW3Y92ipvDjNxYEyaDI=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0 h1:lSZHgNHfbmQTPfuTmWVkEu8J8qXaQwuV30pjCcAUvP8=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw=
-go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 h1:PR9eAf7o0dQs3hshZNZpE9aW2dXWX/KdDf6pJilVD3U=
-go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0/go.mod h1:2Z4KyNdH1uuzivdinyfGsxzNNT/Rl45pwtVwfYVI0xk=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 h1:UGZ1QwZWY67Z6BmckTU+9Rxn04m2bD3gD6Mk0OIOCPk=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0/go.mod h1:fcwWuDuaObkkChiDlhEpSq9+X1C0omv+s5mBtToAQ64=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE=
@@ -2957,10 +2949,6 @@ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0 h1:bl2S7Ubua0Nms+D
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0/go.mod h1:L0hRV50XdVIODHUfWEqGRCXQvj2rV82STVo12FMFBU0=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo=
-go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 h1:KdRxPiAoMptR3vfWzvjjvutTsSiwbC2uG0496rzZNfo=
-go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0/go.mod h1:K/qSA+3G7Eovxi4K09wzrAgkWRnosS0DAOZeEpve7sM=
-go.opentelemetry.io/otel/log v0.22.0 h1:5DBNnfvaJ6CVdkJ+Jle8Tzs50aSSv49TXGj9XRsEYw0=
-go.opentelemetry.io/otel/log v0.22.0/go.mod h1:gzOt/R67vF2GniAqWu8Qv0SXy89f71muHcrkz76PCdc=
go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
@@ -2978,8 +2966,6 @@ go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWm
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
-go.opentelemetry.io/otel/sdk/log v0.22.0 h1:PRL+s6P63XT4E/bheEflopPUpVxuvANqZwtt89yhoGk=
-go.opentelemetry.io/otel/sdk/log v0.22.0/go.mod h1:JNp0sBELrjCTcu5W3GzABVypeU6vDJjBS+X0JISuz+g=
go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
@@ -3492,6 +3478,10 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gorm.io/driver/clickhouse v0.7.0 h1:BCrqvgONayvZRgtuA6hdya+eAW5P2QVagV3OlEp1vtA=
+gorm.io/driver/clickhouse v0.7.0/go.mod h1:TmNo0wcVTsD4BBObiRnCahUgHJHjBIwuRejHwYt3JRs=
+gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
+gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 041663a08..a8e860e1a 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -207,7 +207,7 @@ importers:
vite-plus:
specifier: 'catalog:'
email-templates:
dependencies:
@@ -244,7 +244,7 @@ importers:
vite-plus:
specifier: 'catalog:'
frontend:
dependencies:
@@ -293,12 +293,54 @@ importers:
'@lezer/highlight':
specifier: ^1.2.5
version: 1.2.5
+ '@opentelemetry/api':
+ specifier: ^1.9.1
+ version: 1.9.1
+ '@opentelemetry/api-logs':
+ specifier: ^0.223.0
+ version: 0.223.0
+ '@opentelemetry/exporter-logs-otlp-http':
+ specifier: ^0.223.0
+ version: 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/exporter-metrics-otlp-http':
+ specifier: ^0.223.0
+ version: 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/exporter-trace-otlp-http':
+ specifier: ^0.223.0
+ version: 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/instrumentation':
+ specifier: ^0.223.0
+ version: 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/instrumentation-document-load':
+ specifier: ^0.68.0
+ version: 0.68.0(@opentelemetry/[email protected])
+ '@opentelemetry/instrumentation-fetch':
+ specifier: ^0.223.0
+ version: 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/opentelemetry-browser-detector':
+ specifier: ^0.223.0
+ version: 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources':
+ specifier: ^2.12.0
+ version: 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-logs':
+ specifier: ^0.223.0
+ version: 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-metrics':
+ specifier: ^2.12.0
+ version: 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace-web':
+ specifier: ^2.12.0
+ version: 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions':
+ specifier: ^1.43.0
+ version: 1.43.0
'@simplewebauthn/browser':
specifier: ^14.0.0
version: 14.0.0
'@sveltejs/kit':
specifier: ^3.0.0
'@tanstack/svelte-query':
specifier: ^6.3.1
version: 6.3.1([email protected](@typescript-eslint/[email protected]))
@@ -331,7 +373,7 @@ importers:
version: 0.7.2
bits-ui:
specifier: ^2.19.5
clsx:
specifier: ^2.1.1
version: 2.1.1
@@ -340,7 +382,7 @@ importers:
version: 8.6.0([email protected](@typescript-eslint/[email protected]))
formsnap:
specifier: ^2.0.1
ky:
specifier: ^2.1.0
version: 2.1.0
@@ -352,7 +394,7 @@ importers:
version: 1.1.0([email protected](@typescript-eslint/[email protected]))
runed:
specifier: ^0.37.1
svelte:
specifier: ^5.57.2
version: 5.57.2(@typescript-eslint/[email protected])
@@ -377,6 +419,9 @@ importers:
vaul-svelte:
specifier: 1.0.0-next.7
version: 1.0.0-next.7([email protected](@typescript-eslint/[email protected]))
+ web-vitals:
+ specifier: ^6.2.3
+ version: 6.2.3
yaml:
specifier: ^2.9.1
version: 2.9.1
@@ -395,7 +440,7 @@ importers:
'@sveltejs/adapter-static':
specifier: ^4.0.0
'@sveltejs/vite-plugin-svelte':
specifier: ^7.3.1
@@ -437,10 +482,10 @@ importers:
version: 1.8.1([email protected](@typescript-eslint/[email protected]))
svelte-toolbelt:
specifier: ^0.10.6
sveltekit-superforms:
specifier: ^3.0.0
tailwindcss:
specifier: ^4.3.3
version: 4.3.3
@@ -461,7 +506,7 @@ importers:
vite-plus:
specifier: 'catalog:'
tests:
devDependencies:
@@ -482,7 +527,7 @@ importers:
vite-plus:
specifier: 'catalog:'
packages:
@@ -1139,6 +1184,120 @@ packages:
'@emnapi/core': ^1.7.1 || ^2.0.0-alpha.4
'@emnapi/runtime': ^1.7.1 || ^2.0.0-alpha.4
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-Oepm6etVPFCn0VWB1t833iOTgz1eYBDUPxcrOR/XvwzD1+Ag9VzSPSrrKAFNJ/haatj+8CXbG47pZXvdNQKDJw==}
+ engines: {node: '>=8.0.0'}
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==}
+ engines: {node: '>=8.0.0'}
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-1HsSAuvT4/my0QrXWsyXFdWjKaPeCHsismKZgEEIb8NK13LP6DrMRlsUbvD/0F9mDWK3xDom9AMv3IscxgOnOQ==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.0.0 <1.10.0'
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-YQh9+t/JmV9OeRkoLhPb4yvcrWCX6lAOmXFos/JKoD+KITZYZoxf0acnwAk6YDI9O7fX68O2ka38hx1G9exl8w==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': ^1.3.0
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-hNC8yai/+xCNyTuiOy6JFepXXA/sb/2jCzQRY7c2cUWz6owaidihlHUpO20B9/dj8JKPNo8rcn7ikTy7Cj+JaQ==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': ^1.3.0
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-N8NWZaoQuF9e307S6xoVoI1lvxOOFkmBnynD42r02CPCfxoC95cJExAKBFrTD4C5iuhqthbzOZJN/W29ez4o/A==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': ^1.3.0
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-pkUPmaW0Mlim8Eetfflhc1oSdvs6SGS2oqGw+eOqNUGWqqpOkCzT/s6STvvMk4wzo2L8OIOMFTntDRatVRg71g==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': ^1.3.0
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-D2bRIBvw8IM1AWE/5BZB6kEemmCP4H89Gd2Lg4Yf44rlwwFxJ9Y25GJEwb4slW3r5lPq8YDj0nIU5fGiFCovtQ==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': ^1.3.0
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-tttBo88TbwK1zpsH6VeJoxLQopxOjW3cnrP50fsQJRZftkSKDDwjolXXBvow2jy0LKXlNRkDQh0NT6yI/q+BEQ==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': ^1.3.0
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-96KHQMIpTO6hhFjPKKxnQS+yQCemo2P0BbpVon56X+GZYmk8mPBwn8P0D9lB9RncITb5GQVnVA9jNFu9tgNRlQ==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': ^1.3.0
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-HlJMrJXjOqdQeoYNDHEvFTpaY2p9uWDK1GPl5AZ4wI5easzfoppTduOv47M8eiF+4ew42vfI/Xvt7THYgESnRw==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': ^1.3.0
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-oxmmPL5/MbSsO8K588V9n/XLIe7FZTK30Fo+/U/lAnt0ZIK/hoPM03n2WpSTZVbUBGTUE7aLyxnAzh7nE2PD4Q==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': ^1.3.0
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-+rSRoOdln6NhXTVcBdHSlZxJDNzwxadeweK2ltZO3tVtfp6aklABc06xclDk6ovRR+EiGgIcSbh/6NgjFhnb3A==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.3.0 <1.10.0'
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-ApiAfUiUh6fmJt1GbsBuoeDn8D8IL42UvGyPUr8o1Z89ZeIiOCE1rZ78KkIUjEX3OV8Oorl9ez5Qjo43ZynDyw==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.4.0 <1.10.0'
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-mVMaEHwCUJuY3S4Esn+nT0yQA3qCEGh+SOPJsXygU0Gl453WcsDNjtEa8EfdewMwoLyr1tlofB4fXytVpiXuow==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.9.0 <1.10.0'
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-JOERgZCxRM06FEJ5kEWxlSRChah6MmXDx3KwCruqEv/iEyLkTokCSemf2y1w8oIHtfrr8EVTpzebq/31vn2Cpg==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.3.0 <1.10.0'
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-y0hCzbnBHScmioF0drJOyaPk1sQf2wwbIuy3rWlM0Fcx8mLfPDvFsQ7tJ+zMm8yAqI06rSlFKkL7ZpcrFOqAdQ==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.0.0 <1.10.0'
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-ImVuRQ6faOsTi1Wna88s2avgrrf/iDgXPigIeynkUhjstMuoQUDOJXgkez2j1mlU4btjXcXBGa+q7qKFYR3GsA==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.3.0 <1.10.0'
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==}
+ engines: {node: '>=14'}
+
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-zUx54ocAoj2Pw2vVnUVp5dJgvp5SindWwekmoB2AUjZJGEeBnCcZ1FiqohWLvdqq0xIHm+fQUpPf5hffU3U4kA==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.3.0 <1.10.0'
+
'@oxc-minify/[email protected]':
resolution: {integrity: sha512-gk1jzky1vLeOcK7w9Ib2RT/UooZ7O21s8YSeXCsy4tzsmguKb3Cbsv/Ra2fKhMp4mKeKCzvC1Xa41OfSBWfWdQ==}
engines: {node: ^20.19.0 || >=22.12.0}
@@ -2685,6 +2844,9 @@ packages:
resolution: {integrity: sha512-+6vJA3L98yv+IdfKGZHBNiGW5KHn22e/JwID0Strsz8h4S/csAu/OuICwxrg44k5MRiZHWIo8XXuJgQTriRP4w==}
+ resolution: {integrity: sha512-lmNyBzi6iYiqyrExTctfmjd8tInx+dHu9PBomTfH5RXS53c27JMFJgzNv5TyAnm3PgGf+MLy0+Ws41/HlVxdKQ==}
+
resolution: {integrity: sha512-AwNusCCam51q703dW82x95tOqQp6oC9HNUl724KxJJOfnKscI8dOloXFgyez7LbTTKWuRBA37FScqVbJEoq8Yw==}
@@ -2929,6 +3091,9 @@ packages:
resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==}
+ resolution: {integrity: sha512-yWGe1O/j+almnV4hOYrmZo8u58JdnmXEMneRQAYkWtqSiT86yoMIEKF4rpqEDj52tbYl3xceBD3DQIBwcsC9Pw==}
+
resolution: {integrity: sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==}
engines: {node: '>=18'}
@@ -3125,6 +3290,10 @@ packages:
resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==}
engines: {node: '>= 4'}
+ resolution: {integrity: sha512-WseIA/4o56+GYO3RmlOPvejYi771OFeJEeK/ejROsr9bdjW+E5NXTOD1Di8TL40Qgg3JJX9aa97Hf8iiSVIKZw==}
+ engines: {node: '>=18'}
+
resolution: {integrity: sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==}
@@ -3467,6 +3636,9 @@ packages:
peerDependencies:
svelte: ^5.27.0
+ resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==}
+
resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==}
engines: {node: '>=4'}
@@ -3714,6 +3886,10 @@ packages:
resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==}
engines: {node: '>=0.10.0'}
+ resolution: {integrity: sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==}
+ engines: {node: '>=9.3.0 || >=8.10.0 <9.0.0'}
+
resolution: {integrity: sha512-9q8oUiBYeXIDLWNK5DfCWlkL0EW3oGbk845VdKlPeia28l751VpfesaB/+7pI6rnbx1I6rqoZ2fZxptOJLxILA==}
peerDependencies:
@@ -4250,6 +4426,9 @@ packages:
resolution: {integrity: sha512-4Nsy8K5Tr6SPDH9jhKJOHf7ChDrc1zufZTVSF7x72hwuEXBqxqk9G6cK+K2NRUtB3iELRJqjXb4JPDMBjMTl2Q==}
engines: {node: ^22.22.2 || ^24.15.0 || >=26.0.0}
+ resolution: {integrity: sha512-Dmy+vdO1UPMqXiAtZSNayhl4f794Bxfd94W4HIj2HlYtczEQfEOon969memlcpcHrTuOKLa5baFjX7WbgmOq+A==}
+
resolution: {integrity: sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==}
engines: {node: '>=20'}
@@ -5011,6 +5190,140 @@ snapshots:
'@tybys/wasm-util': 0.10.4
optional: true
+ '@opentelemetry/[email protected]':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+
+ '@opentelemetry/[email protected]': {}
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/semantic-conventions': 1.43.0
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/otlp-exporter-base': 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-logs': 0.223.0(@opentelemetry/[email protected])
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/otlp-exporter-base': 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-metrics': 2.12.0(@opentelemetry/[email protected])
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/otlp-exporter-base': 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace': 2.12.0(@opentelemetry/[email protected])
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/instrumentation': 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace-web': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.43.0
+ transitivePeerDependencies:
+ - supports-color
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/instrumentation': 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.43.0
+ '@opentelemetry/web-common': 0.223.0(@opentelemetry/[email protected])
+ transitivePeerDependencies:
+ - supports-color
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/api-logs': 0.223.0
+ import-in-the-middle: 3.5.2
+ require-in-the-middle: 8.0.1
+ transitivePeerDependencies:
+ - supports-color
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.43.0
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/[email protected])
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/api-logs': 0.223.0
+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-logs': 0.223.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-metrics': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace': 2.12.0(@opentelemetry/[email protected])
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.43.0
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/api-logs': 0.223.0
+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.43.0
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.43.0
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace-base': 2.12.0(@opentelemetry/[email protected])
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.43.0
+
+ '@opentelemetry/[email protected]': {}
+
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.43.0
+
'@oxc-minify/[email protected]':
optional: true
@@ -5394,11 +5707,11 @@ snapshots:
dependencies:
acorn: 8.19.0
dependencies:
dependencies:
'@rolldown/pluginutils': 1.0.1
'@standard-schema/spec': 1.1.0
@@ -5412,6 +5725,7 @@ snapshots:
svelte: 5.57.2(@typescript-eslint/[email protected])
optionalDependencies:
+ '@opentelemetry/api': 1.9.1
typescript: '@typescript/[email protected]'
'@sveltejs/[email protected]': {}
@@ -5797,7 +6111,7 @@ snapshots:
'@testing-library/dom': 10.4.2
'@testing-library/user-event': 14.6.7(@testing-library/[email protected])
transitivePeerDependencies:
- bufferutil
- msw
@@ -5809,7 +6123,7 @@ snapshots:
'@testing-library/dom': 10.4.2
'@testing-library/user-event': 14.6.7(@testing-library/[email protected])
transitivePeerDependencies:
- bufferutil
- msw
@@ -5826,7 +6140,7 @@ snapshots:
pngjs: 7.0.0
sirv: 3.0.2
tinyrainbow: 3.2.0
ws: 8.22.0
transitivePeerDependencies:
- bufferutil
@@ -5844,7 +6158,7 @@ snapshots:
pngjs: 7.0.0
sirv: 3.0.2
tinyrainbow: 3.2.0
ws: 8.22.0
transitivePeerDependencies:
- bufferutil
@@ -5891,7 +6205,7 @@ snapshots:
pathe: 2.0.3
sirv: 3.0.2
tinyrainbow: 3.2.0
'@vitest/[email protected]':
dependencies:
@@ -6145,15 +6459,15 @@ snapshots:
dependencies:
'@floating-ui/core': 1.8.0
'@floating-ui/dom': 1.8.0
'@internationalized/date': 3.12.4
esm-env: 1.2.2
svelte: 5.57.2(@typescript-eslint/[email protected])
tabbable: 6.5.0
transitivePeerDependencies:
- '@sveltejs/kit'
@@ -6185,6 +6499,8 @@ snapshots:
+
dependencies:
'@types/validator': 13.15.10
@@ -6417,6 +6733,8 @@ snapshots:
+
optionalDependencies:
'@esbuild/aix-ppc64': 0.28.2
@@ -6600,11 +6918,11 @@ snapshots:
dependencies:
svelte: 5.57.2(@typescript-eslint/[email protected])
svelte-toolbelt: 0.5.0([email protected](@typescript-eslint/[email protected]))
optional: true
@@ -6658,6 +6976,12 @@ snapshots:
+ dependencies:
+ cjs-module-lexer: 2.3.0
+ es-module-lexer: 3.0.3
+ module-details-from-path: 1.0.4
+
@@ -6938,6 +7262,8 @@ snapshots:
svelte: 5.57.2(@typescript-eslint/[email protected])
svelte-toolbelt: 0.7.1([email protected](@typescript-eslint/[email protected]))
+
@@ -7024,7 +7350,7 @@ snapshots:
'@oxc-parser/binding-win32-x64-msvc': 0.148.0
optional: true
dependencies:
tinypool: 2.2.0
optionalDependencies:
@@ -7048,9 +7374,9 @@ snapshots:
'@oxfmt/binding-win32-ia32-msvc': 0.72.0
'@oxfmt/binding-win32-x64-msvc': 0.72.0
svelte: 5.57.2(@typescript-eslint/[email protected])
dependencies:
tinypool: 2.2.0
optionalDependencies:
@@ -7074,7 +7400,7 @@ snapshots:
'@oxfmt/binding-win32-ia32-msvc': 0.72.0
'@oxfmt/binding-win32-x64-msvc': 0.72.0
svelte: 5.57.2(@typescript-eslint/[email protected])
optionalDependencies:
@@ -7085,7 +7411,7 @@ snapshots:
'@oxlint-tsgolint/win32-arm64': 7.0.2003
'@oxlint-tsgolint/win32-x64': 7.0.2003
optionalDependencies:
'@oxlint/binding-android-arm-eabi': 1.87.0
'@oxlint/binding-android-arm64': 1.87.0
@@ -7107,9 +7433,9 @@ snapshots:
'@oxlint/binding-win32-ia32-msvc': 1.87.0
'@oxlint/binding-win32-x64-msvc': 1.87.0
oxlint-tsgolint: 7.0.2003
optionalDependencies:
'@oxlint/binding-android-arm-eabi': 1.87.0
'@oxlint/binding-android-arm64': 1.87.0
@@ -7131,7 +7457,7 @@ snapshots:
'@oxlint/binding-win32-ia32-msvc': 1.87.0
'@oxlint/binding-win32-x64-msvc': 1.87.0
oxlint-tsgolint: 7.0.2003
dependencies:
@@ -7291,6 +7617,13 @@ snapshots:
+ dependencies:
+ debug: 4.4.3
+ module-details-from-path: 1.0.4
+ transitivePeerDependencies:
+ - supports-color
+
dependencies:
esm-env: 1.2.2
@@ -7311,23 +7644,23 @@ snapshots:
esm-env: 1.2.2
svelte: 5.57.2(@typescript-eslint/[email protected])
dependencies:
dequal: 2.0.3
esm-env: 1.2.2
lz-string: 1.5.0
svelte: 5.57.2(@typescript-eslint/[email protected])
optionalDependencies:
dependencies:
dequal: 2.0.3
esm-env: 1.2.2
lz-string: 1.5.0
svelte: 5.57.2(@typescript-eslint/[email protected])
optionalDependencies:
zod: 4.6.5
@@ -7491,10 +7824,10 @@ snapshots:
runed: 0.28.0([email protected](@typescript-eslint/[email protected]))
svelte: 5.57.2(@typescript-eslint/[email protected])
dependencies:
clsx: 2.1.1
style-to-object: 1.0.14
svelte: 5.57.2(@typescript-eslint/[email protected])
transitivePeerDependencies:
@@ -7540,9 +7873,9 @@ snapshots:
transitivePeerDependencies:
- '@typescript-eslint/types'
dependencies:
devalue: 6.0.2
esm-env: 1.2.2
memoize-weak: 1.0.2
@@ -7762,7 +8095,7 @@ snapshots:
svelte: 5.57.2(@typescript-eslint/[email protected])
svelte-toolbelt: 0.7.1([email protected](@typescript-eslint/[email protected]))
dependencies:
'@oxc-project/types': 0.153.0
'@oxlint/plugins': 1.79.0
@@ -7773,11 +8106,11 @@ snapshots:
'@vitest/snapshot': 5.0.3
'@vitest/spy': 5.0.3
'@vitest/utils': 5.0.3
oxlint-tsgolint: 7.0.2003
optionalDependencies:
'@voidzero-dev/vite-plus-darwin-arm64': 1.1.0
'@voidzero-dev/vite-plus-darwin-x64': 1.1.0
@@ -7822,7 +8155,7 @@ snapshots:
- vue-tsc
- yaml
dependencies:
'@oxc-project/types': 0.153.0
'@oxlint/plugins': 1.79.0
@@ -7833,11 +8166,11 @@ snapshots:
'@vitest/snapshot': 5.0.3
'@vitest/spy': 5.0.3
'@vitest/utils': 5.0.3
oxlint-tsgolint: 7.0.2003
optionalDependencies:
'@voidzero-dev/vite-plus-darwin-arm64': 1.1.0
'@voidzero-dev/vite-plus-darwin-x64': 1.1.0
@@ -7886,7 +8219,7 @@ snapshots:
optionalDependencies:
dependencies:
'@types/chai': 5.2.3
@@ -7903,6 +8236,7 @@ snapshots:
why-is-node-running: 3.2.1
optionalDependencies:
+ '@opentelemetry/api': 1.9.1
'@types/node': 26.6.4
'@vitest/ui': 5.0.3([email protected])
@@ -7910,7 +8244,7 @@ snapshots:
transitivePeerDependencies:
- msw
dependencies:
'@types/chai': 5.2.3
@@ -7927,6 +8261,7 @@ snapshots:
why-is-node-running: 3.2.1
optionalDependencies:
+ '@opentelemetry/api': 1.9.1
'@types/node': 26.6.4
'@vitest/ui': 5.0.3([email protected])
@@ -7940,6 +8275,8 @@ snapshots:
dependencies:
xml-name-validator: 5.0.0
+