Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
feat: add OpenTelemetry metrics, traces, and logs (#4367)
Kyle Mendellf3bfc284b1f0
diff --git a/backend/api/api.go b/backend/api/api.goindex 40a250a31..2ab300eac 100644--- a/backend/api/api.go+++ b/backend/api/api.go@@ -1,11 +1,13 @@package apiimport (+ "cmp""context""encoding/json/v2""io""maps""net/http"+ "path""reflect""strings"@@ -60,13 +62,13 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability""github.com/getarcaneapp/arcane/backend/v2/internal/webhook""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/userctx")const (arcaneTypesPrefix = "github.com/getarcaneapp/arcane/types/v2/"dockerSDKPrefix = "github.com/moby/moby"- // scalarDocsHTML returns the HTML template for Scalar API documentation.scalarDocsHTML = `<!doctype html><html><head>@@ -115,93 +117,46 @@ var jsonV2Format = huma.Format{},}-// customSchemaNamer creates unique schema names using package prefix for types-// from github.com/getarcaneapp/arcane/types/v2 to avoid conflicts between packages that have-// types with the same name (e.g., image.Summary vs env.Summary).+// customSchemaNamer prefixes Arcane and Docker type schemas with their package name+// so same-named types (e.g. image.Summary vs env.Summary) do not collide.func customSchemaNamer(t reflect.Type, hint string) string {name := huma.DefaultSchemaNamer(t, hint)typeStr := t.String()- pkgPath := packagePathForType(t)- shortPkg := shortPackageFromTypeString(typeStr)+ elem := t+ for elem.Kind() == reflect.Pointer {+ elem = elem.Elem()+ }+ pkgPath := elem.PkgPath()+ pkgName := path.Base(pkgPath)+ shortPkg, _, _ := strings.Cut(typeStr, ".")+ isArcaneType := strings.HasPrefix(pkgPath, arcaneTypesPrefix)- if pkgName, ok := arcanePackageName(pkgPath); ok {- name = pkgName + name- } else if dockerPrefix, localOk := dockerSchemaPrefix(pkgPath, shortPkg); localOk {- name = dockerPrefix + name+ if isArcaneType {+ name = kit.Capitalize(pkgName) + name+ } else {+ prefix := dockerSchemaPrefixes[shortPkg]+ if strings.Contains(pkgPath, dockerSDKPrefix) {+ prefix = cmp.Or(dockerSchemaPrefixes[pkgName], prefix)+ }+ name = prefix + name}// Preserve the published name after moving the volume backup response to types.if pkgPath == arcaneTypesPrefix+"volume" && strings.TrimLeft(typeStr, "*") == "volume.Backup" {name = "DockerVolumeVolumeBackup"}- return qualifyGenericArcaneArgumentsInternal(pkgPath, typeStr, name)-}--func packagePathForType(t reflect.Type) string {- for t.Kind() == reflect.Pointer {- t = t.Elem()- }-- return t.PkgPath()-}--func shortPackageFromTypeString(typeStr string) string {- before, _, ok := strings.Cut(typeStr, ".")- return kit.Ternary(!ok, "", before)-}--func arcanePackageName(pkgPath string) (string, bool) {- if !strings.HasPrefix(pkgPath, arcaneTypesPrefix) {- return "", false- }-- parts := strings.Split(pkgPath, "/")- if len(parts) == 0 {- return "", false- }-- pkg := parts[len(parts)-1]- if pkg == "" {- return "", false- }-- return kit.Capitalize(pkg), true-}--func dockerSchemaPrefix(pkgPath, shortPkg string) (string, bool) {- if strings.Contains(pkgPath, dockerSDKPrefix) {- parts := strings.Split(pkgPath, "/")- last := parts[len(parts)-1]- if prefix, ok := dockerSchemaPrefixes[last]; ok {- return prefix, true- }- }-- prefix, ok := dockerSchemaPrefixes[shortPkg]- if !ok {- return "", false- }-- return prefix, true-}--func qualifyGenericArcaneArgumentsInternal(pkgPath, typeName, schemaName string) string {- openBracket := strings.IndexByte(typeName, '[')- if !strings.HasPrefix(pkgPath, arcaneTypesPrefix) || openBracket < 0 {- return schemaName- }- outerPackage := strings.TrimPrefix(pkgPath, arcaneTypesPrefix)- if _, after, found := strings.CutLast(outerPackage, "/"); found {- outerPackage = after+ // Qualify Arcane generic arguments with their package unless it matches the outer type's package.+ openBracket := strings.IndexByte(typeStr, '[')+ if !isArcaneType || openBracket < 0 {+ return name}-- plainSchemaName := huma.DefaultSchemaNamer(reflect.TypeFor[struct{}](), typeName)- schemaPrefix, ok := strings.CutSuffix(schemaName, plainSchemaName)+ anonymous := reflect.TypeFor[struct{}]()+ schemaPrefix, ok := strings.CutSuffix(name, huma.DefaultSchemaNamer(anonymous, typeStr))if !ok {- return schemaName+ return name}- rewrittenTypeName := typeName+ rewrittenTypeName := typeStrsearchOffset := openBracket + 1for {prefixIndex := strings.Index(rewrittenTypeName[searchOffset:], arcaneTypesPrefix)@@ -216,28 +171,18 @@ func qualifyGenericArcaneArgumentsInternal(pkgPath, typeName, schemaName string)break}- innerPackage := afterPrefix[:separator]- if _, after, found := strings.CutLast(innerPackage, "/"); found {- innerPackage = after- }-- replacement := ""- if innerPackage != outerPackage {- replacement = kit.Capitalize(innerPackage)- }-+ innerPackage := path.Base(afterPrefix[:separator])+ replacement := kit.Ternary(innerPackage != pkgName, kit.Capitalize(innerPackage), "")argumentTypeIndex := prefixIndex + len(arcaneTypesPrefix) + separator + 1rewrittenTypeName = rewrittenTypeName[:prefixIndex] + replacement + rewrittenTypeName[argumentTypeIndex:]searchOffset = prefixIndex + len(replacement)}- return schemaPrefix + huma.DefaultSchemaNamer(reflect.TypeFor[struct{}](), rewrittenTypeName)+ return schemaPrefix + huma.DefaultSchemaNamer(anonymous, rewrittenTypeName)}// HandlerDeps contains the services required to register HTTP API handlers.-//-// It intentionally contains only dependencies consumed by SetupAPI,-// registerHandlersInternal, and the authentication middleware.+// It contains only dependencies consumed by SetupAPI and the authentication middleware.type HandlerDeps struct {fx.In@@ -292,22 +237,9 @@ func SetupAPI(e *echo.Echo, apiGroup *echo.Group, appCtx handlerutil.ActivityApphumaConfig.Formats["application/json"] = jsonV2FormathumaConfig.Formats["json"] = jsonV2FormathumaConfig.Info.Description = "Modern Docker Management, Designed for Everyone"-// Disable default docs path - we'll use Scalar insteadhumaConfig.DocsPath = ""-- // Configure servers for OpenAPI spec- if cfg.AppUrl != "" {- humaConfig.Servers = []*huma.Server{- {URL: cfg.AppUrl + "/api"},- }- } else {- humaConfig.Servers = []*huma.Server{- {URL: "/api"},- }- }-- // Configure security schemes+ humaConfig.Servers = []*huma.Server{{URL: cfg.AppUrl + "/api"}}humaConfig.Components.SecuritySchemes = map[string]*huma.SecurityScheme{"BearerAuth": {Type: "http",@@ -326,82 +258,20 @@ func SetupAPI(e *echo.Echo, apiGroup *echo.Group, appCtx handlerutil.ActivityApp{"BearerAuth": {}},{"ApiKeyAuth": {}},}-- // Use custom schema namer to avoid conflicts between types with same name- // from different packages (e.g., image.Summary vs env.Summary)humaConfig.Components.Schemas = huma.NewMapRegistry("#/components/schemas/", customSchemaNamer)- // Create Huma API wrapping the Echo router groupapi := humaecho.NewWithGroup(e, apiGroup, humaConfig)- // Add authentication middlewareapi.UseMiddleware(auth.NewHumaMiddleware(api, deps.Auth.Service(), deps.ApiKey.Service(), deps.Role.Service(), deps.Environment.Service(), cfg))- api.UseMiddleware(middleware.NewActivityBatchID())- registerNormalizationInternal(api)-- // Register all Huma handlers- registerHandlersInternal(api, deps, appCtx, cfg)-- // Register Scalar API docs endpoint with dark mode- registerScalarDocs(apiGroup)-- return api-}--// registerScalarDocs adds the Scalar API documentation endpoint.-func registerScalarDocs(apiGroup *echo.Group) {- apiGroup.GET("/docs", func(c *echo.Context) error {- return c.HTML(http.StatusOK, scalarDocsHTML)+ api.UseMiddleware(func(ctx huma.Context, next func(huma.Context)) {+ if actor, ok := userctx.CurrentUserFromContext(ctx.Context()); ok {+ middleware.RecordAuthenticatedRequest(ctx.Context(), humaecho.Unwrap(ctx).Request().Header, actor)+ }+ next(ctx)})-}--// SetupAPIForSpec creates a Huma API instance for OpenAPI spec generation only.-// No services are required - this is purely for schema generation.-func SetupAPIForSpec() huma.API {- e := echo.New()- apiGroup := e.Group("/api")-- humaConfig := huma.DefaultConfig("Arcane API", config.Version)- humaConfig.Formats = maps.Clone(humaConfig.Formats)- humaConfig.Formats["application/json"] = jsonV2Format- humaConfig.Formats["json"] = jsonV2Format- humaConfig.Info.Description = "Modern Docker Management, Designed for Everyone"- humaConfig.Servers = []*huma.Server{- {URL: "/api"},- }- humaConfig.Components.SecuritySchemes = map[string]*huma.SecurityScheme{- "BearerAuth": {- Type: "http",- Scheme: "bearer",- BearerFormat: "JWT",- Description: "JWT Bearer token authentication",- },- "ApiKeyAuth": {- Type: "apiKey",- In: "header",- Name: "X-API-Key",- Description: "API Key authentication",- },- }- humaConfig.Security = []map[string][]string{- {"BearerAuth": {}},- {"ApiKeyAuth": {}},- }-- // Use custom schema namer to avoid conflicts between types with same name- humaConfig.Components.Schemas = huma.NewMapRegistry("#/components/schemas/", customSchemaNamer)-- api := humaecho.NewWithGroup(e, apiGroup, humaConfig)-- // Register handlers with zero-value dependencies for schema discovery only.- registerNormalizationInternal(api)- registerHandlersInternal(api, HandlerDeps{}, handlerutil.NewActivityAppContext(context.Background()), nil) //nolint:forbidigo // Schema discovery has no running request or application lifecycle.-- return api-}+ api.UseMiddleware(middleware.NewActivityBatchID())+ registerNormalization(api)-// registerHandlersInternal registers all Huma-based API handlers.-func registerHandlersInternal(api huma.API, deps HandlerDeps, handlerAppCtx handlerutil.ActivityAppContext, cfg *config.Config) {health.RegisterRoutes(api)deps.Auth.RegisterRoutes(api)passkey.RegisterPasskeys(api, deps.Passkey, deps.Auth.Service(), deps.User.Service())@@ -410,37 +280,51 @@ func registerHandlersInternal(api huma.API, deps HandlerDeps, handlerAppCtx handdeps.Role.RegisterRoutes(api)appimages.RegisterAppImages(api, deps.AppImages)deps.User.RegisterRoutes(api)- deps.Project.RegisterRoutes(api, handlerAppCtx)+ deps.Project.RegisterRoutes(api, appCtx)version.RegisterVersion(api, deps.Version)deps.Event.RegisterRoutes(api)deps.Activity.RegisterRoutes(api)oidc.RegisterOidc(api, deps.Auth.Service(), deps.Passkey, deps.Oidc, deps.Role.Service(), deps.User.Service(), cfg)- deps.Environment.RegisterRoutes(api, handlerAppCtx)+ deps.Environment.RegisterRoutes(api, appCtx, deps.Version.GetAppVersionInfo)deps.ContainerRegistry.RegisterRoutes(api)deps.Template.RegisterRoutes(api)deps.Variable.RegisterRoutes(api, cfg)- deps.Image.RegisterRoutes(api, handlerAppCtx)+ deps.Image.RegisterRoutes(api, appCtx)deps.Upload.RegisterRoutes(api)build.RegisterBuildWorkspaces(api, deps.Build, deps.Upload.Service())- deps.ImageUpdate.RegisterRoutes(api, handlerAppCtx)+ deps.ImageUpdate.RegisterRoutes(api, appCtx)deps.Settings.RegisterRoutes(api)deps.S3Destination.RegisterRoutes(api)deps.JobSchedule.RegisterRoutes(api)- deps.Volume.RegisterRoutes(api, handlerAppCtx)- deps.Container.RegisterRoutes(api, handlerAppCtx)+ deps.Volume.RegisterRoutes(api, appCtx)+ deps.Container.RegisterRoutes(api, appCtx)port.RegisterPorts(api, deps.Port)- network.RegisterNetworks(api, deps.Network, deps.Docker, deps.Activity.Service(), handlerAppCtx)+ network.RegisterNetworks(api, deps.Network, deps.Docker, deps.Activity.Service(), appCtx)deps.Swarm.RegisterRoutes(api)deps.Notification.RegisterRoutes(api)deps.Apns.RegisterRoutes(api)- deps.Updater.RegisterRoutes(api, handlerAppCtx)+ deps.Updater.RegisterRoutes(api, appCtx)deps.Search.RegisterRoutes(api)- deps.System.RegisterRoutes(api, handlerAppCtx)+ deps.System.RegisterRoutes(api, appCtx)handlers.RegisterDiagnostics(api, deps.Diagnostics)deps.GitRepository.RegisterRoutes(api)deps.GitOpsSync.RegisterRoutes(api)deps.Webhook.RegisterRoutes(api)- deps.Vulnerability.RegisterRoutes(api, handlerAppCtx)+ deps.Vulnerability.RegisterRoutes(api, appCtx)deps.Dashboard.RegisterRoutes(api)handlers.RegisterStream(api, deps.Dashboard.Handler(), deps.Activity.Handler(), deps.Environment.Handler(), deps.Event.Service(), deps.Version)++ apiGroup.GET("/docs", func(c *echo.Context) error {+ return c.HTML(http.StatusOK, scalarDocsHTML)+ })++ return api+}++// SetupAPIForSpec creates a Huma API instance for OpenAPI spec generation only.+// No services are required - this is purely for schema generation.+func SetupAPIForSpec() huma.API {+ e := echo.New()+ appCtx := handlerutil.NewActivityAppContext(context.Background()) //nolint:forbidigo // Schema discovery has no running request or application lifecycle.+ return SetupAPI(e, e.Group("/api"), appCtx, &config.Config{}, HandlerDeps{})}diff --git a/backend/api/api_test.go b/backend/api/api_test.goindex 202c61f23..194782de8 100644--- a/backend/api/api_test.go+++ b/backend/api/api_test.go@@ -385,9 +385,8 @@ func TestEasyJoinRoutesDeclareSwarmJoinPermission(t *testing.T) {}}-// TestEnvScopedOperationsDeclarePermission guards the remote-environment proxy-// authorization model. Every authenticated environment-scoped operation must-// declare the permission that the proxy enforces before forwarding it.+// TestEnvScopedOperationsDeclarePermission ensures every authenticated env-scoped operation+// declares the permission the remote-environment proxy enforces before forwarding.func TestEnvScopedOperationsDeclarePermission(t *testing.T) {api := SetupAPIForSpec()oapi := api.OpenAPI()@@ -400,14 +399,20 @@ func TestEnvScopedOperationsDeclarePermission(t *testing.T) {if !strings.HasPrefix(path, "/environments/{id}/") {continue}- // Upload-session routes derive their permission from the {kind} path- // segment at request time (authz.UploadKindPermission); both the local- // handlers and the proxy's uploadSessionKindInternal special case- // enforce it, so no static mapping exists to declare here.+ // Upload-session routes derive their permission from the {kind} segment at request time.if strings.HasPrefix(path, "/environments/{id}/uploads/{kind}") {continue}- for method, op := range envScopedTestOperationsInternal(item) {+ operations := map[string]*humav2.Operation{+ http.MethodGet: item.Get,+ http.MethodPost: item.Post,+ http.MethodPut: item.Put,+ http.MethodDelete: item.Delete,+ http.MethodPatch: item.Patch,+ http.MethodHead: item.Head,+ http.MethodOptions: item.Options,+ }+ for method, op := range operations {if op == nil {continue}@@ -427,15 +432,3 @@ func TestEnvScopedOperationsDeclarePermission(t *testing.T) {len(missing), strings.Join(missing, "\n "))}}--func envScopedTestOperationsInternal(item *humav2.PathItem) map[string]*humav2.Operation {- return map[string]*humav2.Operation{- http.MethodGet: item.Get,- http.MethodPost: item.Post,- http.MethodPut: item.Put,- http.MethodDelete: item.Delete,- http.MethodPatch: item.Patch,- http.MethodHead: item.Head,- http.MethodOptions: item.Options,- }-}diff --git a/backend/api/handlers/stream.go b/backend/api/handlers/stream.goindex 1de71eb44..74b726cb2 100644--- a/backend/api/handlers/stream.go+++ b/backend/api/handlers/stream.go@@ -17,6 +17,11 @@ import (streamtypes "github.com/getarcaneapp/arcane/types/v2/stream"versiontypes "github.com/getarcaneapp/arcane/types/v2/version""go.getarcane.app/streams/agg"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric"+ "go.opentelemetry.io/otel/propagation"+ "go.opentelemetry.io/otel/trace""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/dashboard"@@ -27,16 +32,14 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")const (clientStreamHeartbeatInterval = 15 * time.SecondclientStreamDeadPeerTimeout = 45 * time.SecondclientStreamEventBuffer = 64- // Per-channel handoff buffer between a channel's producer and the envelope- // forwarder. Small on purpose: the shared writer is the real backpressure- // point, and a deep per-channel buffer would only delay noticing a stalled- // client.+ // Per-channel handoff buffer; small because the shared writer is the real backpressure point.clientStreamChannelBuffer = 16)@@ -55,9 +58,7 @@ type StreamClientInput struct {Limit int `query:"limit" default:"0" doc:"Maximum activities to include in each activities-channel snapshot"`}-// RegisterStream registers the multiplexed client stream. It builds its own-// per-domain handlers from the same services the individual handlers use, so-// each channel's producer stays the single implementation of that feed.+// RegisterStream registers the multiplexed client stream, reusing each domain's producer for its channel.func RegisterStream(api huma.API,dashboardHandler *dashboard.DashboardHandler,@@ -74,6 +75,13 @@ func RegisterStream(version: versionService,}+ meter := otel.Meter(tracing.InstrumentationName)+ connections, err := meter.Int64UpDownCounter("arcane.sse.connections",+ metric.WithDescription("Open client SSE stream connections"), metric.WithUnit("{connection}"))+ if err != nil {+ otel.Handle(err)+ }+sse.Register(api, huma.Operation{OperationID: "streamClient",Method: http.MethodGet,@@ -84,68 +92,53 @@ func RegisterStream(Security: handlerutil.DefaultOperationSecurity(),Middlewares: huma.Middlewares{func(ctx huma.Context, next func(huma.Context)) {httpx.SetStreamHeaders(ctx)+ // The HTTP tracing middleware skips streams, so the lifetime span is a root unless the client sent a traceparent.+ carrier := propagation.MapCarrier{}+ for _, field := range otel.GetTextMapPropagator().Fields() {+ if value := ctx.Header(field); value != "" {+ carrier[field] = value+ }+ }+ parent := otel.GetTextMapPropagator().Extract(ctx.Context(), carrier)+ _, span := otel.Tracer(tracing.InstrumentationName).Start(parent, "sse "+ctx.Operation().Path,+ trace.WithSpanKind(trace.SpanKindServer),+ trace.WithAttributes(+ attribute.String("http.route", ctx.Operation().Path),+ attribute.String("arcane.sse.channels", ctx.Query("channels")),+ attribute.String("client.address", ctx.RemoteAddr()),+ ))+ metricCtx := context.WithoutCancel(ctx.Context())+ connections.Add(metricCtx, 1)+ defer func() {+ connections.Add(metricCtx, -1)+ span.End()+ }()next(ctx)}},- // Ungated: each channel applies its own permission check below, and a- // caller with access to none simply gets a heartbeat-only stream.+ // Ungated: each channel checks its own permission; a caller with none gets a heartbeat-only stream.}, map[string]any{"message": streamtypes.Event{}}, h.StreamClient)}func (h *StreamHandler) StreamClient(ctx context.Context, input *StreamClientInput, send sse.Sender) {ps, _ := middleware.PermissionsFromContext(ctx)- h.streamClientInternal(ctx, ps, input, send)-}--func (h *StreamHandler) streamClientInternal(ctx context.Context, ps *authz.PermissionSet, input *StreamClientInput, send sse.Sender) {releaseDeadPeerTimeout, timeoutErr := httpx.AcquireDeadPeerTimeout(ctx, clientStreamDeadPeerTimeout)if timeoutErr != nil {slog.DebugContext(ctx, "could not bound client stream dead-peer timeout", "error", timeoutErr)}defer releaseDeadPeerTimeout()- streamCtx, cancel := context.WithCancel(ctx)- events := make(chan streamtypes.Event, clientStreamEventBuffer)- var producers sync.WaitGroup- for _, producer := range h.producersForInternal(ps, input) {- producers.Go(func() {- producer(streamCtx, events)- })- }- defer producers.Wait()- defer cancel()-- heartbeat := time.NewTicker(clientStreamHeartbeatInterval)- defer heartbeat.Stop()-- for {- var envelope streamtypes.Event- select {- case <-streamCtx.Done():- return- case envelope = <-events:- case <-heartbeat.C:- envelope = streamtypes.Event{Type: "heartbeat", Timestamp: time.Now()}- }- if streamCtx.Err() != nil {- return- }- if err := send.Data(envelope); err != nil {- slog.DebugContext(ctx, "client stream disconnected", "error", err)- return+ // Select the producers for the requested channels, dropping any the caller lacks permission for.+ requested := make(map[string]bool, 5)+ for name := range strings.SplitSeq(input.Channels, ",") {+ if trimmed := strings.TrimSpace(strings.ToLower(name)); trimmed != "" {+ requested[trimmed] = true}}-}--// producersForInternal selects the producers for the requested channels,-// dropping any the caller lacks permission for.-func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *StreamClientInput) []agg.Producer[streamtypes.Event] {- requested := parseStreamChannelsInternal(input.Channels)producers := make([]agg.Producer[streamtypes.Event], 0, 5)if requested[streamtypes.ChannelEnvironments] {- // Ungated like listEnvironments; the producer filters to the caller's- // accessible environments.- producers = append(producers, forwardStreamChannelInternal(+ // Ungated like listEnvironments; the producer filters to the caller's accessible environments.+ producers = append(producers, forwardStreamChannel(streamtypes.ChannelEnvironments,func(channel string, event environmenttypes.StreamEvent) streamtypes.Event {return streamtypes.Event{Channel: channel, Environment: &event, Timestamp: event.Timestamp}@@ -161,7 +154,7 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Strreturn streamtypes.Event{Channel: channel, Dashboard: &event, Timestamp: event.Timestamp}}if ps.Allows(authz.PermDashboardRead, environment.LocalEnvironmentID) {- producers = append(producers, forwardStreamChannelInternal(+ producers = append(producers, forwardStreamChannel(streamtypes.ChannelDashboard, wrapDashboard,func(producerCtx context.Context, events chan<- dashboardtypes.StreamEvent) {h.dashboard.RunLocalStreamProducer(producerCtx, input.DebugAllGood, events)@@ -169,7 +162,7 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Str))}if ps.AllowsAny(authz.PermDashboardRead) {- producers = append(producers, forwardStreamChannelInternal(+ producers = append(producers, forwardStreamChannel(streamtypes.ChannelDashboard, wrapDashboard,func(producerCtx context.Context, events chan<- dashboardtypes.StreamEvent) {h.dashboard.RunRemoteStreamPollers(producerCtx, ps, input.DebugAllGood, events)@@ -183,7 +176,7 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Strreturn streamtypes.Event{Channel: channel, Activity: &event, Timestamp: event.Timestamp}}if ps.Allows(authz.PermActivitiesRead, environment.LocalEnvironmentID) {- producers = append(producers, forwardStreamChannelInternal(+ producers = append(producers, forwardStreamChannel(streamtypes.ChannelActivities, wrapActivity,func(producerCtx context.Context, events chan<- activitytypes.StreamEvent) {h.activity.RunLocalStreamProducer(producerCtx, input.Limit, events)@@ -191,7 +184,7 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Str))}if ps.AllowsAny(authz.PermActivitiesRead) {- producers = append(producers, forwardStreamChannelInternal(+ producers = append(producers, forwardStreamChannel(streamtypes.ChannelActivities, wrapActivity,func(producerCtx context.Context, events chan<- activitytypes.StreamEvent) {h.activity.RunRemoteStreamPollers(producerCtx, ps, input.Limit, events)@@ -201,7 +194,7 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Str}if requested[streamtypes.ChannelEvents] && ps.Allows(authz.PermEventsRead, "") {- producers = append(producers, forwardStreamChannelInternal(+ producers = append(producers, forwardStreamChannel(streamtypes.ChannelEvents,func(channel string, event eventtypes.StreamEvent) streamtypes.Event {return streamtypes.Event{Channel: channel, EventLog: &event, Timestamp: event.Timestamp}@@ -212,7 +205,7 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Strif requested[streamtypes.ChannelVersion] {// Ungated like GET /app-version.- producers = append(producers, forwardStreamChannelInternal(+ producers = append(producers, forwardStreamChannel(streamtypes.ChannelVersion,func(channel string, event versiontypes.StreamEvent) streamtypes.Event {return streamtypes.Event{Channel: channel, Version: &event, Timestamp: event.Timestamp}@@ -220,23 +213,42 @@ func (h *StreamHandler) producersForInternal(ps *authz.PermissionSet, input *Strh.version.RunStreamProducer,))}- return producers-}-func parseStreamChannelsInternal(raw string) map[string]bool {- channels := make(map[string]bool, 5)- for name := range strings.SplitSeq(raw, ",") {- if trimmed := strings.TrimSpace(strings.ToLower(name)); trimmed != "" {- channels[trimmed] = true+ streamCtx, cancel := context.WithCancel(ctx)+ events := make(chan streamtypes.Event, clientStreamEventBuffer)+ var running sync.WaitGroup+ for _, producer := range producers {+ running.Go(func() {+ producer(streamCtx, events)+ })+ }+ defer running.Wait()+ defer cancel()++ heartbeat := time.NewTicker(clientStreamHeartbeatInterval)+ defer heartbeat.Stop()++ for {+ var envelope streamtypes.Event+ select {+ case <-streamCtx.Done():+ return+ case envelope = <-events:+ case <-heartbeat.C:+ envelope = streamtypes.Event{Type: "heartbeat", Timestamp: time.Now()}+ }+ if streamCtx.Err() != nil {+ return+ }+ if err := send.Data(envelope); err != nil {+ slog.DebugContext(ctx, "client stream disconnected", "error", err)+ return}}- return channels}-// forwardStreamChannelInternal adapts a channel's own producer to the shared-// envelope. Each producer keeps emitting its native event type, so adding a-// channel never means rewriting its feed.-func forwardStreamChannelInternal[T any](+// forwardStreamChannel adapts a channel's producer, which keeps emitting its native event type, to the shared envelope.+func forwardStreamChannel[T any](channel string,wrap func(string, T) streamtypes.Event,producer func(context.Context, chan<- T),diff --git a/backend/api/normalization.go b/backend/api/normalization.goindex f3ef0c3d0..4e6615f7f 100644--- a/backend/api/normalization.go+++ b/backend/api/normalization.go@@ -19,7 +19,7 @@ import ()// Install before routes so Huma validates normalized input.-func registerNormalizationInternal(api huma.API) {+func registerNormalization(api huma.API) {api.OpenAPI().OnAddOperation = append(api.OpenAPI().OnAddOperation, func(_ *huma.OpenAPI, op *huma.Operation) {typ := normalizationBodyTypeInternal(api, op)if typ == nil {diff --git a/backend/api/normalization_test.go b/backend/api/normalization_test.goindex d7ba463ee..b88d33b17 100644--- a/backend/api/normalization_test.go+++ b/backend/api/normalization_test.go@@ -49,7 +49,7 @@ func TestNormalizationBeforeHumaValidation(t *testing.T) {cfg := huma.DefaultConfig("test", "1")cfg.Formats = map[string]huma.Format{"application/json": jsonV2Format, "json": jsonV2Format}api := humaecho.New(router, cfg)- registerNormalizationInternal(api)+ registerNormalization(api)called := falsevar received normalizationTestBodyhuma.Register(api, huma.Operation{@@ -88,7 +88,7 @@ func TestNormalizationBeforeHumaValidation(t *testing.T) {t.Run("identity fields pass through unchanged", func(t *testing.T) {router := echo.New()api := humaecho.New(router, huma.DefaultConfig("test", "1"))- registerNormalizationInternal(api)+ registerNormalization(api)var received usertypes.CreateUserhuma.Register(api, huma.Operation{OperationID: "create",@@ -133,7 +133,7 @@ func TestNormalizationBeforeHumaValidation(t *testing.T) {func TestNormalizationBodyLimits(t *testing.T) {router := echo.New()api := humaecho.New(router, huma.DefaultConfig("test", "1"))- registerNormalizationInternal(api)+ registerNormalization(api)huma.Register(api, huma.Operation{OperationID: "limit",Method: http.MethodPost,@@ -218,7 +218,7 @@ func TestNormalizationReadTimeout(t *testing.T) {logs.Reset()router := echo.New()api := humaecho.New(router, huma.DefaultConfig("test", "1"))- registerNormalizationInternal(api)+ registerNormalization(api)called := falsehuma.Register(api, huma.Operation{OperationID: "timeout",@@ -254,7 +254,7 @@ func TestNormalizationReadTimeout(t *testing.T) {func TestNormalizationRejectsInvalidTagsAtRegistration(t *testing.T) {router := echo.New()api := humaecho.New(router, huma.DefaultConfig("test", "1"))- registerNormalizationInternal(api)+ registerNormalization(api)require.Panics(t, func() {huma.Register(api, huma.Operation{OperationID: "invalid", Method: http.MethodPost, Path: "/invalid"}, func(_ context.Context, _ *struct {Body struct {diff --git a/backend/api/ws/container_stats.go b/backend/api/ws/container_stats.goindex 3c3032a5c..8ddd86e78 100644--- a/backend/api/ws/container_stats.go+++ b/backend/api/ws/container_stats.go@@ -34,7 +34,7 @@ func (h *WebSocketHandler) ContainerStats(c *echo.Context) error {return c.JSON(http.StatusBadRequest, map[string]any{"success": false, "error": "Container ID is required"})}- conn, onRemove, ok := h.acceptWSInternal(c, systemtypes.WSKindContainerStats, containerID)+ conn, onRemove, ok := h.acceptWS(c, systemtypes.WSKindContainerStats, containerID)if !ok {return nil}diff --git a/backend/api/ws/diagnostics.go b/backend/api/ws/diagnostics.goindex 743ebd37c..d9ecc2be6 100644--- a/backend/api/ws/diagnostics.go+++ b/backend/api/ws/diagnostics.go@@ -132,7 +132,7 @@ func (h *WebSocketHandler) ActorDiagnosticsStream(c *echo.Context) error {// streamSnapshotsInternal writes build's result on connect and every interval until// the client disconnects. A non-nil handle answers client commands concurrently.func (h *WebSocketHandler) streamSnapshotsInternal(c *echo.Context, interval time.Duration, build func(context.Context) any, handle func(context.Context, systemtypes.DiagnosticsCommand) any) error {- conn, unregister, accepted := h.acceptWSInternal(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)+ conn, unregister, accepted := h.acceptWS(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)if !accepted {return nil}@@ -190,7 +190,7 @@ func (h *WebSocketHandler) streamSnapshotsInternal(c *echo.Context, interval tim// ServerLogsStream replays the recent backend log backlog then streams new entries live.func (h *WebSocketHandler) ServerLogsStream(c *echo.Context) error {- conn, unregister, accepted := h.acceptWSInternal(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)+ conn, unregister, accepted := h.acceptWS(c, systemtypes.WSKindDiagnostics, c.Request().URL.Path)if !accepted {return nil}diff --git a/backend/api/ws/exec.go b/backend/api/ws/exec.goindex 8b2476c69..4b20554cc 100644--- a/backend/api/ws/exec.go+++ b/backend/api/ws/exec.go@@ -40,7 +40,7 @@ func (h *WebSocketHandler) ContainerExec(c *echo.Context) error {shell := cmp.Or(c.QueryParam("shell"), "/bin/sh")- conn, unregister, ok := h.acceptWSInternal(c, systemtypes.WSKindContainerExec, containerID)+ conn, unregister, ok := h.acceptWS(c, systemtypes.WSKindContainerExec, containerID)if !ok {return nil}@@ -54,7 +54,7 @@ func (h *WebSocketHandler) ContainerExec(c *echo.Context) error {defer cancel(nil)// The pong is serviced by the concurrent stdin reader.- go keepWSConnAliveInternal(ctx, func() { cancel(errors.New("websocket ping failed")) }, conn, 54*time.Second)+ go keepWSConnAlive(ctx, func() { cancel(errors.New("websocket ping failed")) }, conn, 54*time.Second)h.runContainerExecInternal(ctx, cancel, conn, containerID, shell)return nildiff --git a/backend/api/ws/handler.go b/backend/api/ws/handler.goindex 936510b47..b39a94612 100644--- a/backend/api/ws/handler.go+++ b/backend/api/ws/handler.go@@ -13,6 +13,10 @@ import ("github.com/labstack/echo/v5""github.com/samber/hot""go.getarcane.app/sys/cgroup"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/propagation"+ "go.opentelemetry.io/otel/trace""github.com/getarcaneapp/arcane/backend/v2/internal/auth""github.com/getarcaneapp/arcane/backend/v2/internal/config"@@ -26,9 +30,13 @@ import (wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")-var defaultWebSocketMetrics = wshub.NewWebSocketMetrics()+var (+ defaultWebSocketMetrics = wshub.NewWebSocketMetrics()+ observeWebSocketMetricsOnce sync.Once+)// WebSocketHandler consolidates all WebSocket and streaming endpoints.// REST endpoints are handled by Huma handlers.@@ -39,7 +47,6 @@ type WebSocketHandler struct {systemService *system.SystemServicediagnosticsService *diagnostics.DiagnosticsServicecheckWSOrigin func(*http.Request) bool- wsMetrics *wshub.WebSocketMetricsactiveConnectionsMu sync.MutexactiveConnections map[string]intlogStreamsMu sync.Mutex@@ -88,7 +95,6 @@ func NewWebSocketHandler(swarmService: swarmService,systemService: systemService,diagnosticsService: diagnosticsService,- wsMetrics: defaultWebSocketMetrics,logStreams: make(map[string]*wsLogStream),cgroupCache: cgroup.NewCache(cgroupCacheTTL),gpuMonitor: systemlib.NewGPUMonitor(cfg.GPUMonitoringEnabled, cfg.GPUType),@@ -97,6 +103,11 @@ func NewWebSocketHandler(Build(),checkWSOrigin: httpx.ValidateWebSocketOrigin(cfg.GetAppURL()),}+ observeWebSocketMetricsOnce.Do(func() {+ if err := defaultWebSocketMetrics.ObserveConnections(otel.Meter(tracing.InstrumentationName)); err != nil {+ otel.Handle(err)+ }+ })wsGroup := group.Group("/environments/:id/ws", authMiddleware.WithAdminNotRequired().Add())for _, r := range handler.proxiedRoutes() {wsGroup.GET(r.path, r.handler, middleware.RequireEchoPermission(r.perm))@@ -104,37 +115,48 @@ func NewWebSocketHandler(handler.registerDiagnosticsRoutesInternal(group, authMiddleware)}-func buildWSConnectionInfoInternal(c *echo.Context, kind, resourceID string) systemtypes.WebSocketConnectionInfo {+// acceptWS upgrades the request to a WebSocket, registers it with the metrics tracker, and starts its lifetime span.+// The returned unregister func must be called exactly once when the connection ends.+func (h *WebSocketHandler) acceptWS(c *echo.Context, kind, resourceID string) (*websocket.Conn, func(), bool) {+ req := c.Request()+ conn, err := wshub.Accept(c.Response(), req, h.checkWSOrigin)+ if err != nil {+ slog.DebugContext(req.Context(), "websocket accept failed", "kind", kind, "resourceId", resourceID, "error", err)+ return nil, nil, false+ }userID, _ := c.Get("userID").(string)- return systemtypes.WebSocketConnectionInfo{+ info := systemtypes.WebSocketConnectionInfo{Kind: kind,EnvID: c.Param("id"),ResourceID: resourceID,ClientIP: c.RealIP(),UserID: userID,- UserAgent: c.Request().Header.Get("User-Agent"),- }-}--// acceptWSInternal upgrades the request to a WebSocket and registers it with-// the metrics tracker. The returned unregister func must be called exactly-// once when the connection ends.-func (h *WebSocketHandler) acceptWSInternal(c *echo.Context, kind, resourceID string) (*websocket.Conn, func(), bool) {- conn, err := wshub.Accept(c.Response(), c.Request(), h.checkWSOrigin)- if err != nil {- slog.DebugContext(c.Request().Context(), "websocket accept failed", "kind", kind, "resourceId", resourceID, "error", err)- return nil, nil, false+ UserAgent: req.Header.Get("User-Agent"),}- connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, kind, resourceID))- return conn, func() { h.wsMetrics.UnregisterConnection(connID) }, true+ connID := defaultWebSocketMetrics.RegisterConnection(info)+ // The HTTP tracing middleware skips upgrades, so this is a root span unless the client sent a traceparent.+ parent := otel.GetTextMapPropagator().Extract(req.Context(), propagation.HeaderCarrier(req.Header))+ _, span := otel.Tracer(tracing.InstrumentationName).Start(parent, "websocket "+kind,+ trace.WithSpanKind(trace.SpanKindServer),+ trace.WithAttributes(+ attribute.String("arcane.websocket.kind", kind),+ attribute.String("arcane.websocket.connection_id", connID),+ attribute.String("arcane.environment.id", info.EnvID),+ attribute.String("arcane.websocket.resource_id", resourceID),+ attribute.String("http.route", c.Path()),+ attribute.String("client.address", info.ClientIP),+ attribute.String("user.id", userID),+ attribute.String("user_agent.original", info.UserAgent),+ ))+ return conn, func() {+ defaultWebSocketMetrics.UnregisterConnection(connID)+ span.End()+ }, true}-// keepWSConnAliveInternal pings the peer every period. Ping round-trips (the-// pong must be serviced by the connection's concurrent reader) and is safe-// alongside a concurrent writer. A failed ping means the client is gone, so-// it cancels the connection's context — a silently-dead client would-// otherwise keep the session open until a write fails.-func keepWSConnAliveInternal(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn, period time.Duration) {+// keepWSConnAlive pings the peer every period and cancels the connection when a ping fails.+// The pong must be serviced by the connection's concurrent reader.+func keepWSConnAlive(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn, period time.Duration) {ticker := time.NewTicker(period)defer ticker.Stop()for {diff --git a/backend/api/ws/log_streams.go b/backend/api/ws/log_streams.goindex 6c4f275e6..8975492c6 100644--- a/backend/api/ws/log_streams.go+++ b/backend/api/ws/log_streams.go@@ -142,7 +142,7 @@ func (h *WebSocketHandler) serveLogStreamInternal(params logStreamParams,hubBuilder func(streamKey string, onEmpty func(*wsLogStream)) *wsLogStream,) {- conn, unregister, ok := h.acceptWSInternal(c, kind, resourceID)+ conn, unregister, ok := h.acceptWS(c, kind, resourceID)if !ok {return}diff --git a/backend/api/ws/system_stats.go b/backend/api/ws/system_stats.goindex 3446c9005..290519b0b 100644--- a/backend/api/ws/system_stats.go+++ b/backend/api/ws/system_stats.go@@ -428,7 +428,7 @@ func (h *WebSocketHandler) SystemStats(c *echo.Context) error {}defer h.releaseRateLimitInternal(clientIP)- conn, unregister, ok := h.acceptWSInternal(c, systemtypes.WSKindSystemStats, "")+ conn, unregister, ok := h.acceptWS(c, systemtypes.WSKindSystemStats, "")if !ok {return nil}@@ -460,7 +460,7 @@ func (h *WebSocketHandler) SystemStats(c *echo.Context) error {go h.readSystemStatsPumpInternal(ctx, cancel, conn)// The pong is serviced by readSystemStatsPumpInternal.- go keepWSConnAliveInternal(ctx, cancel, conn, 54*time.Second)+ go keepWSConnAlive(ctx, cancel, conn, 54*time.Second)send := func() error {stats, _ := h.systemStatsSampler.latest.Load()diff --git a/backend/go.mod b/backend/go.modindex 2da257e5d..0f36d4037 100644--- a/backend/go.mod+++ b/backend/go.mod@@ -38,6 +38,7 @@ require (github.com/italypaleale/francis v0.1.0github.com/jwx-go/jwkfetch/v4 v4.0.4github.com/klauspost/compress v1.20.1+ github.com/labstack/echo-otel/v5 v5.0.0github.com/labstack/echo/v5 v5.4.0github.com/lestrrat-go/httprc/v3 v3.0.6github.com/lestrrat-go/jwx/v4 v4.5.0@@ -72,6 +73,17 @@ require (go.getarcane.app/sys/cgroup v0.2.4go.getarcane.app/sys/crypto v0.3.0go.getarcane.app/updater v0.11.6+ go.opentelemetry.io/contrib/bridges/otelslog v0.20.1+ go.opentelemetry.io/contrib/exporters/autoexport v0.71.0+ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0+ go.opentelemetry.io/contrib/instrumentation/runtime v0.71.0+ go.opentelemetry.io/otel v1.46.0+ go.opentelemetry.io/otel/log v0.22.0+ go.opentelemetry.io/otel/metric v1.46.0+ go.opentelemetry.io/otel/sdk v1.46.0+ go.opentelemetry.io/otel/sdk/log v0.22.0+ go.opentelemetry.io/otel/sdk/metric v1.46.0+ go.opentelemetry.io/otel/trace v1.46.0go.uber.org/fx v1.24.0go.yaml.in/yaml/v4 v4.0.0-rc.6golang.org/x/crypto v0.57.0@@ -87,6 +99,7 @@ require (google.golang.org/protobuf v1.36.12gorm.io/driver/postgres v1.6.3gorm.io/gorm v1.31.2+ gorm.io/plugin/opentelemetry v0.1.16modernc.org/sqlite v1.60.1)@@ -247,6 +260,7 @@ require (github.com/prometheus/client_golang v1.24.1 // indirectgithub.com/prometheus/client_model v0.6.3 // indirectgithub.com/prometheus/common v0.71.0 // indirect+ github.com/prometheus/otlptranslator v1.0.0 // indirectgithub.com/prometheus/procfs v0.22.0 // indirectgithub.com/quic-go/qpack v0.6.0 // indirectgithub.com/quic-go/quic-go v0.63.0 // indirect@@ -285,19 +299,20 @@ require (github.com/yusufpapurcu/wmi v1.2.4 // indirectgithub.com/zmb3/spotify/v2 v2.3.1 // indirectgo.opentelemetry.io/auto/sdk v1.2.1 // indirect+ go.opentelemetry.io/contrib/bridges/prometheus v0.71.0 // indirectgo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirectgo.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.71.0 // indirect- go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect- go.opentelemetry.io/otel v1.46.0 // indirect+ go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 // indirect+ go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 // indirectgo.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 // indirectgo.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 // indirectgo.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirectgo.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirectgo.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 // indirect- go.opentelemetry.io/otel/metric v1.46.0 // indirect- go.opentelemetry.io/otel/sdk v1.46.0 // indirect- go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect- go.opentelemetry.io/otel/trace v1.46.0 // indirect+ go.opentelemetry.io/otel/exporters/prometheus v0.68.0 // indirect+ go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0 // indirect+ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 // indirect+ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 // indirectgo.opentelemetry.io/proto/otlp v1.11.0 // indirectgo.uber.org/dig v1.19.0 // indirectgo.uber.org/multierr v1.11.0 // indirectdiff --git a/backend/go.sum b/backend/go.sumindex c509cb8bb..a15a1d128 100644--- a/backend/go.sum+++ b/backend/go.sum@@ -454,6 +454,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=+github.com/labstack/echo-otel/v5 v5.0.0 h1:/mcXe6HyOSoGJe8z4WuhWBfK3uFOkBi2oFEvaok1M0Q=+github.com/labstack/echo-otel/v5 v5.0.0/go.mod h1:rMC5ekJzX2voZtHAW7SYMmxWBuTQBobCB+JC2gqfdxE=github.com/labstack/echo/v4 v4.15.4 h1:DL45vVYa+BWE+XuW+zZNd9H0YEdZ80UAWJGcTVW4EVs=github.com/labstack/echo/v4 v4.15.4/go.mod h1:CuMetKIRwsuO/qlAgMq+KTAalwGoB/h4tC+yPdrTj1g=github.com/labstack/echo/v5 v5.4.0 h1:iY674460IvSmUcj7MziL3YrgyTClghz5BpdZoKiGxR4=@@ -603,6 +607,8 @@ github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTgithub.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8=github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=+github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos=+github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM=github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0=@@ -772,14 +778,28 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=+go.opentelemetry.io/contrib/bridges/otelslog v0.20.1 h1:5sHc4ToTFjfSZCtGAAM6jPunICAmJX73htv372T4ipc=+go.opentelemetry.io/contrib/bridges/otelslog v0.20.1/go.mod h1:oa6kgvyz/3GYW04dohd0++xJIH4xdQY8PAbpeCMaM8M=+go.opentelemetry.io/contrib/bridges/prometheus v0.71.0 h1:9qgxsFLskbDMXl8WMqThoF6w8yGJgCumn9qRc67OmnI=+go.opentelemetry.io/contrib/bridges/prometheus v0.71.0/go.mod h1:2rCjF4F2siiTeLCzJsaGZ3CK0XIoimCSKXEBPdv+Je0=+go.opentelemetry.io/contrib/exporters/autoexport v0.71.0 h1:VCsJbp0YLyPtx2tu5Vgv2a2/qLoaMCj8hT2uZ34+Mx0=+go.opentelemetry.io/contrib/exporters/autoexport v0.71.0/go.mod h1:qxZqn7e10f6ajmMCkg/47rMS7qQYfaOl2nj/4aytHUQ=go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s=go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw=go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.71.0 h1:oFNJW32h2SXnET7XXstgT7pVh4vN+jW+GfiIaBguIZE=go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.71.0/go.mod h1:+H3sPOFwag14eMHTPMElZtV0e4YfVZ/85KgrKUCB5FI=go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=+go.opentelemetry.io/contrib/instrumentation/runtime v0.71.0 h1:v4KkRLVvE1cWqTJDfZZkTCG+Z4aolsa6RVos0FX7vqE=+go.opentelemetry.io/contrib/instrumentation/runtime v0.71.0/go.mod h1:g/xbuPC0XbgwMdKuyF5sKOUUEsorSkN6APydyFP/H9E=+go.opentelemetry.io/contrib/propagators/b3 v1.46.0 h1:OFVqWObn7xLIbOjE/koO0LS9fZJNgAyBD0msA+UQAoc=+go.opentelemetry.io/contrib/propagators/b3 v1.46.0/go.mod h1:t/d64xy7xuuEDJN/4ThqohLgRhIuQxL9y7P1v02bYuM=go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 h1:Bu39F5tzJct+f2IZbB8989fwyTps3c8e7EsUQsz+vs8=+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0/go.mod h1:dJUwod88EsFgYCqrDHaSPzhiY9pBUpt0d85/qSfua7k=+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 h1:lYk7RmxdLK865qLwibroNGldHa1U7SWKYYvNjlK7PIo=+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0/go.mod h1:6GvlND0H0xdUJanOtIAn0xfwLkauh1tmsYEEVSMDdqY=go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 h1:qkDYCAFiZXLcs1L4aY+tP2wguQ4kURANqHOQMA2et2s=go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0/go.mod h1:tkipS4DRzmpAmvg+Gw4++O1IdDq6TVDnvnYU6cmbQVs=go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 h1:AP23h/mFgb/lc7tdck1Kfn9qxsM8TAeNPCU5C3pzaps=@@ -790,12 +810,26 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDgo.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY=go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 h1:KrC1YrQeSt46ITMWAbgQx1M1eV1/1TKzttrBzymPmss=go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0/go.mod h1:zDSEzoEqsOrgBeGvH66KRgxh90VonFyJqBHA0Pk3+rM=+go.opentelemetry.io/otel/exporters/prometheus v0.68.0 h1:QOf2IftqQwITVRJpnn0M7M9ZCbgWfxz4P7i9C9yc2N4=+go.opentelemetry.io/otel/exporters/prometheus v0.68.0/go.mod h1:bgSvqu2TWGXiz7yr5UTMfObH8oqxJWHTnubQ3ef9BO4=+go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0 h1:kvMAiLEudKmk+CSG+iYbU8vTUGNNDaf/V09OO5lrTwI=+go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0/go.mod h1:L9Dlksri+MdT1cb2gIiA1cJJYW3Y92ipvDjNxYEyaDI=+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 h1:PR9eAf7o0dQs3hshZNZpE9aW2dXWX/KdDf6pJilVD3U=+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0/go.mod h1:2Z4KyNdH1uuzivdinyfGsxzNNT/Rl45pwtVwfYVI0xk=+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 h1:KdRxPiAoMptR3vfWzvjjvutTsSiwbC2uG0496rzZNfo=+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0/go.mod h1:K/qSA+3G7Eovxi4K09wzrAgkWRnosS0DAOZeEpve7sM=+go.opentelemetry.io/otel/log v0.22.0 h1:5DBNnfvaJ6CVdkJ+Jle8Tzs50aSSv49TXGj9XRsEYw0=+go.opentelemetry.io/otel/log v0.22.0/go.mod h1:gzOt/R67vF2GniAqWu8Qv0SXy89f71muHcrkz76PCdc=go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o=go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU=go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4=go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI=go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM=+go.opentelemetry.io/otel/sdk/log v0.22.0 h1:PRL+s6P63XT4E/bheEflopPUpVxuvANqZwtt89yhoGk=+go.opentelemetry.io/otel/sdk/log v0.22.0/go.mod h1:JNp0sBELrjCTcu5W3GzABVypeU6vDJjBS+X0JISuz+g=+go.opentelemetry.io/otel/sdk/log/logtest v0.22.0 h1:infPnfNrhCNgOUZRs3gWUg8vhoBUHihq02gwK05gzlg=+go.opentelemetry.io/otel/sdk/log/logtest v0.22.0/go.mod h1:gkQZA3z15Bv3KU9vigBTi8dFechSozRP7v94X4VZv+s=go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE=go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4=go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=@@ -1121,6 +1155,8 @@ gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo=gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=+gorm.io/plugin/opentelemetry v0.1.16 h1:Kypj2YYAliJqkIczDZDde6P6sFMhKSlG5IpngMFQGpc=+gorm.io/plugin/opentelemetry v0.1.16/go.mod h1:P3RmTeZXT+9n0F1ccUqR5uuTvEXDxF8k2UpO7mTIB2Y=gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.goindex 295704c20..09eb82b3c 100644--- a/backend/internal/bootstrap/bootstrap.go+++ b/backend/internal/bootstrap/bootstrap.go@@ -16,12 +16,11 @@ import ("github.com/labstack/echo/v5""github.com/moby/moby/client""github.com/subosito/gotenv"- "go.getarcane.app/streams/logs""go.getarcane.app/sys/crypto"+ "go.opentelemetry.io/contrib/bridges/otelslog""go.uber.org/fx""go.uber.org/fx/fxevent"- "github.com/getarcaneapp/arcane/backend/v2/api/ws""github.com/getarcaneapp/arcane/backend/v2/internal/apikey""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -41,6 +40,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/startup""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")func Bootstrap(ctx context.Context) error {@@ -61,9 +61,7 @@ func Bootstrap(ctx context.Context) error {}cfg.DockerConfig = runtimeIdentityCfg.DockerConfig- SetupSlogLogger(cfg)- // Tee all slog output into the in-memory ring buffer that powers the diagnostics live log tail.- slog.SetDefault(slog.New(logs.NewSlogHandler(slog.Default().Handler(), ws.LogBroadcaster())))+ SetupSlogLogger(cfg, nil)database.SetGormLogger(BuildGormLogger(cfg))slog.InfoContext(ctx, "Arcane is starting...", "version", config.Version)startup.ApplyMemoryLimit(ctx)@@ -72,11 +70,12 @@ func Bootstrap(ctx context.Context) error {appCtx, cancelApp := context.WithCancel(ctx)appCtx = utils.WithAppLifecycleContext(appCtx)- db, err := initializeDBAndMigrateInternal(appCtx, cfg)+ db, err := database.Initialize(appCtx, cfg.DatabaseURL, database.MigrationOptions{AllowDowngrade: cfg.AllowDowngrade})if err != nil {cancelApp()return fmt.Errorf("failed to initialize database: %w", err)}+ slog.InfoContext(appCtx, "Database initialized successfully")defer func() {cancelApp()if closeErr := db.Close(); closeErr != nil {@@ -84,6 +83,31 @@ func Bootstrap(ctx context.Context) error {}}()+ // Telemetry identifies this installation by its persisted instance ID, so it starts once the+ // database is open but before Docker clients and services. Exporter errors go to the console-only logger.+ instanceID, err := settings.EnsureInstanceID(appCtx, db)+ if err != nil {+ return err+ }+ providers, err := setupTelemetry(ctx, cfg, slog.Default(), instanceID)+ if err != nil {+ return fmt.Errorf("setup telemetry: %w", err)+ }+ defer func() {+ shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)+ defer cancel()+ if shutdownErr := providers.Shutdown(shutdownCtx); shutdownErr != nil {+ slog.ErrorContext(shutdownCtx, "Error shutting down telemetry", "error", shutdownErr)+ }+ }()+ if providers.loggerProvider != nil {+ SetupSlogLogger(cfg, otelslog.NewHandler(tracing.InstrumentationName, otelslog.WithLoggerProvider(providers.loggerProvider)))+ db.Logger = BuildGormLogger(cfg)+ }+ if providers.tracerProvider != nil || providers.meterProvider != nil || providers.loggerProvider != nil {+ slog.InfoContext(ctx, "OpenTelemetry export enabled", "traces", providers.tracerProvider != nil, "metrics", providers.meterProvider != nil, "logs", providers.loggerProvider != nil)+ }+app := fx.New(applicationOptions(appCtx, cfg, db, cancelApp))startCtx, cancelStart := context.WithTimeout(ctx, 5*time.Minute)@@ -114,7 +138,7 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *databasefx.Supply(cfg, db, cancelApp),fx.Provide(func() context.Context { return appCtx },- newConfiguredHTTPClientInternal,+ newConfiguredHTTPClient,),di.ActorOptions,di.ServiceOptions,@@ -136,11 +160,9 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *database)}-// isWeakProductionEncryptionKeyInternal reports whether an explicit-// ENCRYPTION_KEY is an unprefixed passphrase shorter than 32 characters in-// production. crypto derives a key from any non-empty passphrase, so this-// preserves the historical fail-fast rejection of low-entropy production keys.-func isWeakProductionEncryptionKeyInternal(encryptionKey, localEnvironment string, agentMode bool) bool {+// isWeakProductionEncryptionKey reports whether a production ENCRYPTION_KEY is an+// unprefixed passphrase shorter than 32 characters.+func isWeakProductionEncryptionKey(encryptionKey, localEnvironment string, agentMode bool) bool {if localEnvironment != "production" || agentMode {return false}@@ -151,13 +173,7 @@ func isWeakProductionEncryptionKeyInternal(encryptionKey, localEnvironment strinreturn len(strings.TrimPrefix(key, "raw:")) < 32}-func warnDeprecatedEnvVarsInternal(ctx context.Context, cfg *config.Config) {- for _, envName := range cfg.DeprecatedEnvVarsSet() {- slog.WarnContext(ctx, "Deprecated environment variable is set and no longer used; remove it from your environment", "env", envName)- }-}--func newConfiguredHTTPClientInternal(cfg *config.Config) *http.Client {+func newConfiguredHTTPClient(cfg *config.Config) *http.Client {options := httpxtypes.ClientOptions{Timeout: 10 * time.Second,TLSHandshakeTimeout: 5 * time.Second,@@ -172,9 +188,8 @@ func newConfiguredHTTPClientInternal(cfg *config.Config) *http.Client {type initializeStartupStateParams struct {fx.In- AppCtx context.Context- Config *config.Config- HTTPClient *http.Client+ AppCtx context.Context+ Config *config.ConfigVolume *volume.VolumeServiceSettings *settings.SettingsService@@ -193,7 +208,6 @@ type initializeStartupStateParams struct {func initializeStartupState(p initializeStartupStateParams) {appCtx := p.AppCtxcfg := p.Config- httpClient := p.HTTPClientif p.Volume != nil {startup.CleanupOrphanedVolumeHelpers(appCtx, p.Volume.CleanupOrphanedVolumeHelpers)@@ -208,14 +222,16 @@ func initializeStartupState(p initializeStartupStateParams) {AdminStaticAPIKey: cfg.AdminStaticAPIKey,}- warnDeprecatedEnvVarsInternal(appCtx, cfg)+ for _, envName := range cfg.DeprecatedEnvVarsSet() {+ slog.WarnContext(appCtx, "Deprecated environment variable is set and no longer used; remove it from your environment", "env", envName)+ }startup.LoadAgentToken(appCtx, runtimeCfg, p.Settings.GetStringSetting)startup.EnsureEncryptionKey(appCtx, runtimeCfg, p.Settings.EnsureEncryptionKey)cfg.AgentToken = runtimeCfg.AgentTokencfg.EncryptionKey = runtimeCfg.EncryptionKey- if isWeakProductionEncryptionKeyInternal(cfg.EncryptionKey, string(cfg.Environment), cfg.AgentMode) {+ if isWeakProductionEncryptionKey(cfg.EncryptionKey, string(cfg.Environment), cfg.AgentMode) {panic("ENCRYPTION_KEY passphrase must be at least 32 characters in production (or use a hex:/base64: encoded 32-byte key)")}@@ -237,7 +253,20 @@ func initializeStartupState(p initializeStartupStateParams) {if err := p.Environment.EnsureLocalEnvironment(appCtx, cfg.AppUrl); err != nil {slog.WarnContext(appCtx, "Failed to ensure local environment", "error", err)}- initializeGitOpsStartupStateInternal(appCtx, p.GitOpsSync)+ if p.GitOpsSync != nil {+ if err := p.GitOpsSync.CleanupOrphanedSyncsOnStartup(appCtx); err != nil {+ slog.WarnContext(appCtx, "Failed to clean up orphaned GitOps syncs on startup", "error", err)+ }+ if err := p.GitOpsSync.CleanupLeakedScratchDirsOnStartup(appCtx); err != nil {+ slog.WarnContext(appCtx, "Failed to clean up leaked GitOps scratch directories on startup", "error", err)+ }+ if err := p.GitOpsSync.CleanupLeakedCloneDirsOnStartup(appCtx); err != nil {+ slog.WarnContext(appCtx, "Failed to clean up leaked git clone directories on startup", "error", err)+ }+ if err := p.GitOpsSync.ReconcileDirectorySyncProjectsOnStartup(appCtx); err != nil {+ slog.WarnContext(appCtx, "Failed to reconcile directory GitOps projects on startup", "error", err)+ }+ }p.Vulnerability.ImportLegacyReportFiles(appCtx)p.Vulnerability.BackfillScanItems(appCtx)if p.Project != nil {@@ -251,9 +280,7 @@ func initializeStartupState(p initializeStartupStateParams) {slog.WarnContext(appCtx, "Failed to reconcile edge environment statuses on startup", "error", err)}- // Global variables are a manager resource: import any pre-existing local- // .env.global once, then materialize the effective set everywhere. Agents- // only serve the per-environment variables endpoint the manager pushes to.+ // Global variables are manager-owned: import a legacy local .env.global once, then sync everywhere.p.Environment.SetVariableSyncer(p.Variable)if err := p.Variable.ImportLegacyLocalEnvFile(appCtx); err != nil {slog.WarnContext(appCtx, "Failed to import legacy global variables", "error", err)@@ -285,10 +312,8 @@ func initializeStartupState(p initializeStartupStateParams) {startup.InitializeNonAgentFeatures(appCtx, runtimeCfg,p.Role.EnsureBuiltInRoles,func(ctx context.Context) error {- // Backfill legacy users.roles first so CreateDefaultAdmin's- // zero-global-admin recovery gate sees upgraded assignments.- // Runs once (kv completion marker); runRoleStartupTasks repeats- // the call for agent-mode and error-retry coverage.+ // Backfill legacy users.roles first so CreateDefaultAdmin's zero-global-admin gate sees them;+ // runRoleStartupTasks repeats it for agent mode and retries.if err := p.Role.BackfillLegacyRoleAssignments(ctx); err != nil {slog.WarnContext(ctx, "Failed to backfill legacy role assignments before admin bootstrap", "error", err)}@@ -304,14 +329,11 @@ func initializeStartupState(p initializeStartupStateParams) {)startup.CleanupUnknownSettings(appCtx, p.Settings)- runRoleStartupTasks(appCtx, p.Role, p.ApiKey, cfg, cfg.AgentMode)+ runRoleStartupTasks(appCtx, p.Role, p.ApiKey, cfg)- // Auto-pair only applies in Edge mode (where the agent's outbound tunnel is the- // only path to the manager). Direct mode is passive — the manager dials the agent's- // HTTP server on TCP 3553, and the manager-side health-check promotes the env to- // Online once reachability is confirmed.+ // Only edge agents auto-pair; direct agents are passive and the manager dials them.if cfg.AgentMode && cfg.EdgeAgent && cfg.AgentToken != "" && cfg.ManagerApiUrl != "" {- if err := handleAgentBootstrapPairing(appCtx, cfg, httpClient); err != nil {+ if err := handleAgentBootstrapPairing(appCtx, cfg); err != nil {slog.WarnContext(appCtx, "Failed to auto-pair agent with manager", "error", err)}} else if cfg.AgentMode && !cfg.EdgeAgent {@@ -319,36 +341,14 @@ func initializeStartupState(p initializeStartupStateParams) {}}-func initializeGitOpsStartupStateInternal(appCtx context.Context, gitOpsSync *gitops.GitOpsSyncService) {- if gitOpsSync == nil {- return- }- if err := gitOpsSync.CleanupOrphanedSyncsOnStartup(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to clean up orphaned GitOps syncs on startup", "error", err)- }-- if err := gitOpsSync.CleanupLeakedScratchDirsOnStartup(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to clean up leaked GitOps scratch directories on startup", "error", err)- }- if err := gitOpsSync.CleanupLeakedCloneDirsOnStartup(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to clean up leaked git clone directories on startup", "error", err)- }- if err := gitOpsSync.ReconcileDirectorySyncProjectsOnStartup(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to reconcile directory GitOps projects on startup", "error", err)- }-}--func runRoleStartupTasks(ctx context.Context, roleService *role.RoleService, apiKeyService *apikey.ApiKeyService, cfg *config.Config, agentMode bool) {+func runRoleStartupTasks(ctx context.Context, roleService *role.RoleService, apiKeyService *apikey.ApiKeyService, cfg *config.Config) {if roleService == nil {return}if err := roleService.EnsureBuiltInRoles(ctx); err != nil {slog.ErrorContext(ctx, "Failed to reconcile built-in roles", "error", err)}- // Backfill must run AFTER EnsureBuiltInRoles (it references the role IDs- // seeded there) and BEFORE BackfillApiKeyPermissions / AssertGlobalAdminExists- // (both consult the assignments table this populates). It is a no-op once- // its kv completion marker exists.+ // Runs after EnsureBuiltInRoles seeds role IDs and before the API key backfill and admin guard read assignments.if err := roleService.BackfillLegacyRoleAssignments(ctx); err != nil {slog.ErrorContext(ctx, "Failed to backfill legacy users.roles into user_role_assignments", "error", err)}@@ -357,12 +357,10 @@ func runRoleStartupTasks(ctx context.Context, roleService *role.RoleService, apislog.WarnContext(ctx, "Failed to backfill API key permissions", "error", err)}}- if cfg != nil {- if err := roleService.ReconcileEnvOidcMappings(ctx, cfg.OidcRoleMappings); err != nil {- slog.ErrorContext(ctx, "Failed to reconcile OIDC_ROLE_MAPPINGS", "error", err)- }+ if err := roleService.ReconcileEnvOidcMappings(ctx, cfg.OidcRoleMappings); err != nil {+ slog.ErrorContext(ctx, "Failed to reconcile OIDC_ROLE_MAPPINGS", "error", err)}- if agentMode {+ if cfg.AgentMode {return}if err := roleService.AssertGlobalAdminExists(ctx); err != nil {@@ -374,11 +372,18 @@ func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, cfg *config.var stop func(context.Context) errorlc.Append(fx.Hook{OnStart: func(context.Context) error {- var startErr error- stop, startErr = startEdgeTunnelClientIfConfigured(appCtx, cfg, router)- if startErr != nil {- slog.ErrorContext(appCtx, "Failed to start edge tunnel client", "error", startErr)+ if !cfg.EdgeAgent || cfg.ManagerApiUrl == "" || cfg.AgentToken == "" {+ return nil}+ edgeCfg := buildEdgeRuntimeConfigInternal(cfg)+ slog.InfoContext(appCtx, "Starting edge agent session client", edge.StartupLogAttrs(edgeCfg)...)+ tunnelStop, err := edge.StartTunnelClient(appCtx, edgeCfg, router)+ if err != nil {+ slog.ErrorContext(appCtx, "Failed to start edge tunnel client", "error", err)+ return nil+ }+ stop = tunnelStop+ slog.InfoContext(appCtx, "Edge tunnel client started", "managerUrl", cfg.ManagerApiUrl)return nil},OnStop: func(ctx context.Context) error {@@ -399,40 +404,7 @@ func registerAppCancelHook(lc fx.Lifecycle, cancelApp context.CancelFunc) {})}-func startEdgeTunnelClientIfConfigured(appCtx context.Context, cfg *config.Config, router http.Handler) (func(context.Context) error, error) {- managerEndpointConfigured := cfg.ManagerApiUrl != ""- if !cfg.EdgeAgent || !managerEndpointConfigured || cfg.AgentToken == "" {- return nil, nil- }-- edgeCfg := &edge.Config{- EdgeAgent: cfg.EdgeAgent,- EdgeTransport: cfg.EdgeTransport,- EdgeReconnectInterval: cfg.EdgeReconnectInterval,- EdgeMTLSMode: cfg.EdgeMTLSMode,- EdgeMTLSCAFile: cfg.EdgeMTLSCAFile,- EdgeMTLSCertFile: cfg.EdgeMTLSCertFile,- EdgeMTLSKeyFile: cfg.EdgeMTLSKeyFile,- EdgeMTLSServerName: cfg.EdgeMTLSServerName,- EdgeMTLSAssetsDir: cfg.EdgeMTLSAssetsDir,- AppURL: cfg.GetAppURL(),- ManagerApiUrl: cfg.ManagerApiUrl,- AgentToken: cfg.AgentToken,- Port: cfg.Port,- Listen: cfg.Listen,- }-- slog.InfoContext(appCtx, "Starting edge agent session client", edge.StartupLogAttrs(edgeCfg)...)- stop, err := edge.StartTunnelClient(appCtx, edgeCfg, router)- if err != nil {- return nil, fmt.Errorf("failed to start edge tunnel client: %w", err)- }-- slog.InfoContext(appCtx, "Edge tunnel client started", "managerUrl", cfg.ManagerApiUrl)- return stop, nil-}--func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config, httpClient *http.Client) error {+func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config) error {slog.InfoContext(ctx, "Agent mode detected with token, attempting auto-pairing", "managerUrl", cfg.ManagerApiUrl)pairURL := strings.TrimRight(httpx.ManagerBaseURL(cfg.ManagerApiUrl), "/") + "/api/environments/pair"@@ -447,20 +419,9 @@ func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config, httpClreq.Header.Set("X-Api-Key", cfg.AgentToken)- if cfg.EdgeAgent && strings.TrimSpace(cfg.ManagerApiUrl) != "" {- edgeClient, edgeErr := edge.NewManagerHTTPClient(&edge.Config{- ManagerApiUrl: cfg.ManagerApiUrl,- EdgeMTLSMode: cfg.EdgeMTLSMode,- EdgeMTLSCAFile: cfg.EdgeMTLSCAFile,- EdgeMTLSCertFile: cfg.EdgeMTLSCertFile,- EdgeMTLSKeyFile: cfg.EdgeMTLSKeyFile,- EdgeMTLSServerName: cfg.EdgeMTLSServerName,- EdgeMTLSAssetsDir: cfg.EdgeMTLSAssetsDir,- }, 10*time.Second)- if edgeErr != nil {- return fmt.Errorf("failed to configure edge pairing client: %w", edgeErr)- }- httpClient = edgeClient+ httpClient, err := edge.NewManagerHTTPClient(buildEdgeRuntimeConfigInternal(cfg), 10*time.Second)+ if err != nil {+ return fmt.Errorf("failed to configure edge pairing client: %w", err)}resp, err := httpClient.Do(req)@@ -471,35 +432,17 @@ func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config, httpClbody, _ := io.ReadAll(resp.Body)- switch resp.StatusCode {- case http.StatusOK:+ switch {+ case resp.StatusCode == http.StatusOK:slog.InfoContext(ctx, "Successfully paired agent with manager", "managerUrl", cfg.ManagerApiUrl)return nil- case http.StatusBadRequest:- // Environment is not in pending status - already paired, this is fine- if strings.Contains(string(body), "not in pending status") {- slog.InfoContext(ctx, "Agent already paired with manager", "managerUrl", cfg.ManagerApiUrl)- return nil- }- return fmt.Errorf("pairing failed with status %d: %s", resp.StatusCode, string(body))- case http.StatusUnauthorized:- // Invalid API key - could be already paired with a different key, or key was deleted- // This is not fatal; the agent can still function if it has a valid token configured+ case resp.StatusCode == http.StatusBadRequest && strings.Contains(string(body), "not in pending status"):+ slog.InfoContext(ctx, "Agent already paired with manager", "managerUrl", cfg.ManagerApiUrl)+ return nil+ case resp.StatusCode == http.StatusUnauthorized:+ // Not fatal: the key may belong to an earlier pairing, and a configured token still works.slog.DebugContext(ctx, "Pairing skipped - API key not recognized (agent may already be paired)", "managerUrl", cfg.ManagerApiUrl)return nil- default:- return fmt.Errorf("pairing failed with status %d: %s", resp.StatusCode, string(body))}-}--func initializeDBAndMigrateInternal(ctx context.Context, cfg *config.Config) (*database.DB, error) {- db, err := database.Initialize(ctx, cfg.DatabaseURL, database.MigrationOptions{- AllowDowngrade: cfg.AllowDowngrade,- })- if err != nil {- return nil, fmt.Errorf("failed to initialize database: %w", err)- }-- slog.InfoContext(ctx, "Database initialized successfully")- return db, nil+ return fmt.Errorf("pairing failed with status %d: %s", resp.StatusCode, string(body))}diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.goindex 25e245226..257cea7f0 100644--- a/backend/internal/bootstrap/bootstrap_test.go+++ b/backend/internal/bootstrap/bootstrap_test.go@@ -40,21 +40,21 @@ import (tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1")-type blockingBusWatcherInternal struct {+type blockingBusWatcher struct {started chan struct{}stopped chan struct{}}-func (w *blockingBusWatcherInternal) Name() string { return "blocking" }+func (w *blockingBusWatcher) Name() string { return "blocking" }-func (w *blockingBusWatcherInternal) Start(ctx context.Context) error {+func (w *blockingBusWatcher) Start(ctx context.Context) error {close(w.started)<-ctx.Done()close(w.stopped)return nil}-func (w *blockingBusWatcherInternal) RunNow(context.Context) error { return nil }+func (w *blockingBusWatcher) RunNow(context.Context) error { return nil }func TestNormalizeTunnelGRPCRequestPath(t *testing.T) {fullMethodPath := tunnelpb.TunnelService_Connect_FullMethodName@@ -90,10 +90,8 @@ func TestNormalizeTunnelGRPCRequestPath(t *testing.T) {})t.Run("legacy /api/tunnel/connect is rewritten to gRPC method", func(t *testing.T) {- // Regression: PR #2722 removed this branch, breaking the edge agent's- // gRPC transport. The agent client uses /api/tunnel/connect as its- // gRPC method path so reverse proxies can route tunnel traffic with- // a stable URL instead of the proto-generated gRPC service name.+ // Edge agents use /api/tunnel/connect as their gRPC method path so reverse proxies+ // can route tunnel traffic on a stable URL.req := httptest.NewRequest("POST", "/api/tunnel/connect", http.NoBody)normalized := normalizeTunnelGRPCRequestPathInternal(req)@@ -256,11 +254,8 @@ func TestHTTP2APIResponsesDoNotUseAPIGzip(t *testing.T) {}func TestH2CStreamSurvivesPastReadHeaderTimeout(t *testing.T) {- // Regression guard for the go1.26.6 CVE-2026-56853 backport, which arms- // ReadHeaderTimeout on the raw conn before the h2c preface sniff and never- // clears it on the HTTP/2 handoff. Without ClearReadDeadline in the h2c- // handler wrapper, the connection — and every gRPC tunnel stream on it —- // dies with an i/o timeout exactly ReadHeaderTimeout after accept.+ // go1.26.6 arms ReadHeaderTimeout before the h2c preface sniff and never clears it, so+ // without ClearReadDeadline every h2c stream dies ReadHeaderTimeout after accept.const readHeaderTimeout = 250 * time.Millisecondrelease := make(chan struct{})@@ -300,9 +295,7 @@ func TestH2CStreamSurvivesPastReadHeaderTimeout(t *testing.T) {defer func() { _ = resp.Body.Close() }()require.Equal(t, "HTTP/2.0", resp.Proto)- // Keep the stream idle well past ReadHeaderTimeout before the handler- // writes again; on a connection with the stale deadline still armed the- // read below fails instead of returning the body.+ // Idle past ReadHeaderTimeout; a stale deadline would fail the read below.time.Sleep(4 * readHeaderTimeout)close(release)@@ -423,7 +416,7 @@ func TestJobSchedulerStopCancelsItsPrivateContext(t *testing.T) {executed <- runreturn schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}, nil)- watcher := &blockingBusWatcherInternal{+ watcher := &blockingBusWatcher{started: make(chan struct{}),stopped: make(chan struct{}),}@@ -468,7 +461,7 @@ func TestApplicationOptionsValidate(t *testing.T) {require.NoError(t, err)}-func TestPrepareServerTLSInternal_AgentModeSkipsManagerMTLSValidation(t *testing.T) {+func TestPrepareServerTLS_AgentModeSkipsManagerMTLSValidation(t *testing.T) {cfg := &config.Config{AgentMode: true,EdgeMTLSMode: "required",@@ -484,7 +477,7 @@ func TestPrepareServerTLSInternal_AgentModeSkipsManagerMTLSValidation(t *testingassert.Equal(t, "required", edgeCfg.EdgeMTLSMode)}-func TestPrepareServerTLSInternal_AllowsExternalMTLSTermination(t *testing.T) {+func TestPrepareServerTLS_AllowsExternalMTLSTermination(t *testing.T) {libcrypto.InitEncryption(&libcrypto.Config{EncryptionKey: "test-encryption-key-for-edge-mtls-32bytes-min",Environment: "test",@@ -511,8 +504,8 @@ func TestPrepareServerTLSInternal_AllowsExternalMTLSTermination(t *testing.T) {}func TestIsWeakProductionEncryptionKey(t *testing.T) {- assert.True(t, isWeakProductionEncryptionKeyInternal("short", "production", false))- assert.False(t, isWeakProductionEncryptionKeyInternal("test-encryption-key-for-edge-mtls-32bytes-min", "production", false))- assert.False(t, isWeakProductionEncryptionKeyInternal("hex:abc", "production", false))- assert.False(t, isWeakProductionEncryptionKeyInternal("short", "development", false))+ assert.True(t, isWeakProductionEncryptionKey("short", "production", false))+ assert.False(t, isWeakProductionEncryptionKey("test-encryption-key-for-edge-mtls-32bytes-min", "production", false))+ assert.False(t, isWeakProductionEncryptionKey("hex:abc", "production", false))+ assert.False(t, isWeakProductionEncryptionKey("short", "development", false))}diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex 19143e001..b1fe7e37b 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -393,7 +393,7 @@ func registerDynamicJobs(params dynamicJobsParams) error {params.JobSchedule.RunEnvironmentHealthNow = func(ctx context.Context) error {return params.Environment.RunHealthChecksNow(ctx)}- params.Environment.RegisterHealthJobsOnStartup(params.AppCtx)+ params.Environment.RescheduleHealthJobs(params.AppCtx)}return nil}diff --git a/backend/internal/bootstrap/logger_bootstrap.go b/backend/internal/bootstrap/logger_bootstrap.goindex aff9cf68d..b6f6bf4a7 100644--- a/backend/internal/bootstrap/logger_bootstrap.go+++ b/backend/internal/bootstrap/logger_bootstrap.go@@ -11,12 +11,22 @@ import ("github.com/lmittmann/tint"slogGorm "github.com/orandin/slog-gorm"+ "go.getarcane.app/streams/logs""gorm.io/gorm""gorm.io/gorm/logger"+ "github.com/getarcaneapp/arcane/backend/v2/api/ws""github.com/getarcaneapp/arcane/backend/v2/internal/config")+// logLevels maps LOG_LEVEL values to slog levels; unknown values map to the zero value, slog.LevelInfo.+var logLevels = map[string]slog.Level{+ "debug": slog.LevelDebug,+ "warn": slog.LevelWarn,+ "warning": slog.LevelWarn,+ "error": slog.LevelError,+}+type requestLogHandler struct {handler slog.Handler}@@ -123,7 +133,10 @@ func (h *requestLogHandler) Handle(ctx context.Context, r slog.Record) error {condensedAttrs = append(condensedAttrs, debugAttrs...)for _, a := range attrs {- condensedAttrs = append(condensedAttrs, normalizeRequestLogErrorInternal(a))+ if value, ok := a.Value.Any().(map[string]any); ok && a.Key == "error" {+ a = slog.String("error", fmt.Sprint(value["message"]))+ }+ condensedAttrs = append(condensedAttrs, a)}newRecord := slog.NewRecord(r.Time, r.Level, r.Message, r.PC)@@ -132,25 +145,6 @@ func (h *requestLogHandler) Handle(ctx context.Context, r slog.Record) error {return h.handler.Handle(ctx, newRecord)}-func normalizeRequestLogErrorInternal(a slog.Attr) slog.Attr {- if a.Key != "error" {- return a- }- if value, ok := a.Value.Any().(map[string]any); ok {- return slog.String("error", fmt.Sprint(value["message"]))- }- return a-}--func replaceErrorAttrInternal(_ []string, a slog.Attr) slog.Attr {- if a.Value.Kind() == slog.KindAny {- if err, ok := a.Value.Any().(error); ok {- return slog.String(a.Key, err.Error())- }- }- return a-}-func (h *requestLogHandler) WithAttrs(attrs []slog.Attr) slog.Handler {return &requestLogHandler{handler: h.handler.WithAttrs(attrs)}}@@ -187,43 +181,92 @@ func (h *attrFilterHandler) WithGroup(name string) slog.Handler {return &attrFilterHandler{handler: h.handler.WithGroup(name), dropKeys: h.dropKeys}}-func SetupSlogLogger(cfg *config.Config) {- var lvl slog.Level- switch strings.ToLower(cfg.LogLevel) {- case "debug":- lvl = slog.LevelDebug- case "warn", "warning":- lvl = slog.LevelWarn- case "error":- lvl = slog.LevelError- default:- lvl = slog.LevelInfo+// telemetryLogHandler applies LOG_LEVEL to the OpenTelemetry bridge and strips raw query strings,+// URL queries, and webhook trigger tokens before records are exported.+type telemetryLogHandler struct {+ handler slog.Handler+ level slog.Leveler+}++func (h *telemetryLogHandler) Enabled(ctx context.Context, level slog.Level) bool {+ return level >= h.level.Level() && h.handler.Enabled(ctx, level)+}++func (h *telemetryLogHandler) Handle(ctx context.Context, r slog.Record) error {+ redacted := slog.NewRecord(r.Time, r.Level, redactSecrets(r.Message), r.PC)+ r.Attrs(func(a slog.Attr) bool {+ redacted.AddAttrs(redactLogAttrs([]slog.Attr{a})...)+ return true+ })+ return h.handler.Handle(ctx, redacted)+}++func (h *telemetryLogHandler) WithAttrs(attrs []slog.Attr) slog.Handler {+ return &telemetryLogHandler{handler: h.handler.WithAttrs(redactLogAttrs(attrs)), level: h.level}+}++func (h *telemetryLogHandler) WithGroup(name string) slog.Handler {+ return &telemetryLogHandler{handler: h.handler.WithGroup(name), level: h.level}+}++func redactLogAttrs(attrs []slog.Attr) []slog.Attr {+ redacted := make([]slog.Attr, 0, len(attrs))+ for _, a := range attrs {+ a.Value = a.Value.Resolve()+ if err, ok := a.Value.Any().(error); ok && a.Value.Kind() == slog.KindAny {+ a.Value = slog.StringValue(err.Error())+ }+ switch {+ case a.Key == "query":+ continue+ case a.Value.Kind() == slog.KindString:+ a.Value = slog.StringValue(redactSecrets(a.Value.String()))+ case a.Value.Kind() == slog.KindGroup:+ a.Value = slog.GroupValue(redactLogAttrs(a.Value.Group())...)+ }+ redacted = append(redacted, a)}+ return redacted+}+// SetupSlogLogger installs the console logger and the diagnostics log tee. When+// telemetryHandler is set, records are also sent to it at the same level.+func SetupSlogLogger(cfg *config.Config, telemetryHandler slog.Handler) {lv := new(slog.LevelVar)- lv.Set(lvl)+ lv.Set(logLevels[strings.ToLower(cfg.LogLevel)])++ replaceErrorAttr := func(_ []string, a slog.Attr) slog.Attr {+ if err, ok := a.Value.Any().(error); ok && a.Value.Kind() == slog.KindAny {+ return slog.String(a.Key, err.Error())+ }+ return a+ }var h slog.Handlerif cfg.LogJson {h = slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: lv,- ReplaceAttr: replaceErrorAttrInternal,+ ReplaceAttr: replaceErrorAttr,})} else {h = tint.NewTextHandler(os.Stdout, &tint.Options{Level: lv,TimeFormat: "Jan 02 15:04:05.000",- ReplaceAttr: replaceErrorAttrInternal,+ ReplaceAttr: replaceErrorAttr,})}+ if telemetryHandler != nil {+ h = slog.NewMultiHandler(h, &telemetryLogHandler{handler: telemetryHandler, level: lv})+ }+h = &requestLogHandler{handler: h}- slog.SetDefault(slog.New(h))+ slog.SetDefault(slog.New(logs.NewSlogHandler(h, ws.LogBroadcaster())))}func BuildGormLogger(cfg *config.Config) logger.Interface {- lvl := strings.ToLower(cfg.LogLevel)+ lvl := logLevels[strings.ToLower(cfg.LogLevel)]filteredHandler := &attrFilterHandler{handler: slog.Default().Handler(),@@ -233,27 +276,14 @@ func BuildGormLogger(cfg *config.Config) logger.Interface {opts := []slogGorm.Option{slogGorm.WithHandler(filteredHandler),slogGorm.WithSlowThreshold(200 * time.Millisecond),+ slogGorm.SetLogLevel(slogGorm.DefaultLogType, lvl),+ slogGorm.SetLogLevel(slogGorm.ErrorLogType, slog.LevelError),+ slogGorm.SetLogLevel(slogGorm.SlowQueryLogType, slog.LevelWarn),}-- var defaultTypeLevel slog.Level- switch lvl {- case "debug":- defaultTypeLevel = slog.LevelDebug- // Trace all SQL messages only in debug+ // Trace all SQL messages only in debug.+ if lvl == slog.LevelDebug {opts = append(opts, slogGorm.WithTraceAll())- case "warn", "warning":- defaultTypeLevel = slog.LevelWarn- case "error":- defaultTypeLevel = slog.LevelError- default:- defaultTypeLevel = slog.LevelInfo}- opts = append(opts,- slogGorm.SetLogLevel(slogGorm.DefaultLogType, defaultTypeLevel),- slogGorm.SetLogLevel(slogGorm.ErrorLogType, slog.LevelError),- slogGorm.SetLogLevel(slogGorm.SlowQueryLogType, slog.LevelWarn),- )-return gormLogger{Interface: slogGorm.New(opts...)}}diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.goindex c8d08e39a..cbbb3dffe 100644--- a/backend/internal/bootstrap/router_bootstrap.go+++ b/backend/internal/bootstrap/router_bootstrap.go@@ -7,10 +7,12 @@ import ("net""net/http""path"+ "slices""strings""github.com/getarcaneapp/arcane/types/v2"usertypes "github.com/getarcaneapp/arcane/types/v2/user"+ "github.com/labstack/echo-otel/v5""github.com/labstack/echo/v5"echomiddleware "github.com/labstack/echo/v5/middleware""github.com/samber/slog-echo/v2"@@ -24,6 +26,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/federated""github.com/getarcaneapp/arcane/backend/v2/internal/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/internal/telemetry""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/cookie"@@ -44,6 +47,7 @@ var loggerSkipPatterns = []string{"GET /api/environments/*/ws/projects/*/logs","GET /api/environments/*/ws/system/stats","GET /api/stream",+ "POST /api/telemetry/*","GET /_app/*","GET /img","GET /api/health",@@ -52,52 +56,7 @@ var loggerSkipPatterns = []string{"PUT /api/environments/*/uploads/*/*/chunks/*",}-func shouldLogRequestInternal(c *echo.Context, _ error) bool {- mp := c.Request().Method + " " + c.Request().URL.Path- for _, pat := range loggerSkipPatterns {- if pat == mp {- return false- }- if before, ok := strings.CutSuffix(pat, "/*"); ok {- if strings.HasPrefix(mp, before) {- return false- }- }- if ok, _ := path.Match(pat, mp); ok {- return false- }- if strings.HasSuffix(pat, "/") && strings.HasPrefix(mp, pat) {- return false- }- }- return true-}--// requestLoggerMiddlewareInternal wraps slog-echo and filters out internal-// edge tunnel requests plus high-volume endpoints (health, WS, static).-func requestLoggerMiddlewareInternal() echo.MiddlewareFunc {- loggerMiddleware := slogecho.NewWithConfig(slog.Default(), slogecho.Config{- DefaultLevel: slog.LevelInfo,- ClientErrorLevel: slog.LevelWarn,- ServerErrorLevel: slog.LevelError,- Filters: []slogecho.Filter{shouldLogRequestInternal},- WithRequestID: true,- })-- return func(next echo.HandlerFunc) echo.HandlerFunc {- return func(c *echo.Context) error {- if edge.IsInternalTunnelRequest(c.Request().Context()) {- return next(c)- }- if c.Request().Body == nil {- c.Request().Body = http.NoBody- }- return loggerMiddleware(next)(c)- }- }-}--func createAuthValidatorInternal(deps api.HandlerDeps) middleware.AuthValidator {+func createAuthValidator(deps api.HandlerDeps) middleware.AuthValidator {resolveUser := func(ctx context.Context, user *usertypes.Actor) *authz.PermissionSet {ps, err := deps.Role.Service().ResolvePermissions(ctx, user.ID)if err != nil || ps == nil {@@ -134,26 +93,21 @@ func createAuthValidatorInternal(deps api.HandlerDeps) middleware.AuthValidatorreturn nil, nil, false}- user, ok := verifyRouterRequestUserInternal(ctx, deps.Auth.Service(), req)- if !ok {+ // Bearer tokens take precedence over the browser token cookie.+ verify := deps.Auth.Service().VerifyBrowserToken+ token, err := cookie.GetTokenCookie(req)+ if bearer, ok := strings.CutPrefix(req.Header.Get("Authorization"), "Bearer "); ok {+ verify, token, err = deps.Auth.Service().VerifyToken, bearer, nil+ }+ if err != nil || token == "" {return nil, nil, false}- return resolveUser(ctx, user), user, true- }-}--func verifyRouterRequestUserInternal(ctx context.Context, authService *auth.AuthService, req *http.Request) (*usertypes.Actor, bool) {- if authHeader := req.Header.Get("Authorization"); strings.HasPrefix(authHeader, "Bearer ") {- user, _, err := authService.VerifyToken(ctx, strings.TrimPrefix(authHeader, "Bearer "))- return user.Actor(), err == nil && user != nil- }-- browserToken, err := cookie.GetTokenCookie(req)- if err != nil || browserToken == "" {- return nil, false+ user, _, err := verify(ctx, token)+ if err != nil || user == nil {+ return nil, nil, false+ }+ return resolveUser(ctx, user.Actor()), user.Actor(), true}- user, _, err := authService.VerifyBrowserToken(ctx, browserToken)- return user.Actor(), err == nil && user != nil}type RouterParams struct {@@ -167,45 +121,84 @@ type RouterParams struct {TunnelRegistry *edge.TunnelRegistry}-// newEchoInternal builds the Echo instance with path parameter values-// percent-decoded before they reach handlers (RFC 3986 §6.2.2.2); route-// matching still happens on the escaped path, so an encoded "/" cannot-// change which route matches.-func newEchoInternal() *echo.Echo {- return echo.NewWithConfig(echo.Config{- Router: echo.NewRouter(echo.RouterConfig{- AllowOverwritingRoute: true,- UnescapePathParamValues: true,- }),- })-}-func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {ctx := p.Contextcfg := p.Configdeps := p.HandlerDeps- e := newEchoInternal()+ // Path params are decoded after matching on the escaped path, so an encoded "/" cannot change the route.+ e := echo.NewWithConfig(echo.Config{+ Router: echo.NewRouter(echo.RouterConfig{AllowOverwritingRoute: true, UnescapePathParamValues: true}),+ })- trustedProxyNets := parseTrustedProxyCIDRsInternal(cfg.TrustedProxies)+ var trustedProxyNets []*net.IPNet+ // Trust only the configured ranges, not Echo's built-in loopback/link-local/private defaults.+ trustOpts := []echo.TrustOption{echo.TrustLoopback(false), echo.TrustLinkLocal(false), echo.TrustPrivateNet(false)}+ for cidr := range strings.SplitSeq(cfg.TrustedProxies, ",") {+ cidr = strings.TrimSpace(cidr)+ if cidr == "" {+ continue+ }+ _, ipnet, err := net.ParseCIDR(cidr)+ if err != nil {+ slog.WarnContext(ctx, "invalid TRUSTED_PROXIES CIDR, ignoring", "cidr", cidr, "error", err)+ continue+ }+ trustedProxyNets = append(trustedProxyNets, ipnet)+ trustOpts = append(trustOpts, echo.TrustIPRange(ipnet))+ }if cfg.TrustedProxies != "" && len(trustedProxyNets) == 0 {slog.WarnContext(ctx, "TRUSTED_PROXIES set but no valid CIDRs found; falling back to direct IP extraction")}if len(trustedProxyNets) == 0 {e.IPExtractor = echo.ExtractIPDirect()} else {- // Trust only the configured ranges, not Echo's built-in loopback/link-local/private defaults.- opts := []echo.TrustOption{echo.TrustLoopback(false), echo.TrustLinkLocal(false), echo.TrustPrivateNet(false)}- for _, ipnet := range trustedProxyNets {- opts = append(opts, echo.TrustIPRange(ipnet))- }- e.IPExtractor = echo.ExtractIPFromXFFHeader(opts...)+ e.IPExtractor = echo.ExtractIPFromXFFHeader(trustOpts...)}e.Use(echomiddleware.Recover())e.Use(echomiddleware.RequestID())- e.Use(requestLoggerMiddlewareInternal())- e.Use(secureCookieContextMiddlewareInternal(trustedProxyNets))+ // Trace REST API calls only, skipping long-lived streams, tunnels, health probes, and the browser trace relay.+ e.Use(echootel.NewMiddlewareWithConfig(echootel.Config{Skipper: func(c *echo.Context) bool {+ req := c.Request()+ p := req.URL.Path+ return !strings.HasPrefix(p, "/api/") || p == "/api/health" ||+ strings.HasPrefix(p, "/api/telemetry/") || strings.HasPrefix(p, "/api/tunnel/") ||+ httpx.IsWebSocketUpgradeRequest(req) || strings.Contains(req.Header.Get("Accept"), "text/event-stream")+ }}))+ e.Use(middleware.SpanEnvironment(deps.Environment.Service().ResolveEnvironmentName))+ requestLogger := slogecho.NewWithConfig(slog.Default(), slogecho.Config{+ DefaultLevel: slog.LevelInfo,+ ClientErrorLevel: slog.LevelWarn,+ ServerErrorLevel: slog.LevelError,+ Filters: []slogecho.Filter{func(c *echo.Context, _ error) bool {+ mp := c.Request().Method + " " + c.Request().URL.Path+ return !slices.ContainsFunc(loggerSkipPatterns, func(pat string) bool {+ if prefix, ok := strings.CutSuffix(pat, "/*"); ok && strings.HasPrefix(mp, prefix) {+ return true+ }+ matched, _ := path.Match(pat, mp)+ return matched+ })+ }},+ WithRequestID: true,+ WithTraceID: true,+ WithSpanID: true,+ })+ // Internal edge tunnel requests are never logged.+ e.Use(func(next echo.HandlerFunc) echo.HandlerFunc {+ logged := requestLogger(next)+ return func(c *echo.Context) error {+ if edge.IsInternalTunnelRequest(c.Request().Context()) {+ return next(c)+ }+ if c.Request().Body == nil {+ c.Request().Body = http.NoBody+ }+ return logged(c)+ }+ })+ e.Use(secureCookieContextMiddleware(trustedProxyNets))authMiddleware := p.AuthMiddlewaree.Use(middleware.NewCORSMiddleware(cfg).Add())@@ -234,9 +227,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {apiGroup.Use(middleware.PerTokenRateLimitForPaths([]string{"/api/webhooks/trigger/:token"}, 60, 10, deps.Webhook.Service().IsKnownToken,))- // Agent event ingestion authenticates on the agent token alone and sits- // outside the auth middleware, so it needs its own brute-force ceiling.- // The allowance is generous because busy agents legitimately batch events.+ // Agent event ingestion sits outside the auth middleware, so it needs its own ceiling; agents batch events.apiGroup.Use(middleware.PerIPRateLimitForPaths([]string{"/api/events"}, 60, 30,))@@ -253,6 +244,9 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {// Register public webhook trigger endpoint before auth middleware (token in URL is the sole auth)api.RegisterWebhookTrigger(apiGroup, deps.Webhook.Service(), handlerAppCtx)federated.RegisterFederatedTokenExchange(apiGroup, deps.Federated)+ if !cfg.AgentMode {+ telemetry.RegisterRoutes(apiGroup, telemetry.NewService())+ }deps.Event.RegisterAgentRoutes(apiGroup, func(ctx context.Context, token string) (string, error) {env, err := deps.Environment.Service().ResolveEnvironmentByAccessToken(ctx, token)if err != nil {@@ -270,7 +264,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {types.LocalDockerEnvironmentID,"id",envResolver,- createAuthValidatorInternal(deps),+ createAuthValidator(deps),permissionMatcher,p.TunnelRegistry,httpx.ValidateWebSocketOrigin(cfg.GetAppURL()),@@ -279,10 +273,8 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {humaAPI := api.SetupAPI(e, apiGroup, handlerAppCtx, cfg, deps)- // Populate the proxy permission matcher from the registered API surface so- // remote-environment requests are authorized with the same permissions- // enforced locally. The matcher pointer is shared with the proxy middleware- // constructed above; population completes here, before the server serves traffic.+ // Fill the proxy's shared matcher from the local API surface before serving so remote+ // requests are authorized with the same permissions.permissionMatcher.CollectFromHumaAPI(humaAPI)ws.AddProxiedPermissions(permissionMatcher)@@ -293,8 +285,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {// Remaining echo handlers (WebSocket/streaming)ws.NewWebSocketHandler(apiGroup, deps.Project.Service(), deps.Container.Service(), deps.Swarm.Service(), deps.System.Service(), deps.Diagnostics, authMiddleware, cfg)- // Register edge tunnel endpoint for manager to accept agent connections- // This is only registered when NOT in agent mode (i.e., running as manager)+ // Managers accept agent connections on the edge tunnel endpoint.var tunnelServer *edge.TunnelServerif !cfg.AgentMode {tunnelServer = registerEdgeTunnelRoutes(ctx, p.Lifecycle, cfg, apiGroup, deps.Environment.Service(), deps.Event.Service(), deps.Notification.Service(), p.TunnelRegistry)@@ -327,42 +318,22 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {return e, tunnelServer}-// parseTrustedProxyCIDRsInternal parses TRUSTED_PROXIES into a list of-// validated networks. Invalid entries are logged and skipped.-func parseTrustedProxyCIDRsInternal(raw string) []*net.IPNet {- if raw == "" {- return nil- }- var nets []*net.IPNet- for cidr := range strings.SplitSeq(raw, ",") {- cidr = strings.TrimSpace(cidr)- if cidr == "" {- continue- }- _, ipnet, err := net.ParseCIDR(cidr)- if err != nil {- slog.WarnContext(context.Background(), "invalid TRUSTED_PROXIES CIDR, ignoring", "cidr", cidr, "error", err) //nolint:forbidigo // Startup configuration parsing has no request context.- continue- }- nets = append(nets, ipnet)- }- return nets-}--// secureCookieContextMiddlewareInternal records whether the request should be-// treated as HTTPS for cookie-emitting handlers. X-Forwarded-Proto is honored-// ONLY when the direct TCP peer is in TRUSTED_PROXIES — an untrusted client-// setting the header directly cannot trick the server into issuing Secure /-// __Host- cookies over plain HTTP.-func secureCookieContextMiddlewareInternal(trustedProxyNets []*net.IPNet) echo.MiddlewareFunc {+// secureCookieContextMiddleware marks requests as HTTPS for cookies. X-Forwarded-Proto is+// honored only when the direct TCP peer is in TRUSTED_PROXIES.+func secureCookieContextMiddleware(trustedProxyNets []*net.IPNet) echo.MiddlewareFunc {return func(next echo.HandlerFunc) echo.HandlerFunc {return func(c *echo.Context) error {req := c.Request()secure := req.TLS != nil- if !secure && len(trustedProxyNets) > 0 &&- strings.EqualFold(req.Header.Get("X-Forwarded-Proto"), "https") &&- remoteAddrInTrustedProxiesInternal(req.RemoteAddr, trustedProxyNets) {- secure = true+ if !secure && len(trustedProxyNets) > 0 && strings.EqualFold(req.Header.Get("X-Forwarded-Proto"), "https") {+ host, _, err := net.SplitHostPort(req.RemoteAddr)+ if err != nil {+ host = req.RemoteAddr+ }+ // Unparseable remote addresses are untrusted.+ if ip := net.ParseIP(host); ip != nil {+ secure = slices.ContainsFunc(trustedProxyNets, func(n *net.IPNet) bool { return n.Contains(ip) })+ }}if secure {c.SetRequest(req.WithContext(context.WithValue(req.Context(), cookie.SecureCookieContextKey{}, true)))@@ -371,23 +342,3 @@ func secureCookieContextMiddlewareInternal(trustedProxyNets []*net.IPNet) echo.M}}}--// remoteAddrInTrustedProxiesInternal reports whether the direct TCP peer-// address of the request falls within any of the configured trusted-proxy-// networks. Unparseable remote addresses are treated as untrusted.-func remoteAddrInTrustedProxiesInternal(remoteAddr string, nets []*net.IPNet) bool {- host, _, err := net.SplitHostPort(remoteAddr)- if err != nil {- host = remoteAddr- }- ip := net.ParseIP(host)- if ip == nil {- return false- }- for _, n := range nets {- if n.Contains(ip) {- return true- }- }- return false-}diff --git a/backend/internal/bootstrap/router_bootstrap_test.go b/backend/internal/bootstrap/router_bootstrap_test.goindex 092882afa..657499458 100644--- a/backend/internal/bootstrap/router_bootstrap_test.go+++ b/backend/internal/bootstrap/router_bootstrap_test.go@@ -13,15 +13,39 @@ import ("github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "github.com/getarcaneapp/arcane/backend/v2/api"+ "github.com/getarcaneapp/arcane/backend/v2/internal/auth"+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"+ "github.com/getarcaneapp/arcane/backend/v2/internal/container"+ "github.com/getarcaneapp/arcane/backend/v2/internal/project"+ "github.com/getarcaneapp/arcane/backend/v2/internal/swarm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/system"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/cookie")-func TestNewEchoInternal_DecodesPathParams(t *testing.T) {- router := newEchoInternal()- api := humaecho.NewWithGroup(router, router.Group("/api"), huma.DefaultConfig("test", "1.0.0"))+func TestNewRouter_DecodesPathParams(t *testing.T) {+ cfg := &config.Config{+ AgentMode: true,+ AppUrl: "http://localhost:3552",+ Environment: config.AppEnvironmentTest,+ }+ router, _ := newRouter(RouterParams{+ Context: t.Context(),+ Config: cfg,+ HandlerDeps: api.HandlerDeps{+ Project: project.New(&project.ProjectService{}, nil),+ Container: container.New(&container.ContainerService{}, nil, nil, nil),+ Swarm: swarm.New(&swarm.SwarmService{}, nil, nil, nil),+ System: system.New(&system.SystemService{}, nil, nil, nil),+ },+ AuthMiddleware: auth.NewAuthMiddleware(nil, cfg),+ TunnelRegistry: edge.NewTunnelRegistry(),+ })+ humaAPI := humaecho.NewWithGroup(router, router.Group("/api"), huma.DefaultConfig("test", "1.0.0"))var got string- huma.Register(api, huma.Operation{+ huma.Register(humaAPI, huma.Operation{OperationID: "get-image",Method: http.MethodGet,Path: "/environments/{id}/images/{imageId}",@@ -66,7 +90,7 @@ func TestSecureCookieContextMiddleware_TrustGating(t *testing.T) {c := e.NewContext(req, rec)var observed bool- handler := secureCookieContextMiddlewareInternal(nets)(func(c *echo.Context) error {+ handler := secureCookieContextMiddleware(nets)(func(c *echo.Context) error {observed = cookie.SecureCookieFromContext(c.Request().Context())return nil})diff --git a/backend/internal/bootstrap/telemetry_bootstrap.go b/backend/internal/bootstrap/telemetry_bootstrap.gonew file mode 100644index 000000000..a5626a6b4--- /dev/null+++ b/backend/internal/bootstrap/telemetry_bootstrap.go@@ -0,0 +1,195 @@+package bootstrap++import (+ "context"+ "errors"+ "fmt"+ "log/slog"+ "regexp"+ "slices"+ "sync"++ "go.getarcane.app/kit/pkg"+ "go.opentelemetry.io/contrib/exporters/autoexport"+ "go.opentelemetry.io/contrib/instrumentation/runtime"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/log/global"+ "go.opentelemetry.io/otel/propagation"+ "go.opentelemetry.io/otel/sdk/log"+ "go.opentelemetry.io/otel/sdk/metric"+ "go.opentelemetry.io/otel/sdk/resource"+ "go.opentelemetry.io/otel/sdk/trace"+ "go.opentelemetry.io/otel/semconv/v1.40.0"++ "github.com/getarcaneapp/arcane/backend/v2/internal/config"+ "github.com/getarcaneapp/arcane/backend/v2/internal/telemetry"+)++// telemetryProviders owns the providers enabled through the standard OTEL_*+// exporter variables. A nil provider means that signal is off.+type telemetryProviders struct {+ tracerProvider *trace.TracerProvider+ meterProvider *metric.MeterProvider+ loggerProvider *log.LoggerProvider++ shutdownOnce sync.Once+ shutdownErr error+}++// setupTelemetry installs global providers for each OTEL_*_EXPORTER that is set+// to something other than none. Exporter failures are reported to errorLogger.+func setupTelemetry(ctx context.Context, cfg *config.Config, errorLogger *slog.Logger, instanceID string) (*telemetryProviders, error) {+ tel := &telemetryProviders{}+ if telemetry.SDKDisabled() {+ return tel, nil+ }+ fail := func(err error) (*telemetryProviders, error) {+ return nil, errors.Join(err, tel.Shutdown(context.WithoutCancel(ctx)))+ }++ mode := "manager"+ if cfg.AgentMode {+ mode = kit.Ternary(cfg.EdgeAgent, "edge", "agent")+ }+ // OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES may override the defaults, but never the+ // persisted instance ID or mode that identify this installation.+ res, err := resource.New(ctx,+ resource.WithTelemetrySDK(),+ resource.WithAttributes(semconv.ServiceName("arcane"), semconv.ServiceVersion(config.Version)),+ resource.WithFromEnv(),+ resource.WithAttributes(semconv.ServiceInstanceID(instanceID), attribute.String("arcane.mode", mode)),+ )+ if err != nil {+ return fail(fmt.Errorf("create telemetry resource: %w", err))+ }++ // The fallbacks return nil so an unset OTEL_*_EXPORTER means off, not otlp.+ spanExporter, err := autoexport.NewSpanExporter(ctx, autoexport.WithFallbackSpanExporter(func(context.Context) (trace.SpanExporter, error) {+ return nil, nil //nolint:nilnil // nil means tracing is off.+ }))+ if err != nil {+ return fail(fmt.Errorf("create trace exporter: %w", err))+ }+ if spanExporter != nil && !autoexport.IsNoneSpanExporter(spanExporter) {+ tel.tracerProvider = trace.NewTracerProvider(trace.WithResource(res), trace.WithBatcher(redactingExporter{spanExporter}))+ }++ // Adds go.schedule.duration, which runtime.Start does not report; OTEL_METRICS_PRODUCERS overrides it.+ autoexport.WithFallbackMetricProducer(func(context.Context) (metric.Producer, error) { return runtime.NewProducer(), nil })+ metricReader, err := autoexport.NewMetricReader(ctx, autoexport.WithFallbackMetricReader(func(context.Context) (metric.Reader, error) {+ return nil, nil //nolint:nilnil // nil means metrics are off.+ }))+ if err != nil {+ return fail(fmt.Errorf("create metric reader: %w", err))+ }+ if metricReader != nil && !autoexport.IsNoneMetricReader(metricReader) {+ tel.meterProvider = metric.NewMeterProvider(metric.WithResource(res), metric.WithReader(metricReader))+ if err = runtime.Start(runtime.WithMeterProvider(tel.meterProvider)); err != nil {+ return fail(fmt.Errorf("start runtime metrics: %w", err))+ }+ }++ logExporter, err := autoexport.NewLogExporter(ctx, autoexport.WithFallbackLogExporter(func(context.Context) (log.Exporter, error) {+ return nil, nil //nolint:nilnil // nil means log export is off.+ }))+ if err != nil {+ return fail(fmt.Errorf("create log exporter: %w", err))+ }+ if logExporter != nil && !autoexport.IsNoneLogExporter(logExporter) {+ tel.loggerProvider = log.NewLoggerProvider(log.WithResource(res), log.WithProcessor(log.NewBatchProcessor(logExporter)))+ }++ if tel.tracerProvider == nil && tel.meterProvider == nil && tel.loggerProvider == nil {+ return tel, nil+ }+ otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {+ errorLogger.WarnContext(ctx, "OpenTelemetry export failed", "error", err)+ }))+ if tel.tracerProvider != nil {+ otel.SetTracerProvider(tel.tracerProvider)+ otel.SetTextMapPropagator(propagation.TraceContext{})+ }+ if tel.meterProvider != nil {+ otel.SetMeterProvider(tel.meterProvider)+ }+ if tel.loggerProvider != nil {+ global.SetLoggerProvider(tel.loggerProvider)+ }+ return tel, nil+}++// Shutdown flushes and stops every provider. It is safe to call more than once.+func (t *telemetryProviders) Shutdown(ctx context.Context) error {+ t.shutdownOnce.Do(func() {+ var errs []error+ if t.tracerProvider != nil {+ errs = append(errs, t.tracerProvider.Shutdown(ctx))+ }+ if t.meterProvider != nil {+ errs = append(errs, t.meterProvider.Shutdown(ctx))+ }+ if t.loggerProvider != nil {+ errs = append(errs, t.loggerProvider.Shutdown(ctx))+ }+ t.shutdownErr = errors.Join(errs...)+ })+ return t.shutdownErr+}++var (+ urlUserinfo = regexp.MustCompile(`(https?://)[^@/\s"']*@`)+ urlQuery = regexp.MustCompile(`(https?://[^\s?#"']*)[?#][^\s"']*`)+ webhookToken = regexp.MustCompile(`(/webhooks/trigger/)[^/?#\s"']+`)+)++// redactSecrets strips URL credentials, queries, and fragments, and masks webhook trigger tokens, which travel in the path.+func redactSecrets(s string) string {+ s = urlUserinfo.ReplaceAllString(s, "$1")+ return webhookToken.ReplaceAllString(urlQuery.ReplaceAllString(s, "$1"), "${1}:token")+}++// redactingExporter strips URL queries and fragments from span attributes, events, and statuses before export.+type redactingExporter struct{ trace.SpanExporter }++func (e redactingExporter) ExportSpans(ctx context.Context, spans []trace.ReadOnlySpan) error {+ redacted := make([]trace.ReadOnlySpan, len(spans))+ for i, span := range spans {+ redacted[i] = redactedSpan{span}+ }+ return e.SpanExporter.ExportSpans(ctx, redacted)+}++type redactedSpan struct{ trace.ReadOnlySpan }++func (s redactedSpan) Attributes() []attribute.KeyValue {+ return redactAttributes(s.ReadOnlySpan.Attributes())+}++func (s redactedSpan) Events() []trace.Event {+ events := slices.Clone(s.ReadOnlySpan.Events())+ for i := range events {+ events[i].Attributes = redactAttributes(events[i].Attributes)+ }+ return events+}++func (s redactedSpan) Status() trace.Status {+ status := s.ReadOnlySpan.Status()+ status.Description = redactSecrets(status.Description)+ return status+}++func redactAttributes(attrs []attribute.KeyValue) []attribute.KeyValue {+ redacted := make([]attribute.KeyValue, 0, len(attrs))+ for _, kv := range attrs {+ switch {+ case kv.Key == "url.query":+ continue+ case kv.Value.Type() == attribute.STRING:+ kv.Value = attribute.StringValue(redactSecrets(kv.Value.AsString()))+ }+ redacted = append(redacted, kv)+ }+ return redacted+}diff --git a/backend/internal/container/children/stats/service.go b/backend/internal/container/children/stats/service.goindex 6948eca13..30f4e0617 100644--- a/backend/internal/container/children/stats/service.go+++ b/backend/internal/container/children/stats/service.go@@ -1,4 +1,4 @@-// Package stats owns container resource sampling, resource sorting and the+// Package stats owns container resource sampling, resource metrics, resource sorting and the// live stats stream.package stats@@ -10,6 +10,8 @@ import ("fmt""io""log/slog"+ "maps"+ "slices""strings""sync""time"@@ -18,8 +20,11 @@ import ("github.com/moby/moby/api/types/container""github.com/moby/moby/client""github.com/samber/hot"+ "go.getarcane.app/docker""go.getarcane.app/kit/pkg""go.getarcane.app/streams/stats"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric""golang.org/x/sync/errgroup""golang.org/x/sync/singleflight"@@ -33,6 +38,7 @@ const (ContainerResourceSampleCacheSize = 4096ContainerResourceCollectConcurrency = 16ContainerResourceCollectTimeout = 3 * time.Second+ ContainerMetricsCollectTimeout = 20 * time.Second)// Service collects container stats through the parent's Docker client.@@ -40,7 +46,8 @@ type Service struct {getClient func(context.Context) (*client.Client, error)dockerHost func() stringhistory stats.Store- cache *hot.HotCache[string, containertypes.ResourceSample]+ cache *hot.HotCache[string, container.StatsResponse]+ inspectCache *hot.HotCache[string, container.InspectResponse]flight singleflight.GroupsampleTimeout time.DurationbatchTimeout time.Duration@@ -57,10 +64,15 @@ func New(getClient func(context.Context) (*client.Client, error), dockerHost funreturn &Service{getClient: getClient,dockerHost: dockerHost,- cache: hot.NewHotCache[string, containertypes.ResourceSample](hot.LRU, ContainerResourceSampleCacheSize).+ cache: hot.NewHotCache[string, container.StatsResponse](hot.LRU, ContainerResourceSampleCacheSize).WithTTL(cacheTTL).WithJanitor().Build(),+ // Restart count and start time only change on restart, so metrics reuse inspects for a minute.+ inspectCache: hot.NewHotCache[string, container.InspectResponse](hot.LRU, ContainerResourceSampleCacheSize).+ WithTTL(time.Minute).+ WithJanitor().+ Build(),sampleTimeout: sampleTimeout,batchTimeout: batchTimeout,}@@ -107,12 +119,7 @@ func (s *Service) Stream(ctx context.Context, containerID string, statsChan chanStatsResponse: statsData,CurrentHistorySample: stats.BuildSample(statsData),}- payload.StatsHistory = s.history.Record(- containerID,- payload.CurrentHistorySample,- !historySent,- recordedAt,- )+ payload.StatsHistory = s.history.Record(containerID, payload.CurrentHistorySample, !historySent, recordedAt)historySent = trueselect {@@ -123,26 +130,45 @@ func (s *Service) Stream(ctx context.Context, containerID string, statsChan chan}}-// Collect fetches one sample per running container. Individual failures leave-// the container unsampled; a batch timeout or cancellation fails the request-// so partial lists never sort as global.+// Collect fetches one sample per running container. Individual failures leave the container+// unsampled; a batch timeout or cancellation fails so partial lists never sort as global.func (s *Service) Collect(ctx context.Context, items []containertypes.Summary) (map[string]*containertypes.ResourceSample, error) {- batchCtx, cancel := context.WithTimeout(ctx, s.batchTimeout)+ ids := make([]string, 0, len(items))+ for i := range items {+ if items[i].State == string(container.StateRunning) {+ ids = append(ids, items[i].ID)+ }+ }+ raw, err := collect(ctx, s.batchTimeout, ids, s.fetch)+ if err != nil {+ return nil, fmt.Errorf("failed to collect container resource samples: %w", err)+ }+ samples := make(map[string]*containertypes.ResourceSample, len(raw))+ for id, statsData := range raw {+ built := stats.BuildSample(statsData)+ samples[id] = &containertypes.ResourceSample{+ CPUPercent: float64(built.CPUTenths) / 10,+ MemoryUsageBytes: built.MemoryUsageBytes,+ MemoryLimitBytes: statsData.MemoryStats.Limit,+ SampleTime: statsData.Read,+ }+ }+ return samples, nil+}++// collect runs fetch for each ID with bounded concurrency under timeout. Individual failures are+// skipped; on batch timeout or cancellation it returns the partial result with the error.+func collect[T any](ctx context.Context, timeout time.Duration, ids []string, fetch func(context.Context, string) (T, error)) (map[string]T, error) {+ batchCtx, cancel := context.WithTimeout(ctx, timeout)defer cancel()g, groupCtx := errgroup.WithContext(batchCtx)g.SetLimit(ContainerResourceCollectConcurrency)-- samples := make(map[string]*containertypes.ResourceSample, len(items))+ results := make(map[string]T, len(ids))var mu sync.Mutex-- for i := range items {- if items[i].State != string(container.StateRunning) {- continue- }- id := items[i].ID+ for _, id := range ids {g.Go(func() error {- sample, err := s.fetchInternal(groupCtx, id)+ result, err := fetch(groupCtx, id)if err != nil {if groupCtx.Err() != nil {return groupCtx.Err()@@ -151,93 +177,200 @@ func (s *Service) Collect(ctx context.Context, items []containertypes.Summary) (return nil}mu.Lock()- samples[id] = sample+ results[id] = resultmu.Unlock()return nil})}-- if err := g.Wait(); err != nil {- return nil, fmt.Errorf("failed to collect container resource samples: %w", err)- }- return samples, nil-}--// cacheKeyInternal isolates cache and coalescing entries by Docker daemon and-// container ID.-func (s *Service) cacheKeyInternal(containerID string) string {- return s.dockerHost() + "\x00" + containerID+ return results, g.Wait()}-// fetchInternal returns a cached sample when fresh and coalesces concurrent-// fetches for the same container.-func (s *Service) fetchInternal(ctx context.Context, containerID string) (*containertypes.ResourceSample, error) {- key := s.cacheKeyInternal(containerID)+// fetch returns a cached stats sample when fresh and coalesces concurrent fetches per daemon and+// container. IncludePreviousSample gives the CPU delta a valid previous sample.+func (s *Service) fetch(ctx context.Context, containerID string) (container.StatsResponse, error) {+ key := s.dockerHost() + "\x00" + containerIDif sample, ok, _ := s.cache.Get(key); ok {- return &sample, nil+ return sample, nil}ch := s.flight.DoChan(key, func() (any, error) {flightCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.sampleTimeout)defer cancel()- return s.sampleInternal(flightCtx, containerID)++ dockerClient, err := s.getClient(flightCtx)+ if err != nil {+ return nil, fmt.Errorf("failed to connect to Docker: %w", err)+ }+ statsResponse, err := dockerClient.ContainerStats(flightCtx, containerID, client.ContainerStatsOptions{Stream: false, IncludePreviousSample: true})+ if err != nil {+ return nil, fmt.Errorf("failed to fetch container stats: %w", err)+ }+ defer func() { _ = statsResponse.Body.Close() }()++ var statsData container.StatsResponse+ if decodeErr := json.UnmarshalDecode(jsontext.NewDecoder(statsResponse.Body), &statsData); decodeErr != nil {+ return nil, fmt.Errorf("failed to decode container stats: %w", decodeErr)+ }+ if statsData.Read.IsZero() {+ statsData.Read = time.Now()+ }+ s.cache.Set(key, statsData)+ return statsData, nil})fetchCtx, cancel := context.WithTimeout(ctx, s.sampleTimeout)defer cancel()-select {case <-fetchCtx.Done():- return nil, fetchCtx.Err()+ return container.StatsResponse{}, fetchCtx.Err()case res := <-ch:if res.Err != nil {- return nil, res.Err+ return container.StatsResponse{}, res.Err}- sample, ok := res.Val.(*containertypes.ResourceSample)+ statsData, ok := res.Val.(container.StatsResponse)if !ok {- return nil, errors.New("resource sample flight returned unexpected type")+ return container.StatsResponse{}, errors.New("resource sample flight returned unexpected type")}- return sample, nil+ return statsData, nil}}-// sampleInternal performs the non-streaming Docker stats call.-// IncludePreviousSample gives the CPU delta a valid previous sample.-func (s *Service) sampleInternal(ctx context.Context, containerID string) (*containertypes.ResourceSample, error) {- dockerClient, err := s.getClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }+type resourceSample struct {+ stats container.StatsResponse+ inspect *container.InspectResponse+}- statsResponse, err := dockerClient.ContainerStats(ctx, containerID, client.ContainerStatsOptions{Stream: false, IncludePreviousSample: true})- if err != nil {- return nil, fmt.Errorf("failed to fetch container stats: %w", err)+type resourceInstruments struct {+ cpuTime metric.Float64ObservableCounter+ cpuUtilization, uptime metric.Float64ObservableGauge+ memoryUsage, memoryLimit metric.Int64ObservableGauge+ pids metric.Int64ObservableGauge+ networkIO, diskIO metric.Int64ObservableCounter+ restarts metric.Int64ObservableCounter+}++// ObserveResources reports arcane.container.* metrics for running containers from list, sharing the+// sample cache and flights with Collect so overlapping collections reuse Docker calls.+func (s *Service) ObserveResources(meter metric.Meter, list func(context.Context) ([]container.Summary, error)) error {+ var r resourceInstruments+ var errs [9]error+ r.cpuTime, errs[0] = meter.Float64ObservableCounter("arcane.container.cpu.time",+ metric.WithDescription("Cumulative CPU time consumed by a running container"), metric.WithUnit("s"))+ r.cpuUtilization, errs[1] = meter.Float64ObservableGauge("arcane.container.cpu.utilization",+ metric.WithDescription("Container CPU usage as a fraction of total host CPU capacity"), metric.WithUnit("1"))+ r.memoryUsage, errs[2] = meter.Int64ObservableGauge("arcane.container.memory.usage",+ metric.WithDescription("Container memory usage excluding inactive page cache"), metric.WithUnit("By"))+ r.memoryLimit, errs[3] = meter.Int64ObservableGauge("arcane.container.memory.limit",+ metric.WithDescription("Container memory limit, or host memory when unlimited"), metric.WithUnit("By"))+ r.pids, errs[4] = meter.Int64ObservableGauge("arcane.container.pids",+ metric.WithDescription("Processes and threads running in a container"), metric.WithUnit("{process}"))+ r.networkIO, errs[5] = meter.Int64ObservableCounter("arcane.container.network.io",+ metric.WithDescription("Bytes received and transmitted across all container network interfaces"), metric.WithUnit("By"))+ r.diskIO, errs[6] = meter.Int64ObservableCounter("arcane.container.disk.io",+ metric.WithDescription("Bytes read from and written to block devices by a container"), metric.WithUnit("By"))+ r.restarts, errs[7] = meter.Int64ObservableCounter("arcane.container.restarts",+ metric.WithDescription("Times Docker restarted the container under its restart policy"), metric.WithUnit("{restart}"))+ r.uptime, errs[8] = meter.Float64ObservableGauge("arcane.container.uptime",+ metric.WithDescription("Time since the container last started"), metric.WithUnit("s"))+ if err := errors.Join(errs[:]...); err != nil {+ return err}- defer func() { _ = statsResponse.Body.Close() }()- var statsData container.StatsResponse- if unmarshalDecodeErr := json.UnmarshalDecode(jsontext.NewDecoder(statsResponse.Body), &statsData); unmarshalDecodeErr != nil {- return nil, fmt.Errorf("failed to decode container stats: %w", unmarshalDecodeErr)+ _, err := meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {+ containers, err := list(ctx)+ if err != nil {+ slog.DebugContext(ctx, "Skipping container resource metrics", "error", err)+ return nil+ }+ running := make(map[string]container.Summary, len(containers))+ for _, c := range containers {+ if c.State == container.StateRunning {+ running[c.ID] = c+ }+ }+ samples, err := collect(ctx, ContainerMetricsCollectTimeout, slices.Collect(maps.Keys(running)), func(ctx context.Context, id string) (resourceSample, error) {+ statsData, fetchErr := s.fetch(ctx, id)+ if fetchErr != nil {+ return resourceSample{}, fetchErr+ }+ sample := resourceSample{stats: statsData}+ if cached, ok, _ := s.inspectCache.Get(id); ok {+ sample.inspect = &cached+ } else if dockerClient, clientErr := s.getClient(ctx); clientErr == nil {+ if inspect, inspectErr := dockerClient.ContainerInspect(ctx, id, client.ContainerInspectOptions{}); inspectErr == nil {+ s.inspectCache.Set(id, inspect.Container)+ sample.inspect = &inspect.Container+ }+ }+ return sample, nil+ })+ if err != nil {+ slog.DebugContext(ctx, "Container resource metrics are partial", "error", err)+ }+ for id, sample := range samples {+ r.observe(o, running[id], sample)+ }+ return nil+ }, r.cpuTime, r.cpuUtilization, r.memoryUsage, r.memoryLimit, r.pids, r.networkIO, r.diskIO, r.restarts, r.uptime)+ return err+}++// observe records one container's sample, labelled by name, image and Compose project/service.+func (r *resourceInstruments) observe(o metric.Observer, c container.Summary, sample resourceSample) {+ attrs := []attribute.KeyValue{+ attribute.String("container.name", docker.ContainerNameFromNames(c.Names)),+ attribute.String("container.image.name", c.Image),+ }+ if project := docker.ComposeProjectLabel(c.Labels); project != "" {+ attrs = append(attrs, attribute.String("compose.project", project))+ }+ if service := docker.ComposeServiceLabel(c.Labels); service != "" {+ attrs = append(attrs, attribute.String("compose.service", service))+ }+ labels := metric.WithAttributes(attrs...)+ with := func(kv attribute.KeyValue) metric.MeasurementOption {+ return metric.WithAttributes(append(attrs[:len(attrs):len(attrs)], kv)...)}+ statsData := sample.statsbuilt := stats.BuildSample(statsData)- sample := &containertypes.ResourceSample{- CPUPercent: float64(built.CPUTenths) / 10,- MemoryUsageBytes: built.MemoryUsageBytes,- MemoryLimitBytes: statsData.MemoryStats.Limit,- SampleTime: statsData.Read,+ o.ObserveFloat64(r.cpuTime, float64(statsData.CPUStats.CPUUsage.TotalUsage)/float64(time.Second), labels)+ o.ObserveFloat64(r.cpuUtilization, float64(built.CPUTenths)/1000, labels)+ o.ObserveInt64(r.memoryUsage, int64(built.MemoryUsageBytes), labels)+ o.ObserveInt64(r.memoryLimit, int64(statsData.MemoryStats.Limit), labels)+ o.ObserveInt64(r.pids, int64(statsData.PidsStats.Current), labels)++ var rx, tx, read, write uint64+ for _, network := range statsData.Networks {+ rx += network.RxBytes+ tx += network.TxBytes}- if sample.SampleTime.IsZero() {- sample.SampleTime = time.Now()+ for _, entry := range statsData.BlkioStats.IoServiceBytesRecursive {+ switch strings.ToLower(entry.Op) {+ case "read":+ read += entry.Value+ case "write":+ write += entry.Value+ }}+ o.ObserveInt64(r.networkIO, int64(rx), with(attribute.String("network.io.direction", "receive")))+ o.ObserveInt64(r.networkIO, int64(tx), with(attribute.String("network.io.direction", "transmit")))+ o.ObserveInt64(r.diskIO, int64(read), with(attribute.String("disk.io.direction", "read")))+ o.ObserveInt64(r.diskIO, int64(write), with(attribute.String("disk.io.direction", "write")))- s.cache.Set(s.cacheKeyInternal(containerID), *sample)- return sample, nil+ if sample.inspect == nil {+ return+ }+ o.ObserveInt64(r.restarts, int64(sample.inspect.RestartCount), labels)+ if sample.inspect.State != nil {+ if startedAt, err := time.Parse(time.RFC3339Nano, sample.inspect.State.StartedAt); err == nil {+ o.ObserveFloat64(r.uptime, time.Since(startedAt).Seconds(), labels)+ }+ }}-// ContainerResourceSampleSort orders summaries by their collected sample-// value, keeping unsampled containers last in both directions and breaking-// ties by name, then ID. Valid zero values sort as zero, not as unavailable.+// ContainerResourceSampleSort orders summaries by sample value with unsampled containers last in both+// directions, breaking ties by name then ID. Valid zero values sort as zero, not as unavailable.func ContainerResourceSampleSort(sort string, descending bool) pagination.SortOption[containertypes.Summary] {value := kit.Ternary(sort == containertypes.SortCPUUsage,diff --git a/backend/internal/container/module.go b/backend/internal/container/module.goindex 9e46bc57e..45238a572 100644--- a/backend/internal/container/module.go+++ b/backend/internal/container/module.go@@ -6,6 +6,7 @@ import ("net/http""github.com/danielgtaylor/huma/v2"+ "go.opentelemetry.io/otel""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/docker"@@ -13,6 +14,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")// Module wires the container domain and mounts its routes.@@ -23,8 +25,13 @@ type Module struct {activityService *activity.ActivityService}-// New wires container routes around an existing service.+// New wires container routes around an existing service and registers per-container resource metrics once.func New(service *ContainerService, dockerService *docker.DockerClientService, settingsService *settings.SettingsService, activityService *activity.ActivityService) *Module {+ if service != nil && service.stats != nil && service.dockerService != nil {+ if err := service.stats.ObserveResources(otel.Meter(tracing.InstrumentationName), service.dockerService.ListContainers); err != nil {+ otel.Handle(err)+ }+ }return &Module{service: service, dockerService: dockerService, settingsService: settingsService, activityService: activityService}}diff --git a/backend/internal/dashboard/module.go b/backend/internal/dashboard/module.goindex f27cc6d80..767fe7232 100644--- a/backend/internal/dashboard/module.go+++ b/backend/internal/dashboard/module.go@@ -1,17 +1,18 @@-// Package dashboard owns the dashboard snapshot: the service that assembles it-// from the container, project, image, volume and vulnerability domains, and the-// HTTP surface that serves it.+// Package dashboard owns the dashboard snapshot assembled from the container, project, image,+// volume and vulnerability domains, the HTTP surface that serves it, and local inventory metrics.package dashboardimport ("net/http""github.com/danielgtaylor/huma/v2"+ "go.opentelemetry.io/otel""github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")// Module is the dashboard domain's wiring seam: it owns the service and the@@ -21,8 +22,13 @@ type Module struct {handler *DashboardHandler}-// New wires dashboard routes around an existing service.+// New wires dashboard routes around an existing service and registers its inventory metrics once.func New(service *DashboardService, environmentService *environment.EnvironmentService) *Module {+ if service != nil {+ if err := service.ObserveInventory(otel.Meter(tracing.InstrumentationName)); err != nil {+ otel.Handle(err)+ }+ }return &Module{service: service, handler: NewHandler(service, environmentService)}}diff --git a/backend/internal/dashboard/service.go b/backend/internal/dashboard/service.goindex 54e427f1b..676c72e5a 100644--- a/backend/internal/dashboard/service.go+++ b/backend/internal/dashboard/service.go@@ -1,25 +1,30 @@package dashboardimport (+ "cmp""context""errors""fmt"+ "log/slog""sort""strconv""sync/atomic""time"- "github.com/getarcaneapp/arcane/types/v2/base"containertypes "github.com/getarcaneapp/arcane/types/v2/container""github.com/getarcaneapp/arcane/types/v2/dashboard"imagetypes "github.com/getarcaneapp/arcane/types/v2/image"versiontypes "github.com/getarcaneapp/arcane/types/v2/version"volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"dockercontainer "github.com/moby/moby/api/types/container"+ "github.com/moby/moby/api/types/swarm"+ "github.com/moby/moby/client""go.getarcane.app/docker""go.getarcane.app/kit/pkg""go.getarcane.app/sys/cgroup""go.getarcane.app/updater/labels"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric""golang.org/x/sync/errgroup""golang.org/x/sync/singleflight"@@ -34,7 +39,9 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/version""github.com/getarcaneapp/arcane/backend/v2/internal/volume""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/iconcatalog")@@ -43,6 +50,7 @@ const (dashboardSnapshotPreloadLimit = 50dashboardSnapshotCacheTTL = 5 * time.SeconddashboardSnapshotBuildTimeout = 30 * time.Second+ dashboardInventoryCollectTimeout = 15 * time.Second)type DashboardService struct {@@ -57,18 +65,13 @@ type DashboardService struct {versionService *version.VersionServicevolumeService *volume.VolumeService- // snapshotFlight + snapshotCache share one snapshot build across all- // concurrent dashboard consumers (HTTP first paint + every stream- // subscriber). Cached snapshots are shared pointers and must be treated- // as immutable by callers. Indexed by- // [includeTables][debugAllGood][iconCatalog]: full snapshots resolve- // container icons against the requesting user's icon catalog, so each- // catalog gets its own entry.+ // Shared, immutable snapshot builds indexed by [includeTables][debugAllGood][iconCatalog];+ // full snapshots resolve icons per catalog, so each catalog gets its own entry.snapshotFlight singleflight.Group- snapshotCache [2][2][2]atomic.Pointer[dashboardSnapshotCacheEntryInternal]+ snapshotCache [2][2][2]atomic.Pointer[dashboardSnapshotCacheEntry]}-type dashboardSnapshotCacheEntryInternal struct {+type dashboardSnapshotCacheEntry struct {snapshot *dashboard.SnapshotbuiltAt time.Time}@@ -103,16 +106,10 @@ func NewDashboardService(}}-// GetSnapshot returns the dashboard snapshot, shared across all concurrent-// consumers (HTTP first paint + every stream subscriber) through a short-TTL-// cache; the result is a shared pointer and must not be mutated.-// includeTables controls whether the first-page container/image tables are-// built: stream subscribers pass false since the all-environments dashboard-// only reads the aggregate counters, skipping the per-container DTO builds,-// sorting, and icon resolution entirely.+// GetSnapshot returns a short-TTL cached snapshot shared by all consumers; it must not be mutated.+// includeTables builds the first-page tables; stream subscribers pass false and read only counters.func (s *DashboardService) GetSnapshot(ctx context.Context, options DashboardActionItemsOptions, includeTables bool) (*dashboard.Snapshot, error) {- // Trimmed snapshots skip icon resolution entirely, so every consumer- // shares one entry regardless of the requesting user's catalog.+ // Trimmed snapshots skip icon resolution, so every consumer shares one entry.catalog := iconcatalog.DefaultCatalogif includeTables {catalog = iconcatalog.Normalize(project.IconCatalogForContext(ctx))@@ -128,8 +125,7 @@ func (s *DashboardService) GetSnapshot(ctx context.Context, options DashboardActkey := strconv.FormatBool(includeTables) + ":" + strconv.FormatBool(options.DebugAllGood) + ":" + catalogresult, err, _ := s.snapshotFlight.Do(key, func() (any, error) {- // Re-check under the flight: a caller that queued behind the winner- // finds the fresh entry here instead of rebuilding.+ // Re-check so a caller queued behind the winner reuses its fresh entry.if entry := slot.Load(); entry != nil && time.Since(entry.builtAt) < dashboardSnapshotCacheTTL {return entry.snapshot, nil}@@ -137,11 +133,11 @@ func (s *DashboardService) GetSnapshot(ctx context.Context, options DashboardActbuildCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dashboardSnapshotBuildTimeout)defer cancel()- snapshot, err := s.buildSnapshotInternal(buildCtx, options, includeTables)+ snapshot, err := s.buildSnapshot(buildCtx, options, includeTables)if err != nil {return nil, err}- slot.Store(&dashboardSnapshotCacheEntryInternal{snapshot: snapshot, builtAt: time.Now()})+ slot.Store(&dashboardSnapshotCacheEntry{snapshot: snapshot, builtAt: time.Now()})return snapshot, nil})if err != nil {@@ -154,7 +150,7 @@ func (s *DashboardService) GetSnapshot(ctx context.Context, options DashboardActreturn snapshot, nil}-func (s *DashboardService) buildSnapshotInternal(ctx context.Context, options DashboardActionItemsOptions, includeTables bool) (*dashboard.Snapshot, error) {+func (s *DashboardService) buildSnapshot(ctx context.Context, options DashboardActionItemsOptions, includeTables bool) (*dashboard.Snapshot, error) {if s.dockerService == nil {return nil, errors.New("docker service not available")}@@ -228,15 +224,14 @@ func (s *DashboardService) buildSnapshotInternal(ctx context.Context, options DaimageUsageCounts := dockerInternal.CountImageUsage(dockerImages, imageConsumers)- // Uses the unfiltered container list so a volume mounted only by an internal- // container still counts as in use, matching the volumes page.+ // Unfiltered containers keep volumes mounted only by internal containers in use, matching the volumes page.var volumeUsageCounts *volumetypes.UsageCountsif s.volumeService != nil && dockerSnapshot.Volumes != nil {counts := s.volumeService.CountUsageFromSnapshot(dockerSnapshot.Volumes.Items, dockerContainers)volumeUsageCounts = &counts}- actionItems, err := s.buildActionItemsForSnapshotInternal(ctx, options, filteredContainers, dockerContainers)+ actionItems, err := s.buildActionItems(ctx, options, containerCounts.StoppedContainers, dockerContainers)if err != nil {return nil, err}@@ -246,15 +241,17 @@ func (s *DashboardService) buildSnapshotInternal(ctx context.Context, options DaversionInfo = s.versionService.GetAppVersionInfo(ctx)}+ preloadParams := pagination.QueryParams{Params: pagination.Params{Limit: dashboardSnapshotPreloadLimit}}+return &dashboard.Snapshot{Containers: dashboard.SnapshotContainers{Data: containerPage,Counts: containerCounts,- Pagination: buildDashboardPaginationResponseInternal(len(filteredContainers), dashboardSnapshotPreloadLimit),+ Pagination: handlerutil.PaginationResponse(pagination.BuildResponse(int64(len(filteredContainers)), int64(len(filteredContainers)), preloadParams)),},Images: dashboard.SnapshotImages{Data: imagePage,- Pagination: buildDashboardPaginationResponseInternal(len(dockerImages), dashboardSnapshotPreloadLimit),+ Pagination: handlerutil.PaginationResponse(pagination.BuildResponse(int64(len(dockerImages)), int64(len(dockerImages)), preloadParams)),},ImageUsageCounts: imageUsageCounts,VolumeUsageCounts: volumeUsageCounts,@@ -264,15 +261,9 @@ func (s *DashboardService) buildSnapshotInternal(ctx context.Context, options Da}, nil}-// buildActionItemsForSnapshotInternal derives the dashboard's action badges.-// filteredContainers drives the stopped-container badge; allContainers is the raw-// snapshot list, passed to the update count so it need not re-list containers.-func (s *DashboardService) buildActionItemsForSnapshotInternal(- ctx context.Context,- options DashboardActionItemsOptions,- filteredContainers []dockercontainer.Summary,- allContainers []dockercontainer.Summary,-) (*dashboard.ActionItems, error) {+// buildActionItems derives the dashboard's action badges; allContainers is the raw snapshot+// list, reused so the update count need not re-list containers.+func (s *DashboardService) buildActionItems(ctx context.Context, options DashboardActionItemsOptions, stoppedContainers int, allContainers []dockercontainer.Summary) (*dashboard.ActionItems, error) {if options.DebugAllGood {return &dashboard.ActionItems{Items: []dashboard.ActionItem{}}, nil}@@ -288,33 +279,56 @@ func (s *DashboardService) buildActionItemsForSnapshotInternal(g.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "dashboard action item worker")- count, err := s.getPendingResourceUpdatesCountInternal(groupCtx, allContainers)+ if s.db == nil || s.dockerService == nil {+ return nil+ }+ standaloneContainers := make([]dockercontainer.Summary, 0, len(allContainers))+ for _, c := range container.FilterExcludedContainers(allContainers, false, false) {+ if docker.ComposeProjectLabel(c.Labels) == "" {+ standaloneContainers = append(standaloneContainers, c)+ }+ }+ containerCount, err := s.getPendingContainerUpdatesCount(groupCtx, standaloneContainers)if err != nil {return err}- pendingResourceUpdates = count+ // Hidden service identities stay so project counting can exclude their Compose and cached image records.+ projectCount := 0+ if s.projectService != nil {+ if projectCount, err = s.projectService.CountProjectsWithPendingUpdates(groupCtx, allContainers); err != nil {+ return fmt.Errorf("failed to count projects with updates: %w", err)+ }+ }+ pendingResourceUpdates = containerCount + projectCountreturn nil})g.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "dashboard action item worker")- count, err := s.getActionableVulnerabilitiesCountInternal(groupCtx)- if err != nil {- return err+ if s.vulnerabilityService == nil {+ return nil}- actionableVulnerabilities = count- return nil+ actionableVulnerabilities, workerErr = s.vulnerabilityService.ActionableCountExcludingIgnored(groupCtx)+ return workerErr})g.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "dashboard action item worker")- count, err := s.getExpiringAPIKeysCountInternal(groupCtx)+ if s.db == nil {+ return nil+ }+ var count int64+ err := s.db.WithContext(groupCtx).+ Model(&apikey.ApiKey{}).+ Where("expires_at IS NOT NULL").+ Where("expires_at <= ?", time.Now().Add(defaultDashboardAPIKeyExpiryWindow)).+ Count(&count).Errorif err != nil {- return err+ return fmt.Errorf("failed to count expiring API keys: %w", err)}- expiringAPIKeys = count+ expiringAPIKeys = int(count)return nil})@@ -322,99 +336,24 @@ func (s *DashboardService) buildActionItemsForSnapshotInternal(return nil, err}- stoppedContainers := 0- for _, container := range filteredContainers {- if container.State != "running" {- stoppedContainers++- }- }-- return buildDashboardActionItemsInternal(stoppedContainers, pendingResourceUpdates, actionableVulnerabilities, expiringAPIKeys), nil-}--func buildDashboardActionItemsInternal(- stoppedContainers int,- pendingResourceUpdates int,- actionableVulnerabilities int,- expiringAPIKeys int,-) *dashboard.ActionItems {- actionItems := make([]dashboard.ActionItem, 0, 4)-- if stoppedContainers > 0 {- actionItems = append(actionItems, dashboard.ActionItem{- Kind: dashboard.ActionItemKindStoppedContainers,- Count: stoppedContainers,- Severity: dashboard.ActionItemSeverityWarning,- })- }-- if pendingResourceUpdates > 0 {- actionItems = append(actionItems, dashboard.ActionItem{- Kind: dashboard.ActionItemKindImageUpdates,- Count: pendingResourceUpdates,- Severity: dashboard.ActionItemSeverityWarning,- })- }-- if actionableVulnerabilities > 0 {- actionItems = append(actionItems, dashboard.ActionItem{- Kind: dashboard.ActionItemKindActionableVulnerabilities,- Count: actionableVulnerabilities,- Severity: dashboard.ActionItemSeverityCritical,- })- }-- if expiringAPIKeys > 0 {- actionItems = append(actionItems, dashboard.ActionItem{- Kind: dashboard.ActionItemKindExpiringKeys,- Count: expiringAPIKeys,- Severity: dashboard.ActionItemSeverityWarning,- })- }-- return &dashboard.ActionItems{Items: actionItems}-}--// getPendingResourceUpdatesCountInternal counts standalone containers and projects-// with a pending image update. allContainers is the snapshot's container list,-// reused rather than re-listed: this used to issue its own GetAllContainers on-// top of the two the project count triggered, for a single badge number.-func (s *DashboardService) getPendingResourceUpdatesCountInternal(ctx context.Context, allContainers []dockercontainer.Summary) (int, error) {- if s.db == nil || s.dockerService == nil {- return 0, nil- }-- filteredContainers := container.FilterExcludedContainers(allContainers, false, false)- standaloneContainers := filterStandaloneDockerContainersInternal(filteredContainers)- containerCount, err := s.getPendingContainerUpdatesCountInternal(ctx, standaloneContainers)- if err != nil {- return 0, err- }-- // Keep hidden service identities so project counting can exclude their Compose and cached image records.- projectCount, err := s.getPendingProjectUpdatesCountInternal(ctx, allContainers)- if err != nil {- return 0, err+ candidates := []dashboard.ActionItem{+ {Kind: dashboard.ActionItemKindStoppedContainers, Count: stoppedContainers, Severity: dashboard.ActionItemSeverityWarning},+ {Kind: dashboard.ActionItemKindImageUpdates, Count: pendingResourceUpdates, Severity: dashboard.ActionItemSeverityWarning},+ {Kind: dashboard.ActionItemKindActionableVulnerabilities, Count: actionableVulnerabilities, Severity: dashboard.ActionItemSeverityCritical},+ {Kind: dashboard.ActionItemKindExpiringKeys, Count: expiringAPIKeys, Severity: dashboard.ActionItemSeverityWarning},}-- return containerCount + projectCount, nil-}--func filterStandaloneDockerContainersInternal(containers []dockercontainer.Summary) []dockercontainer.Summary {- filtered := make([]dockercontainer.Summary, 0, len(containers))- for _, c := range containers {- if docker.ComposeProjectLabel(c.Labels) != "" {- continue+ items := make([]dashboard.ActionItem, 0, len(candidates))+ for _, item := range candidates {+ if item.Count > 0 {+ items = append(items, item)}- filtered = append(filtered, c)}- return filtered+ return &dashboard.ActionItems{Items: items}, nil}-// getPendingContainerUpdatesCountInternal counts standalone containers whose-// stored check reports an update, through the same lookup the container list-// uses. Containers sharing one image each count once.-func (s *DashboardService) getPendingContainerUpdatesCountInternal(ctx context.Context, containers []dockercontainer.Summary) (int, error) {+// getPendingContainerUpdatesCount counts standalone containers whose stored check reports an+// update, using the container list's lookup. Containers sharing one image each count once.+func (s *DashboardService) getPendingContainerUpdatesCount(ctx context.Context, containers []dockercontainer.Summary) (int, error) {if s.imageService == nil || len(containers) == 0 {return 0, nil}@@ -431,60 +370,134 @@ func (s *DashboardService) getPendingContainerUpdatesCountInternal(ctx context.Creturn count, nil}-func (s *DashboardService) getPendingProjectUpdatesCountInternal(ctx context.Context, allContainers []dockercontainer.Summary) (int, error) {- if s.projectService == nil {- return 0, nil- }+type inventoryGauges struct {+ containers, health, images, imageSize, volumes, networks, projects metric.Int64ObservableGauge+ swarmNodes, swarmServices, swarmTasks metric.Int64ObservableGauge+}- count, err := s.projectService.CountProjectsWithPendingUpdates(ctx, allContainers)- if err != nil {- return 0, fmt.Errorf("failed to count projects with updates: %w", err)- }+// ObserveInventory reports the local Docker engine's inventory through arcane.docker.* and arcane.projects gauges.+func (s *DashboardService) ObserveInventory(meter metric.Meter) error {+ var g inventoryGauges+ var errs [10]error+ g.containers, errs[0] = meter.Int64ObservableGauge("arcane.docker.containers",+ metric.WithDescription("Containers on the local Docker engine by state"), metric.WithUnit("{container}"))+ g.health, errs[1] = meter.Int64ObservableGauge("arcane.docker.containers.health",+ metric.WithDescription("Containers with a healthcheck by health status"), metric.WithUnit("{container}"))+ g.images, errs[2] = meter.Int64ObservableGauge("arcane.docker.images",+ metric.WithDescription("Images on the local Docker engine by usage"), metric.WithUnit("{image}"))+ g.imageSize, errs[3] = meter.Int64ObservableGauge("arcane.docker.images.size",+ metric.WithDescription("Total size of images on the local Docker engine"), metric.WithUnit("By"))+ g.volumes, errs[4] = meter.Int64ObservableGauge("arcane.docker.volumes",+ metric.WithDescription("Volumes on the local Docker engine by usage, excluding Arcane-internal volumes"), metric.WithUnit("{volume}"))+ g.networks, errs[5] = meter.Int64ObservableGauge("arcane.docker.networks",+ metric.WithDescription("Networks on the local Docker engine by usage; built-in networks report as default"), metric.WithUnit("{network}"))+ g.projects, errs[6] = meter.Int64ObservableGauge("arcane.projects",+ metric.WithDescription("Arcane Compose projects by status"), metric.WithUnit("{project}"))+ g.swarmNodes, errs[7] = meter.Int64ObservableGauge("arcane.docker.swarm.nodes",+ metric.WithDescription("Swarm nodes by role and state, reported by swarm managers"), metric.WithUnit("{node}"))+ g.swarmServices, errs[8] = meter.Int64ObservableGauge("arcane.docker.swarm.services",+ metric.WithDescription("Swarm services, reported by swarm managers"), metric.WithUnit("{service}"))+ g.swarmTasks, errs[9] = meter.Int64ObservableGauge("arcane.docker.swarm.service.tasks",+ metric.WithDescription("Running and desired tasks per swarm service, reported by swarm managers"), metric.WithUnit("{task}"))+ if err := errors.Join(errs[:]...); err != nil {+ return err+ }++ _, err := meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {+ ctx, cancel := context.WithTimeout(ctx, dashboardInventoryCollectTimeout)+ defer cancel()+ snapshot, err := s.dockerService.GetSnapshot(ctx, environment.LocalEnvironmentID)+ if err != nil {+ slog.DebugContext(ctx, "Skipping Docker inventory metrics", "error", err)+ return nil+ }- return count, nil-}+ states := map[dockercontainer.ContainerState]int64{+ dockercontainer.StateCreated: 0, dockercontainer.StateRunning: 0, dockercontainer.StatePaused: 0, dockercontainer.StateRestarting: 0,+ dockercontainer.StateRemoving: 0, dockercontainer.StateExited: 0, dockercontainer.StateDead: 0,+ }+ health := map[dockercontainer.HealthStatus]int64{dockercontainer.Starting: 0, dockercontainer.Healthy: 0, dockercontainer.Unhealthy: 0}+ for _, c := range snapshot.Containers {+ states[c.State]+++ if c.Health != nil && c.Health.Status != dockercontainer.NoHealthcheck {+ health[c.Health.Status]+++ }+ }+ for state, n := range states {+ o.ObserveInt64(g.containers, n, metric.WithAttributes(attribute.String("container.state", string(state))))+ }+ for status, n := range health {+ o.ObserveInt64(g.health, n, metric.WithAttributes(attribute.String("container.health.status", string(status))))+ }-func (s *DashboardService) getActionableVulnerabilitiesCountInternal(ctx context.Context) (int, error) {- if s.vulnerabilityService == nil {- return 0, nil- }+ images := dockerInternal.CountImageUsage(snapshot.Images, container.FilterExcludedContainers(snapshot.Containers, false, true))+ o.ObserveInt64(g.images, int64(images.Inuse), metric.WithAttributes(attribute.String("usage", "in_use")))+ o.ObserveInt64(g.images, int64(images.Unused), metric.WithAttributes(attribute.String("usage", "unused")))+ o.ObserveInt64(g.imageSize, images.TotalSize)- return s.vulnerabilityService.ActionableCountExcludingIgnored(ctx)-}+ if s.volumeService != nil && snapshot.Volumes != nil {+ volumes := s.volumeService.CountUsageFromSnapshot(snapshot.Volumes.Items, snapshot.Containers)+ o.ObserveInt64(g.volumes, int64(volumes.Inuse), metric.WithAttributes(attribute.String("usage", "in_use")))+ o.ObserveInt64(g.volumes, int64(volumes.Unused), metric.WithAttributes(attribute.String("usage", "unused")))+ }-func (s *DashboardService) getExpiringAPIKeysCountInternal(ctx context.Context) (int, error) {- if s.db == nil {- return 0, nil- }+ if len(snapshot.Networks) > 0 {+ inuse, unused, total := dockerInternal.CountNetworkUsage(snapshot.Networks, snapshot.Containers)+ o.ObserveInt64(g.networks, int64(inuse), metric.WithAttributes(attribute.String("usage", "in_use")))+ o.ObserveInt64(g.networks, int64(unused), metric.WithAttributes(attribute.String("usage", "unused")))+ o.ObserveInt64(g.networks, int64(total-inuse-unused), metric.WithAttributes(attribute.String("usage", "default")))+ }- var count int64- err := s.db.WithContext(ctx).- Model(&apikey.ApiKey{}).- Where("expires_at IS NOT NULL").- Where("expires_at <= ?", time.Now().Add(defaultDashboardAPIKeyExpiryWindow)).- Count(&count).Error- if err != nil {- return 0, fmt.Errorf("failed to count expiring API keys: %w", err)- }+ if s.projectService != nil {+ if counts, countErr := s.projectService.GetProjectStatusCounts(ctx); countErr != nil {+ slog.DebugContext(ctx, "Skipping project inventory metrics", "error", countErr)+ } else {+ other := counts.TotalProjects - counts.RunningProjects - counts.StoppedProjects - counts.ArchivedProjects+ for status, n := range map[string]int{"running": counts.RunningProjects, "stopped": counts.StoppedProjects, "archived": counts.ArchivedProjects, "unknown": other} {+ o.ObserveInt64(g.projects, int64(n), metric.WithAttributes(attribute.String("project.status", status)))+ }+ }+ }- return int(count), nil+ s.observeSwarm(ctx, o, &g)+ return nil+ }, g.containers, g.health, g.images, g.imageSize, g.volumes, g.networks, g.projects, g.swarmNodes, g.swarmServices, g.swarmTasks)+ return err}-func buildDashboardPaginationResponseInternal(totalItems, limit int) base.PaginationResponse {- if limit <= 0 {- limit = dashboardSnapshotPreloadLimit+// observeSwarm reports nodes and services only when the local engine is a swarm manager; the ping keeps non-swarm hosts cheap.+func (s *DashboardService) observeSwarm(ctx context.Context, o metric.Observer, g *inventoryGauges) {+ cli, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return+ }+ ping, err := cli.Ping(ctx, client.PingOptions{})+ if err != nil || ping.SwarmStatus == nil || !ping.SwarmStatus.ControlAvailable {+ return}- totalPages := 1- if totalItems > 0 {- totalPages = (totalItems + limit - 1) / limit+ if nodes, nodeErr := cli.NodeList(ctx, client.NodeListOptions{}); nodeErr != nil {+ slog.DebugContext(ctx, "Skipping swarm node metrics", "error", nodeErr)+ } else {+ counts := map[[2]string]int64{}+ for _, node := range nodes.Items {+ counts[[2]string{string(node.Spec.Role), string(node.Status.State)}]+++ }+ for key, n := range counts {+ o.ObserveInt64(g.swarmNodes, n, metric.WithAttributes(attribute.String("swarm.node.role", key[0]), attribute.String("swarm.node.state", key[1])))+ }}- return base.PaginationResponse{- TotalPages: int64(totalPages),- TotalItems: int64(totalItems),- CurrentPage: 1,- ItemsPerPage: limit,- GrandTotalItems: int64(totalItems),+ services, err := cli.ServiceList(ctx, client.ServiceListOptions{Status: true})+ if err != nil {+ slog.DebugContext(ctx, "Skipping swarm service metrics", "error", err)+ return+ }+ o.ObserveInt64(g.swarmServices, int64(len(services.Items)))+ for _, service := range services.Items {+ status := cmp.Or(service.ServiceStatus, &swarm.ServiceStatus{})+ name := attribute.String("swarm.service.name", service.Spec.Name)+ o.ObserveInt64(g.swarmTasks, int64(status.RunningTasks), metric.WithAttributes(name, attribute.String("swarm.task.state", "running")))+ o.ObserveInt64(g.swarmTasks, int64(status.DesiredTasks), metric.WithAttributes(name, attribute.String("swarm.task.state", "desired")))}}diff --git a/backend/internal/dashboard/service_test.go b/backend/internal/dashboard/service_test.goindex 7a676e062..336d5be7a 100644--- a/backend/internal/dashboard/service_test.go+++ b/backend/internal/dashboard/service_test.go@@ -47,8 +47,9 @@ func setupDashboardServiceTestDB(t *testing.T) (*database.DB, *settings.Settingsrequire.NoError(t, db.AutoMigrate(&apikey.ApiKey{}, &environment.Environment{}, &imageupdate.ImageUpdateRecord{}, &project.Project{}, &settings.SettingVariable{}))databaseDB := &database.DB{DB: db}- settingsSvc, err := newSettingsServiceForTestInternal(t.Context(), t, databaseDB)+ settingsSvc, err := settings.NewSettingsService(t.Context(), databaseDB)require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, settingsSvc.Stop(context.WithoutCancel(t.Context()))) })return databaseDB, settingsSvc}@@ -185,7 +186,8 @@ func TestDashboardService_GetSnapshot_ReturnsDashboardSnapshot(t *testing.T) {t.Setenv("PROJECTS_DIRECTORY", projectsDir)require.NoError(t, settingsSvc.SetStringSetting(t.Context(), "projectsDirectory", projectsDir))require.NoError(t, settingsSvc.SetStringSetting(t.Context(), "autoUpdateExcludedContainers", "stopped-app"))- projectPath := createComposeProjectDirInternal(t, projectsDir, "project-with-update")+ projectPath := filepath.Join(projectsDir, "project-with-update")+ require.NoError(t, os.MkdirAll(projectPath, 0o755))require.NoError(t, os.WriteFile(filepath.Join(projectPath, "compose.yaml"), []byte("services:\n app:\n image: repo/worker:latest\n"), 0o644))dirName := "project-with-update"require.NoError(t, db.WithContext(t.Context()).Create(&project.Project{@@ -266,7 +268,7 @@ func TestDashboardService_GetSnapshot_DebugAllGoodOnlyClearsActionItems(t *testirequire.Empty(t, snapshot.ActionItems.Items)}-func TestDashboardService_GetSnapshot_EnrichesPinnedReferencesInternal(t *testing.T) {+func TestDashboardService_GetSnapshot_EnrichesPinnedReferences(t *testing.T) {db, settingsSvc := setupDashboardServiceTestDB(t)pinnedRef := "ghcr.io/syncthing/syncthing:2.1.3@sha256:8c8ff37ab6aa8be23b700648a90fa9412e214852e9fd6ea8477c8334792daec0"@@ -306,29 +308,6 @@ func TestDashboardService_GetSnapshot_EnrichesPinnedReferencesInternal(t *testinassert.Equal(t, []string{pinnedRef}, snapshot.Images.Data[0].PinnedReferences)}-// createComposeProjectDirInternal writes a minimal single-service compose project under-// root and returns its path.-func createComposeProjectDirInternal(t *testing.T, root, name string) string {- t.Helper()-- projectPath := filepath.Join(root, name)- require.NoError(t, os.MkdirAll(projectPath, 0o755))- require.NoError(t, os.WriteFile(filepath.Join(projectPath, "compose.yaml"), []byte("services:\n app:\n image: nginx:alpine\n"), 0o600))-- return projectPath-}--// newSettingsServiceForTestInternal builds a SettingsService backed by its own actor runtime,-// stopped when the test ends.-func newSettingsServiceForTestInternal(ctx context.Context, t testing.TB, db *database.DB) (*settings.SettingsService, error) {- t.Helper()- svc, err := settings.NewSettingsService(ctx, db)- if err == nil {- t.Cleanup(func() { require.NoError(t, svc.Stop(context.WithoutCancel(t.Context()))) })- }- return svc, err-}-func TestDashboardService_GetSnapshot_TrimmedOmitsTablesAndSharesBuilds(t *testing.T) {db, settingsSvc := setupDashboardServiceTestDB(t)@@ -463,27 +442,27 @@ func TestPendingContainerCountUsesScopedTagRecords(t *testing.T) {{ID: "first", Image: "app:1.2.3", ImageID: "shared", Labels: map[string]string{labels.LabelUpdateStrategy: "auto"}},{ID: "unchecked", Image: "app:1.2.3", ImageID: "shared", Labels: map[string]string{labels.LabelUpdateStrategy: "auto"}},}- count, err := service.getPendingContainerUpdatesCountInternal(t.Context(), containers)+ count, err := service.getPendingContainerUpdatesCount(t.Context(), containers)require.NoError(t, err)require.Equal(t, 1, count)containers = append(containers, dockercontainer.Summary{ID: "digest", Image: "app:latest", ImageID: "shared"})- count, err = service.getPendingContainerUpdatesCountInternal(t.Context(), containers)+ count, err = service.getPendingContainerUpdatesCount(t.Context(), containers)require.NoError(t, err)require.Equal(t, 2, count, "a digest record under another tag of the running image counts")containers = append(containers, dockercontainer.Summary{ID: "digest-twin", Image: "app:latest", ImageID: "shared"})- count, err = service.getPendingContainerUpdatesCountInternal(t.Context(), containers)+ count, err = service.getPendingContainerUpdatesCount(t.Context(), containers)require.NoError(t, err)require.Equal(t, 3, count, "separate containers sharing one image each count once")containers[0].Labels[labels.LabelUpdater] = "off"- count, err = service.getPendingContainerUpdatesCountInternal(t.Context(), containers)+ count, err = service.getPendingContainerUpdatesCount(t.Context(), containers)require.NoError(t, err)require.Equal(t, 3, count, "disabling automatic installation keeps the pending update visible")containers[0].Labels[imageref.UpdateCheckLabel] = "false"- count, err = service.getPendingContainerUpdatesCountInternal(t.Context(), containers)+ count, err = service.getPendingContainerUpdatesCount(t.Context(), containers)require.NoError(t, err)require.Equal(t, 2, count, "disabling update checks suppresses the result")containers[2].ImageID = ""- count, err = service.getPendingContainerUpdatesCountInternal(t.Context(), containers)+ count, err = service.getPendingContainerUpdatesCount(t.Context(), containers)require.NoError(t, err)require.Equal(t, 1, count, "a missing runtime image ID inherits no digest check")}diff --git a/backend/internal/database/database.go b/backend/internal/database/database.goindex 9c57638ce..7aa4135fd 100644--- a/backend/internal/database/database.go+++ b/backend/internal/database/database.go@@ -23,6 +23,7 @@ import ("gorm.io/driver/postgres""gorm.io/gorm""gorm.io/gorm/logger"+ "gorm.io/plugin/opentelemetry/metrics"sqliteutil "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/sqlite""github.com/getarcaneapp/arcane/backend/v2/resources"@@ -137,11 +138,30 @@ func Initialize(ctx context.Context, databaseURL string, options MigrationOptionif cancellationErr := ctx.Err(); cancellationErr != nil {return nil, cancellationErr}-sqlDB, err := db.SQLDB()if err != nil {return nil, fmt.Errorf("failed to get sql.DB: %w", err)}+ // Statement spans and pool metrics are no-ops until telemetry is enabled.+ cb := gormDB.Callback()+ err = errors.Join(+ cb.Create().Before("gorm:create").Register("arcane:otel_before_create", startStatementSpan),+ cb.Create().After("gorm:create").Register("arcane:otel_after_create", endStatementSpan),+ cb.Query().Before("gorm:query").Register("arcane:otel_before_query", startStatementSpan),+ cb.Query().After("gorm:query").Register("arcane:otel_after_query", endStatementSpan),+ cb.Update().Before("gorm:update").Register("arcane:otel_before_update", startStatementSpan),+ cb.Update().After("gorm:update").Register("arcane:otel_after_update", endStatementSpan),+ cb.Delete().Before("gorm:delete").Register("arcane:otel_before_delete", startStatementSpan),+ cb.Delete().After("gorm:delete").Register("arcane:otel_after_delete", endStatementSpan),+ cb.Row().Before("gorm:row").Register("arcane:otel_before_row", startStatementSpan),+ cb.Row().After("gorm:row").Register("arcane:otel_after_row", endStatementSpan),+ cb.Raw().Before("gorm:raw").Register("arcane:otel_before_raw", startStatementSpan),+ cb.Raw().After("gorm:raw").Register("arcane:otel_after_raw", endStatementSpan),+ )+ if err != nil {+ return nil, fmt.Errorf("failed to register database instrumentation: %w", err)+ }+ metrics.ReportDBStatsMetrics(sqlDB)if migrationErr := migrateDatabase(ctx, sqlDB, dbProvider, options, latestMigrationVersion); migrationErr != nil {slog.ErrorContext(ctx, "Failed to run migrations", "error", migrationErr)return nil, fmt.Errorf("failed to run migrations: %w", migrationErr)diff --git a/backend/internal/database/tracing.go b/backend/internal/database/tracing.gonew file mode 100644index 000000000..326da6a42--- /dev/null+++ b/backend/internal/database/tracing.go@@ -0,0 +1,57 @@+package database++import (+ "database/sql"+ "errors"+ "strings"++ "go.getarcane.app/kit/pkg"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/trace"+ "gorm.io/gorm"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"+)++const statementSpanKey = "arcane:otel_span"++var statementTracer = otel.Tracer(tracing.InstrumentationName)++// startStatementSpan opens a client span only inside an existing trace, so+// background queries do not create root traces.+func startStatementSpan(tx *gorm.DB) {+ ctx := tx.Statement.Context+ if !trace.SpanContextFromContext(ctx).IsValid() {+ return+ }+ _, span := statementTracer.Start(ctx, "db", trace.WithSpanKind(trace.SpanKindClient))+ tx.InstanceSet(statementSpanKey, span)+}++// endStatementSpan names and ends the span with the placeholder SQL; parameter values are never recorded.+func endStatementSpan(tx *gorm.DB) {+ value, _ := tx.InstanceGet(statementSpanKey)+ span, ok := value.(trace.Span)+ if !ok {+ return+ }++ query := tx.Statement.SQL.String()+ operation, _, _ := strings.Cut(strings.TrimSpace(query), " ")+ operation = strings.ToLower(operation)+ if name := strings.TrimSpace(operation + " " + tx.Statement.Table); name != "" {+ span.SetName(name)+ }+ span.SetAttributes(+ attribute.String("db.system.name", kit.Ternary(tx.Name() == dbProviderPostgres, "postgresql", "sqlite")),+ attribute.String("db.query.text", query),+ attribute.String("db.operation.name", operation),+ attribute.String("db.collection.name", tx.Statement.Table),+ )+ err := tx.Error+ if errors.Is(err, gorm.ErrRecordNotFound) || errors.Is(err, sql.ErrNoRows) {+ err = nil+ }+ tracing.End(span, err)+}diff --git a/backend/internal/docker/service.go b/backend/internal/docker/service.goindex 26f10d5ee..5c351238f 100644--- a/backend/internal/docker/service.go+++ b/backend/internal/docker/service.go@@ -14,11 +14,8 @@ import ("github.com/getarcaneapp/arcane/types/v2/dashboard"imagetypes "github.com/getarcaneapp/arcane/types/v2/image""github.com/moby/moby/api/types/container"- "github.com/moby/moby/api/types/events""github.com/moby/moby/api/types/image"- "github.com/moby/moby/api/types/mount""github.com/moby/moby/api/types/network"- "github.com/moby/moby/api/types/volume""github.com/moby/moby/client""go.getarcane.app/docker""go.getarcane.app/docker/compat"@@ -53,10 +50,8 @@ type DockerClientService struct {mu sync.MutexeventBus *bus.DockerEventBus- // Coalesce concurrent full-inventory list calls so overlapping requests- // (e.g. the Homepage widget's simultaneous counts endpoints) decode one- // Docker response instead of one per caller. Per-instance on purpose:- // a WithClient copy talks to a different daemon and must not share flights.+ // Coalesce concurrent full-inventory lists into one Docker response; per-instance+ // because a WithClient copy talks to a different daemon.imageListFlight singleflight.GroupcontainerListFlight singleflight.Group}@@ -82,28 +77,12 @@ func (s *DockerClientService) WithClient(dockerClient *client.Client) *DockerCli}}-func newDockerClientInternal(ctx context.Context, host string) (*client.Client, error) {- apiVersion, err := detectDockerAPIVersionInternal(ctx, host)- if err != nil {- return nil, err- }-- configuredClient, err := newDockerClientWithAPIVersionInternal(host, apiVersion)- if err != nil {- return nil, err- }-- return configuredClient, nil-}--func detectDockerAPIVersionInternal(ctx context.Context, host string) (string, error) {- probeClient, err := client.New(- client.WithHost(host),- )+func detectDockerAPIVersion(ctx context.Context, host string) (string, error) {+ probeClient, err := client.New(client.WithHost(host))if err != nil {return "", fmt.Errorf("failed to create Docker probe client: %w", err)}- defer closeDockerClientInternal(ctx, probeClient, "probe")+ defer closeDockerClient(ctx, probeClient, "probe")ctx, cancel := context.WithTimeout(ctx, dockerClientNegotiationTimeout)defer cancel()@@ -122,19 +101,7 @@ func detectDockerAPIVersionInternal(ctx context.Context, host string) (string, ereturn apiVersion, nil}-func newDockerClientWithAPIVersionInternal(host, apiVersion string) (*client.Client, error) {- configuredClient, err := client.New(- client.WithHost(host),- client.WithAPIVersion(apiVersion),- )- if err != nil {- return nil, fmt.Errorf("failed to configure Docker client API version %s: %w", apiVersion, err)- }-- return configuredClient, nil-}--func closeDockerClientInternal(ctx context.Context, cli *client.Client, purpose string) {+func closeDockerClient(ctx context.Context, cli *client.Client, purpose string) {if cli == nil {return}@@ -155,16 +122,20 @@ func (s *DockerClientService) GetClient(ctx context.Context) (*client.Client, er}s.mu.Unlock()- cli, err := newDockerClientInternal(ctx, s.config.DockerHost)+ apiVersion, err := detectDockerAPIVersion(ctx, s.config.DockerHost)if err != nil {return nil, fmt.Errorf("failed to create Docker client: %w", err)}+ cli, err := client.New(client.WithHost(s.config.DockerHost), client.WithAPIVersion(apiVersion))+ if err != nil {+ return nil, fmt.Errorf("failed to create Docker client: failed to configure Docker client API version %s: %w", apiVersion, err)+ }s.mu.Lock()if s.Client != nil {existingClient := s.Clients.mu.Unlock()- closeDockerClientInternal(ctx, cli, "unused after concurrent initialization")+ closeDockerClient(ctx, cli, "unused after concurrent initialization")return existingClient, nil}@@ -179,7 +150,7 @@ func (s *DockerClientService) GetClient(ctx context.Context) (*client.Client, er// RefreshClient probes the Docker daemon and recreates the cached client when// the daemon's effective API version changed.func (s *DockerClientService) RefreshClient(ctx context.Context) error {- apiVersion, err := detectDockerAPIVersionInternal(ctx, s.config.DockerHost)+ apiVersion, err := detectDockerAPIVersion(ctx, s.config.DockerHost)if err != nil {return fmt.Errorf("failed to refresh Docker client: %w", err)}@@ -192,16 +163,16 @@ func (s *DockerClientService) RefreshClient(ctx context.Context) error {}s.mu.Unlock()- cli, err := newDockerClientWithAPIVersionInternal(s.config.DockerHost, apiVersion)+ cli, err := client.New(client.WithHost(s.config.DockerHost), client.WithAPIVersion(apiVersion))if err != nil {- return fmt.Errorf("failed to refresh Docker client: %w", err)+ return fmt.Errorf("failed to refresh Docker client: failed to configure Docker client API version %s: %w", apiVersion, err)}s.mu.Lock()if s.Client != nil && apiVersion == s.clientVersion {s.clientLastProbe = time.Now()s.mu.Unlock()- closeDockerClientInternal(ctx, cli, "unused after concurrent refresh")+ closeDockerClient(ctx, cli, "unused after concurrent refresh")return nil}@@ -211,7 +182,7 @@ func (s *DockerClientService) RefreshClient(ctx context.Context) error {s.clientLastProbe = time.Now()s.mu.Unlock()- closeDockerClientInternal(ctx, oldClient, "replaced")+ closeDockerClient(ctx, oldClient, "replaced")return nil}@@ -231,7 +202,7 @@ func (s *DockerClientService) Close() {s.Client = nils.mu.Unlock()- closeDockerClientInternal(context.Background(), oldClient, "cached") //nolint:forbidigo // Shutdown close runs without a request context.+ closeDockerClient(context.Background(), oldClient, "cached") //nolint:forbidigo // Shutdown close runs without a request context.}func (s *DockerClientService) EventBus() *bus.DockerEventBus {@@ -249,77 +220,53 @@ func (s *DockerClientService) WatchEvents(ctx context.Context) {eventBackoff.MaxInterval = 30 * time.Secondfor ctx.Err() == nil {- dockerClient, err := s.GetClient(ctx)- if err != nil {+ if dockerClient, err := s.GetClient(ctx); err != nil {slog.WarnContext(ctx, "failed to connect to Docker event stream", "error", err)- if !sleepDockerEventBackoffInternal(ctx, eventBackoff) {- return+ } else {+ result := dockerClient.Events(ctx, client.EventsListOptions{})+ streamStart := time.Now()+ var streamErr error+ stream:+ for {+ select {+ case <-ctx.Done():+ return+ case msg, ok := <-result.Messages:+ if !ok {+ break stream+ }+ // Stamp missing daemon timestamps once, before normal and overflow delivery diverge.+ if msg.TimeNano == 0 {+ msg.TimeNano = time.Now().UnixNano()+ }+ s.EventBus().Publish(msg)+ case recvErr, ok := <-result.Err:+ streamErr = kit.Ternary(ok, recvErr, nil)+ break stream+ }+ }+ if streamErr != nil && !errors.Is(streamErr, context.Canceled) && !errors.Is(streamErr, io.EOF) {+ slog.WarnContext(ctx, "Docker event stream stopped", "error", streamErr)+ }+ // Only a stream that stayed up counts as recovery, so a daemon that drops connections immediately keeps backing off.+ if time.Since(streamStart) >= dockerEventStreamHealthyAfter {+ eventBackoff.Reset()}- continue- }-- result := dockerClient.Events(ctx, client.EventsListOptions{})- streamStart := time.Now()- err = s.consumeEventsInternal(ctx, result.Messages, result.Err)- if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, io.EOF) {- slog.WarnContext(ctx, "Docker event stream stopped", "error", err)- }-- // Only a stream that actually stayed up counts as recovery. Resetting- // unconditionally before Events() meant a daemon that accepts the- // connection and drops it immediately was reconnected at the floor- // interval forever.- if time.Since(streamStart) >= dockerEventStreamHealthyAfter {- eventBackoff.Reset()- }-- if !sleepDockerEventBackoffInternal(ctx, eventBackoff) {- return}- }-}-func (s *DockerClientService) consumeEventsInternal(ctx context.Context, messages <-chan events.Message, errs <-chan error) error {- for {+ timer := time.NewTimer(eventBackoff.NextBackOff())select {case <-ctx.Done():- return ctx.Err()- case msg, ok := <-messages:- if !ok {- return nil- }- // Stamp missing daemon timestamps once, before normal and overflow delivery diverge.- if msg.TimeNano == 0 {- msg.TimeNano = time.Now().UnixNano()- }- s.EventBus().Publish(msg)- case err, ok := <-errs:- return kit.Ternary(!ok, nil, err)+ timer.Stop()+ return+ case <-timer.C:}}}-func sleepDockerEventBackoffInternal(ctx context.Context, eventBackoff *backoff.ExponentialBackOff) bool {- delay := eventBackoff.NextBackOff()- timer := time.NewTimer(delay)- defer timer.Stop()- select {- case <-ctx.Done():- return false- case <-timer.C:- return true- }-}--// listCoalescedInternal runs op through the given singleflight group so-// concurrent callers share one Docker request and one decoded result. The-// flight runs on a context detached from the initiating caller, bounded only-// by op's own Docker API timeout, so one canceled waiter cannot cancel work-// other waiters still need; each waiter stops waiting when its own context-// ends. Results and errors are shared only among concurrent callers —-// singleflight drops the key once the flight completes, so nothing is cached.-// Callers must treat the shared result as immutable.-func listCoalescedInternal[T any](ctx context.Context, group *singleflight.Group, op func(context.Context) (T, error)) (T, error) {+// listCoalesced shares one detached op among concurrent callers; each waiter stops on its own+// context, nothing is cached, and the shared result must be treated as immutable.+func listCoalesced[T any](ctx context.Context, group *singleflight.Group, op func(context.Context) (T, error)) (T, error) {var zero Tch := group.DoChan("list", func() (any, error) {@@ -341,81 +288,56 @@ func listCoalescedInternal[T any](ctx context.Context, group *singleflight.Group}}-func (s *DockerClientService) apiTimeoutInternal() time.Duration {- if s.settingsService == nil {- return timeouts.DefaultDockerAPI+// callAPI runs op against the cached client under the configured Docker API timeout.+func callAPI[T any](ctx context.Context, s *DockerClientService, op func(context.Context, *client.Client) (T, error)) (T, error) {+ dockerClient, err := s.GetClient(ctx)+ if err != nil {+ var zero T+ return zero, fmt.Errorf("failed to connect to Docker: %w", err)}- return timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI)+ timeout := timeouts.DefaultDockerAPI+ if s.settingsService != nil {+ timeout = timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI)+ }+ apiCtx, cancel := context.WithTimeout(ctx, timeout)+ defer cancel()+ return op(apiCtx, dockerClient)}func (s *DockerClientService) ListContainers(ctx context.Context) ([]container.Summary, error) {- return listCoalescedInternal(ctx, &s.containerListFlight, func(ctx context.Context) ([]container.Summary, error) {- dockerClient, err := s.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- apiCtx, cancel := context.WithTimeout(ctx, s.apiTimeoutInternal())- defer cancel()-- containerList, err := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})- if err != nil {- return nil, fmt.Errorf("failed to list Docker containers: %w", err)- }- return containerList.Items, nil+ return listCoalesced(ctx, &s.containerListFlight, func(ctx context.Context) ([]container.Summary, error) {+ return callAPI(ctx, s, func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {+ containerList, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ if err != nil {+ return nil, fmt.Errorf("failed to list Docker containers: %w", err)+ }+ return containerList.Items, nil+ })})}func (s *DockerClientService) ListImages(ctx context.Context) ([]image.Summary, error) {- return listCoalescedInternal(ctx, &s.imageListFlight, func(ctx context.Context) ([]image.Summary, error) {- dockerClient, err := s.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- apiCtx, cancel := context.WithTimeout(ctx, s.apiTimeoutInternal())- defer cancel()+ return listCoalesced(ctx, &s.imageListFlight, func(ctx context.Context) ([]image.Summary, error) {+ return callAPI(ctx, s, func(ctx context.Context, dockerClient *client.Client) ([]image.Summary, error) {+ imageList, err := dockerClient.ImageList(ctx, client.ImageListOptions{All: true})+ if err != nil {+ return nil, fmt.Errorf("failed to list Docker images: %w", err)+ }+ return imageList.Items, nil+ })+ })+}- imageList, err := dockerClient.ImageList(apiCtx, client.ImageListOptions{All: true})+func (s *DockerClientService) listNetworks(ctx context.Context) ([]network.Summary, error) {+ return callAPI(ctx, s, func(ctx context.Context, dockerClient *client.Client) ([]network.Summary, error) {+ networkList, err := compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})if err != nil {- return nil, fmt.Errorf("failed to list Docker images: %w", err)+ return nil, fmt.Errorf("failed to list Docker networks: %w", err)}- return imageList.Items, nil+ return networkList.Items, nil})}-func (s *DockerClientService) listNetworksInternal(ctx context.Context) ([]network.Summary, error) {- dockerClient, err := s.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- apiCtx, cancel := context.WithTimeout(ctx, s.apiTimeoutInternal())- defer cancel()-- networkList, err := compat.NetworkListWithCompatibility(apiCtx, dockerClient, client.NetworkListOptions{})- if err != nil {- return nil, fmt.Errorf("failed to list Docker networks: %w", err)- }- return networkList.Items, nil-}--func (s *DockerClientService) listVolumesInternal(ctx context.Context) (*client.VolumeListResult, error) {- dockerClient, err := s.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- apiCtx, cancel := context.WithTimeout(ctx, s.apiTimeoutInternal())- defer cancel()-- volResp, err := dockerClient.VolumeList(apiCtx, client.VolumeListOptions{})- if err != nil {- return nil, fmt.Errorf("failed to list Docker volumes: %w", err)- }- return &volResp, nil-}-func (s *DockerClientService) GetSnapshot(ctx context.Context, envID string) (*dashboard.DockerSnapshot, error) {g, groupCtx := errgroup.WithContext(ctx)@@ -442,7 +364,7 @@ func (s *DockerClientService) GetSnapshot(ctx context.Context, envID string) (*ddefer utils.RecoverToError(&workerErr, "docker snapshot worker")var err error- networks, err = s.listNetworksInternal(groupCtx)+ networks, err = s.listNetworks(groupCtx)if err != nil {slog.WarnContext(groupCtx, "failed to list Docker networks for snapshot", "error", err)}@@ -452,7 +374,13 @@ func (s *DockerClientService) GetSnapshot(ctx context.Context, envID string) (*ddefer utils.RecoverToError(&workerErr, "docker snapshot worker")var err error- volumes, err = s.listVolumesInternal(groupCtx)+ volumes, err = callAPI(groupCtx, s, func(ctx context.Context, dockerClient *client.Client) (*client.VolumeListResult, error) {+ volResp, listErr := dockerClient.VolumeList(ctx, client.VolumeListOptions{})+ if listErr != nil {+ return nil, fmt.Errorf("failed to list Docker volumes: %w", listErr)+ }+ return &volResp, nil+ })if err != nil {slog.WarnContext(groupCtx, "failed to list Docker volumes for snapshot", "error", err)}@@ -518,9 +446,8 @@ func (s *DockerClientService) GetAllImages(ctx context.Context) ([]image.Summaryreturn images, CountImageUsage(images, containers), nil}-// CountImageUsage tallies in-use, unused, and total image counts, plus the-// summed size of all images. An image is in use when a container references-// its ID.+// CountImageUsage tallies in-use, unused, and total image counts plus the summed+// image size. An image is in use when a container references its ID.func CountImageUsage(images []image.Summary, containers []container.Summary) imagetypes.UsageCounts {inUseImageIDs := make(map[string]struct{}, len(containers))for _, c := range containers {@@ -549,65 +476,27 @@ func (s *DockerClientService) GetAllNetworks(ctx context.Context) ([]network.Sumif err != nil {return nil, 0, 0, 0, err}- inUseByID, inUseByName := docker.BuildNetworkUsageMaps(containers)-- networks, err := s.listNetworksInternal(ctx)+ networks, err := s.listNetworks(ctx)if err != nil {return nil, 0, 0, 0, err}-- var inuse, unused, total int- for _, n := range networks {- total++ // total includes all networks (including defaults)-- // Only count non-default networks towards in-use/unused breakdown- if !docker.IsDefaultNetwork(n.Name) {- used := inUseByID[n.ID] || inUseByName[n.Name]- if used {- inuse++- } else {- unused++- }- }- }-- // Return order: inuse, unused, total (matches handler expectations)+ inuse, unused, total := CountNetworkUsage(networks, containers)return networks, inuse, unused, total, nil}-func (s *DockerClientService) GetAllVolumes(ctx context.Context) ([]*volume.Volume, int, int, int, error) {- containers, err := s.ListContainers(ctx)- if err != nil {- return nil, 0, 0, 0, err- }- ref := make(map[string]int64, len(containers))- for _, c := range containers {- for _, m := range c.Mounts {- if m.Type == mount.TypeVolume && m.Name != "" {- ref[m.Name]++- }- }- }-- volResp, err := s.listVolumesInternal(ctx)- if err != nil {- return nil, 0, 0, 0, err- }- volumeItems := volResp.Items- volumes := make([]*volume.Volume, 0, len(volumeItems))- for i := range volumeItems {- volumes = append(volumes, &volumeItems[i])- }-- var inuse, unused, total int- for _, v := range volumes {+// CountNetworkUsage returns in-use and unused counts for non-default networks, and the total of all networks.+func CountNetworkUsage(networks []network.Summary, containers []container.Summary) (inuse, unused, total int) {+ inUseByID, inUseByName := docker.BuildNetworkUsageMaps(containers)+ for _, n := range networks {total++- if ref[v.Name] > 0 {+ if docker.IsDefaultNetwork(n.Name) {+ continue+ }+ if inUseByID[n.ID] || inUseByName[n.Name] {inuse++} else {unused++}}-- return volumes, inuse, unused, total, nil+ return inuse, unused, total}diff --git a/backend/internal/docker/service_test.go b/backend/internal/docker/service_test.goindex 894b9e5a4..e1e537ceb 100644--- a/backend/internal/docker/service_test.go+++ b/backend/internal/docker/service_test.go@@ -2,10 +2,11 @@ package dockerimport ("context"+ "encoding/json""net""net/http""net/http/httptest"- "regexp"+ "path""sync/atomic""testing""time"@@ -22,7 +23,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency")-func TestNewDockerClient_PinsEffectiveAPIVersion(t *testing.T) {+func TestDockerClientService_GetClientPinsEffectiveAPIVersion(t *testing.T) {t.Setenv("DOCKER_API_VERSION", "1.54")t.Setenv("DOCKER_HOST", "tcp://docker-from-env:2375")@@ -45,13 +46,12 @@ func TestNewDockerClient_PinsEffectiveAPIVersion(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- server := newDockerPingTestServerInternal(t, tt.pingAPIVersion)+ server := newDockerPingTestServer(t, func() string { return tt.pingAPIVersion })+ svc := newDockerClientServiceForTest(t, server.URL)+ t.Cleanup(svc.Close)- cli, err := newDockerClientInternal(t.Context(), server.URL)+ cli, err := svc.GetClient(t.Context())require.NoError(t, err)- t.Cleanup(func() {- _ = cli.Close()- })assert.Equal(t, server.URL, cli.DaemonHost())assert.Equal(t, tt.expectedVersion, cli.ClientVersion())@@ -61,8 +61,8 @@ func TestNewDockerClient_PinsEffectiveAPIVersion(t *testing.T) {}func TestDockerClientService_GetClientReturnsCachedClientUntilRefresh(t *testing.T) {- server := newDockerPingTestServerInternal(t, "1.41")- svc := newDockerClientServiceForTestInternal(t, server.URL)+ server := newDockerPingTestServer(t, func() string { return "1.41" })+ svc := newDockerClientServiceForTest(t, server.URL)firstClient, err := svc.GetClient(t.Context())require.NoError(t, err)@@ -80,10 +80,10 @@ func TestDockerClientService_GetClientReturnsCachedClientUntilRefresh(t *testingfunc TestDockerClientService_RefreshClientRecreatesCachedClientAfterAPIVersionChange(t *testing.T) {apiVersion := atomic.Value{}apiVersion.Store("1.41")- server := newDockerPingTestServerWithVersionInternal(t, func() string {+ server := newDockerPingTestServer(t, func() string {return apiVersion.Load().(string)})- svc := newDockerClientServiceForTestInternal(t, server.URL)+ svc := newDockerClientServiceForTest(t, server.URL)firstClient, err := svc.GetClient(t.Context())require.NoError(t, err)@@ -107,13 +107,23 @@ func TestDockerClientService_RefreshClientRecreatesCachedClientAfterAPIVersionChfunc TestDockerClientService_RefreshClientClosesOldCachedClientWhenReplaced(t *testing.T) {var oldServerClosedConnections atomic.Int32- oldServer := newDockerPingTestServerWithConnStateInternal(t, "1.41", func(_ net.Conn, state http.ConnState) {+ oldServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/_ping" {+ http.NotFound(w, r)+ return+ }+ w.Header().Set("Api-Version", "1.41")+ w.WriteHeader(http.StatusOK)+ }))+ oldServer.Config.ConnState = func(_ net.Conn, state http.ConnState) {if state == http.StateClosed {oldServerClosedConnections.Add(1)}- })- newServer := newDockerPingTestServerInternal(t, "1.42")- svc := newDockerClientServiceForTestInternal(t, oldServer.URL)+ }+ oldServer.Start()+ t.Cleanup(oldServer.Close)+ newServer := newDockerPingTestServer(t, func() string { return "1.42" })+ svc := newDockerClientServiceForTest(t, oldServer.URL)firstClient, err := svc.GetClient(t.Context())require.NoError(t, err)@@ -143,7 +153,7 @@ func TestDockerClientService_RefreshClientClosesOldCachedClientWhenReplaced(t *tfunc TestDockerClientService_RefreshClientProbeFailureKeepsCachedClient(t *testing.T) {var failProbe atomic.Bool- server := newDockerPingTestServerWithHandlerInternal(t, func(w http.ResponseWriter, r *http.Request) {+ server := newDockerPingTestServerWithHandler(t, func(w http.ResponseWriter, r *http.Request) {if r.URL.Path != "/_ping" {http.NotFound(w, r)return@@ -155,7 +165,7 @@ func TestDockerClientService_RefreshClientProbeFailureKeepsCachedClient(t *testiw.Header().Set("Api-Version", "1.41")w.WriteHeader(http.StatusOK)})- svc := newDockerClientServiceForTestInternal(t, server.URL)+ svc := newDockerClientServiceForTest(t, server.URL)firstClient, err := svc.GetClient(t.Context())require.NoError(t, err)@@ -172,33 +182,50 @@ func TestDockerClientService_RefreshClientProbeFailureKeepsCachedClient(t *testiassert.False(t, svc.clientLastProbe.IsZero())}-func TestDockerClientService_PublishesDaemonImageEventsInternal(t *testing.T) {- svc := NewDockerClientService(t.Context(), nil, &config.Config{}, nil)+func TestDockerClientService_PublishesDaemonImageEvents(t *testing.T) {+ expected := events.Message{Type: events.ImageEventType, Action: events.ActionPull, TimeNano: 123, Actor: events.Actor{ID: "nginx:latest"}}+ payload, err := json.Marshal(expected)+ require.NoError(t, err)+ server := newDockerPingTestServerWithHandler(t, func(w http.ResponseWriter, r *http.Request) {+ switch path.Base(r.URL.Path) {+ case "_ping":+ w.Header().Set("Api-Version", "1.41")+ w.WriteHeader(http.StatusOK)+ case "events":+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write(payload)+ default:+ http.NotFound(w, r)+ }+ })+ svc := newDockerClientServiceForTest(t, server.URL)t.Cleanup(svc.Close)eventCh, unsubscribe := svc.EventBus().Subscribe(events.ImageEventType)t.Cleanup(unsubscribe)- messages := make(chan events.Message, 1)- expected := events.Message{Type: events.ImageEventType, Action: events.ActionPull, TimeNano: 123, Actor: events.Actor{ID: "nginx:latest"}}- messages <- expected- close(messages)- require.NoError(t, svc.consumeEventsInternal(t.Context(), messages, nil))+ stop, err := concurrency.StartSupervised(t.Context(), "Docker event watcher", func(ctx context.Context) error {+ svc.WatchEvents(ctx)+ return nil+ })+ require.NoError(t, err)+ t.Cleanup(func() { _ = stop(context.WithoutCancel(t.Context())) })+select {case message := <-eventCh:require.Equal(t, expected, message)- default:+ case <-time.After(time.Second):t.Fatal("daemon image event was not published")}}-func TestDockerClientService_EventActorStopCancelsAndJoinsStreamInternal(t *testing.T) {+func TestDockerClientService_EventActorStopCancelsAndJoinsStream(t *testing.T) {streamStarted := make(chan struct{})streamStopped := make(chan struct{})- server := newDockerPingTestServerWithHandlerInternal(t, func(w http.ResponseWriter, r *http.Request) {- switch dockerTestPathInternal(r.URL.Path) {- case "/_ping":+ server := newDockerPingTestServerWithHandler(t, func(w http.ResponseWriter, r *http.Request) {+ switch path.Base(r.URL.Path) {+ case "_ping":w.Header().Set("Api-Version", "1.41")w.WriteHeader(http.StatusOK)- case "/events":+ case "events":w.Header().Set("Content-Type", "application/json")w.WriteHeader(http.StatusOK)if flusher, ok := w.(http.Flusher); ok {@@ -212,7 +239,7 @@ func TestDockerClientService_EventActorStopCancelsAndJoinsStreamInternal(t *test}})- service := newDockerClientServiceForTestInternal(t, server.URL)+ service := newDockerClientServiceForTest(t, server.URL)t.Cleanup(service.Close)eventsCh, unsubscribe := service.EventBus().Subscribe(events.ImageEventType)@@ -275,26 +302,14 @@ func TestCountImageUsage_NoImages(t *testing.T) {assert.Equal(t, imagetypes.UsageCounts{}, counts)}-func newDockerClientServiceForTestInternal(t *testing.T, host string) *DockerClientService {+func newDockerClientServiceForTest(t *testing.T, host string) *DockerClientService {return NewDockerClientService(t.Context(), nil, &config.Config{DockerHost: host}, nil)}-func dockerTestPathInternal(path string) string {- return regexp.MustCompile(`^/v\d+\.\d+`).ReplaceAllString(path, "")-}--func newDockerPingTestServerInternal(t *testing.T, apiVersion string) *httptest.Server {- t.Helper()-- return newDockerPingTestServerWithVersionInternal(t, func() string {- return apiVersion- })-}--func newDockerPingTestServerWithVersionInternal(t *testing.T, apiVersion func() string) *httptest.Server {+func newDockerPingTestServer(t *testing.T, apiVersion func() string) *httptest.Server {t.Helper()- return newDockerPingTestServerWithHandlerInternal(t, func(w http.ResponseWriter, r *http.Request) {+ return newDockerPingTestServerWithHandler(t, func(w http.ResponseWriter, r *http.Request) {if r.URL.Path != "/_ping" {http.NotFound(w, r)return@@ -307,26 +322,7 @@ func newDockerPingTestServerWithVersionInternal(t *testing.T, apiVersion func()})}-func newDockerPingTestServerWithConnStateInternal(t *testing.T, apiVersion string, connState func(net.Conn, http.ConnState)) *httptest.Server {- t.Helper()-- server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- if r.URL.Path != "/_ping" {- http.NotFound(w, r)- return- }-- w.Header().Set("Api-Version", apiVersion)- w.WriteHeader(http.StatusOK)- }))- server.Config.ConnState = connState- server.Start()- t.Cleanup(server.Close)-- return server-}--func newDockerPingTestServerWithHandlerInternal(t *testing.T, handler http.HandlerFunc) *httptest.Server {+func newDockerPingTestServerWithHandler(t *testing.T, handler http.HandlerFunc) *httptest.Server {t.Helper()server := httptest.NewServer(handler)diff --git a/backend/internal/environment/handler.go b/backend/internal/environment/handler.goindex 856440a48..4a174aedd 100644--- a/backend/internal/environment/handler.go+++ b/backend/internal/environment/handler.go@@ -6,7 +6,6 @@ import ("cmp""context""crypto/x509"- "encoding/json/v2""encoding/pem""errors""fmt"@@ -24,7 +23,6 @@ import ("github.com/danielgtaylor/huma/v2""github.com/getarcaneapp/arcane/types/v2/base""github.com/getarcaneapp/arcane/types/v2/environment"- usertypes "github.com/getarcaneapp/arcane/types/v2/user""github.com/getarcaneapp/arcane/types/v2/version"kit "go.getarcane.app/kit/pkg""go.getarcane.app/kit/pkg/mapping"@@ -42,20 +40,16 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/userctx")const (- localDockerEnvironmentID = "0"-- // Re-applies in-memory runtime state to cached rows, covering poll-mode TTL- // expiry; CRUD, tunnel and health-check changes arrive on the runtime-change signal.+ // Re-applies runtime state to cached rows to cover poll-mode TTL expiry.environmentStreamRuntimeInterval = 5 * time.Second// Safety net for writes that bypass the runtime-change signal.environmentStreamReloadInterval = 60 * time.SecondenvironmentStreamRefreshFloor = 30 * time.Second- environmentStreamHubKeyInternal = "environments"+ environmentStreamHubKey = "environments")// EnvironmentHandler handles environment management endpoints.@@ -68,6 +62,7 @@ type EnvironmentHandler struct {streamHub *agg.Hub[[]environment.Environment]activityService activity.ServiceappCtx context.Context+ localVersion func(context.Context) *version.Info}type ListEnvironmentsInput struct {@@ -171,19 +166,11 @@ func NewHandler(// ListEnvironments returns a paginated list of environments.func (h *EnvironmentHandler) ListEnvironments(ctx context.Context, input *ListEnvironmentsInput) (*handlerutil.Page[environment.Environment], error) {- // The list endpoint backs both the environments management page and the- // environment switcher, so any authenticated caller may reach it. Global- // listers (sudo, global admins, or holders of the org-level- // environments:list permission) see every environment; environment-scoped- // callers see only the environments they hold at least one permission on.ps, ok := middleware.PermissionsFromContext(ctx)if !ok {return nil, huma.Error403Forbidden("permission denied")}- var accessibleEnvIDs []string // nil = no restriction- if !environmentListerSeesAllInternal(ps) {- accessibleEnvIDs = accessibleEnvironmentIDsInternal(ps)- }+ accessibleEnvIDs := accessibleEnvironmentIDs(ps)params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)if input.Type != "" {@@ -195,7 +182,7 @@ func (h *EnvironmentHandler) ListEnvironments(ctx context.Context, input *ListEnreturn nil, huma.Error500InternalServerError("Failed to fetch environments")}for i := range envs {- h.applyEdgeRuntimeStateInternal(&envs[i])+ ApplyEnvironmentRuntimeState(&envs[i])}return &handlerutil.Page[environment.Environment]{@@ -207,60 +194,36 @@ func (h *EnvironmentHandler) ListEnvironments(ctx context.Context, input *ListEn}, nil}-// environmentListerSeesAllInternal reports whether the caller may list every-// environment. True for sudo callers, global admins, and holders of the-// org-level environments:list permission (Allows short-circuits on sudo and-// treats global admins as holding every permission).-func environmentListerSeesAllInternal(ps *authz.PermissionSet) bool {- return ps != nil && ps.Allows(authz.PermEnvironmentsList, "")-}--// accessibleEnvironmentIDsInternal returns the sorted set of environment IDs the-// caller holds at least one environment-scoped permission on. A non-nil result-// (possibly empty) restricts the environment list for non-global callers; an-// empty result yields no environments.-func accessibleEnvironmentIDsInternal(ps *authz.PermissionSet) []string {- if ps == nil {+// accessibleEnvironmentIDs returns nil when the caller may list every environment,+// otherwise the sorted IDs it holds at least one permission on.+func accessibleEnvironmentIDs(ps *authz.PermissionSet) []string {+ switch {+ case ps == nil:return []string{}+ case ps.Allows(authz.PermEnvironmentsList, ""):+ return nil+ default:+ // Never nil: callers treat nil as unrestricted, so a user without grants must get an empty list.+ ids := slices.AppendSeq(make([]string, 0, len(ps.PerEnv)), maps.Keys(ps.PerEnv))+ slices.Sort(ids)+ return ids}- ids := slices.Sorted(maps.Keys(ps.PerEnv))- return ids-}--// visibleEnvironmentsForInternal filters the shared environment snapshot down-// to what the caller may see. It reuses the same access rules as-// ListEnvironments so the stream and the REST list can never disagree about-// which environments a caller has. The input slice is shared across stream-// subscribers and must not be mutated.-func visibleEnvironmentsForInternal(envs []environment.Environment, ps *authz.PermissionSet) []environment.Environment {- if environmentListerSeesAllInternal(ps) {- return envs- }-- allowed := make(map[string]struct{}, len(ps.PerEnv))- for _, envID := range accessibleEnvironmentIDsInternal(ps) {- allowed[envID] = struct{}{}- }-- filtered := make([]environment.Environment, 0, len(allowed))- for _, env := range envs {- if _, ok := allowed[env.ID]; ok {- filtered = append(filtered, env)- }- }- return filtered}func (h *EnvironmentHandler) RunStreamProducer(ctx context.Context, ps *authz.PermissionSet, events chan<- environment.StreamEvent) {var lastFingerprint uint64var haveFingerprint boolvar lastSentAt time.Time+ accessibleEnvIDs := accessibleEnvironmentIDs(ps)- h.streamHub.Subscribe(ctx, environmentStreamHubKeyInternal, h.runEnvironmentStreamListerInternal, func(all []environment.Environment) bool {- envs := visibleEnvironmentsForInternal(all, ps)+ h.streamHub.Subscribe(ctx, environmentStreamHubKey, h.runEnvironmentStreamLister, func(all []environment.Environment) bool {+ // all is shared across subscribers, so filter a copy.+ envs := all+ if accessibleEnvIDs != nil {+ envs = slices.DeleteFunc(slices.Clone(all), func(env environment.Environment) bool { return !slices.Contains(accessibleEnvIDs, env.ID) })+ }fingerprint := kit.Fingerprint(envs)- // Re-send unchanged state on a floor so relative timestamps in the UI- // ("last seen 2 minutes ago") keep advancing.+ // Re-send unchanged state on a floor so relative UI timestamps keep advancing.if haveFingerprint && fingerprint == lastFingerprint && time.Since(lastSentAt) < environmentStreamRefreshFloor {return true}@@ -276,7 +239,7 @@ func (h *EnvironmentHandler) RunStreamProducer(ctx context.Context, ps *authz.Pe})}-func (h *EnvironmentHandler) runEnvironmentStreamListerInternal(ctx context.Context, publish func([]environment.Environment)) {+func (h *EnvironmentHandler) runEnvironmentStreamLister(ctx context.Context, publish func([]environment.Environment)) {changes, unsubscribe := h.environmentService.SubscribeRuntimeChanges()defer unsubscribe()@@ -351,100 +314,57 @@ func (h *EnvironmentHandler) CreateEnvironment(ctx context.Context, input *Creatif input.Body.IsEdge != nil {env.IsEdge = *input.Body.IsEdge}-- // Determine pairing methoduseApiKey := input.Body.UseApiKey != nil && *input.Body.UseApiKey-if useApiKey {- return h.createEnvironmentWithApiKeyInternal(ctx, env, user)+ env.Status = string(EnvironmentStatusPending)+ } else if input.Body.AccessToken != nil && *input.Body.AccessToken != "" {+ env.AccessToken = input.Body.AccessToken}- return h.createEnvironmentLegacyInternal(ctx, env, user, input.Body)-}--func (h *EnvironmentHandler) createEnvironmentWithApiKeyInternal(ctx context.Context, env *Environment, user *usertypes.Actor) (*handlerutil.Out[EnvironmentWithApiKey], error) {- // New API key-based pairing flow- env.Status = string(EnvironmentStatusPending)-created, err := h.environmentService.CreateEnvironment(ctx, env, &user.ID, &user.Username)if err != nil {return nil, huma.Error500InternalServerError("Failed to create environment: " + err.Error())}- // Generate API key for environment- apiKeyDto, err := h.apiKeyService.CreateEnvironmentApiKey(ctx, created.ID)- if err != nil {- slog.ErrorContext(ctx, "Failed to create environment API key", "environmentId", created.ID, "error", err.Error())- return nil, huma.Error500InternalServerError("Failed to create environment API key")- }-- // Store the full API key in AccessToken for manager-to-agent auth.- apiKey := apiKeyDto.Key-- // Link the API key to the environment for manager use.- updates := map[string]any{- "api_key_id": apiKeyDto.ID,- "access_token": apiKey,- }- updated, err := h.environmentService.UpdateEnvironment(ctx, created.ID, updates, &user.ID, &user.Username)- if err != nil {- // Remove the key so a failed create does not leave an orphaned row- // behind. ErrApiKeyProtected means the link actually landed and the- // update failed afterwards, so the key legitimately belongs to the- // environment and must survive; deleting the environment cascades it.- if delErr := h.apiKeyService.DeleteApiKey(ctx, apiKeyDto.ID); delErr != nil &&- !errors.Is(delErr, apikey.ErrApiKeyNotFound) && !errors.Is(delErr, apikey.ErrApiKeyProtected) {- slog.ErrorContext(ctx, "Failed to clean up unlinked environment API key", "environmentId", created.ID, "error", delErr.Error())+ var apiKey *string+ if useApiKey {+ apiKeyDto, keyErr := h.apiKeyService.CreateEnvironmentApiKey(ctx, created.ID)+ if keyErr != nil {+ slog.ErrorContext(ctx, "Failed to create environment API key", "environmentId", created.ID, "error", keyErr.Error())+ return nil, huma.Error500InternalServerError("Failed to create environment API key")+ }++ // The full key doubles as the manager-to-agent access token.+ updates := map[string]any{+ "api_key_id": apiKeyDto.ID,+ "access_token": apiKeyDto.Key,+ }+ updated, updateErr := h.environmentService.UpdateEnvironment(ctx, created.ID, updates, &user.ID, &user.Username)+ if updateErr != nil {+ // ErrApiKeyProtected means the link landed, so the key belongs to the environment and must survive.+ if delErr := h.apiKeyService.DeleteApiKey(ctx, apiKeyDto.ID); delErr != nil &&+ !errors.Is(delErr, apikey.ErrApiKeyNotFound) && !errors.Is(delErr, apikey.ErrApiKeyProtected) {+ slog.ErrorContext(ctx, "Failed to clean up unlinked environment API key", "environmentId", created.ID, "error", delErr.Error())+ }+ slog.ErrorContext(ctx, "Failed to link API key to environment", "environmentId", created.ID, "error", updateErr.Error())+ return nil, huma.Error500InternalServerError("Failed to link API key")}- slog.ErrorContext(ctx, "Failed to link API key to environment", "environmentId", created.ID, "error", err.Error())- return nil, huma.Error500InternalServerError("Failed to link API key")+ created = updated+ apiKey = new(apiKeyDto.Key)+ } else if created.AccessToken != nil && *created.AccessToken != "" {+ h.triggerEnvironmentResourceSync(ctx, created.ID, created.Name, "environment creation")}- created = updatedout, mapErr := mapping.MapOne[*Environment, environment.Environment](created)if mapErr != nil {return nil, huma.Error500InternalServerError("Failed to map environment")}- h.applyEdgeRuntimeStateInternal(&out)+ ApplyEnvironmentRuntimeState(&out)return &handlerutil.Out[EnvironmentWithApiKey]{Body: base.ApiResponse[EnvironmentWithApiKey]{Success: true,- Data: EnvironmentWithApiKey{- Environment: out,- ApiKey: new(apiKeyDto.Key),- },- },- }, nil-}--func (h *EnvironmentHandler) createEnvironmentLegacyInternal(ctx context.Context, env *Environment, user *usertypes.Actor, body environment.Create) (*handlerutil.Out[EnvironmentWithApiKey], error) {- if body.AccessToken != nil && *body.AccessToken != "" {- env.AccessToken = body.AccessToken- }-- created, err := h.environmentService.CreateEnvironment(ctx, env, &user.ID, &user.Username)- if err != nil {- return nil, huma.Error500InternalServerError("Failed to create environment: " + err.Error())- }-- // Sync registries and git repositories in background (intentionally detached from request context)- if created.AccessToken != nil && *created.AccessToken != "" {- h.triggerEnvironmentResourceSyncInternal(ctx, created.ID, created.Name, "environment creation")- }-- out, mapErr := mapping.MapOne[*Environment, environment.Environment](created)- if mapErr != nil {- return nil, huma.Error500InternalServerError("Failed to map environment")- }- h.applyEdgeRuntimeStateInternal(&out)-- return &handlerutil.Out[EnvironmentWithApiKey]{- Body: base.ApiResponse[EnvironmentWithApiKey]{- Success: true,- Data: EnvironmentWithApiKey{- Environment: out,- },+ Data: EnvironmentWithApiKey{Environment: out, ApiKey: apiKey},},}, nil}@@ -460,9 +380,9 @@ func (h *EnvironmentHandler) GetEnvironment(ctx context.Context, input *GetEnvirif mapErr != nil {return nil, huma.Error500InternalServerError("Failed to map environment")}- h.applyEdgeRuntimeStateInternal(&out)+ ApplyEnvironmentRuntimeState(&out)if env.IsEdge {- if certInfo, certErr := readGeneratedEdgeMTLSCertificateInfoInternal(h.cfg, env.ID); certErr == nil {+ if certInfo, certErr := readGeneratedEdgeMTLSCertificateInfo(h.edgeMTLSConfig(), env.ID); certErr == nil {out.EdgeMTLSCertificate = certInfo}}@@ -477,17 +397,24 @@ func (h *EnvironmentHandler) GetEnvironment(ctx context.Context, input *GetEnvir// UpdateEnvironment updates an environment.func (h *EnvironmentHandler) UpdateEnvironment(ctx context.Context, input *UpdateEnvironmentInput) (*handlerutil.Out[environment.Environment], error) {- isLocalEnv := input.ID == localDockerEnvironmentID- updates := h.buildUpdateMapInternal(&input.Body, isLocalEnv)-- h.handleEnvironmentPairingInternal(ctx, input.ID, &input.Body, updates, isLocalEnv)-- user, _ := userctx.CurrentUserFromContext(ctx)- var userID, username *string- if user != nil {- userID = new(user.ID)- username = new(user.Username)+ req := &input.Body+ updates := map[string]any{}+ if input.ID != LocalEnvironmentID {+ if req.ApiUrl != nil {+ updates["api_url"] = *req.ApiUrl+ }+ if req.Enabled != nil {+ updates["enabled"] = *req.Enabled+ }+ if req.AccessToken != nil {+ updates["access_token"] = *req.AccessToken+ }+ }+ if req.Name != nil {+ updates["name"] = *req.Name}++ userID, username := currentUserRefs(ctx)updated, updateErr := h.environmentService.UpdateEnvironment(ctx, input.ID, updates, userID, username)if updateErr != nil {apiErr := common.ToAPIError(updateErr)@@ -497,17 +424,21 @@ func (h *EnvironmentHandler) UpdateEnvironment(ctx context.Context, input *Updatreturn nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)}- h.triggerPostUpdateTasksInternal(ctx, input.ID, updated, &input.Body)-- out, mapErr := mapping.MapOne[*Environment, environment.Environment](updated)- if mapErr != nil {- return nil, huma.Error500InternalServerError("Failed to map environment")+ if updated.Enabled {+ go func(syncCtx context.Context, envID, envName string) {+ status, err := h.environmentService.TestConnection(syncCtx, envID, nil)+ if err != nil {+ slog.WarnContext(syncCtx, "Failed to test connection after environment update",+ "environmentId", envID, "environmentName", envName, "status", status, "error", err)+ }+ }(context.WithoutCancel(ctx), input.ID, updated.Name)+ }+ if updated.AccessToken != nil && *updated.AccessToken != "" && ((req.AccessToken != nil && *req.AccessToken != "") || req.Name != nil) {+ h.triggerEnvironmentResourceSync(ctx, input.ID, updated.Name, "environment update")}- h.applyEdgeRuntimeStateInternal(&out)- // If regenerating API key, return the new keyvar newApiKey *string- if input.Body.RegenerateApiKey != nil && *input.Body.RegenerateApiKey {+ if req.RegenerateApiKey != nil && *req.RegenerateApiKey {localUser, err := handlerutil.RequireUser(ctx)if err != nil {return nil, err@@ -519,24 +450,19 @@ func (h *EnvironmentHandler) UpdateEnvironment(ctx context.Context, input *Updatreturn nil, huma.Error500InternalServerError("Failed to regenerate API key")}- // Fetch updated environmentupdated, err = h.environmentService.GetEnvironmentByID(ctx, input.ID)if err != nil {slog.ErrorContext(ctx, "Failed to fetch updated environment", "environmentId", input.ID, "error", err.Error())return nil, huma.Error500InternalServerError("Failed to fetch updated environment")}-- // Re-map with updated environment data- out, mapErr = mapping.MapOne[*Environment, environment.Environment](updated)- if mapErr != nil {- return nil, huma.Error500InternalServerError("Failed to map environment")- }- h.applyEdgeRuntimeStateInternal(&out)-newApiKey = new(apiKey)}- // Set the API key on the response if regenerated+ out, mapErr := mapping.MapOne[*Environment, environment.Environment](updated)+ if mapErr != nil {+ return nil, huma.Error500InternalServerError("Failed to map environment")+ }+ ApplyEnvironmentRuntimeState(&out)out.ApiKey = newApiKeyreturn &handlerutil.Out[environment.Environment]{@@ -547,22 +473,21 @@ func (h *EnvironmentHandler) UpdateEnvironment(ctx context.Context, input *Updat}, nil}-func (h *EnvironmentHandler) applyEdgeRuntimeStateInternal(env *environment.Environment) {- ApplyEnvironmentRuntimeState(env)+// currentUserRefs returns the request user's ID and username, or nils without a user.+func currentUserRefs(ctx context.Context) (userID, username *string) {+ if user, _ := userctx.CurrentUserFromContext(ctx); user != nil {+ return new(user.ID), new(user.Username)+ }+ return nil, nil}// DeleteEnvironment deletes an environment.func (h *EnvironmentHandler) DeleteEnvironment(ctx context.Context, input *DeleteEnvironmentInput) (*handlerutil.Out[base.MessageResponse], error) {- if input.ID == localDockerEnvironmentID {+ if input.ID == LocalEnvironmentID {return nil, huma.Error400BadRequest("Cannot delete local environment")}- user, _ := userctx.CurrentUserFromContext(ctx)- var userID, username *string- if user != nil {- userID = new(user.ID)- username = new(user.Username)- }+ userID, username := currentUserRefs(ctx)if err := h.environmentService.DeleteEnvironment(ctx, input.ID, userID, username); err != nil {return nil, huma.Error500InternalServerError("Failed to delete environment: " + err.Error())}@@ -619,7 +544,7 @@ func (h *EnvironmentHandler) UpdateHeartbeat(ctx context.Context, input *UpdateH// PairAgent generates or rotates the local agent pairing token.func (h *EnvironmentHandler) PairAgent(ctx context.Context, input *PairAgentInput) (*handlerutil.Out[environment.AgentPairResponse], error) {- if input.ID != localDockerEnvironmentID {+ if input.ID != LocalEnvironmentID {return nil, huma.Error404NotFound("Not found")}@@ -658,93 +583,20 @@ func (h *EnvironmentHandler) SyncEnvironment(ctx context.Context, input *SyncEnvreturn handlerutil.MessageOutput("Environment synced successfully", activityID), nil}-func (h *EnvironmentHandler) buildUpdateMapInternal(req *environment.Update, isLocalEnv bool) map[string]any {- updates := map[string]any{}-- if !isLocalEnv {- if req.ApiUrl != nil {- updates["api_url"] = *req.ApiUrl- }- if req.Enabled != nil {- updates["enabled"] = *req.Enabled- }- }-- if req.Name != nil {- updates["name"] = *req.Name- }-- return updates-}--func (h *EnvironmentHandler) handleEnvironmentPairingInternal(ctx context.Context, environmentID string, req *environment.Update, updates map[string]any, isLocalEnv bool) {- _ = ctx- _ = environmentID- if isLocalEnv {- return- }-- if req.AccessToken != nil {- updates["access_token"] = *req.AccessToken- }-}--func (h *EnvironmentHandler) triggerPostUpdateTasksInternal(ctx context.Context, environmentID string, updated *Environment, req *environment.Update) {- if updated.Enabled {- detachedCtx := context.WithoutCancel(ctx)- go func(syncCtx context.Context, envID, envName string) {- status, err := h.environmentService.TestConnection(syncCtx, envID, nil)- if err != nil {- slog.WarnContext(syncCtx, "Failed to test connection after environment update",- "environmentId", envID, "environmentName", envName, "status", status, "error", err)- }- }(detachedCtx, environmentID, updated.Name)- }-- if updated.AccessToken != nil && *updated.AccessToken != "" && ((req.AccessToken != nil && *req.AccessToken != "") || req.Name != nil) {- h.triggerEnvironmentResourceSyncInternal(ctx, environmentID, updated.Name, "environment update")- }-}--func (h *EnvironmentHandler) triggerEnvironmentResourceSyncInternal(ctx context.Context, environmentID, environmentName, reason string) {+func (h *EnvironmentHandler) triggerEnvironmentResourceSync(ctx context.Context, environmentID, environmentName, reason string) {h.environmentService.ForgetSyncState(environmentID)detachedCtx := context.WithoutCancel(ctx)- go func(syncCtx context.Context, envID, envName, syncReason string) {- syncCtx, cancel := context.WithTimeout(syncCtx, edge.DefaultProxyTimeout)- defer cancel()- if err := h.environmentService.SyncRegistriesToEnvironment(syncCtx, envID); err != nil {- slog.WarnContext(syncCtx, "Failed to sync registries to environment",- "environmentId", envID,- "environmentName", envName,- "reason", syncReason,- "error", err.Error())- }- }(detachedCtx, environmentID, environmentName, reason)-- go func(syncCtx context.Context, envID, envName, syncReason string) {- syncCtx, cancel := context.WithTimeout(syncCtx, edge.DefaultProxyTimeout)- defer cancel()- if err := h.environmentService.SyncS3DestinationsToEnvironment(syncCtx, envID); err != nil {- slog.WarnContext(syncCtx, "Failed to sync S3 destinations to environment",- "environmentId", envID,- "environmentName", envName,- "reason", syncReason,- "error", err.Error())- }- }(detachedCtx, environmentID, environmentName, reason)-- go func(syncCtx context.Context, envID, envName, syncReason string) {- syncCtx, cancel := context.WithTimeout(syncCtx, edge.DefaultProxyTimeout)- defer cancel()- if err := h.environmentService.SyncRepositoriesToEnvironment(syncCtx, envID); err != nil {- slog.WarnContext(syncCtx, "Failed to sync git repositories to environment",- "environmentId", envID,- "environmentName", envName,- "reason", syncReason,- "error", err.Error())- }- }(detachedCtx, environmentID, environmentName, reason)+ for _, resource := range h.environmentService.resourceSyncs() {+ go func() {+ syncCtx, cancel := context.WithTimeout(detachedCtx, edge.DefaultProxyTimeout)+ defer cancel()+ if err := resource.sync(syncCtx, environmentID); err != nil {+ slog.WarnContext(syncCtx, "Failed to sync resources to environment",+ "kind", resource.kind, "environmentId", environmentID, "environmentName", environmentName, "reason", reason, "error", err.Error())+ }+ }()+ }}// PairEnvironment handles agent pairing callback with API key.@@ -783,7 +635,7 @@ func (h *EnvironmentHandler) PairEnvironment(ctx context.Context, input *PairEnv}slog.InfoContext(ctx, "Environment pairing completed", "environmentId", *envID, "environmentName", env.Name)- h.triggerEnvironmentResourceSyncInternal(ctx, *envID, env.Name, "environment pairing")+ h.triggerEnvironmentResourceSync(ctx, *envID, env.Name, "environment pairing")return handlerutil.MessageOutput("Environment pairing completed successfully", ""), nil}@@ -803,16 +655,9 @@ func (h *EnvironmentHandler) GetDeploymentSnippets(ctx context.Context, input *Greturn nil, huma.Error400BadRequest("Environment is missing access token")}- // Generate snippets with API key- // Use edge snippets for edge environmentsvar snippets *DeploymentSnippetsif env.IsEdge {- snippets, err = h.environmentService.GenerateEdgeDeploymentSnippets(ctx, env.ID, h.cfg.GetAppURL(), *env.AccessToken, &edge.Config{- EdgeMTLSMode: h.cfg.EdgeMTLSMode,- EdgeMTLSCAFile: h.cfg.EdgeMTLSCAFile,- EdgeMTLSAssetsDir: h.cfg.EdgeMTLSAssetsDir,- AppURL: h.cfg.GetAppURL(),- })+ snippets, err = h.environmentService.GenerateEdgeDeploymentSnippets(ctx, env.ID, h.cfg.GetAppURL(), *env.AccessToken, h.edgeMTLSConfig())} else {snippets, err = h.environmentService.GenerateDeploymentSnippets(ctx, env.ID, h.cfg.GetAppURL(), env.ApiUrl, *env.AccessToken)}@@ -825,14 +670,14 @@ func (h *EnvironmentHandler) GetDeploymentSnippets(ctx context.Context, input *Gif snippets.MTLS != nil {files := make([]DeploymentSnippetFile, 0, len(snippets.MTLS.Files))for _, file := range snippets.MTLS.Files {- sensitive := isSensitiveMTLSAssetNameInternal(file.Name)entry := DeploymentSnippetFile{Name: file.Name,ContainerPath: file.ContainerPath,Permissions: file.Permissions,DownloadURL: fmt.Sprintf("/api/environments/%s/deployment/mtls/%s", env.ID, file.Name),}- if sensitive {+ // Secret material is only served through the download endpoint.+ if isSensitiveMTLSAssetName(file.Name) {entry.Sensitive = true} else {entry.Content = file.Content@@ -859,7 +704,7 @@ func (h *EnvironmentHandler) GetDeploymentSnippets(ctx context.Context, input *G}, nil}-// GetEnvironmentVersion returns the version of a remote environment.+// GetEnvironmentVersion returns the version of the local or a remote environment.func (h *EnvironmentHandler) GetEnvironmentVersion(ctx context.Context, input *GetEnvironmentVersionInput) (*handlerutil.Out[version.Info], error) {env, err := h.environmentService.GetEnvironmentByID(ctx, input.ID)if err != nil {@@ -870,61 +715,15 @@ func (h *EnvironmentHandler) GetEnvironmentVersion(ctx context.Context, input *Gdefer cancel()var versionInfo version.Info-- // For edge environments, route through the tunnel- if env.IsEdge {- if !edge.HasActiveTunnel(input.ID) {- if _, ok := edge.RequestTunnelAndWait(reqCtx, input.ID, edge.DefaultTunnelDemandTTL, edge.DefaultTunnelAcquireTimeout()).Get(); !ok {- return nil, huma.Error503ServiceUnavailable("Edge agent is not connected")- }- }-- statusCode, respBody, doRequestErr := edge.DoRequest(reqCtx, input.ID, http.MethodGet, "/api/app-version", nil)- if doRequestErr != nil {- return nil, huma.Error500InternalServerError("Request via tunnel failed: " + doRequestErr.Error())- }- if statusCode != http.StatusOK {- return nil, huma.Error500InternalServerError(fmt.Sprintf("Unexpected status code: %d", statusCode))- }-- if unmarshalErr := json.Unmarshal(respBody, &versionInfo); unmarshalErr != nil {- return nil, huma.Error500InternalServerError("Failed to decode version response")- }- } else {- // Direct HTTP request for non-edge environments- validatedURL, validateErr := httpx.ValidateOutboundHTTPURL(env.ApiUrl)- if validateErr != nil {- return nil, huma.Error400BadRequest("Invalid environment API URL")- }- validatedURL.RawQuery = ""- validatedURL.Fragment = ""- validatedURL.Path = strings.TrimRight(validatedURL.Path, "/") + "/api/app-version"-- req, newRequestWithContextErr := http.NewRequestWithContext(reqCtx, http.MethodGet, validatedURL.String(), http.NoBody)- if newRequestWithContextErr != nil {- return nil, huma.Error500InternalServerError("Failed to create request")- }-- client := &http.Client{Timeout: 15 * time.Second}- resp, newRequestWithContextErr := client.Do(req)- if newRequestWithContextErr != nil {- return nil, huma.Error500InternalServerError("Request failed: " + newRequestWithContextErr.Error())- }- defer func() { _ = resp.Body.Close() }()-- if resp.StatusCode != http.StatusOK {- return nil, huma.Error500InternalServerError(fmt.Sprintf("Unexpected status code: %d", resp.StatusCode))- }-- if unmarshalReadErr := json.UnmarshalRead(resp.Body, &versionInfo); unmarshalReadErr != nil {- return nil, huma.Error500InternalServerError("Failed to decode version response")- }+ if env.ID == LocalEnvironmentID && h.localVersion != nil {+ versionInfo = *h.localVersion(reqCtx)+ } else if proxyErr := h.environmentService.ProxyJSONRequestForEnvironment(reqCtx, *env, http.MethodGet, "/api/app-version", nil, &versionInfo); proxyErr != nil {+ return nil, handlerutil.TranslateRemoteProxyError(proxyErr)}- // Update environment status to online since we successfully contacted it+ // A successful reply proves the environment is online.if updateErr := h.environmentService.UpdateEnvironmentHeartbeat(ctx, input.ID); updateErr != nil {slog.WarnContext(ctx, "Failed to update environment heartbeat", "environmentId", input.ID, "error", updateErr)- // Don't fail the request if heartbeat update fails}return &handlerutil.Out[version.Info]{@@ -937,55 +736,26 @@ func (h *EnvironmentHandler) GetEnvironmentVersion(ctx context.Context, input *G// DownloadEdgeMTLSCA downloads the Arcane-managed edge mTLS CA certificate.func (h *EnvironmentHandler) DownloadEdgeMTLSCA(ctx context.Context, _ *DownloadEdgeMTLSCAInput) (*huma.StreamResponse, error) {- var edgeCfg *edge.Config- if h.cfg != nil {- edgeCfg = &edge.Config{- EdgeMTLSMode: h.cfg.EdgeMTLSMode,- EdgeMTLSCAFile: h.cfg.EdgeMTLSCAFile,- EdgeMTLSAssetsDir: h.cfg.EdgeMTLSAssetsDir,- }- }- caPath, err := edge.AvailableManagerMTLSCAPath(edgeCfg)+ caPath, err := edge.AvailableManagerMTLSCAPath(h.edgeMTLSConfig())if err != nil {return nil, huma.Error404NotFound("Arcane-managed edge mTLS CA is not available")}- // os.* rather than acfs: the CA path may resolve to a user-configured- // location anywhere on the host, so no confinement root exists for it.+ // os rather than acfs: the CA path may be configured anywhere on the host.caPEM, err := os.ReadFile(caPath)if err != nil {slog.ErrorContext(ctx, "Failed to read generated edge mTLS CA", "path", caPath, "error", err.Error())return nil, huma.Error500InternalServerError("Failed to read Arcane-generated edge mTLS CA")}- fileName := filepath.Base(caPath)- if strings.TrimSpace(fileName) == "" {- fileName = "ca.crt"- }-- return &huma.StreamResponse{- Body: func(humaCtx huma.Context) { //nolint:contextcheck // context is obtained from humaCtx.Context()- humaCtx.SetHeader("Content-Type", "application/x-pem-file")- humaCtx.SetHeader("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName))- humaCtx.SetHeader("Content-Length", strconv.Itoa(len(caPEM)))-- if written, writeErr := humaCtx.BodyWriter().Write(bytes.Clone(caPEM)); writeErr != nil || written != len(caPEM) {- slog.WarnContext(humaCtx.Context(), "Failed to stream edge mTLS CA download", "fileName", fileName, "bytesWritten", written, "bytesExpected", len(caPEM), "error", writeErr)- return- }- h.logMTLSAuditEventInternal(humaCtx.Context(), nil, event.EventTypeEnvironmentMTLSDownload,- "mTLS CA downloaded",- fmt.Sprintf("Administrator downloaded edge mTLS CA %q", fileName),- database.JSON{- "fileName": fileName,- "kind": "ca",- })- },- }, nil+ fileName := cmp.Or(strings.TrimSpace(filepath.Base(caPath)), "ca.crt")+ return h.mtlsDownloadResponse(nil, "application/x-pem-file", fileName, caPEM,+ "mTLS CA downloaded", fmt.Sprintf("Administrator downloaded edge mTLS CA %q", fileName),+ database.JSON{"fileName": fileName, "kind": "ca"}), nil}func (h *EnvironmentHandler) DownloadEnvironmentMTLSBundle(ctx context.Context, input *DownloadEnvironmentMTLSBundleInput) (*huma.StreamResponse, error) {- env, files, err := h.loadEnvironmentMTLSFilesInternal(ctx, input.ID)+ env, files, err := h.loadEnvironmentMTLSFiles(ctx, input.ID)if err != nil {return nil, err}@@ -994,12 +764,16 @@ func (h *EnvironmentHandler) DownloadEnvironmentMTLSBundle(ctx context.Context,zipWriter := zip.NewWriter(&archive)for _, file := range files {- downloadName := environmentMTLSAssetDownloadNameInternal(env, file.Name)+ downloadName := environmentMTLSAssetDownloadName(env, file.Name)header := &zip.FileHeader{Name: downloadName,Method: zip.Deflate,}- header.SetMode(environmentMTLSAssetFileModeInternal(file))+ mode := kit.Ternary[os.FileMode](isSensitiveMTLSAssetName(file.Name), 0o600, 0o644)+ if parsed, parseErr := strconv.ParseUint(strings.TrimSpace(file.Permissions), 8, 32); parseErr == nil && parsed != 0 {+ mode = os.FileMode(parsed)+ }+ header.SetMode(mode)entry, createErr := zipWriter.CreateHeader(header)if createErr != nil {@@ -1018,122 +792,57 @@ func (h *EnvironmentHandler) DownloadEnvironmentMTLSBundle(ctx context.Context,return nil, huma.Error500InternalServerError("Failed to build mTLS bundle")}- fileName := environmentMTLSDownloadBaseNameInternal(env) + "-mtls.zip"-- return &huma.StreamResponse{- Body: func(humaCtx huma.Context) { //nolint:contextcheck // context is obtained from humaCtx.Context()- humaCtx.SetHeader("Content-Type", "application/zip")- humaCtx.SetHeader("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName))- humaCtx.SetHeader("Content-Length", strconv.Itoa(archive.Len()))-- if written, writeErr := humaCtx.BodyWriter().Write(archive.Bytes()); writeErr != nil || written != archive.Len() {- slog.WarnContext(- humaCtx.Context(),- "Failed to stream edge mTLS bundle download",- "environmentId",- input.ID,- "fileName",- fileName,- "bytesWritten",- written,- "bytesExpected",- archive.Len(),- "error",- writeErr,- )- return- }- h.logMTLSAuditEventInternal(humaCtx.Context(), env, event.EventTypeEnvironmentMTLSDownload,- "mTLS bundle downloaded",- fmt.Sprintf("Administrator downloaded edge mTLS bundle %q (%d files)", fileName, len(files)),- database.JSON{- "fileName": fileName,- "kind": "bundle",- "fileCount": len(files),- })- },- }, nil+ fileName := environmentMTLSDownloadBaseName(env) + "-mtls.zip"+ return h.mtlsDownloadResponse(env, "application/zip", fileName, archive.Bytes(),+ "mTLS bundle downloaded", fmt.Sprintf("Administrator downloaded edge mTLS bundle %q (%d files)", fileName, len(files)),+ database.JSON{"fileName": fileName, "kind": "bundle", "fileCount": len(files)}), nil}func (h *EnvironmentHandler) DownloadEnvironmentMTLSFile(ctx context.Context, input *DownloadEnvironmentMTLSFileInput) (*huma.StreamResponse, error) {- env, file, err := h.loadEnvironmentMTLSFileInternal(ctx, input.ID, input.FileName)+ env, files, err := h.loadEnvironmentMTLSFiles(ctx, input.ID)if err != nil {return nil, err}+ index := slices.IndexFunc(files, func(file DeploymentSnippetFile) bool { return file.Name == input.FileName })+ if index < 0 {+ return nil, huma.Error404NotFound("Requested mTLS asset was not found")+ }+ file := files[index]- fileContent := []byte(file.Content)- downloadName := environmentMTLSAssetDownloadNameInternal(env, file.Name)+ return h.mtlsDownloadResponse(env, "application/x-pem-file", environmentMTLSAssetDownloadName(env, file.Name), []byte(file.Content),+ "mTLS asset downloaded", fmt.Sprintf("Administrator downloaded edge mTLS asset %q", file.Name),+ database.JSON{"fileName": file.Name, "kind": "file", "sensitive": isSensitiveMTLSAssetName(file.Name)}), nil+}- return &huma.StreamResponse{- Body: func(humaCtx huma.Context) { //nolint:contextcheck // context is obtained from humaCtx.Context()- humaCtx.SetHeader("Content-Type", "application/x-pem-file")- humaCtx.SetHeader("Content-Disposition", fmt.Sprintf("attachment; filename=%q", downloadName))- humaCtx.SetHeader("Content-Length", strconv.Itoa(len(fileContent)))-- if written, writeErr := humaCtx.BodyWriter().Write(fileContent); writeErr != nil || written != len(fileContent) {- slog.WarnContext(- humaCtx.Context(),- "Failed to stream edge mTLS asset download",- "environmentId",- input.ID,- "fileName",- file.Name,- "bytesWritten",- written,- "bytesExpected",- len(- fileContent,- ),- "error",- writeErr,- )- return- }- h.logMTLSAuditEventInternal(humaCtx.Context(), env, event.EventTypeEnvironmentMTLSDownload,- "mTLS asset downloaded",- fmt.Sprintf("Administrator downloaded edge mTLS asset %q", file.Name),- database.JSON{- "fileName": file.Name,- "kind": "file",- "sensitive": isSensitiveMTLSAssetNameInternal(file.Name),- })- },- }, nil+// edgeMTLSConfig maps the handler config to edge mTLS settings, or nil without config.+func (h *EnvironmentHandler) edgeMTLSConfig() *edge.Config {+ if h.cfg == nil {+ return nil+ }+ return &edge.Config{+ EdgeMTLSMode: h.cfg.EdgeMTLSMode,+ EdgeMTLSCAFile: h.cfg.EdgeMTLSCAFile,+ EdgeMTLSAssetsDir: h.cfg.EdgeMTLSAssetsDir,+ AppURL: h.cfg.GetAppURL(),+ }}-func (h *EnvironmentHandler) loadEnvironmentMTLSEnvironmentInternal(ctx context.Context, environmentID string) (*Environment, error) {+func (h *EnvironmentHandler) loadEnvironmentMTLSFiles(ctx context.Context, environmentID string) (*Environment, []DeploymentSnippetFile, error) {env, err := h.environmentService.GetEnvironmentByID(ctx, environmentID)if err != nil {- return nil, huma.Error404NotFound("Environment not found")+ return nil, nil, huma.Error404NotFound("Environment not found")}-if !env.IsEdge {- return nil, huma.Error400BadRequest("Environment is not an edge agent")+ return nil, nil, huma.Error400BadRequest("Environment is not an edge agent")}-if env.ApiKeyID == nil {- return nil, huma.Error400BadRequest("Environment does not have an API key configured")+ return nil, nil, huma.Error400BadRequest("Environment does not have an API key configured")}-if env.AccessToken == nil || *env.AccessToken == "" {- return nil, huma.Error400BadRequest("Environment is missing access token")+ return nil, nil, huma.Error400BadRequest("Environment is missing access token")}- return env, nil-}--func (h *EnvironmentHandler) loadEnvironmentMTLSFilesInternal(ctx context.Context, environmentID string) (*Environment, []DeploymentSnippetFile, error) {- env, err := h.loadEnvironmentMTLSEnvironmentInternal(ctx, environmentID)- if err != nil {- return nil, nil, err- }-- snippets, err := h.environmentService.GenerateEdgeDeploymentSnippets(ctx, env.ID, h.cfg.GetAppURL(), *env.AccessToken, &edge.Config{- EdgeMTLSMode: h.cfg.EdgeMTLSMode,- EdgeMTLSCAFile: h.cfg.EdgeMTLSCAFile,- EdgeMTLSAssetsDir: h.cfg.EdgeMTLSAssetsDir,- AppURL: h.cfg.GetAppURL(),- })+ snippets, err := h.environmentService.GenerateEdgeDeploymentSnippets(ctx, env.ID, h.cfg.GetAppURL(), *env.AccessToken, h.edgeMTLSConfig())if err != nil {slog.ErrorContext(ctx, "Failed to generate environment mTLS assets", "environmentId", environmentID, "error", err.Error())return nil, nil, huma.Error500InternalServerError("Failed to generate environment mTLS assets")@@ -1146,150 +855,95 @@ func (h *EnvironmentHandler) loadEnvironmentMTLSFilesInternal(ctx context.Contexreturn env, snippets.MTLS.Files, nil}-func (h *EnvironmentHandler) loadEnvironmentMTLSFileInternal(ctx context.Context, environmentID, fileName string) (*Environment, DeploymentSnippetFile, error) {- env, files, err := h.loadEnvironmentMTLSFilesInternal(ctx, environmentID)- if err != nil {- return nil, DeploymentSnippetFile{}, err- }+// mtlsDownloadResponse streams an mTLS asset and audits the download; the audit+// must never include certificate or key material.+func (h *EnvironmentHandler) mtlsDownloadResponse(env *Environment, contentType, fileName string, content []byte, title, description string, metadata database.JSON) *huma.StreamResponse {+ return &huma.StreamResponse{+ Body: func(humaCtx huma.Context) { //nolint:contextcheck // context is obtained from humaCtx.Context()+ ctx := humaCtx.Context()+ humaCtx.SetHeader("Content-Type", contentType)+ humaCtx.SetHeader("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName))+ humaCtx.SetHeader("Content-Length", strconv.Itoa(len(content)))- for _, file := range files {- if file.Name == fileName {- return env, file, nil- }- }+ if written, writeErr := humaCtx.BodyWriter().Write(content); writeErr != nil || written != len(content) {+ slog.WarnContext(ctx, "Failed to stream edge mTLS download", "fileName", fileName, "bytesWritten", written, "bytesExpected", len(content), "error", writeErr)+ return+ }+ if h.eventService == nil {+ return+ }- return nil, DeploymentSnippetFile{}, huma.Error404NotFound("Requested mTLS asset was not found")+ if remoteAddr := strings.TrimSpace(middleware.GetRemoteAddrFromContext(ctx)); remoteAddr != "" {+ metadata["remoteAddr"] = remoteAddr+ }+ userID, username := currentUserRefs(ctx)+ req := event.CreateEventRequest{+ Type: event.EventTypeEnvironmentMTLSDownload,+ Severity: event.EventSeverityInfo,+ Title: title,+ Description: description,+ UserID: userID,+ Username: username,+ Metadata: metadata,+ }+ if env != nil {+ req.ResourceType = new("environment")+ req.ResourceID = new(env.ID)+ req.ResourceName = new(env.Name)+ req.EnvironmentID = new(env.ID)+ }+ if _, err := h.eventService.CreateEvent(ctx, req); err != nil {+ slog.WarnContext(ctx, "Failed to record mTLS audit event", "type", string(req.Type), "error", err)+ }+ },+ }}-// isSensitiveMTLSAssetNameInternal reports whether the given generated asset-// filename contains secret material (currently just the agent private key).-// Sensitive asset contents must not be returned inline in JSON responses; the-// client should fetch them via the admin-only download endpoint instead.-func isSensitiveMTLSAssetNameInternal(fileName string) bool {+// isSensitiveMTLSAssetName reports whether a generated asset holds secret material+// (the agent private key), which must never be returned inline.+func isSensitiveMTLSAssetName(fileName string) bool {name := strings.ToLower(strings.TrimSpace(fileName))return strings.HasSuffix(name, ".key") || strings.HasSuffix(name, "-key.pem") || strings.HasSuffix(name, "_key.pem")}-func environmentMTLSDownloadBaseNameInternal(env *Environment) string {- baseName := cmp.Or(strings.TrimSpace(env.Name), "environment")-- baseName = strings.Map(func(r rune) rune {+func environmentMTLSDownloadBaseName(env *Environment) string {+ baseName := strings.Map(func(r rune) rune {switch {- case r >= 'a' && r <= 'z':- return r- case r >= 'A' && r <= 'Z':- return r- case r >= '0' && r <= '9':+ case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':return rdefault:return '-'}- }, baseName)-- baseName = cmp.Or(strings.Trim(baseName, "-"), "environment")-- return baseName + "-" + env.ID+ }, strings.TrimSpace(env.Name))+ return cmp.Or(strings.Trim(baseName, "-"), "environment") + "-" + env.ID}-func environmentMTLSAssetDownloadNameInternal(env *Environment, fileName string) string {- baseName := environmentMTLSDownloadBaseNameInternal(env)-+func environmentMTLSAssetDownloadName(env *Environment, fileName string) string {switch fileName {case "agent.crt":- return baseName + ".pem"+ return environmentMTLSDownloadBaseName(env) + ".pem"case "agent.key":- return baseName + ".key"+ return environmentMTLSDownloadBaseName(env) + ".key"default:return fileName}}-func environmentMTLSAssetFileModeInternal(file DeploymentSnippetFile) os.FileMode {- if parsed, err := strconv.ParseUint(strings.TrimSpace(file.Permissions), 8, 32); err == nil && parsed != 0 {- return os.FileMode(parsed)- }- return kit.Ternary[os.FileMode](isSensitiveMTLSAssetNameInternal(file.Name), 0o600, 0o644)-}--// logMTLSAuditEventInternal records an audit event for administrator-triggered-// edge mTLS actions (downloads, bundle exports). Must never include raw-// certificate content or private key material.-func (h *EnvironmentHandler) logMTLSAuditEventInternal(ctx context.Context, env *Environment, eventType event.EventType, title, description string, extra database.JSON) {- if h == nil || h.eventService == nil {- return- }-- user, _ := userctx.CurrentUserFromContext(ctx)- var userID, username *string- if user != nil {- userID = new(user.ID)- username = new(user.Username)- }-- if extra == nil {- extra = database.JSON{}- }- if remoteAddr := strings.TrimSpace(middleware.GetRemoteAddrFromContext(ctx)); remoteAddr != "" {- extra["remoteAddr"] = remoteAddr- }-- req := event.CreateEventRequest{- Type: eventType,- Severity: event.EventSeverityInfo,- Title: title,- Description: description,- UserID: userID,- Username: username,- Metadata: extra,- }- if env != nil {- envID := env.ID- req.ResourceType = new("environment")- req.ResourceID = &envID- req.ResourceName = new(env.Name)- req.EnvironmentID = &envID- }-- if _, err := h.eventService.CreateEvent(ctx, req); err != nil {- slog.WarnContext(ctx, "Failed to record mTLS audit event", "type", string(eventType), "error", err)- }-}-const edgeMTLSCertificateExpiryWarningWindow = 30 * 24 * time.Hour-func generatedEdgeMTLSClientCertPathInternal(cfg *config.Config, envID string) (string, error) {- if cfg == nil {- return "", errors.New("config not available")- }- if edge.NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == edge.EdgeMTLSModeDisabled {- return "", errors.New("edge mTLS is disabled")+func readGeneratedEdgeMTLSCertificateInfo(edgeCfg *edge.Config, envID string) (*environment.EdgeMTLSCertificate, error) {+ if edgeCfg == nil {+ return nil, errors.New("config not available")}-- edgeCfg := &edge.Config{- EdgeMTLSAssetsDir: cfg.EdgeMTLSAssetsDir,+ if edge.NormalizeEdgeMTLSMode(edgeCfg.EdgeMTLSMode) == edge.EdgeMTLSModeDisabled {+ return nil, errors.New("edge mTLS is disabled")}certPath, err := edge.GeneratedManagerClientMTLSCertPath(edgeCfg, envID)if err != nil {- return "", fmt.Errorf("resolve generated edge mTLS client certificate path: %w", err)- }- // os.* rather than acfs: the assets dir may be user-configured to anywhere- // on the host, so no confinement root exists for this path.- if _, statErr := os.Stat(certPath); statErr != nil {- return "", fmt.Errorf("stat generated edge mTLS client certificate: %w", statErr)+ return nil, fmt.Errorf("resolve generated edge mTLS client certificate path: %w", err)}-- return certPath, nil-}--func readGeneratedEdgeMTLSCertificateInfoInternal(cfg *config.Config, envID string) (*environment.EdgeMTLSCertificate, error) {- certPath, err := generatedEdgeMTLSClientCertPathInternal(cfg, envID)- if err != nil {- return nil, err- }-- // os.* rather than acfs: the assets dir may be user-configured to anywhere- // on the host, so no confinement root exists for this path.+ // os rather than acfs: the assets dir may be configured anywhere on the host.certPEM, err := os.ReadFile(certPath)if err != nil {return nil, fmt.Errorf("read generated edge mTLS client certificate: %w", err)@@ -1310,7 +964,7 @@ func readGeneratedEdgeMTLSCertificateInfoInternal(cfg *config.Config, envID striremaining := expiresAt.Sub(now)info := &environment.EdgeMTLSCertificate{ExpiresAt: &expiresAt,- DaysRemaining: new(edgeMTLSCertificateDaysRemainingInternal(now, expiresAt)),+ DaysRemaining: new(int(math.Ceil(max(remaining, 0).Hours() / 24))),Expired: now.After(expiresAt),ExpiringSoon: now.Before(expiresAt) && remaining <= edgeMTLSCertificateExpiryWarningWindow,}@@ -1321,11 +975,3 @@ func readGeneratedEdgeMTLSCertificateInfoInternal(cfg *config.Config, envID strireturn info, nil}--func edgeMTLSCertificateDaysRemainingInternal(now, expiresAt time.Time) int {- remaining := expiresAt.Sub(now)- if remaining <= 0 {- return 0- }- return int(math.Ceil(remaining.Hours() / 24))-}diff --git a/backend/internal/environment/helpers.go b/backend/internal/environment/helpers.goindex 8b087ae95..11336bde9 100644--- a/backend/internal/environment/helpers.go+++ b/backend/internal/environment/helpers.go@@ -5,34 +5,30 @@ import ("strings""sync"- "github.com/getarcaneapp/arcane/types/v2/environment""github.com/samber/hot""github.com/samber/mo"-- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge")-// edgeTokenCacheInternal maps an edge agent token to its environment ID. The-// reverse index lets an environment's entry be dropped when its token rotates,-// which the TTL cache alone cannot do.-type edgeTokenCacheInternal struct {+// edgeTokenCache maps agent tokens to environment IDs; the reverse index drops+// an environment's entry when its token rotates.+type edgeTokenCache struct {mu sync.RWMutexbyToken *hot.HotCache[string, string]byEnvID map[string]string}-// remoteEnvSnapshotCacheInternal holds the latest in-process copy of every-// enabled, visible, non-local environment, so hot paths avoid a DB round trip.-type remoteEnvSnapshotCacheInternal struct {+// remoteEnvSnapshotCache holds every enabled, visible, non-local environment+// in memory so hot paths avoid a DB round trip.+type remoteEnvSnapshotCache struct {mu sync.RWMutexenvs map[string]Environmentseeded boolrevision uint64}-// environmentCacheKeyInternal scopes cached records to a cache generation so-// loads that started before a write can never be served after it.-type environmentCacheKeyInternal struct {+// environmentCacheKey scopes cached records to a generation so loads that+// started before a write are never served after it.+type environmentCacheKey struct {gen uint64id string}@@ -43,43 +39,14 @@ const environmentCacheStripes = 64var environmentCacheSeed = maphash.MakeSeed()-func environmentCacheStripeInternal(id string) int {- return int(maphash.String(environmentCacheSeed, id) % environmentCacheStripes)-}--// runtimeWatchersInternal fans a coalesced wake-up out to everyone watching for-// environment liveness changes.-type runtimeWatchersInternal struct {+// runtimeWatchers fans a coalesced wake-up out to environment liveness watchers.+type runtimeWatchers struct {mu sync.Mutexchans map[int]chan struct{}seq int}-func newEdgeTokenCacheInternal() *edgeTokenCacheInternal {- return &edgeTokenCacheInternal{- byToken: hot.NewHotCache[string, string](hot.LRU, 1024).- WithTTL(edgeTokenCacheTTL).- WithJanitor().- Build(),- byEnvID: make(map[string]string),- }-}--// newEnvironmentCacheInternal caches environment records by ID, including-// misses, for hot paths that only read CRUD-managed fields.-func newEnvironmentCacheInternal() *hot.HotCache[environmentCacheKeyInternal, Environment] {- return hot.NewHotCache[environmentCacheKeyInternal, Environment](hot.LRU, 256).- WithTTL(environmentCacheTTL).- WithMissingSharedCache().- WithJanitor().- Build()-}--func newRemoteEnvSnapshotCacheInternal() *remoteEnvSnapshotCacheInternal {- return &remoteEnvSnapshotCacheInternal{envs: make(map[string]Environment)}-}--func (c *edgeTokenCacheInternal) environmentID(token string) mo.Option[string] {+func (c *edgeTokenCache) environmentID(token string) mo.Option[string] {if c == nil || c.byToken == nil || token == "" {return mo.None[string]()}@@ -99,7 +66,7 @@ func (c *edgeTokenCacheInternal) environmentID(token string) mo.Option[string] {return mo.None[string]()}-func (c *edgeTokenCacheInternal) put(envID, token string) {+func (c *edgeTokenCache) put(envID, token string) {if c == nil || c.byToken == nil || envID == "" || token == "" {return}@@ -115,7 +82,7 @@ func (c *edgeTokenCacheInternal) put(envID, token string) {c.byToken.Set(token, envID)}-func (c *edgeTokenCacheInternal) invalidate(envID string) {+func (c *edgeTokenCache) invalidate(envID string) {if c == nil || envID == "" {return}@@ -131,9 +98,8 @@ func (c *edgeTokenCacheInternal) invalidate(envID string) {}}-// sync replaces an environment's cached token, dropping the entry entirely when-// the new token is blank.-func (c *edgeTokenCacheInternal) sync(envID, token string) {+// sync replaces an environment's cached token, dropping it when the new token is blank.+func (c *edgeTokenCache) sync(envID, token string) {if c == nil || envID == "" {return}@@ -145,11 +111,11 @@ func (c *edgeTokenCacheInternal) sync(envID, token string) {}}-func isActiveRemoteEnvironmentInternal(env Environment) bool {+func isActiveRemoteEnvironment(env Environment) bool {return env.ID != "" && env.ID != "0" && env.Enabled && !env.Hidden}-func (c *remoteEnvSnapshotCacheInternal) get(environmentID string) mo.Option[Environment] {+func (c *remoteEnvSnapshotCache) get(environmentID string) mo.Option[Environment] {if c == nil || environmentID == "" {return mo.None[Environment]()}@@ -157,18 +123,18 @@ func (c *remoteEnvSnapshotCacheInternal) get(environmentID string) mo.Option[Envc.mu.RLock()envRecord, ok := c.envs[environmentID]c.mu.RUnlock()- if !ok || !isActiveRemoteEnvironmentInternal(envRecord) {+ if !ok || !isActiveRemoteEnvironment(envRecord) {return mo.None[Environment]()}return mo.Some(envRecord)}-func (c *remoteEnvSnapshotCacheInternal) put(env Environment) {+func (c *remoteEnvSnapshotCache) put(env Environment) {if c == nil {return}- if !isActiveRemoteEnvironmentInternal(env) {+ if !isActiveRemoteEnvironment(env) {c.remove(env.ID)return}@@ -179,7 +145,7 @@ func (c *remoteEnvSnapshotCacheInternal) put(env Environment) {c.mu.Unlock()}-func (c *remoteEnvSnapshotCacheInternal) remove(environmentID string) {+func (c *remoteEnvSnapshotCache) remove(environmentID string) {if c == nil || environmentID == "" {return}@@ -190,7 +156,7 @@ func (c *remoteEnvSnapshotCacheInternal) remove(environmentID string) {c.mu.Unlock()}-func (c *remoteEnvSnapshotCacheInternal) update(environmentID string, update func(*Environment)) {+func (c *remoteEnvSnapshotCache) update(environmentID string, update func(*Environment)) {if c == nil || environmentID == "" || update == nil {return}@@ -204,60 +170,16 @@ func (c *remoteEnvSnapshotCacheInternal) update(environmentID string, update funreturn}update(&envRecord)- if isActiveRemoteEnvironmentInternal(envRecord) {+ if isActiveRemoteEnvironment(envRecord) {c.envs[environmentID] = envRecord} else {delete(c.envs, environmentID)}}-// filterEnvironmentsByIDInternal returns only the environments whose ID is in-// allowedIDs. A non-nil but empty allowedIDs yields an empty slice. Used to-// restrict the runtime-filtered list path to a caller's accessible environments.-func filterEnvironmentsByIDInternal(items []environment.Environment, allowedIDs []string) []environment.Environment {- allowed := make(map[string]struct{}, len(allowedIDs))- for _, id := range allowedIDs {- allowed[id] = struct{}{}- }- out := make([]environment.Environment, 0, len(items))- for _, item := range items {- if _, ok := allowed[item.ID]; ok {- out = append(out, item)- }- }- return out-}--func environmentTypeMatchesInternal(env environment.Environment, filterValue string) bool {- return environmentTypeKeyInternal(env) == strings.ToLower(strings.TrimSpace(filterValue))-}--func environmentTypeKeyInternal(env environment.Environment) string {- if !env.IsEdge {- return "http"- }- transport := ""- if env.Connected != nil && *env.Connected && env.EdgeTransport != nil {- transport = *env.EdgeTransport- } else if env.LastEdgeTransport != nil {- // Disconnected or poll-only agents classify by the transport they- // last used rather than collapsing into the generic edge bucket.- transport = *env.LastEdgeTransport- }- switch strings.ToLower(strings.TrimSpace(transport)) {- case edge.EdgeTransportWebSocket:- return "websocket"- case edge.EdgeTransportGRPC:- return "grpc"- default:- return "edge"- }-}--// subscribe returns a channel that receives a coalesced wake-up whenever-// environment liveness may have changed, plus a function to release it. The-// channel is never closed; callers select on it alongside their own context.-func (w *runtimeWatchersInternal) subscribe() (<-chan struct{}, func()) {+// subscribe returns a never-closed channel that receives a coalesced wake-up on+// liveness changes, plus a function to release it.+func (w *runtimeWatchers) subscribe() (<-chan struct{}, func()) {w.mu.Lock()defer w.mu.Unlock()@@ -267,8 +189,7 @@ func (w *runtimeWatchersInternal) subscribe() (<-chan struct{}, func()) {w.seq++id := w.seq- // Capacity 1: a second signal arriving before the watcher wakes is- // redundant, because the watcher reads live state rather than a queue.+ // Capacity 1: watchers read live state, so a second pending signal is redundant.ch := make(chan struct{}, 1)w.chans[id] = ch@@ -279,9 +200,8 @@ func (w *runtimeWatchersInternal) subscribe() (<-chan struct{}, func()) {}}-// notify wakes every watcher. It never blocks, so it is safe to call from-// connection callbacks on the tunnel hot path.-func (w *runtimeWatchersInternal) notify() {+// notify wakes every watcher without blocking, so tunnel callbacks may call it.+func (w *runtimeWatchers) notify() {if w == nil {return}diff --git a/backend/internal/environment/module.go b/backend/internal/environment/module.goindex 7cefa1057..50e15fe1d 100644--- a/backend/internal/environment/module.go+++ b/backend/internal/environment/module.go@@ -3,7 +3,10 @@package environmentimport (+ "context"+"github.com/danielgtaylor/huma/v2"+ "github.com/getarcaneapp/arcane/types/v2/version""github.com/getarcaneapp/arcane/backend/v2/internal/apikey""github.com/getarcaneapp/arcane/backend/v2/internal/config"@@ -48,12 +51,14 @@ func (m *Module) Handler() *EnvironmentHandler {return m.handler}-func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContext) {+// RegisterRoutes registers the environment endpoints; localVersion answers version requests for the local environment.+func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContext, localVersion func(context.Context) *version.Info) {if m == nil {RegisterEnvironments(api, NewHandler(nil, nil, nil, nil, nil, nil))return}m.handler.appCtx = appCtx.Context()+ m.handler.localVersion = localVersionRegisterEnvironments(api, m.handler)}diff --git a/backend/internal/environment/service.go b/backend/internal/environment/service.goindex 2adeba10b..2dff0d14b 100644--- a/backend/internal/environment/service.go+++ b/backend/internal/environment/service.go@@ -5,7 +5,9 @@ import ("context""errors""fmt"+ "hash/maphash""log/slog"+ "maps""net/http""net/url""slices"@@ -25,6 +27,9 @@ import ("go.getarcane.app/kit/pkg""go.getarcane.app/kit/pkg/mapping""go.getarcane.app/sys/crypto"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/apikey"@@ -47,36 +52,28 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation")type EnvironmentService struct {db *database.DB- httpClient *http.Client- dockerService *docker.DockerClientServiceeventService *event.EventServicesettingsService *settings.SettingsService- edgeTokens *edgeTokenCacheInternal- remoteEnvs *remoteEnvSnapshotCacheInternal- environmentCache *hot.HotCache[environmentCacheKeyInternal, Environment]+ edgeTokens *edgeTokenCache+ remoteEnvs *remoteEnvSnapshotCache+ environmentCache *hot.HotCache[environmentCacheKey, Environment]environmentGens [environmentCacheStripes]atomic.Uint64- // jobs carries the scheduler and app lifecycle context, injected- // post-construction via SetScheduler (manager-only). Each enabled environment- // gets its own health-check job; this replaces the single global- // environment-health job.+ // jobs holds one health-check job per enabled environment; SetScheduler wires it on the manager.jobs *entityjobs.Registry- // variableSyncer is injected post-construction via SetVariableSyncer- // (manager-only) to avoid a wire cycle with variable.VariableService.+ // variableSyncer is set by SetVariableSyncer on the manager to avoid a wire cycle.variableSyncer VariableSyncer- // runtimeWatchers receive a coalesced wake-up whenever an environment's- // liveness changes. See environment_runtime_notify.go.- runtimeWatchers runtimeWatchersInternal+ runtimeWatchers runtimeWatchers- // syncGate skips the periodic registry/S3/repository pushes to an agent- // while the payload it last accepted is unchanged.+ // syncGate skips agent config pushes while the last accepted payload is unchanged.syncGate utils.SyncGateproxy *proxy.Service@@ -87,12 +84,11 @@ type EnvironmentService struct {const (// LocalEnvironmentID is the reserved ID of the environment Arcane manages directly.- LocalEnvironmentID = "0"- localEnvironmentFallbackNameInternal = "Local"+ LocalEnvironmentID = "0"+ localEnvironmentFallbackName = "Local"- // SyncDeliveryExpiry bounds how long an accepted config push is trusted. An- // agent rebuilt with a fresh data volume while its status stayed online gets- // everything again within this window without operator action.+ // SyncDeliveryExpiry bounds how long an accepted config push is trusted, so a+ // rebuilt agent that stayed online is resynced without operator action.SyncDeliveryExpiry = time.Hour)@@ -103,14 +99,12 @@ var ()// VariableSyncer pushes the effective global-variable set to one environment.-// Implemented by variable.VariableService.type VariableSyncer interface {SyncEnvironment(ctx context.Context, envID string) errorForgetSyncState(envID string)}-// ForgetSyncState makes the next sync of every resource group resend to the-// environment regardless of whether the payload changed.+// ForgetSyncState makes the next sync of every resource group resend regardless of payload changes.func (s *EnvironmentService) ForgetSyncState(environmentID string) {s.syncGate.Forget(environmentID)if s.variableSyncer != nil {@@ -130,26 +124,70 @@ func NewEnvironmentService(httpClient = http.DefaultClient}s := &EnvironmentService{- syncGate: utils.SyncGate{Expiry: SyncDeliveryExpiry},- db: db,- httpClient: httpClient,- dockerService: dockerService,- eventService: eventService,- settingsService: settingsService,- edgeTokens: newEdgeTokenCacheInternal(),- remoteEnvs: newRemoteEnvSnapshotCacheInternal(),- environmentCache: newEnvironmentCacheInternal(),- jobs: entityjobs.New(environmentHealthJobPrefix, environmentHealthAdmissionScopeInternal),- health: health.New(httpClient, dockerService),- snippets: snippets.New(eventService),- swarm: swarm.New(apiKeyService),+ syncGate: utils.SyncGate{Expiry: SyncDeliveryExpiry},+ db: db,+ eventService: eventService,+ settingsService: settingsService,+ edgeTokens: &edgeTokenCache{+ byToken: hot.NewHotCache[string, string](hot.LRU, 1024).WithTTL(edgeTokenCacheTTL).WithJanitor().Build(),+ byEnvID: make(map[string]string),+ },+ remoteEnvs: &remoteEnvSnapshotCache{envs: make(map[string]Environment)},+ // Caches records by ID, including misses, for hot paths that read only CRUD-managed fields.+ environmentCache: hot.NewHotCache[environmentCacheKey, Environment](hot.LRU, 256).+ WithTTL(environmentCacheTTL).+ WithMissingSharedCache().+ WithJanitor().+ Build(),+ jobs: entityjobs.New(environmentHealthJobPrefix, environmentHealthAdmissionScope),+ health: health.New(httpClient, dockerService),+ snippets: snippets.New(eventService),+ swarm: swarm.New(apiKeyService),}s.proxy = proxy.New(db, httpClient, settingsService, &s.syncGate)++ meter := otel.Meter(tracing.InstrumentationName)+ statusGauge, err := meter.Int64ObservableGauge("arcane.environments",+ metric.WithDescription("Visible environments by persisted status"),+ metric.WithUnit("{environment}"),+ )+ if err == nil {+ _, err = meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {+ var rows []struct {+ Status string+ Count int64+ }+ if scanErr := s.db.WithContext(ctx).Model(&Environment{}).+ Select("status, COUNT(*) AS count").+ Where("hidden = ?", false).+ Group("status").+ Scan(&rows).Error; scanErr != nil {+ return fmt.Errorf("failed to count environments by status: %w", scanErr)+ }+ // Known statuses report zero so their series drop instead of going stale.+ counts := map[string]int64{+ string(EnvironmentStatusOnline): 0,+ string(EnvironmentStatusStandby): 0,+ string(EnvironmentStatusOffline): 0,+ string(EnvironmentStatusError): 0,+ string(EnvironmentStatusPending): 0,+ }+ for _, row := range rows {+ counts[row.Status] += row.Count+ }+ for status, count := range counts {+ o.ObserveInt64(statusGauge, count, metric.WithAttributes(attribute.String("status", status)))+ }+ return nil+ }, statusGauge)+ }+ if err != nil {+ otel.Handle(err)+ }return s}-// SetVariableSyncer injects the global-variable syncer. Called during-// bootstrap on the manager only; agents leave it nil.+// SetVariableSyncer injects the global-variable syncer on the manager; agents leave it nil.func (s *EnvironmentService) SetVariableSyncer(syncer VariableSyncer) {s.variableSyncer = syncer}@@ -171,7 +209,7 @@ func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context,Where("access_token = ?", token).First(&env).Error; err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {- s.logEdgeTokenResolveMissInternal(ctx, token)+ s.logEdgeTokenResolveMiss(ctx, token)return "", errors.New("invalid agent token")}return "", fmt.Errorf("failed to resolve edge environment by token: %w", err)@@ -181,16 +219,9 @@ func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context,return env.ID, nil}-// logEdgeTokenResolveMissInternal emits a debug log diagnosing why an agent-// token failed to resolve to an edge environment. Counts existing edge-// environments (by access_token presence) so operators can distinguish-// "no edge envs configured" from "token does not match any configured env".-// Token contents are never logged in full — only length and a short-// fingerprint that cannot be reversed.-func (s *EnvironmentService) logEdgeTokenResolveMissInternal(ctx context.Context, token string) {- if s == nil || s.db == nil {- return- }+// logEdgeTokenResolveMiss logs edge environment counts at debug level so a token+// miss can be diagnosed; only the token's length and irreversible fingerprint are logged.+func (s *EnvironmentService) logEdgeTokenResolveMiss(ctx context.Context, token string) {if !slog.Default().Enabled(ctx, slog.LevelDebug) {return}@@ -217,9 +248,8 @@ func (s *EnvironmentService) logEdgeTokenResolveMissInternal(ctx context.Contextslog.DebugContext(ctx, "Edge agent token did not match any environment", args...)}-// ResolveEnvironmentName looks up an environment and returns the label to show for it.-// Use this instead of hardcoding a name for a known ID: names are user-editable, so-// even the local environment's is not fixed.+// ResolveEnvironmentName returns an environment's display label; names are+// user-editable, so never hardcode one for a known ID.func (s *EnvironmentService) ResolveEnvironmentName(ctx context.Context, environmentID string) string {if strings.TrimSpace(environmentID) == "" {environmentID = LocalEnvironmentID@@ -239,12 +269,11 @@ func (s *EnvironmentService) EnsureLocalEnvironment(ctx context.Context, appUrlerr := s.db.WithContext(ctx).Where("id = ?", LocalEnvironmentID).First(&existingEnv).Errorif err == nil {- // Local environment already exists, ensure ApiUrl matches current appUrlif existingEnv.ApiUrl != appUrl {if updateLocalURLErr := s.db.WithContext(ctx).Model(&existingEnv).Update("api_url", appUrl).Error; updateLocalURLErr != nil {return fmt.Errorf("failed to update local environment api url: %w", updateLocalURLErr)}- s.invalidateEnvironmentCacheInternal(LocalEnvironmentID)+ s.invalidateEnvironmentCache(LocalEnvironmentID)slog.InfoContext(ctx, "updated local environment api url", "id", LocalEnvironmentID, "url", appUrl)}return nil@@ -254,7 +283,6 @@ func (s *EnvironmentService) EnsureLocalEnvironment(ctx context.Context, appUrlreturn fmt.Errorf("failed to check for local environment: %w", err)}- // Create the local environmentnow := time.Now()localEnv := &Environment{ID: LocalEnvironmentID,@@ -269,7 +297,7 @@ func (s *EnvironmentService) EnsureLocalEnvironment(ctx context.Context, appUrlif createLocalEnvironmentErr := s.db.WithContext(ctx).Create(localEnv).Error; createLocalEnvironmentErr != nil {return fmt.Errorf("failed to create local environment: %w", createLocalEnvironmentErr)}- s.invalidateEnvironmentCacheInternal(LocalEnvironmentID)+ s.invalidateEnvironmentCache(LocalEnvironmentID)slog.InfoContext(ctx, "created local environment record", "id", LocalEnvironmentID)return nil@@ -294,29 +322,14 @@ func (s *EnvironmentService) CreateEnvironment(ctx context.Context, env *Environreturn nil, fmt.Errorf("failed to create environment: %w", err)}- // Create event in background- go s.createEnvironmentEvent(- context.WithoutCancel(- ctx,- ),- env.ID,- env.Name,- event.EventTypeEnvironmentCreate,- "Environment Created",- fmt.Sprintf(- "Environment '%s' was created",- env.Name,- ),- event.EventSeveritySuccess,- userID,- username,- )+ go s.createEnvironmentEvent(context.WithoutCancel(ctx), env.ID, env.Name, event.EventTypeEnvironmentCreate,+ "Environment Created", fmt.Sprintf("Environment '%s' was created", env.Name), event.EventSeveritySuccess, userID, username)if env.Enabled {- s.registerHealthJobInternal(ctx, env.ID)+ s.registerHealthJob(ctx, env.ID)}s.remoteEnvs.put(*env)- s.invalidateEnvironmentCacheInternal(env.ID)+ s.invalidateEnvironmentCache(env.ID)s.NotifyRuntimeStateChanged()return env, nil@@ -371,7 +384,7 @@ func (s *EnvironmentService) UpdateEnvironment(ctx context.Context, id string, uif updateEnvironmentErr := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", id).Updates(updates).Error; updateEnvironmentErr != nil {return nil, fmt.Errorf("failed to update environment: %w", updateEnvironmentErr)}- s.invalidateEnvironmentCacheInternal(id)+ s.invalidateEnvironmentCache(id)s.NotifyRuntimeStateChanged()updated, err := s.GetEnvironmentByID(ctx, id)@@ -389,45 +402,29 @@ func (s *EnvironmentService) UpdateEnvironment(ctx context.Context, id string, uif rawEnabled, ok := updates["enabled"]; ok {if enabled, isBool := rawEnabled.(bool); isBool {if enabled {- s.registerHealthJobInternal(ctx, id)+ s.registerHealthJob(ctx, id)} else {- s.removeHealthJobInternal(ctx, id)+ s.jobs.Unregister(ctx, id)}}}- // Create event in background (skip for local environment)- if id != "0" {- go s.createEnvironmentEvent(- context.WithoutCancel(- ctx,- ),- id,- updated.Name,- event.EventTypeEnvironmentUpdate,- "Environment Updated",- fmt.Sprintf(- "Environment '%s' was updated",- updated.Name,- ),- event.EventSeverityInfo,- userID,- username,- )+ if id != LocalEnvironmentID {+ go s.createEnvironmentEvent(context.WithoutCancel(ctx), id, updated.Name, event.EventTypeEnvironmentUpdate,+ "Environment Updated", fmt.Sprintf("Environment '%s' was updated", updated.Name), event.EventSeverityInfo, userID, username)}return updated, nil}func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, userID, username *string) error {- // Get environment details before deletionenv, err := s.GetEnvironmentByID(ctx, id)if err != nil {return err}// Stop the per-environment health job before the row is removed.- s.removeHealthJobInternal(ctx, id)+ s.jobs.Unregister(ctx, id)var syncIDs []stringif transactionErr := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {@@ -453,13 +450,12 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, ureturn nil}); transactionErr != nil {if env.Enabled {- s.registerHealthJobInternal(ctx, env.ID)+ s.registerHealthJob(ctx, env.ID)}return transactionErr}- // Deleting an environment orphans its GitOps syncs, whose jobs belong to- // gitops.GitOpsSyncService's own registry — remove them by name here.+ // The deleted GitOps syncs' jobs live in the gitops registry, so remove them by name.if jobScheduler := s.jobs.Scheduler(); jobScheduler != nil {schedulerCtx := s.jobs.Context(ctx)for _, syncID := range syncIDs {@@ -470,33 +466,16 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, us.edgeTokens.invalidate(id)s.ForgetSyncState(id)s.remoteEnvs.remove(id)- s.invalidateEnvironmentCacheInternal(id)+ s.invalidateEnvironmentCache(id)s.NotifyRuntimeStateChanged()- // Create event in background- go s.createEnvironmentEvent(- context.WithoutCancel(- ctx,- ),- id,- env.Name,- event.EventTypeEnvironmentDelete,- "Environment Deleted",- fmt.Sprintf(- "Environment '%s' was deleted",- env.Name,- ),- event.EventSeverityWarning,- userID,- username,- )+ go s.createEnvironmentEvent(context.WithoutCancel(ctx), id, env.Name, event.EventTypeEnvironmentDelete,+ "Environment Deleted", fmt.Sprintf("Environment '%s' was deleted", env.Name), event.EventSeverityWarning, userID, username)return nil}-func (- s *EnvironmentService,-) createEnvironmentEvent(+func (s *EnvironmentService) createEnvironmentEvent(ctx context.Context,envID, envName string,eventType event.EventType,@@ -504,7 +483,7 @@ func (severity event.EventSeverity,userID, username *string,) {- if s == nil || s.eventService == nil {+ if s.eventService == nil {return}@@ -534,17 +513,14 @@ func (s *EnvironmentService) RegenerateEnvironmentApiKey(ctx context.Context, en}func (s *EnvironmentService) linkEnvironmentApiKey(ctx context.Context, envID, newApiKeyID, apiKey, userID, username, envName string) error {- // Trim once at the boundary so the value persisted, the value cached,- // and the value returned by callers (which already TrimSpace before- // returning) all stay byte-identical. Any divergence here would surface- // as a 401 "invalid agent token" because lookup is direct equality.+ // Persisted, cached, and returned keys must stay byte-identical: token lookup is direct equality.apiKey = strings.TrimSpace(apiKey)updates := map[string]any{"api_key_id": newApiKeyID,"access_token": apiKey,"status": string(EnvironmentStatusPending),- "last_seen": nil, // Clear last seen time+ "last_seen": nil,}result := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", envID).Updates(updates)@@ -552,11 +528,10 @@ func (s *EnvironmentService) linkEnvironmentApiKey(ctx context.Context, envID, nreturn fmt.Errorf("failed to update environment with new API key: %w", result.Error)}if result.RowsAffected == 0 {- // A zero-row update would otherwise report a successful rotation while- // the new key was never linked to anything.+ // Otherwise a rotation would succeed without linking the new key.return ErrEnvironmentNotFound}- s.invalidateEnvironmentCacheInternal(envID)+ s.invalidateEnvironmentCache(envID)s.NotifyRuntimeStateChanged()s.edgeTokens.sync(envID, apiKey)@@ -569,24 +544,8 @@ func (s *EnvironmentService) linkEnvironmentApiKey(ctx context.Context, envID, nenv.UpdatedAt = &now})- // Create event log in background- go s.createEnvironmentEvent(- context.WithoutCancel(- ctx,- ),- envID,- envName,- event.EventTypeEnvironmentApiKeyRegenerated,- "API Key Regenerated",- "Environment API key was regenerated and status set to pending",- event.EventSeverityInfo,- new(- userID,- ),- new(- username,- ),- )+ go s.createEnvironmentEvent(context.WithoutCancel(ctx), envID, envName, event.EventTypeEnvironmentApiKeyRegenerated,+ "API Key Regenerated", "Environment API key was regenerated and status set to pending", event.EventSeverityInfo, new(userID), new(username))return nil}@@ -660,19 +619,11 @@ func (s *EnvironmentService) SyncResourcesToEnvironment(ctx context.Context, envvar failedGroups []strings.ForgetSyncState(environmentID)- if err := s.SyncRegistriesToEnvironment(ctx, environmentID); err != nil {- slog.WarnContext(ctx, "Failed to sync registries", "environmentId", environmentID, "error", err.Error())- failedGroups = append(failedGroups, "container registries")- }-- if err := s.SyncS3DestinationsToEnvironment(ctx, environmentID); err != nil {- slog.WarnContext(ctx, "Failed to sync S3 destinations", "environmentId", environmentID, "error", err.Error())- failedGroups = append(failedGroups, "S3 destinations")- }-- if err := s.SyncRepositoriesToEnvironment(ctx, environmentID); err != nil {- slog.WarnContext(ctx, "Failed to sync git repositories", "environmentId", environmentID, "error", err.Error())- failedGroups = append(failedGroups, "git repositories")+ for _, resource := range s.resourceSyncs() {+ if err := resource.sync(ctx, environmentID); err != nil {+ slog.WarnContext(ctx, "Failed to sync resources", "kind", resource.kind, "environmentId", environmentID, "error", err.Error())+ failedGroups = append(failedGroups, resource.kind)+ }}if len(failedGroups) > 0 {@@ -683,13 +634,27 @@ func (s *EnvironmentService) SyncResourcesToEnvironment(ctx context.Context, env})}+type resourceSync struct {+ kind string+ sync func(ctx context.Context, environmentID string) error+}++// resourceSyncs lists the manager-owned resource groups pushed to remote environments.+func (s *EnvironmentService) resourceSyncs() []resourceSync {+ return []resourceSync{+ {"container registries", s.SyncRegistriesToEnvironment},+ {"S3 destinations", s.SyncS3DestinationsToEnvironment},+ {"git repositories", s.SyncRepositoriesToEnvironment},+ }+}+// DisplayName returns the stored environment name or its readable fallback.func DisplayName(environmentID, storedName string) string {if name := strings.TrimSpace(storedName); name != "" {return name}id := strings.TrimSpace(environmentID)- return kit.Ternary(id == "" || id == LocalEnvironmentID, localEnvironmentFallbackNameInternal, id)+ return kit.Ternary(id == "" || id == LocalEnvironmentID, localEnvironmentFallbackName, id)}const (@@ -707,15 +672,11 @@ func (s *EnvironmentService) GetActiveRemoteEnvironmentSnapshot(environmentID st}// ListActiveRemoteEnvironments returns every enabled, visible, non-local-// environment from memory, sorted by ID. The first call loads them from the-// database.+// environment from memory, sorted by ID; the first call loads them from the database.func (s *EnvironmentService) ListActiveRemoteEnvironments(ctx context.Context) ([]Environment, error) {s.remoteEnvs.mu.RLock()seeded := s.remoteEnvs.seeded- environments := make([]Environment, 0, len(s.remoteEnvs.envs))- for _, envRecord := range s.remoteEnvs.envs {- environments = append(environments, envRecord)- }+ environments := slices.AppendSeq(make([]Environment, 0, len(s.remoteEnvs.envs)), maps.Values(s.remoteEnvs.envs))s.remoteEnvs.mu.RUnlock()if !seeded {@@ -732,11 +693,11 @@ func (s *EnvironmentService) ListActiveRemoteEnvironments(ctx context.Context) (}// GetEnvironmentByIDCached is GetEnvironmentByID behind a short TTL cache that-// also remembers unknown IDs. Status and heartbeat fields may be stale.+// remembers unknown IDs; status and heartbeat fields may be stale.func (s *EnvironmentService) GetEnvironmentByIDCached(ctx context.Context, id string) (*Environment, error) {- key := environmentCacheKeyInternal{gen: s.environmentGens[environmentCacheStripeInternal(id)].Load(), id: id}- envRecord, found, err := s.environmentCache.GetWithLoaders(key, func(keys []environmentCacheKeyInternal) (map[environmentCacheKeyInternal]Environment, error) {- found := make(map[environmentCacheKeyInternal]Environment, len(keys))+ key := environmentCacheKey{gen: s.environmentGens[maphash.String(environmentCacheSeed, id)%environmentCacheStripes].Load(), id: id}+ envRecord, found, err := s.environmentCache.GetWithLoaders(key, func(keys []environmentCacheKey) (map[environmentCacheKey]Environment, error) {+ found := make(map[environmentCacheKey]Environment, len(keys))for _, k := range keys {loaded, loadErr := s.GetEnvironmentByID(ctx, k.id)if errors.Is(loadErr, ErrEnvironmentNotFound) {@@ -758,36 +719,24 @@ func (s *EnvironmentService) GetEnvironmentByIDCached(ctx context.Context, id streturn &envRecord, nil}-// invalidateEnvironmentCacheInternal starts a new generation for id's stripe-// after a committed write, orphaning its cached record and in-flight loads.-func (s *EnvironmentService) invalidateEnvironmentCacheInternal(id string) {+// invalidateEnvironmentCache starts a new generation for id's stripe after a+// committed write, orphaning its cached record and in-flight loads.+func (s *EnvironmentService) invalidateEnvironmentCache(id string) {if s == nil || s.environmentCache == nil {return}- previous := s.environmentGens[environmentCacheStripeInternal(id)].Add(1) - 1- s.environmentCache.Delete(environmentCacheKeyInternal{gen: previous, id: id})-}--// invalidateAllEnvironmentCacheInternal covers writes whose affected IDs are unknown.-func (s *EnvironmentService) invalidateAllEnvironmentCacheInternal() {- if s == nil || s.environmentCache == nil {- return- }- for i := range s.environmentGens {- s.environmentGens[i].Add(1)- }- s.environmentCache.Purge()+ previous := s.environmentGens[maphash.String(environmentCacheSeed, id)%environmentCacheStripes].Add(1) - 1+ s.environmentCache.Delete(environmentCacheKey{gen: previous, id: id})}func (s *EnvironmentService) ListEnvironmentsPaginated(ctx context.Context, params pagination.QueryParams, accessibleEnvIDs []string) ([]environment.Environment, pagination.Response, error) {if strings.TrimSpace(params.Filters["type"]) != "" {- return s.listEnvironmentsPaginatedWithRuntimeFiltersInternal(ctx, params, accessibleEnvIDs)+ return s.listEnvironmentsPaginatedWithRuntimeFilters(ctx, params, accessibleEnvIDs)}var envs []Environmentq := s.db.WithContext(ctx).Model(&Environment{}).Where("hidden = ?", false)- // accessibleEnvIDs == nil means "no restriction". A non-nil slice limits the- // result to those environment IDs; an empty slice therefore matches nothing.+ // nil means no restriction; a non-nil slice limits results, so an empty one matches nothing.switch {case accessibleEnvIDs == nil:// no restriction@@ -821,17 +770,11 @@ func (s *EnvironmentService) ListEnvironmentsPaginated(ctx context.Context, parareturn out, paginationResp, nil}-func (- s *EnvironmentService,-) listEnvironmentsPaginatedWithRuntimeFiltersInternal(+func (s *EnvironmentService) listEnvironmentsPaginatedWithRuntimeFilters(ctx context.Context,params pagination.QueryParams,accessibleEnvIDs []string,-) (- []environment.Environment,- pagination.Response,- error,-) {+) ([]environment.Environment, pagination.Response, error) {var envs []Environmentif err := s.db.WithContext(ctx).Model(&Environment{}).@@ -845,9 +788,9 @@ func (return nil, pagination.Response{}, fmt.Errorf("failed to map environments: %w", mapErr)}- // nil = no restriction; non-nil restricts to the caller's accessible envs.+ // nil means no restriction; non-nil restricts to the caller's accessible environments.if accessibleEnvIDs != nil {- items = filterEnvironmentsByIDInternal(items, accessibleEnvIDs)+ items = slices.DeleteFunc(items, func(item environment.Environment) bool { return !slices.Contains(accessibleEnvIDs, item.ID) })}for i := range items {@@ -910,7 +853,27 @@ func (},{Key: "type",- Fn: environmentTypeMatchesInternal,+ Fn: func(item environment.Environment, filterValue string) bool {+ typeKey := "http"+ if item.IsEdge {+ // Disconnected or poll-only agents classify by the transport they last used.+ transport := ""+ if item.Connected != nil && *item.Connected && item.EdgeTransport != nil {+ transport = *item.EdgeTransport+ } else if item.LastEdgeTransport != nil {+ transport = *item.LastEdgeTransport+ }+ switch strings.ToLower(strings.TrimSpace(transport)) {+ case edge.EdgeTransportWebSocket:+ typeKey = "websocket"+ case edge.EdgeTransportGRPC:+ typeKey = "grpc"+ default:+ typeKey = "edge"+ }+ }+ return typeKey == strings.ToLower(strings.TrimSpace(filterValue))+ },},},}@@ -964,7 +927,7 @@ func (s *EnvironmentService) ListRemoteEnvironments(ctx context.Context) ([]Envivar envs []Environmenterr := s.db.WithContext(ctx).Model(&Environment{}).- Where("id != ?", "0").+ Where("id != ?", LocalEnvironmentID).Where("enabled = ?", true).Where("hidden = ?", false).Find(&envs).Error@@ -1004,39 +967,33 @@ func (s *EnvironmentService) NotifyRuntimeStateChanged() {}const (- defaultEnvironmentHealthInterval = "0 */2 * * * *"- environmentHealthCheckTimeout = 90 * time.Second- environmentHealthJobPrefix = "environment-health:"- environmentHealthAdmissionScopeInternal = "environment-health"+ defaultEnvironmentHealthInterval = "0 */2 * * * *"+ environmentHealthCheckTimeout = 90 * time.Second+ environmentHealthJobPrefix = "environment-health:"+ environmentHealthAdmissionScope = "environment-health")-// SetScheduler injects the job scheduler and app lifecycle context. Called during-// bootstrap on the manager only (agent mode leaves scheduler nil, so all health-job-// registration becomes a no-op).+// SetScheduler injects the job scheduler and app lifecycle context on the manager;+// agents leave it nil, making health-job registration a no-op.func (s *EnvironmentService) SetScheduler(ctx context.Context, jobScheduler scheduler.DynamicScheduler, admissionGate *runs.Admission) error {return s.jobs.SetScheduler(ctx, jobScheduler, admissionGate)}-// registerHealthJobInternal schedules one environment's health check on the-// global environmentHealthInterval (environment health has no per-entity-// interval); the run body self-cancels cleanly when the environment is gone.-func (s *EnvironmentService) registerHealthJobInternal(ctx context.Context, envID string) {+// registerHealthJob schedules one environment's health check on the global+// environmentHealthInterval; the run self-cancels once the environment is gone.+func (s *EnvironmentService) registerHealthJob(ctx context.Context, envID string) {s.jobs.Register(ctx, envID,func(ctx context.Context) string {sched := s.settingsService.GetStringSetting(ctx, "environmentHealthInterval", defaultEnvironmentHealthInterval)return kit.Ternary(sched == "", defaultEnvironmentHealthInterval, sched)},- func(ctx context.Context) (scheduler.Outcome, error) { return s.runHealthCheckInternal(ctx, envID) },+ func(ctx context.Context) (scheduler.Outcome, error) { return s.runHealthCheck(ctx, envID) },func(context.Context, scheduler.Run) (scheduler.Outcome, error) {return scheduler.Outcome{Status: scheduler.Retrying, Message: "Connection checks can safely resume"}, nil},)}-func (s *EnvironmentService) removeHealthJobInternal(ctx context.Context, envID string) {- s.jobs.Unregister(ctx, envID)-}-func (s *EnvironmentService) ListEnabledEnvironmentIDs(ctx context.Context) ([]string, error) {var ids []stringif err := s.db.WithContext(ctx).@@ -1048,35 +1005,21 @@ func (s *EnvironmentService) ListEnabledEnvironmentIDs(ctx context.Context) ([]sreturn ids, nil}-func (s *EnvironmentService) registerAllEnabledHealthJobsInternal(ctx context.Context) int {+// RescheduleHealthJobs re-registers every enabled environment's health job,+// picking up a changed global interval.+func (s *EnvironmentService) RescheduleHealthJobs(ctx context.Context) {+ if !s.jobs.Enabled() {+ return+ }ids, err := s.ListEnabledEnvironmentIDs(ctx)if err != nil {slog.ErrorContext(ctx, "failed to list environments for health jobs", "error", err)- return 0- }- for _, id := range ids {- s.registerHealthJobInternal(ctx, id)- }- return len(ids)-}--// RegisterHealthJobsOnStartup registers a health-check job for every enabled-// environment. Replaces the old global environment-health job.-func (s *EnvironmentService) RegisterHealthJobsOnStartup(ctx context.Context) {- if !s.jobs.Enabled() {return}- n := s.registerAllEnabledHealthJobsInternal(ctx)- slog.InfoContext(ctx, "Registered environment health jobs on startup", "count", n)-}--// RescheduleHealthJobs re-registers all enabled environments' health jobs, picking-// up a changed global interval. Wired from the Jobs UI via job.JobService.-func (s *EnvironmentService) RescheduleHealthJobs(ctx context.Context) {- if !s.jobs.Enabled() {- return+ for _, id := range ids {+ s.registerHealthJob(ctx, id)}- s.registerAllEnabledHealthJobsInternal(ctx)+ slog.InfoContext(ctx, "Registered environment health jobs", "count", len(ids))}// RunHealthChecksNow queues each enabled environment's health check.@@ -1090,7 +1033,7 @@ func (s *EnvironmentService) RunHealthChecksNow(ctx context.Context) error {return err}parent, hasParent := jobcontext.Run(ctx)- request := scheduler.Request{EnvironmentID: "0", Trigger: "internal"}+ request := scheduler.Request{EnvironmentID: LocalEnvironmentID, Trigger: "internal"}if hasParent {request.Trigger = parent.Triggerrequest.RequestedBy = parent.RequestedBy@@ -1105,9 +1048,9 @@ func (s *EnvironmentService) RunHealthChecksNow(ctx context.Context) error {return nil}-// runHealthCheckInternal tests one environment's connection (updating its DB status)-// and, for online remotes, syncs registries and repositories to it.-func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID string) (scheduler.Outcome, error) {+// runHealthCheck tests one environment's connection, persisting its status, and+// syncs manager-owned resources to online remotes.+func (s *EnvironmentService) runHealthCheck(ctx context.Context, envID string) (scheduler.Outcome, error) {lease, admitted, err := s.jobs.TryAcquire(ctx, envID)if err != nil {slog.ErrorContext(ctx, "environment health check admission failed", "environmentId", envID, "error", err)@@ -1136,13 +1079,12 @@ func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID sreturn scheduler.Outcome{Status: scheduler.Retrying, Message: "Environment is offline"}, errors.New("environment is offline")}- // Local environment (ID "0") has no registries/repositories to push.- if envID == "0" {+ // The local environment has no resources to push.+ if envID == LocalEnvironmentID {return scheduler.Outcome{Status: scheduler.Succeeded}, nil}- // An agent that just came back may have restarted with different state;- // resend everything once instead of trusting the last delivered payload.+ // An agent that just came back may have restarted with different state, so resend everything.if !wasOnline {s.ForgetSyncState(envID)}@@ -1150,22 +1092,16 @@ func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID svar syncErrors []errorsyncCtx, cancel := context.WithTimeout(ctx, environmentHealthCheckTimeout)defer cancel()- if syncRegistriesToEnvironmentErr := s.SyncRegistriesToEnvironment(syncCtx, envID); syncRegistriesToEnvironmentErr != nil {- slog.WarnContext(syncCtx, "failed to sync registries during health check", "environmentId", envID, "error", syncRegistriesToEnvironmentErr)- syncErrors = append(syncErrors, syncRegistriesToEnvironmentErr)- }- if syncS3DestinationsToEnvironmentErr := s.SyncS3DestinationsToEnvironment(syncCtx, envID); syncS3DestinationsToEnvironmentErr != nil {- slog.WarnContext(syncCtx, "failed to sync S3 destinations during health check", "environmentId", envID, "error", syncS3DestinationsToEnvironmentErr)- syncErrors = append(syncErrors, syncS3DestinationsToEnvironmentErr)- }- if syncRepositoriesToEnvironmentErr := s.SyncRepositoriesToEnvironment(syncCtx, envID); syncRepositoriesToEnvironmentErr != nil {- slog.WarnContext(syncCtx, "failed to sync git repositories during health check", "environmentId", envID, "error", syncRepositoriesToEnvironmentErr)- syncErrors = append(syncErrors, syncRepositoriesToEnvironmentErr)+ for _, resource := range s.resourceSyncs() {+ if syncErr := resource.sync(syncCtx, envID); syncErr != nil {+ slog.WarnContext(syncCtx, "failed to sync resources during health check", "kind", resource.kind, "environmentId", envID, "error", syncErr)+ syncErrors = append(syncErrors, syncErr)+ }}if s.variableSyncer != nil {- if syncEnvironmentErr := s.variableSyncer.SyncEnvironment(syncCtx, envID); syncEnvironmentErr != nil {- slog.WarnContext(syncCtx, "failed to sync global variables during health check", "environmentId", envID, "error", syncEnvironmentErr)- syncErrors = append(syncErrors, syncEnvironmentErr)+ if syncErr := s.variableSyncer.SyncEnvironment(syncCtx, envID); syncErr != nil {+ slog.WarnContext(syncCtx, "failed to sync global variables during health check", "environmentId", envID, "error", syncErr)+ syncErrors = append(syncErrors, syncErr)}}if joinErr := errors.Join(syncErrors...); joinErr != nil {@@ -1176,7 +1112,7 @@ func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID s// SyncRegistriesToEnvironment syncs all registries from this manager to a remote environment.func (s *EnvironmentService) SyncRegistriesToEnvironment(ctx context.Context, environmentID string) error {- target, err := s.resolveRemoteEnvironmentTargetInternal(ctx, environmentID)+ target, err := s.resolveRemoteEnvironmentTarget(ctx, environmentID)if err != nil {return err}@@ -1185,7 +1121,7 @@ func (s *EnvironmentService) SyncRegistriesToEnvironment(ctx context.Context, en// SyncS3DestinationsToEnvironment sends manager-owned destinations to one remote environment.func (s *EnvironmentService) SyncS3DestinationsToEnvironment(ctx context.Context, environmentID string) error {- target, err := s.resolveRemoteEnvironmentTargetInternal(ctx, environmentID)+ target, err := s.resolveRemoteEnvironmentTarget(ctx, environmentID)if err != nil {return err}@@ -1194,7 +1130,7 @@ func (s *EnvironmentService) SyncS3DestinationsToEnvironment(ctx context.Context// SyncRepositoriesToEnvironment syncs all git repositories from this manager to a remote environment.func (s *EnvironmentService) SyncRepositoriesToEnvironment(ctx context.Context, environmentID string) error {- target, err := s.resolveRemoteEnvironmentTargetInternal(ctx, environmentID)+ target, err := s.resolveRemoteEnvironmentTarget(ctx, environmentID)if err != nil {return err}@@ -1213,7 +1149,7 @@ func (s *EnvironmentService) TestConnection(ctx context.Context, id string, cust}status, err := s.health.Probe(ctx, id, apiURL, envRecord.IsEdge, customApiUrl != nil)if customApiUrl == nil {- if updateErr := s.updateEnvironmentStatusInternal(ctx, id, status); updateErr != nil {+ if updateErr := s.updateEnvironmentStatus(ctx, id, status); updateErr != nil {slog.WarnContext(ctx, "Failed to persist environment status", "environmentId", id, "status", status, "error", updateErr)}return status, err@@ -1225,18 +1161,15 @@ func (s *EnvironmentService) TestConnection(ctx context.Context, id string, custreturn status, nil}-func (s *EnvironmentService) updateEnvironmentStatusInternal(ctx context.Context, id, status string) error {+func (s *EnvironmentService) updateEnvironmentStatus(ctx context.Context, id, status string) error {var currentEnv Environmentif err := s.db.WithContext(ctx).Select("status", "is_edge").Where("id = ?", id).First(¤tEnv).Error; err != nil {return fmt.Errorf("failed to check environment status: %w", err)}if currentEnv.Status == string(EnvironmentStatusPending) {- // Edge envs must complete pairing via the agent's outbound tunnel — manager- // can't dial them directly, so a manager-side reachability check means nothing.- // Direct envs are reachable from the manager, so a successful health check IS- // the pairing signal. Don't promote on offline/error ticks though, or a transient- // blip during initial setup would flip the env out of pending.+ // Only a successful check pairs a direct environment; edge environments pair+ // through the agent's tunnel, so manager reachability means nothing for them.if currentEnv.IsEdge || status != string(EnvironmentStatusOnline) {slog.DebugContext(ctx, "skipping status update for pending environment", "environmentId", id)return nil@@ -1253,8 +1186,7 @@ func (s *EnvironmentService) updateEnvironmentStatusInternal(ctx context.Contextif err := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", id).Updates(updates).Error; err != nil {return fmt.Errorf("failed to update environment status: %w", err)}- // Direct environments have no tunnel callback, so this health-check write is- // the only moment their liveness changes.+ // Direct environments have no tunnel callback, so this is their only liveness change.s.NotifyRuntimeStateChanged()return nil}@@ -1262,8 +1194,7 @@ func (s *EnvironmentService) updateEnvironmentStatusInternal(ctx context.Contextfunc (s *EnvironmentService) UpdateEnvironmentHeartbeat(ctx context.Context, id string) error {now := time.Now()- // Use Exec with raw SQL for better performance- // Throttle unchanged online status to one write every 30 seconds.+ // Throttle unchanged online status to one write every 30 seconds; this doubles as the notify throttle.result := s.db.WithContext(ctx).Exec(`UPDATE environmentsSET last_seen = ?, status = ?, updated_at = ?@@ -1272,8 +1203,6 @@ func (s *EnvironmentService) UpdateEnvironmentHeartbeat(ctx context.Context, id`, new(now), string(EnvironmentStatusOnline), new(now), id, string(EnvironmentStatusOnline), now.Add(-30*time.Second))if result.Error != nil {- // The 30s throttle above doubles as the notify throttle: a no-op heartbeat- // changed nothing worth waking a stream for.return fmt.Errorf("failed to update environment heartbeat: %w", result.Error)}@@ -1284,8 +1213,8 @@ func (s *EnvironmentService) UpdateEnvironmentHeartbeat(ctx context.Context, idreturn nil}-// UpdateEnvironmentConnectionState updates runtime connectivity status without creating-// a generic "environment updated" event. This is used for edge tunnel connect/disconnect.+// UpdateEnvironmentConnectionState records an edge tunnel connect or disconnect+// without emitting an "environment updated" event.func (s *EnvironmentService) UpdateEnvironmentConnectionState(ctx context.Context, id string, connected bool) error {now := time.Now()@@ -1295,8 +1224,7 @@ func (s *EnvironmentService) UpdateEnvironmentConnectionState(ctx context.Contexif connected {updates["status"] = string(EnvironmentStatusOnline)updates["last_seen"] = &now- // Remember the tunnel transport so the UI can keep showing it after- // the tunnel drops or while the agent is poll-only.+ // Remember the transport so the UI can show it after the tunnel drops or while poll-only.if state, ok := edge.GetTunnelRuntimeState(id).Get(); ok && state.Transport != "" {updates["last_edge_transport"] = state.Transport}@@ -1313,8 +1241,8 @@ func (s *EnvironmentService) UpdateEnvironmentConnectionState(ctx context.Contexreturn nil}-// ApplyEnvironmentRuntimeState normalizes edge environment runtime status using-// in-memory tunnel and poll registries without mutating persisted state.+// ApplyEnvironmentRuntimeState overlays in-memory tunnel and poll state onto an+// edge environment without mutating persisted state.func ApplyEnvironmentRuntimeState(env *environment.Environment) {if env == nil || !env.IsEdge {return@@ -1371,9 +1299,8 @@ func ApplyEnvironmentRuntimeState(env *environment.Environment) {}}-// ReconcileEdgeStatusesOnStartup resets edge environments to offline when the manager starts.-// Live edge tunnels are process-local runtime state, so persisted "online" flags can be stale-// after a restart until agents reconnect. Pending environments are left untouched.+// ReconcileEdgeStatusesOnStartup marks non-pending edge environments offline at+// startup, since tunnels are process-local and persisted "online" flags are stale.func (s *EnvironmentService) ReconcileEdgeStatusesOnStartup(ctx context.Context) error {result := s.db.WithContext(ctx).Model(&Environment{}).Where("is_edge = ?", true).@@ -1394,97 +1321,58 @@ func (s *EnvironmentService) ReconcileEdgeStatusesOnStartup(ctx context.Context)return nil}-// SyncRegistriesToRemoteEnvironments syncs container registries to all eligible remote environments.-// Eligibility requires a non-local, enabled environment with a configured access token.+// SyncRegistriesToRemoteEnvironments syncs container registries to every remote environment with an access token.func (s *EnvironmentService) SyncRegistriesToRemoteEnvironments(ctx context.Context) error {- envs, err := s.ListRemoteEnvironments(ctx)- if err != nil {- return fmt.Errorf("failed to list remote environments for registry sync: %w", err)- }-- if len(envs) == 0 {- return nil- }-- var failedCount int- for _, env := range envs {- if env.AccessToken == nil || *env.AccessToken == "" {- slog.DebugContext(ctx, "Skipping registry sync for environment without access token",- "environmentId", env.ID,- "environmentName", env.Name)- continue- }-- if syncRegistriesToEnvironmentErr := s.SyncRegistriesToEnvironment(ctx, env.ID); syncRegistriesToEnvironmentErr != nil {- failedCount++- slog.WarnContext(ctx, "Failed to sync registries to remote environment",- "environmentId", env.ID,- "environmentName", env.Name,- "error", syncRegistriesToEnvironmentErr.Error())- }- }-- if failedCount > 0 {- return fmt.Errorf("failed to sync registries to %d remote environment(s)", failedCount)- }-- return nil+ return s.syncToRemoteEnvironments(ctx, "registries", s.SyncRegistriesToEnvironment)}-// SyncS3DestinationsToRemoteEnvironments refreshes the destination cache on every enabled remote environment.+// SyncS3DestinationsToRemoteEnvironments refreshes the destination cache on every remote environment with an access token.func (s *EnvironmentService) SyncS3DestinationsToRemoteEnvironments(ctx context.Context) error {+ return s.syncToRemoteEnvironments(ctx, "S3 destinations", s.SyncS3DestinationsToEnvironment)+}++func (s *EnvironmentService) syncToRemoteEnvironments(ctx context.Context, kind string, sync func(context.Context, string) error) error {envs, err := s.ListRemoteEnvironments(ctx)if err != nil {- return fmt.Errorf("failed to list remote environments for S3 destination sync: %w", err)+ return fmt.Errorf("failed to list remote environments for %s sync: %w", kind, err)}var failedCount intfor _, env := range envs {if env.AccessToken == nil || strings.TrimSpace(*env.AccessToken) == "" {- slog.DebugContext(ctx, "Skipping S3 destination sync for environment without access token", "environmentId", env.ID, "environmentName", env.Name)+ slog.DebugContext(ctx, "Skipping sync for environment without access token", "kind", kind, "environmentId", env.ID, "environmentName", env.Name)continue}- if syncS3DestinationsToEnvironmentErr := s.SyncS3DestinationsToEnvironment(ctx, env.ID); syncS3DestinationsToEnvironmentErr != nil {+ if syncErr := sync(ctx, env.ID); syncErr != nil {failedCount++- slog.WarnContext(ctx, "Failed to sync S3 destinations to remote environment", "environmentId", env.ID, "environmentName", env.Name, "error", syncS3DestinationsToEnvironmentErr)+ slog.WarnContext(ctx, "Failed to sync to remote environment", "kind", kind, "environmentId", env.ID, "environmentName", env.Name, "error", syncErr)}}if failedCount > 0 {- return fmt.Errorf("failed to sync S3 destinations to %d remote environment(s)", failedCount)+ return fmt.Errorf("failed to sync %s to %d remote environment(s)", kind, failedCount)}return nil}-// CheckS3DestinationReferences returns an error while any managed environment-// still references the destination, or when a synced environment cannot be-// checked conclusively. Deleting credentials that a remote environment still-// needs would strand its policies and retained backups.+// CheckS3DestinationReferences fails while any synced environment references the+// destination or cannot be checked, since deleting it would strand remote backups.func (s *EnvironmentService) CheckS3DestinationReferences(ctx context.Context, destinationID string) error {envs, err := s.ListRemoteEnvironments(ctx)if err != nil {return fmt.Errorf("failed to list remote environments for S3 destination reference check: %w", err)}for _, env := range envs {+ // Environments without an access token never receive destination syncs.if env.AccessToken == nil || strings.TrimSpace(*env.AccessToken) == "" {- // Environments without an access token never receive destination- // syncs, so they cannot hold a reference.continue}var result struct {InUse bool `json:"inUse"`}- if proxyJSONRequestForEnvironmentErr := s.ProxyJSONRequestForEnvironment(- ctx,- env,- http.MethodGet,- "/api/backups/s3/"+url.PathEscape(- destinationID,- )+"/in-use",- nil,- &result,- ); proxyJSONRequestForEnvironmentErr != nil {- return fmt.Errorf("cannot verify S3 destination references on environment %s; restore connectivity before deleting: %w", env.Name, proxyJSONRequestForEnvironmentErr)+ path := "/api/backups/s3/" + url.PathEscape(destinationID) + "/in-use"+ if proxyErr := s.ProxyJSONRequestForEnvironment(ctx, env, http.MethodGet, path, nil, &result); proxyErr != nil {+ return fmt.Errorf("cannot verify S3 destination references on environment %s; restore connectivity before deleting: %w", env.Name, proxyErr)}if result.InUse {return fmt.Errorf("still referenced by environment %s", env.Name)@@ -1493,7 +1381,7 @@ func (s *EnvironmentService) CheckS3DestinationReferences(ctx context.Context, dreturn nil}-func (s *EnvironmentService) resolveRemoteEnvironmentTargetInternal(ctx context.Context, envID string) (*proxy.Target, error) {+func (s *EnvironmentService) resolveRemoteEnvironmentTarget(ctx context.Context, envID string) (*proxy.Target, error) {envRecord, err := s.GetEnvironmentByID(ctx, envID)if err != nil {return nil, fmt.Errorf("failed to get environment: %w", err)@@ -1503,7 +1391,7 @@ func (s *EnvironmentService) resolveRemoteEnvironmentTargetInternal(ctx context.}func (s *EnvironmentService) ExecuteRemoteRequest(ctx context.Context, envID, method, path string, body []byte) (*remenv.Response, error) {- target, err := s.resolveRemoteEnvironmentTargetInternal(ctx, envID)+ target, err := s.resolveRemoteEnvironmentTarget(ctx, envID)if err != nil {return nil, err}@@ -1515,7 +1403,7 @@ func (s *EnvironmentService) ProxyJSONRequest(ctx context.Context, envID, methodproxyCtx, cancel := s.proxy.Context(ctx)defer cancel()- target, err := s.resolveRemoteEnvironmentTargetInternal(proxyCtx, envID)+ target, err := s.resolveRemoteEnvironmentTarget(proxyCtx, envID)if err != nil {return err}@@ -1523,8 +1411,7 @@ func (s *EnvironmentService) ProxyJSONRequest(ctx context.Context, envID, methodreturn s.proxy.JSON(proxyCtx, target, method, path, body, out)}-// ProxyJSONRequestForEnvironment sends a JSON request using an already-loaded-// environment row, avoiding an extra environment lookup on hot stream paths.+// ProxyJSONRequestForEnvironment sends a JSON request for an already-loaded environment row.func (s *EnvironmentService) ProxyJSONRequestForEnvironment(ctx context.Context, env Environment, method, path string, body []byte, out any) error {proxyCtx, cancel := s.proxy.Context(ctx)defer cancel()@@ -1582,8 +1469,8 @@ func (s *EnvironmentService) GenerateDeploymentSnippets(ctx context.Context, env}, nil}-// GenerateEdgeDeploymentSnippets generates Docker deployment snippets for an edge agent.-// Edge agents connect outbound to the manager and don't require exposed ports.+// GenerateEdgeDeploymentSnippets generates Docker deployment snippets for an+// edge agent, which dials the manager and exposes no ports.func (s *EnvironmentService) GenerateEdgeDeploymentSnippets(ctx context.Context, envID, managerURL, apiKey string, edgeCfg *edge.Config) (*DeploymentSnippets, error) {managerURL = strings.TrimRight(managerURL, "/")dockerRun, dockerCompose := snippets.Edge(managerURL, apiKey)@@ -1636,15 +1523,14 @@ func (s *EnvironmentService) ListSwarmNodeAgentEnvironments(ctx context.Context,return envs, nil}-// ListSwarmNodeCandidateEnvironments returns enabled visible environments that-// can provide swarm-node coverage for a manager environment.+// ListSwarmNodeCandidateEnvironments returns enabled visible environments that can cover a swarm node.func (s *EnvironmentService) ListSwarmNodeCandidateEnvironments(ctx context.Context) ([]Environment, error) {var envs []Environmentif err := s.db.WithContext(ctx).Model(&Environment{}).Where("hidden = ?", false).Where("enabled = ?", true).- Where("id <> ?", "0").+ Where("id <> ?", LocalEnvironmentID).Order("name ASC").Find(&envs).Error; err != nil {return nil, fmt.Errorf("failed to list swarm node candidate environments: %w", err)@@ -1652,8 +1538,7 @@ func (s *EnvironmentService) ListSwarmNodeCandidateEnvironments(ctx context.Contreturn envs, nil}-// BindSwarmNodeEnvironment binds an existing visible environment to a swarm-// node without modifying its connection details or agent token.+// BindSwarmNodeEnvironment binds a visible environment to a swarm node, keeping its connection and token.func (s *EnvironmentService) BindSwarmNodeEnvironment(ctx context.Context,parentEnvironmentID, nodeID, environmentID string,@@ -1701,11 +1586,14 @@ func (s *EnvironmentService) BindSwarmNodeEnvironment(}s.remoteEnvs.put(envRecord)- // A rebind also clears the binding on other environments.+ // A rebind also clears the binding on other environments, whose IDs are unknown.if rebind {- s.invalidateAllEnvironmentCacheInternal()+ for i := range s.environmentGens {+ s.environmentGens[i].Add(1)+ }+ s.environmentCache.Purge()} else {- s.invalidateEnvironmentCacheInternal(envRecord.ID)+ s.invalidateEnvironmentCache(envRecord.ID)}s.NotifyRuntimeStateChanged()return &envRecord, nil@@ -1719,14 +1607,17 @@ func (s *EnvironmentService) DetachSwarmNodeEnvironment(ctx context.Context, parUpdates(map[string]any{"parent_environment_id": nil, "swarm_node_id": nil, "updated_at": &now}).Error; err != nil {return fmt.Errorf("failed to detach swarm node environment: %w", err)}- s.invalidateAllEnvironmentCacheInternal()+ // The detached environment's ID is unknown, so invalidate every stripe.+ for i := range s.environmentGens {+ s.environmentGens[i].Add(1)+ }+ s.environmentCache.Purge()s.NotifyRuntimeStateChanged()return nil}-// DeleteSwarmNodeAgentDeployment removes a dedicated hidden agent registration-// while leaving visible remote environments untouched.+// DeleteSwarmNodeAgentDeployment removes a dedicated hidden agent registration, leaving visible ones untouched.func (s *EnvironmentService) DeleteSwarmNodeAgentDeployment(ctx context.Context, parentEnvironmentID, nodeID string, userID, username *string) error {var envRecord Environmentif err := s.db.WithContext(ctx).@@ -1752,9 +1643,8 @@ func (s *EnvironmentService) EnsureSwarmNodeAgentEnvironment(}var env Environment- // Prefer an existing visible binding. Legacy hidden registrations remain- // reusable, but all newly provisioned node agents are normal Remote- // Environments so one token and one agent can serve both use cases.+ // Prefer a visible binding; legacy hidden registrations stay reusable, but new+ // node agents are normal remote environments so one token serves both uses.err = s.db.WithContext(ctx).Where("parent_environment_id = ?", parentEnvironmentID).Where("swarm_node_id = ?", nodeID).@@ -1806,7 +1696,7 @@ func (s *EnvironmentService) UpdateSwarmNodeIdentity(ctx context.Context, envID,if err := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", envID).Updates(updates).Error; err != nil {return fmt.Errorf("failed to update swarm node identity: %w", err)}- s.invalidateEnvironmentCacheInternal(envID)+ s.invalidateEnvironmentCache(envID)s.NotifyRuntimeStateChanged()return nildiff --git a/backend/internal/environment/service_test.go b/backend/internal/environment/service_test.goindex eb31ce23a..36b6496c2 100644--- a/backend/internal/environment/service_test.go+++ b/backend/internal/environment/service_test.go@@ -45,42 +45,42 @@ import (francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing")-func TestEnvironmentService_OverlappingHealthCheckIsSkippedInternal(t *testing.T) {- gate := newAdmissionGateForEnvironmentTestInternal(t)+func TestEnvironmentService_OverlappingHealthCheckIsSkipped(t *testing.T) {+ gate := newAdmissionGateForEnvironmentTest(t)- key := scheduler.AdmissionKey{Scope: environmentHealthAdmissionScopeInternal, ID: "environment-id"}+ key := scheduler.AdmissionKey{Scope: environmentHealthAdmissionScope, ID: "environment-id"}lease, admitted, err := gate.TryAcquire(t.Context(), key)require.NoError(t, err)require.True(t, admitted)- service := &EnvironmentService{jobs: entityjobs.New(environmentHealthJobPrefix, environmentHealthAdmissionScopeInternal)}- require.NoError(t, service.SetScheduler(t.Context(), &environmentTestSchedulerInternal{}, gate))- outcome, runErr := service.runHealthCheckInternal(t.Context(), "environment-id")+ service := &EnvironmentService{jobs: entityjobs.New(environmentHealthJobPrefix, environmentHealthAdmissionScope)}+ require.NoError(t, service.SetScheduler(t.Context(), &environmentTestScheduler{}, gate))+ outcome, runErr := service.runHealthCheck(t.Context(), "environment-id")require.NoError(t, runErr)require.Equal(t, scheduler.Skipped, outcome.Status)lease.Release(t.Context())}-type environmentTestSchedulerInternal struct {+type environmentTestScheduler struct {submitted []scheduler.Requestadded []stringremoved []string}-func (s *environmentTestSchedulerInternal) AddJob(_ context.Context, job scheduler.Job) error {+func (s *environmentTestScheduler) AddJob(_ context.Context, job scheduler.Job) error {s.added = append(s.added, job.Name())return nil}-func (s *environmentTestSchedulerInternal) RemoveJob(_ context.Context, name string) {+func (s *environmentTestScheduler) RemoveJob(_ context.Context, name string) {s.removed = append(s.removed, name)}-func (s *environmentTestSchedulerInternal) HasJob(_ string) bool {+func (s *environmentTestScheduler) HasJob(_ string) bool {return false}-func newAdmissionGateForEnvironmentTestInternal(t testing.TB) *runs.Admission {+func newAdmissionGateForEnvironmentTest(t testing.TB) *runs.Admission {t.Helper()runtime := francistest.New(t)gate := runs.NewAdmission(runtime.Service(), t.Name())@@ -162,10 +162,10 @@ func createTestEnvironmentServiceUser(t *testing.T, ctx context.Context, userSerfunc createTestEnvironment(t *testing.T, db *database.DB, id, apiURL string, accessToken *string) {t.Helper()- createNamedTestEnvironmentInternal(t, db, id, "env-"+id, apiURL, accessToken)+ createNamedTestEnvironment(t, db, id, "env-"+id, apiURL, accessToken)}-func createNamedTestEnvironmentInternal(t *testing.T, db *database.DB, id, name, apiURL string, accessToken *string) {+func createNamedTestEnvironment(t *testing.T, db *database.DB, id, name, apiURL string, accessToken *string) {t.Helper()now := time.Now()@@ -226,9 +226,9 @@ func TestEnvironmentService_DeleteEnvironment_CascadesGitOpsSyncs(t *testing.T)GitOpsManagedBy: &syncID,}).Error)- jobScheduler := &environmentTestSchedulerInternal{}+ jobScheduler := &environmentTestScheduler{}svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)- require.NoError(t, svc.SetScheduler(ctx, jobScheduler, newAdmissionGateForEnvironmentTestInternal(t)))+ require.NoError(t, svc.SetScheduler(ctx, jobScheduler, newAdmissionGateForEnvironmentTest(t)))require.NoError(t, svc.DeleteEnvironment(ctx, "env-delete-gitops", nil, nil))@@ -661,14 +661,14 @@ func TestEnvironmentService_UpdateEnvironmentConnectionState(t *testing.T) {require.Equal(t, *lastSeen, *env.LastSeen)}-func TestEnvironmentService_UpdateEnvironmentStatusInternal_PromotesPendingDirectEnv(t *testing.T) {+func TestEnvironmentService_UpdateEnvironmentStatus_PromotesPendingDirectEnv(t *testing.T) {ctx := t.Context()db := setupEnvironmentServiceTestDB(t)svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)createTestEnvironmentWithState(t, db, "direct-pending", "http://agent:3553", string(EnvironmentStatusPending), false, nil)- require.NoError(t, svc.updateEnvironmentStatusInternal(ctx, "direct-pending", string(EnvironmentStatusOnline)))+ require.NoError(t, svc.updateEnvironmentStatus(ctx, "direct-pending", string(EnvironmentStatusOnline)))var env Environmentrequire.NoError(t, db.WithContext(ctx).Where("id = ?", "direct-pending").First(&env).Error)@@ -676,18 +676,16 @@ func TestEnvironmentService_UpdateEnvironmentStatusInternal_PromotesPendingDirecrequire.NotNil(t, env.LastSeen)}-func TestEnvironmentService_UpdateEnvironmentStatusInternal_DoesNotDemotePendingDirectEnvOnFailedTick(t *testing.T) {+func TestEnvironmentService_UpdateEnvironmentStatus_DoesNotDemotePendingDirectEnvOnFailedTick(t *testing.T) {ctx := t.Context()db := setupEnvironmentServiceTestDB(t)svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)createTestEnvironmentWithState(t, db, "direct-pending", "http://agent:3553", string(EnvironmentStatusPending), false, nil)- // A transient health-check failure before pairing completes must NOT flip a- // pending Direct env to offline/error — the env should stay pending so a later- // successful tick can still promote it to online.- require.NoError(t, svc.updateEnvironmentStatusInternal(ctx, "direct-pending", string(EnvironmentStatusOffline)))- require.NoError(t, svc.updateEnvironmentStatusInternal(ctx, "direct-pending", string(EnvironmentStatusError)))+ // A transient failure before pairing must leave a pending direct env pending.+ require.NoError(t, svc.updateEnvironmentStatus(ctx, "direct-pending", string(EnvironmentStatusOffline)))+ require.NoError(t, svc.updateEnvironmentStatus(ctx, "direct-pending", string(EnvironmentStatusError)))var env Environmentrequire.NoError(t, db.WithContext(ctx).Where("id = ?", "direct-pending").First(&env).Error)@@ -695,7 +693,7 @@ func TestEnvironmentService_UpdateEnvironmentStatusInternal_DoesNotDemotePendingrequire.Nil(t, env.LastSeen)}-func TestEnvironmentService_UpdateEnvironmentStatusInternal_LeavesPendingEdgeEnvAlone(t *testing.T) {+func TestEnvironmentService_UpdateEnvironmentStatus_LeavesPendingEdgeEnvAlone(t *testing.T) {ctx := t.Context()db := setupEnvironmentServiceTestDB(t)svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)@@ -704,7 +702,7 @@ func TestEnvironmentService_UpdateEnvironmentStatusInternal_LeavesPendingEdgeEnv// Edge envs in pending must complete pairing via the agent's outbound tunnel;// a manager-side reachability tick must NOT promote them.- require.NoError(t, svc.updateEnvironmentStatusInternal(ctx, "edge-pending", string(EnvironmentStatusOnline)))+ require.NoError(t, svc.updateEnvironmentStatus(ctx, "edge-pending", string(EnvironmentStatusOnline)))var env Environmentrequire.NoError(t, db.WithContext(ctx).Where("id = ?", "edge-pending").First(&env).Error)@@ -774,7 +772,7 @@ func TestEnvironmentService_UpdateEnvironment_ClearingAccessTokenInvalidatesCachrequire.Contains(t, err.Error(), "invalid agent token")}-func TestEnvironmentServiceUpdateEnvironmentRejectsTargetChangeWithStoredTokenInternal(t *testing.T) {+func TestEnvironmentServiceUpdateEnvironmentRejectsTargetChangeWithStoredToken(t *testing.T) {ctx := t.Context()db := setupEnvironmentServiceTestDB(t)svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)@@ -793,7 +791,7 @@ func TestEnvironmentServiceUpdateEnvironmentRejectsTargetChangeWithStoredTokenInrequire.Equal(t, oldToken, *stored.AccessToken)}-func TestEnvironmentServiceUpdateEnvironmentAllowsTargetChangeWithExplicitTokenInternal(t *testing.T) {+func TestEnvironmentServiceUpdateEnvironmentAllowsTargetChangeWithExplicitToken(t *testing.T) {for _, tc := range []struct {name stringtoken string@@ -825,7 +823,7 @@ func TestEnvironmentServiceUpdateEnvironmentAllowsTargetChangeWithExplicitTokenI}}-func TestEnvironmentService_getCachedEnvironmentIDForTokenInternal_ExpiresAndCleansReverseIndex(t *testing.T) {+func TestEnvironmentService_getCachedEnvironmentIDForToken_ExpiresAndCleansReverseIndex(t *testing.T) {svc := NewEnvironmentService(nil, nil, nil, nil, nil, nil)svc.edgeTokens.mu.Lock()svc.edgeTokens.byEnvID["env-expired"] = "expired-token"@@ -852,7 +850,7 @@ func TestEnvironmentService_ResolveEnvironmentByAccessToken(t *testing.T) {svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)accessToken := "remote-token"- createNamedTestEnvironmentInternal(t, db, "env-remote", "Remote Alpha", "http://remote.example", &accessToken)+ createNamedTestEnvironment(t, db, "env-remote", "Remote Alpha", "http://remote.example", &accessToken)env, err := svc.ResolveEnvironmentByAccessToken(ctx, accessToken)require.NoError(t, err)@@ -925,7 +923,7 @@ func TestEnvironmentService_GenerateDeploymentSnippets_PublishesAgentURLPort(t *}}-func TestAgentHostPortInternal(t *testing.T) {+func TestAgentHostPort(t *testing.T) {t.Parallel()tests := []struct {@@ -1082,14 +1080,8 @@ func TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_ReusesLegacyHiddenRe"environment count = %d, want 1", len(environments))}-// TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_TokenResolvesEndToEnd-// pins the agent token round-trip: the token returned from the swarm node-// agent provisioning flow must resolve back to the same environment via-// ResolveEdgeEnvironmentByToken, and rotation must invalidate the previous-// token while making the new one resolvable. This is the end-to-end gap that-// would have caught the v1.18.1 "invalid agent token" bug if any silent-// transformation (trim, encode, hash) were ever introduced between the-// command-generation path and the poll-validation path.+// The provisioned agent token must resolve back to its environment, and rotation+// must retire the old token, with no transformation between issue and validation.func TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_TokenResolvesEndToEnd(t *testing.T) {ctx := t.Context()db := setupEnvironmentServiceTestDB(t)@@ -1188,8 +1180,8 @@ func TestEnvironmentService_ListMethods_ExcludeHiddenEnvironments(t *testing.T)svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)createTestEnvironment(t, db, "0", "http://localhost:3552", nil)- createNamedTestEnvironmentInternal(t, db, "env-visible", "Visible Remote", "http://visible.example", new("visible-token"))- createNamedTestEnvironmentInternal(t, db, "env-hidden", "Hidden Node Agent", "edge://swarm-node-hidden", new("hidden-token"))+ createNamedTestEnvironment(t, db, "env-visible", "Visible Remote", "http://visible.example", new("visible-token"))+ createNamedTestEnvironment(t, db, "env-hidden", "Hidden Node Agent", "edge://swarm-node-hidden", new("hidden-token"))require.NoError(t, db.WithContext(ctx).Model(&Environment{}).@@ -1223,8 +1215,8 @@ func TestEnvironmentService_ListEnvironmentsPaginated_FiltersByAccessibleEnvIDs(svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)createTestEnvironment(t, db, "0", "http://localhost:3552", nil)- createNamedTestEnvironmentInternal(t, db, "env-a", "Env A", "http://a.example", new("token-a"))- createNamedTestEnvironmentInternal(t, db, "env-b", "Env B", "http://b.example", new("token-b"))+ createNamedTestEnvironment(t, db, "env-a", "Env A", "http://a.example", new("token-a"))+ createNamedTestEnvironment(t, db, "env-b", "Env B", "http://b.example", new("token-b"))newParams := func(typeFilter string) pagination.QueryParams {filters := map[string]string{}@@ -1238,17 +1230,17 @@ func TestEnvironmentService_ListEnvironmentsPaginated_FiltersByAccessibleEnvIDs(// nil = no restriction: every non-hidden environment is returned.all, _, err := svc.ListEnvironmentsPaginated(ctx, newParams(""), nil)require.NoError(t, err)- require.ElementsMatch(t, []string{"0", "env-a", "env-b"}, environmentIDsInternal(all))+ require.ElementsMatch(t, []string{"0", "env-a", "env-b"}, environmentIDs(all))// A non-nil allow-list restricts the result on the DB path.scoped, _, err := svc.ListEnvironmentsPaginated(ctx, newParams(""), []string{"env-a"})require.NoError(t, err)- require.ElementsMatch(t, []string{"env-a"}, environmentIDsInternal(scoped))+ require.ElementsMatch(t, []string{"env-a"}, environmentIDs(scoped))// The runtime-filter path (type filter) honors the allow-list too.scopedTyped, _, err := svc.ListEnvironmentsPaginated(ctx, newParams("http"), []string{"env-a"})require.NoError(t, err)- require.ElementsMatch(t, []string{"env-a"}, environmentIDsInternal(scopedTyped))+ require.ElementsMatch(t, []string{"env-a"}, environmentIDs(scopedTyped))// An empty (non-nil) allow-list yields no environments on either path.none, _, err := svc.ListEnvironmentsPaginated(ctx, newParams(""), []string{})@@ -1330,7 +1322,7 @@ func TestEnvironmentService_ListEnvironmentsPaginated_FiltersByRuntimeType(t *tegrpcTunnel := edge.NewAgentTunnelWithConn("env-grpc", edge.NewGRPCManagerTunnelConn(nil))edge.GetRegistry().Register("env-grpc", grpcTunnel)- websocketTunnel, closeWebSocketTunnel := newTestWebSocketTunnelInternal(t, "env-websocket")+ websocketTunnel, closeWebSocketTunnel := newTestWebSocketTunnel(t, "env-websocket")defer closeWebSocketTunnel()edge.GetRegistry().Register("env-websocket", websocketTunnel)@@ -1357,12 +1349,12 @@ func TestEnvironmentService_ListEnvironmentsPaginated_FiltersByRuntimeType(t *teFilters: map[string]string{"type": tt.typeFilter},}, nil)require.NoError(t, err)- require.ElementsMatch(t, tt.wantIDs, environmentIDsInternal(listedEnvironments))+ require.ElementsMatch(t, tt.wantIDs, environmentIDs(listedEnvironments))})}}-func environmentIDsInternal(environments []environment.Environment) []string {+func environmentIDs(environments []environment.Environment) []string {ids := make([]string, 0, len(environments))for _, env := range environments {ids = append(ids, env.ID)@@ -1370,7 +1362,7 @@ func environmentIDsInternal(environments []environment.Environment) []string {return ids}-func newTestWebSocketTunnelInternal(t *testing.T, envID string) (*edge.AgentTunnel, func()) {+func newTestWebSocketTunnel(t *testing.T, envID string) (*edge.AgentTunnel, func()) {t.Helper()connCh := make(chan *websocket.Conn, 1)@@ -1459,7 +1451,7 @@ func TestEnvironmentService_TestConnection_RejectsInvalidCustomURL(t *testing.T)require.EqualError(t, err, "Environment connection test failed")}-func TestEnvironmentServiceTestConnectionHidesCustomURLFailureInternal(t *testing.T) {+func TestEnvironmentServiceTestConnectionHidesCustomURLFailure(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {w.WriteHeader(http.StatusTeapot)}))@@ -1476,7 +1468,7 @@ func TestEnvironmentServiceTestConnectionHidesCustomURLFailureInternal(t *testinrequire.NotContains(t, err.Error(), "418")}-func TestEnvironmentServiceTestConnectionAllowsStoredPrivateURLInternal(t *testing.T) {+func TestEnvironmentServiceTestConnectionAllowsStoredPrivateURL(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {w.WriteHeader(http.StatusOK)}))@@ -1504,12 +1496,12 @@ func TestEnvironmentService_ExecuteRemoteRequest_RejectsInvalidEnvironmentURL(trequire.Contains(t, err.Error(), "invalid environment API URL")}-func (s *environmentTestSchedulerInternal) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {+func (s *environmentTestScheduler) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {s.submitted = append(s.submitted, request)return scheduler.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: scheduler.Queued}, nil}-func setupSyncDBInternal(t *testing.T) *database.DB {+func setupSyncDB(t *testing.T) *database.DB {t.Helper()db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "sync.db")), &gorm.Config{})require.NoError(t, err)@@ -1532,7 +1524,7 @@ func setupSyncDBInternal(t *testing.T) *database.DB {func TestSyncCredentialDecryptionAbortsSnapshot(t *testing.T) {for _, kind := range []string{"registry token", "ECR secret", "repository token", "repository SSH key", "S3 secret"} {t.Run(kind, func(t *testing.T) {- db := setupSyncDBInternal(t)+ db := setupSyncDB(t)var calls atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {calls.Add(1)@@ -1574,7 +1566,7 @@ func TestSyncCredentialDecryptionAbortsSnapshot(t *testing.T) {}func TestSyncRepositoriesPreservesEmptyCredentials(t *testing.T) {- db := setupSyncDBInternal(t)+ db := setupSyncDB(t)require.NoError(t, db.Create(&gitrepo.GitRepository{ID: "public-repo", Name: "Public", URL: "https://example.com/public.git", AuthType: "none"}).Error)server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {var payload gitops.RepositorySyncRequest@@ -1595,7 +1587,7 @@ func TestSyncRepositoriesPreservesEmptyCredentials(t *testing.T) {}func TestSyncSkipsUnchangedPayloadUntilForgotten(t *testing.T) {- db := setupSyncDBInternal(t)+ db := setupSyncDB(t)var calls atomic.Int32var reject atomic.Boolserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {@@ -1644,7 +1636,7 @@ func TestSyncSkipsUnchangedPayloadUntilForgotten(t *testing.T) {// A watcher must see a change signal, and a burst must not block the notifier —// it runs on the edge tunnel's connect/disconnect path.-func TestEnvironmentServiceRuntimeChangeSignalCoalescesInternal(t *testing.T) {+func TestEnvironmentServiceRuntimeChangeSignalCoalesces(t *testing.T) {s := &EnvironmentService{}changes, unsubscribe := s.SubscribeRuntimeChanges()diff --git a/backend/internal/federated/service.go b/backend/internal/federated/service.goindex a0c8a96b5..4ce38e86f 100644--- a/backend/internal/federated/service.go+++ b/backend/internal/federated/service.go@@ -8,6 +8,7 @@ import ("log/slog""net/http""regexp"+ "slices""strings""sync""time"@@ -18,6 +19,10 @@ import (httpxtypes "github.com/getarcaneapp/arcane/types/v2/httpx""github.com/samber/mo""go.getarcane.app/kit/pkg"+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric""golang.org/x/sync/singleflight""gorm.io/gorm"@@ -34,6 +39,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/jwtclaims""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/oidcjwk"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")const (@@ -41,6 +47,8 @@ const (defaultFederatedSubjectClaim = "sub")+var errInvalidGrant = common.Classify(common.ErrFederatedCredentialInvalidGrant, errors.New("invalid federated token grant"))+type FederatedCredentialService struct {db *database.DBauthService *auth.AuthService@@ -53,6 +61,7 @@ type FederatedCredentialService struct {providerMu sync.RWMutexkeySets map[string]oidc.KeySetproviderGroup singleflight.Group+ exchanges metric.Int64Counter}func NewFederatedCredentialService(@@ -65,8 +74,17 @@ func NewFederatedCredentialService(keySetManager *oidcjwk.KeySetManager,roleService *role.RoleService,) *FederatedCredentialService {- if httpClient == nil {- httpClient = httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: 15 * time.Second, TLSHandshakeTimeout: 10 * time.Second})+ client := httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: 15 * time.Second, TLSHandshakeTimeout: 10 * time.Second})+ if httpClient != nil {+ copied := *httpClient+ client = &copied+ }+ client.Transport = otelhttp.NewTransport(client.Transport)++ exchanges, err := otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.federated.token_exchanges",+ metric.WithDescription("Federated credential token exchanges by outcome"), metric.WithUnit("{exchange}"))+ if err != nil {+ otel.Handle(err)}return &FederatedCredentialService{@@ -75,10 +93,11 @@ func NewFederatedCredentialService(userService: userService,settingsService: settingsService,eventService: eventService,- httpClient: httpClient,+ httpClient: client,keySetManager: keySetManager,roleService: roleService,keySets: make(map[string]oidc.KeySet),+ exchanges: exchanges,}}@@ -139,11 +158,7 @@ func (s *FederatedCredentialService) Create(ctx context.Context, callerUserID sts.roleService.InvalidateUser(created.IdentityUserID)}- reloaded, err := s.Get(ctx, created.ID)- if err != nil {- return nil, err- }- return reloaded, nil+ return s.Get(ctx, created.ID)}func (s *FederatedCredentialService) List(ctx context.Context, params pagination.QueryParams) ([]federated.FederatedCredential, pagination.Response, error) {@@ -172,28 +187,28 @@ func (s *FederatedCredentialService) List(ctx context.Context, params pagination}func (s *FederatedCredentialService) Get(ctx context.Context, id string) (*federated.FederatedCredential, error) {- var credential FederatedCredential- if err := s.db.WithContext(ctx).- Preload("IdentityUser").- Preload("Role").- Preload("Environment").- Where("id = ?", id).- First(&credential).Error; err != nil {- if errors.Is(err, gorm.ErrRecordNotFound) {- return nil, common.Classify(common.ErrFederatedCredentialNotFound, errors.New("federated credential not found"))- }- return nil, fmt.Errorf("failed to get federated credential: %w", err)+ credential, err := findCredential(s.db.WithContext(ctx).Preload("IdentityUser").Preload("Role").Preload("Environment"), id)+ if err != nil {+ return nil, err}return new(toFederatedCredentialDTOInternal(&credential)), nil}-func (s *FederatedCredentialService) Update(ctx context.Context, callerUserID, id string, req federated.UpdateFederatedCredential) (*federated.FederatedCredential, error) {+func findCredential(query *gorm.DB, id string) (FederatedCredential, error) {var credential FederatedCredential- if err := s.db.WithContext(ctx).Where("id = ?", id).First(&credential).Error; err != nil {+ if err := query.Where("id = ?", id).First(&credential).Error; err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {- return nil, common.Classify(common.ErrFederatedCredentialNotFound, errors.New("federated credential not found"))+ return credential, common.Classify(common.ErrFederatedCredentialNotFound, errors.New("federated credential not found"))}- return nil, fmt.Errorf("failed to load federated credential: %w", err)+ return credential, fmt.Errorf("failed to load federated credential: %w", err)+ }+ return credential, nil+}++func (s *FederatedCredentialService) Update(ctx context.Context, callerUserID, id string, req federated.UpdateFederatedCredential) (*federated.FederatedCredential, error) {+ credential, err := findCredential(s.db.WithContext(ctx), id)+ if err != nil {+ return nil, err}updated, roleChanged, err := applyFederatedCredentialUpdateInternal(credential, req)@@ -249,20 +264,17 @@ func (s *FederatedCredentialService) Update(ctx context.Context, callerUserID, i}func (s *FederatedCredentialService) Delete(ctx context.Context, id string) error {- var credential FederatedCredential- if err := s.db.WithContext(ctx).Where("id = ?", id).First(&credential).Error; err != nil {- if errors.Is(err, gorm.ErrRecordNotFound) {- return common.Classify(common.ErrFederatedCredentialNotFound, errors.New("federated credential not found"))- }- return fmt.Errorf("failed to load federated credential: %w", err)+ credential, err := findCredential(s.db.WithContext(ctx), id)+ if err != nil {+ return err}- err := dbutil.WithTx(ctx, s.db.DB, func(tx *gorm.DB) error {- if err := tx.Delete(&FederatedCredential{}, "id = ?", credential.ID).Error; err != nil {- return fmt.Errorf("failed to delete federated credential: %w", err)+ err = dbutil.WithTx(ctx, s.db.DB, func(tx *gorm.DB) error {+ if deleteErr := tx.Delete(&FederatedCredential{}, "id = ?", credential.ID).Error; deleteErr != nil {+ return fmt.Errorf("failed to delete federated credential: %w", deleteErr)}- if err := tx.Delete(&user.User{}, "id = ?", credential.IdentityUserID).Error; err != nil {- return fmt.Errorf("failed to delete federated service user: %w", err)+ if deleteErr := tx.Delete(&user.User{}, "id = ?", credential.IdentityUserID).Error; deleteErr != nil {+ return fmt.Errorf("failed to delete federated service user: %w", deleteErr)}return nil})@@ -278,7 +290,8 @@ func (s *FederatedCredentialService) Delete(ctx context.Context, id string) erroreturn nil}-func (s *FederatedCredentialService) ExchangeToken(ctx context.Context, req federated.TokenExchangeRequest) (*federated.FederatedTokenResponse, error) {+func (s *FederatedCredentialService) ExchangeToken(ctx context.Context, req federated.TokenExchangeRequest) (_ *federated.FederatedTokenResponse, err error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "federated.token_exchange")claims := jwtclaims.ParseJWTClaims(req.SubjectToken)issuer := ""subject := ""@@ -291,54 +304,89 @@ func (s *FederatedCredentialService) ExchangeToken(ctx context.Context, req fedelogResult := "failure"logReason := ""- var matchedCredential *FederatedCredential- var matchedUser *user.User+ var credentialID, credentialName, userID, username stringdefer func() {- s.logExchangeInternal(ctx, logResult, logReason, issuer, subject, audiences, matchedCredential, matchedUser)+ outcome := []attribute.KeyValue{attribute.String("arcane.federated.outcome", logResult), attribute.String("arcane.federated.reason", logReason)}+ s.exchanges.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes(outcome...))+ span.SetAttributes(append(outcome, attribute.String("arcane.federated.issuer", issuer), attribute.String("arcane.federated.credential_id", credentialID))...)+ tracing.End(span, err)++ slog.InfoContext(ctx, "Federated credential token exchange",+ "result", logResult,+ "reason", logReason,+ "issuer", issuer,+ "subject", subject,+ "audiences", audiences,+ "credentialId", credentialID,+ )+ if s.eventService == nil {+ return+ }+ severity := event.EventSeverityInfo+ title := "Federated credential token exchange"+ if logResult != "success" {+ severity = event.EventSeverityWarning+ title = "Federated credential token exchange rejected"+ }+ eventRequest := event.CreateEventRequest{+ Type: event.EventTypeFederatedExchange,+ Severity: severity,+ Title: title,+ Description: "Workload identity federation token exchange",+ ResourceType: new("federated_credential"),+ ResourceID: mo.EmptyableToOption(strings.TrimSpace(credentialID)).ToPointer(),+ ResourceName: mo.EmptyableToOption(strings.TrimSpace(credentialName)).ToPointer(),+ UserID: mo.EmptyableToOption(strings.TrimSpace(userID)).ToPointer(),+ Username: mo.EmptyableToOption(strings.TrimSpace(username)).ToPointer(),+ Metadata: database.JSON{+ "action": "federated_token_exchange",+ "result": logResult,+ "reason": logReason,+ "issuer": issuer,+ "subject": subject,+ "audiences": audiences,+ "credentialId": credentialID,+ },+ }+ go func() {+ bgCtx := context.WithoutCancel(ctx)+ if _, eventErr := s.eventService.CreateEvent(bgCtx, eventRequest); eventErr != nil {+ slog.WarnContext(bgCtx, "failed to audit federated credential token exchange", "error", eventErr)+ }+ }()}()- if req.GrantType != federated.TokenExchangeGrantType || strings.TrimSpace(req.SubjectToken) == "" {- logReason = "invalid_request"- return nil, common.Classify(common.ErrFederatedCredentialInvalidRequest, errors.New("invalid federated token exchange request"))- }- switch req.SubjectTokenType {- case federated.SubjectTokenTypeJWT, federated.SubjectTokenTypeIDToken:- default:- logReason = "invalid_request"- return nil, common.Classify(common.ErrFederatedCredentialInvalidRequest, errors.New("invalid federated token exchange request"))- }- if req.RequestedTokenType != "" && req.RequestedTokenType != federated.RequestedTokenTypeAccessJWT {+ invalidRequest := req.GrantType != federated.TokenExchangeGrantType || strings.TrimSpace(req.SubjectToken) == "" ||+ (req.SubjectTokenType != federated.SubjectTokenTypeJWT && req.SubjectTokenType != federated.SubjectTokenTypeIDToken) ||+ (req.RequestedTokenType != "" && req.RequestedTokenType != federated.RequestedTokenTypeAccessJWT)+ if invalidRequest {logReason = "invalid_request"return nil, common.Classify(common.ErrFederatedCredentialInvalidRequest, errors.New("invalid federated token exchange request"))}if issuer == "" {logReason = "missing_issuer"- return nil, common.Classify(common.ErrFederatedCredentialInvalidGrant, errors.New("invalid federated token grant"))+ return nil, errInvalidGrant}var credentials []FederatedCredential- if err := s.db.WithContext(ctx).+ if findErr := s.db.WithContext(ctx).Where("issuer_url = ? AND enabled = ?", issuer, true).Order("created_at ASC").Order("id ASC").- Find(&credentials).Error; err != nil {+ Find(&credentials).Error; findErr != nil {logReason = "credential_lookup_failed"- return nil, fmt.Errorf("failed to list federated credentials for issuer: %w", err)+ return nil, fmt.Errorf("failed to list federated credentials for issuer: %w", findErr)}now := time.Now()- active := credentials[:0]- for _, credential := range credentials {- if credential.ExpiresAt == nil || !now.After(*credential.ExpiresAt) {- active = append(active, credential)- }- }- credentials = active+ credentials = slices.DeleteFunc(credentials, func(credential FederatedCredential) bool {+ return credential.ExpiresAt != nil && now.After(*credential.ExpiresAt)+ })if len(credentials) == 0 {logReason = "issuer_not_allowed"- return nil, common.Classify(common.ErrFederatedCredentialInvalidGrant, errors.New("invalid federated token grant"))+ return nil, errInvalidGrant}- verifiedToken, verifiedClaims, err := s.verifySubjectTokenInternal(ctx, issuer, req.SubjectToken)+ verifiedToken, verifiedClaims, err := s.verifySubjectToken(ctx, issuer, req.SubjectToken)if err != nil {logReason = "token_verification_failed"return nil, common.Classify(common.ErrFederatedCredentialInvalidGrant, fmt.Errorf("invalid federated token grant: %w", err))@@ -350,15 +398,39 @@ func (s *FederatedCredentialService) ExchangeToken(ctx context.Context, req fedeaudiences = append([]string{}, verifiedToken.Audience...)}- credential := selectMatchingCredentialInternal(credentials, verifiedToken.Audience, verifiedClaims)- if credential == nil {+ matchIndex := slices.IndexFunc(credentials, func(credential FederatedCredential) bool {+ return credentialMatchesToken(&credential, verifiedToken.Audience, verifiedClaims)+ })+ if matchIndex < 0 {logReason = "no_matching_credential"- return nil, common.Classify(common.ErrFederatedCredentialInvalidGrant, errors.New("invalid federated token grant"))+ return nil, errInvalidGrant+ }+ credential := &credentials[matchIndex]+ credentialID, credentialName = credential.ID, credential.Name+ logReason = "token_replay_rejected"+ if verifiedToken.Expiry.IsZero() || time.Now().After(verifiedToken.Expiry) {+ return nil, errInvalidGrant+ }+ if pruneErr := s.db.WithContext(ctx).Where("expires_at < ?", time.Now()).Delete(&FederatedTokenReplay{}).Error; pruneErr != nil {+ return nil, fmt.Errorf("failed to prune federated token replay records: %w", pruneErr)+ }+ tokenID := strings.TrimSpace(kit.ToString(jwtclaims.GetByPath(verifiedClaims, "jti").OrEmpty()))+ tokenKind := "jti"+ if tokenID == "" {+ tokenID = req.SubjectToken+ tokenKind = "token"}- matchedCredential = credential- if recordTokenReplayGuardErr := s.recordTokenReplayGuardInternal(ctx, issuer, req.SubjectToken, verifiedClaims, verifiedToken.Expiry); recordTokenReplayGuardErr != nil {- logReason = "token_replay_rejected"- return nil, recordTokenReplayGuardErr+ replay := FederatedTokenReplay{+ TokenHash: kit.SHA256Hex(issuer + "\x00" + tokenKind + "\x00" + tokenID),+ IssuerURL: issuer,+ ExpiresAt: verifiedToken.Expiry,+ }+ if replayErr := s.db.WithContext(ctx).Create(&replay).Error; replayErr != nil {+ message := strings.ToLower(replayErr.Error())+ if strings.Contains(message, "unique") || strings.Contains(message, "duplicate key") {+ return nil, errInvalidGrant+ }+ return nil, fmt.Errorf("failed to record federated token replay guard: %w", replayErr)}identityUser, err := s.userService.GetUserByID(ctx, credential.IdentityUserID)@@ -366,7 +438,7 @@ func (s *FederatedCredentialService) ExchangeToken(ctx context.Context, req fedelogReason = "identity_user_missing"return nil, common.Classify(common.ErrFederatedCredentialInvalidGrant, fmt.Errorf("invalid federated token grant: %w", err))}- matchedUser = identityUser+ userID, username = identityUser.ID, identityUser.UsernametokenPair, err := s.authService.IssueFederatedToken(ctx, identityUser, credential.ID, credential.TokenTTLSeconds)if err != nil {@@ -396,18 +468,54 @@ func (s *FederatedCredentialService) ExchangeToken(ctx context.Context, req fede}, nil}-func (s *FederatedCredentialService) verifySubjectTokenInternal(ctx context.Context, issuer, rawToken string) (*oidc.IDToken, map[string]any, error) {- keySet, err := s.keySetForIssuerInternal(ctx, issuer)- if err != nil {- return nil, nil, err+// verifySubjectToken discovers the issuer's JWK set once and verifies rawToken against it.+func (s *FederatedCredentialService) verifySubjectToken(ctx context.Context, issuer, rawToken string) (*oidc.IDToken, map[string]any, error) {+ s.providerMu.RLock()+ keySet := s.keySets[issuer]+ s.providerMu.RUnlock()+ if keySet == nil {+ value, err, _ := s.providerGroup.Do(issuer, func() (any, error) {+ providerCtx := oidc.ClientContext(context.WithoutCancel(ctx), s.httpClient)+ provider, err := oidc.NewProvider(providerCtx, issuer)+ if err != nil {+ return nil, fmt.Errorf("failed to discover federated issuer: %w", err)+ }++ var metadata struct {+ JWKSURL string `json:"jwks_uri"`+ }+ if claimsErr := provider.Claims(&metadata); claimsErr != nil {+ return nil, fmt.Errorf("failed to read federated issuer metadata: %w", claimsErr)+ }+ if metadata.JWKSURL == "" {+ return nil, errors.New("federated issuer metadata is missing jwks_uri")+ }+ if s.keySetManager == nil {+ return nil, errors.New("JWK set manager is not configured")+ }++ discovered, err := s.keySetManager.KeySet(context.WithoutCancel(ctx), s.httpClient, metadata.JWKSURL)+ if err != nil {+ return nil, fmt.Errorf("failed to configure federated issuer JWK set: %w", err)+ }+ s.providerMu.Lock()+ s.keySets[issuer] = discovered+ s.providerMu.Unlock()+ return discovered, nil+ })+ if err != nil {+ return nil, nil, err+ }+ if keySet, _ = value.(oidc.KeySet); keySet == nil {+ return nil, nil, errors.New("federated issuer discovery returned invalid key set")+ }}- providerCtx := oidc.ClientContext(ctx, s.httpClient)verifier := oidc.NewVerifier(issuer, keySet, &oidc.Config{SkipClientIDCheck: true,SupportedSigningAlgs: oidcjwk.SupportedSigningAlgs(),})- idToken, err := verifier.Verify(providerCtx, rawToken)+ idToken, err := verifier.Verify(oidc.ClientContext(ctx, s.httpClient), rawToken)if err != nil {return nil, nil, err}@@ -419,110 +527,11 @@ func (s *FederatedCredentialService) verifySubjectTokenInternal(ctx context.Contreturn idToken, claims, nil}-func (s *FederatedCredentialService) recordTokenReplayGuardInternal(ctx context.Context, issuer, rawToken string, claims map[string]any, expiresAt time.Time) error {- if expiresAt.IsZero() || time.Now().After(expiresAt) {- return common.Classify(common.ErrFederatedCredentialInvalidGrant, errors.New("invalid federated token grant"))- }-- now := time.Now()- if err := s.db.WithContext(ctx).- Where("expires_at < ?", now).- Delete(&FederatedTokenReplay{}).Error; err != nil {- return fmt.Errorf("failed to prune federated token replay records: %w", err)- }-- tokenID := strings.TrimSpace(kit.ToString(jwtclaims.GetByPath(claims, "jti").OrEmpty()))- tokenKind := "jti"- if tokenID == "" {- tokenID = rawToken- tokenKind = "token"- }- replay := FederatedTokenReplay{- TokenHash: kit.SHA256Hex(issuer + "\x00" + tokenKind + "\x00" + tokenID),- IssuerURL: issuer,- ExpiresAt: expiresAt,- }- if err := s.db.WithContext(ctx).Create(&replay).Error; err != nil {- message := strings.ToLower(err.Error())- if strings.Contains(message, "unique") || strings.Contains(message, "duplicate key") {- return common.Classify(common.ErrFederatedCredentialInvalidGrant, errors.New("invalid federated token grant"))- }- return fmt.Errorf("failed to record federated token replay guard: %w", err)- }- return nil-}--func (s *FederatedCredentialService) keySetForIssuerInternal(ctx context.Context, issuer string) (oidc.KeySet, error) {- s.providerMu.RLock()- if keySet := s.keySets[issuer]; keySet != nil {- s.providerMu.RUnlock()- return keySet, nil- }- s.providerMu.RUnlock()-- value, err, _ := s.providerGroup.Do(issuer, func() (any, error) {- providerCtx := oidc.ClientContext(context.WithoutCancel(ctx), s.httpClient)- provider, err := oidc.NewProvider(providerCtx, issuer)- if err != nil {- return nil, fmt.Errorf("failed to discover federated issuer: %w", err)- }-- var metadata struct {- JWKSURL string `json:"jwks_uri"`- }- if claimsErr := provider.Claims(&metadata); claimsErr != nil {- return nil, fmt.Errorf("failed to read federated issuer metadata: %w", claimsErr)- }- if metadata.JWKSURL == "" {- return nil, errors.New("federated issuer metadata is missing jwks_uri")- }- if s.keySetManager == nil {- return nil, errors.New("JWK set manager is not configured")- }-- keySet, err := s.keySetManager.KeySet(context.WithoutCancel(ctx), s.httpClient, metadata.JWKSURL)- if err != nil {- return nil, fmt.Errorf("failed to configure federated issuer JWK set: %w", err)- }- s.providerMu.Lock()- s.keySets[issuer] = keySet- s.providerMu.Unlock()- return keySet, nil+func credentialMatchesToken(credential *FederatedCredential, tokenAudiences []string, claims map[string]any) bool {+ audienceMatched := slices.ContainsFunc(credential.Audiences, func(audience string) bool {+ audience = strings.TrimSpace(audience)+ return audience != "" && slices.Contains(tokenAudiences, audience)})- if err != nil {- return nil, err- }-- keySet, ok := value.(oidc.KeySet)- if !ok || keySet == nil {- return nil, errors.New("federated issuer discovery returned invalid key set")- }- return keySet, nil-}--func selectMatchingCredentialInternal(credentials []FederatedCredential, tokenAudiences []string, claims map[string]any) *FederatedCredential {- for i := range credentials {- credential := &credentials[i]- if credentialMatchesTokenInternal(credential, tokenAudiences, claims) {- return credential- }- }- return nil-}--func credentialMatchesTokenInternal(credential *FederatedCredential, tokenAudiences []string, claims map[string]any) bool {- audiences := make(map[string]struct{}, len(credential.Audiences))- for _, audience := range credential.Audiences {- if audience = strings.TrimSpace(audience); audience != "" {- audiences[audience] = struct{}{}- }- }- audienceMatched := false- for _, audience := range tokenAudiences {- if _, audienceMatched = audiences[audience]; audienceMatched {- break- }- }if !audienceMatched {return false}@@ -552,66 +561,3 @@ func credentialMatchesTokenInternal(credential *FederatedCredential, tokenAudienmatched, err := regexp.MatchString(expression.String(), subject)return err == nil && matched}--func (s *FederatedCredentialService) logExchangeInternal(ctx context.Context, result, reason, issuer, subject string, audiences []string, credential *FederatedCredential, identityUser *user.User) {- credentialID := ""- credentialName := ""- if credential != nil {- credentialID = credential.ID- credentialName = credential.Name- }- slog.InfoContext(ctx, "Federated credential token exchange",- "result", result,- "reason", reason,- "issuer", issuer,- "subject", subject,- "audiences", audiences,- "credentialId", credentialID,- )-- if s.eventService == nil {- return- }-- metadata := database.JSON{- "action": "federated_token_exchange",- "result": result,- "reason": reason,- "issuer": issuer,- "subject": subject,- "audiences": audiences,- "credentialId": credentialID,- }-- userID := ""- username := ""- if identityUser != nil {- userID = identityUser.ID- username = identityUser.Username- }- severity := event.EventSeverityInfo- title := "Federated credential token exchange"- if result != "success" {- severity = event.EventSeverityWarning- title = "Federated credential token exchange rejected"- }-- go func() {- bgCtx := context.WithoutCancel(ctx)- _, err := s.eventService.CreateEvent(bgCtx, event.CreateEventRequest{- Type: event.EventTypeFederatedExchange,- Severity: severity,- Title: title,- Description: "Workload identity federation token exchange",- ResourceType: mo.EmptyableToOption(strings.TrimSpace("federated_credential")).ToPointer(),- ResourceID: mo.EmptyableToOption(strings.TrimSpace(credentialID)).ToPointer(),- ResourceName: mo.EmptyableToOption(strings.TrimSpace(credentialName)).ToPointer(),- UserID: mo.EmptyableToOption(strings.TrimSpace(userID)).ToPointer(),- Username: mo.EmptyableToOption(strings.TrimSpace(username)).ToPointer(),- Metadata: metadata,- })- if err != nil {- slog.WarnContext(bgCtx, "failed to audit federated credential token exchange", "error", err)- }- }()-}diff --git a/backend/internal/federated/service_test.go b/backend/internal/federated/service_test.goindex 32325e873..a0947fc86 100644--- a/backend/internal/federated/service_test.go+++ b/backend/internal/federated/service_test.go@@ -35,20 +35,20 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils/oidcjwk")-type federatedTestIssuerInternal struct {+type federatedTestIssuer struct {IssuerURL stringprivate *mldsa.PrivateKeykeyID stringserver *httptest.Server}-func newFederatedTestIssuerInternal(t *testing.T) *federatedTestIssuerInternal {+func newFederatedTestIssuer(t *testing.T) *federatedTestIssuer {t.Helper()privateKey, err := mldsa.GenerateKey(mldsa.MLDSA87())require.NoError(t, err)- issuer := &federatedTestIssuerInternal{+ issuer := &federatedTestIssuer{private: privateKey,keyID: "federated-test-key",}@@ -90,7 +90,7 @@ func newFederatedTestIssuerInternal(t *testing.T) *federatedTestIssuerInternal {return issuer}-func (i *federatedTestIssuerInternal) tokenInternal(t *testing.T, subject string, audience []string) string {+func (i *federatedTestIssuer) token(t *testing.T, subject string, audience []string) string {t.Helper()now := time.Now()@@ -110,7 +110,7 @@ func (i *federatedTestIssuerInternal) tokenInternal(t *testing.T, subject stringreturn string(signed)}-func setupFederatedCredentialServiceTestDBInternal(t *testing.T) *database.DB {+func setupFederatedCredentialServiceTestDB(t *testing.T) *database.DB {t.Helper()dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.NewReplacer("/", "_", " ", "_").Replace(t.Name()))@@ -135,15 +135,15 @@ func setupFederatedCredentialServiceTestDBInternal(t *testing.T) *database.DB {return &database.DB{DB: db}}-func setupFederatedCredentialServiceInternal(t *testing.T, issuer *federatedTestIssuerInternal) (*FederatedCredentialService, *auth.AuthService, *database.DB) {+func setupFederatedCredentialService(t *testing.T, issuer *federatedTestIssuer) (*FederatedCredentialService, *auth.AuthService, *database.DB) {t.Helper()ctx := t.Context()- db := setupFederatedCredentialServiceTestDBInternal(t)+ db := setupFederatedCredentialServiceTestDB(t)roleSvc := role.NewRoleService(db)userSvc := user.NewUserService(db, roleSvc, session.RevokeAllUserSessionsExceptInDB)sessionSvc := session.NewSessionService(db)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)eventSvc := event.NewEventService(db, &config.Config{}, nil)signingKey, err := mldsa.GenerateKey(mldsa.MLDSA87())@@ -197,8 +197,8 @@ func setupFederatedCredentialServiceInternal(t *testing.T, issuer *federatedTest}func TestFederatedCredentialServiceExchangeToken(t *testing.T) {- issuer := newFederatedTestIssuerInternal(t)- service, authSvc, db := setupFederatedCredentialServiceInternal(t, issuer)+ issuer := newFederatedTestIssuer(t)+ service, authSvc, db := setupFederatedCredentialService(t, issuer)ctx := t.Context()tests := []struct {@@ -208,18 +208,18 @@ func TestFederatedCredentialServiceExchangeToken(t *testing.T) {}{{name: "issues an Arcane bearer token for a matching issuer audience and subject",- token: issuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),+ token: issuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),},{name: "rejects audience mismatch",- token: issuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"other-audience"}),+ token: issuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"other-audience"}),wantError: func(err error) bool {return errors.Is(err, common.ErrFederatedCredentialInvalidGrant)},},{name: "rejects subject mismatch",- token: issuer.tokenInternal(t, "repo:other/repo:ref:refs/heads/main", []string{"arcane-ci"}),+ token: issuer.token(t, "repo:other/repo:ref:refs/heads/main", []string{"arcane-ci"}),wantError: func(err error) bool {return errors.Is(err, common.ErrFederatedCredentialInvalidGrant)},@@ -261,14 +261,14 @@ func TestFederatedCredentialServiceExchangeToken(t *testing.T) {}}-func TestFederatedCredentialServiceExchangeTokenRejectsIssuerWithoutCredentialInternal(t *testing.T) {- issuer := newFederatedTestIssuerInternal(t)- otherIssuer := newFederatedTestIssuerInternal(t)- service, _, _ := setupFederatedCredentialServiceInternal(t, issuer)+func TestFederatedCredentialServiceExchangeTokenRejectsIssuerWithoutCredential(t *testing.T) {+ issuer := newFederatedTestIssuer(t)+ otherIssuer := newFederatedTestIssuer(t)+ service, _, _ := setupFederatedCredentialService(t, issuer)resp, err := service.ExchangeToken(t.Context(), federated.TokenExchangeRequest{GrantType: federated.TokenExchangeGrantType,- SubjectToken: otherIssuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),+ SubjectToken: otherIssuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),SubjectTokenType: federated.SubjectTokenTypeJWT,Audience: "https://arcane.example.com",})@@ -278,14 +278,14 @@ func TestFederatedCredentialServiceExchangeTokenRejectsIssuerWithoutCredentialInrequire.Nil(t, resp)}-func TestFederatedCredentialServiceExchangeTokenDoesNotRequireGlobalFeatureFlagInternal(t *testing.T) {- issuer := newFederatedTestIssuerInternal(t)- service, _, _ := setupFederatedCredentialServiceInternal(t, issuer)+func TestFederatedCredentialServiceExchangeTokenDoesNotRequireGlobalFeatureFlag(t *testing.T) {+ issuer := newFederatedTestIssuer(t)+ service, _, _ := setupFederatedCredentialService(t, issuer)service.settingsService = nilresp, err := service.ExchangeToken(t.Context(), federated.TokenExchangeRequest{GrantType: federated.TokenExchangeGrantType,- SubjectToken: issuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),+ SubjectToken: issuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),SubjectTokenType: federated.SubjectTokenTypeJWT,Audience: "https://arcane.example.com",})@@ -295,9 +295,9 @@ func TestFederatedCredentialServiceExchangeTokenDoesNotRequireGlobalFeatureFlagIrequire.NotEmpty(t, resp.AccessToken)}-func TestFederatedCredentialServiceExchangeTokenRejectsExpiredCredentialInternal(t *testing.T) {- issuer := newFederatedTestIssuerInternal(t)- service, _, db := setupFederatedCredentialServiceInternal(t, issuer)+func TestFederatedCredentialServiceExchangeTokenRejectsExpiredCredential(t *testing.T) {+ issuer := newFederatedTestIssuer(t)+ service, _, db := setupFederatedCredentialService(t, issuer)expiredAt := time.Now().Add(-time.Minute)require.NoError(t, db.WithContext(t.Context()).Model(&FederatedCredential{}).@@ -306,7 +306,7 @@ func TestFederatedCredentialServiceExchangeTokenRejectsExpiredCredentialInternalresp, err := service.ExchangeToken(t.Context(), federated.TokenExchangeRequest{GrantType: federated.TokenExchangeGrantType,- SubjectToken: issuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),+ SubjectToken: issuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),SubjectTokenType: federated.SubjectTokenTypeJWT,Audience: "https://arcane.example.com",})@@ -316,14 +316,14 @@ func TestFederatedCredentialServiceExchangeTokenRejectsExpiredCredentialInternalrequire.Nil(t, resp)}-func TestFederatedCredentialServiceUpdateDisableRevokesIssuedSessionsInternal(t *testing.T) {- issuer := newFederatedTestIssuerInternal(t)- service, authSvc, _ := setupFederatedCredentialServiceInternal(t, issuer)+func TestFederatedCredentialServiceUpdateDisableRevokesIssuedSessions(t *testing.T) {+ issuer := newFederatedTestIssuer(t)+ service, authSvc, _ := setupFederatedCredentialService(t, issuer)ctx := t.Context()resp, err := service.ExchangeToken(ctx, federated.TokenExchangeRequest{GrantType: federated.TokenExchangeGrantType,- SubjectToken: issuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),+ SubjectToken: issuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"}),SubjectTokenType: federated.SubjectTokenTypeJWT,Audience: "https://arcane.example.com",})@@ -340,11 +340,11 @@ func TestFederatedCredentialServiceUpdateDisableRevokesIssuedSessionsInternal(trequire.ErrorIs(t, err, common.ErrSessionRevoked, "unexpected error: %v", err)}-func TestFederatedCredentialServiceRejectsReplayedSubjectTokenInternal(t *testing.T) {- issuer := newFederatedTestIssuerInternal(t)- service, _, _ := setupFederatedCredentialServiceInternal(t, issuer)+func TestFederatedCredentialServiceRejectsReplayedSubjectToken(t *testing.T) {+ issuer := newFederatedTestIssuer(t)+ service, _, _ := setupFederatedCredentialService(t, issuer)ctx := t.Context()- subjectToken := issuer.tokenInternal(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"})+ subjectToken := issuer.token(t, "repo:getarcaneapp/arcane:ref:refs/heads/main", []string{"arcane-ci"})req := federated.TokenExchangeRequest{GrantType: federated.TokenExchangeGrantType,SubjectToken: subjectToken,@@ -363,8 +363,8 @@ func TestFederatedCredentialServiceRejectsReplayedSubjectTokenInternal(t *testin}func TestFederatedCredentialServiceCreateRejectsBareWildcardGlob(t *testing.T) {- issuer := newFederatedTestIssuerInternal(t)- service, _, _ := setupFederatedCredentialServiceInternal(t, issuer)+ issuer := newFederatedTestIssuer(t)+ service, _, _ := setupFederatedCredentialService(t, issuer)_, err := service.Create(t.Context(), "admin-user", federated.CreateFederatedCredential{Name: "Unsafe wildcard",@@ -380,7 +380,7 @@ func TestFederatedCredentialServiceCreateRejectsBareWildcardGlob(t *testing.T) {require.ErrorIs(t, err, common.ErrFederatedCredentialInvalid, "unexpected error: %v", err)}-func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {+func newSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {t.Helper()svc, err := settings.NewSettingsService(ctx, db)if err == nil {diff --git a/backend/internal/gitops/service.go b/backend/internal/gitops/service.goindex 81fbe4d10..e36e58eeb 100644--- a/backend/internal/gitops/service.go+++ b/backend/internal/gitops/service.go@@ -23,6 +23,10 @@ import ("go.getarcane.app/acfs""go.getarcane.app/kit/pkg""go.getarcane.app/kit/pkg/mapping"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric"+ "go.opentelemetry.io/otel/trace""gorm.io/gorm""gorm.io/gorm/clause"@@ -35,14 +39,25 @@ import (projectpkg "github.com/getarcaneapp/arcane/backend/v2/internal/project""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/swarm"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/gitutil""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/projects""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")+var syncRuns = func() metric.Int64Counter {+ counter, err := otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.gitops.syncs",+ metric.WithDescription("GitOps sync runs by outcome"), metric.WithUnit("{sync}"))+ if err != nil {+ otel.Handle(err)+ }+ return counter+}()+const (defaultGitSyncTimeout = 5 * time.Minute@@ -749,7 +764,22 @@ func (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id s// performSyncAdmitted runs one sync under its admission lease and dispatches by mode. backupAdopt makes a// backup replace whatever the remote holds in its backup directory.-func (s *GitOpsSyncService) performSyncAdmitted(ctx context.Context, environmentID, id string, actor user.Actor, backupAdopt bool) (*gitops.SyncResult, error) {+func (s *GitOpsSyncService) performSyncAdmitted(ctx context.Context, environmentID, id string, actor user.Actor, backupAdopt bool) (syncResult *gitops.SyncResult, syncErr error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "gitops.sync", trace.WithAttributes(+ attribute.String("arcane.gitops.sync_id", id),+ attribute.String("arcane.environment.id", environmentID),+ ))+ counted := false+ defer func() {+ success := syncErr == nil && syncResult != nil && syncResult.Success+ if counted {+ outcome := attribute.String("arcane.gitops.outcome", kit.Ternary(success, "success", "failure"))+ span.SetAttributes(outcome)+ syncRuns.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes(outcome))+ }+ tracing.End(span, syncErr)+ }()+// Overlapping runs of one sync (schedule, kick, manual, webhook) must not race the clone and redeploy.lease, admitted, err := s.jobs.TryAcquire(ctx, id)if err != nil {@@ -757,9 +787,11 @@ func (s *GitOpsSyncService) performSyncAdmitted(ctx context.Context, environment}if !admitted {slog.InfoContext(ctx, "GitOps sync already in progress; skipping", "syncId", id)+ span.SetAttributes(attribute.Bool("arcane.gitops.skipped", true))return &gitops.SyncResult{Success: false, Message: "sync already in progress", SyncedAt: time.Now()}, nil}defer lease.Release(ctx)+ counted = truesyncCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)defer cancel()@@ -768,6 +800,14 @@ func (s *GitOpsSyncService) performSyncAdmitted(ctx context.Context, environmentif err != nil {return nil, err}+ span.SetAttributes(+ attribute.String("arcane.git.branch", syncRecord.Branch),+ attribute.String("arcane.gitops.mode", syncRecord.Mode),+ attribute.String("arcane.gitops.target_type", syncRecord.TargetType),+ )+ if syncRecord.Repository != nil {+ span.SetAttributes(git.RepositoryAttribute(syncRecord.Repository.URL))+ }result := &gitops.SyncResult{Success: false,diff --git a/backend/internal/image/children/patch/service.go b/backend/internal/image/children/patch/service.goindex 226449fd7..e90233796 100644--- a/backend/internal/image/children/patch/service.go+++ b/backend/internal/image/children/patch/service.go@@ -32,6 +32,10 @@ import ("github.com/project-copacetic/copacetic/pkg/types""github.com/samber/mo""go.getarcane.app/acfs"+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/trace""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -45,6 +49,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/logging"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")// scheduledPatchUser is the actor recorded for scheduled patches and their verification scans.@@ -52,6 +57,8 @@ var (scheduledPatchUser = usertypes.Actor{Username: "System"}// errPatchInterrupted fails a patch cut off mid-run; its record keeps the message so it never reruns.errPatchInterrupted = errors.New("image patch was interrupted before its result was recorded; start a new patch")+ // registryTransport traces platform-manifest lookups; ggcr adds its retry wrappers on top.+ registryTransport = otelhttp.NewTransport(remote.DefaultTransport))// Target is one image a scheduled patch run covers.@@ -276,7 +283,16 @@ func (s *Service) startPatch(}// runPatch executes one recorded patch with Copacetic and settles its record and activity.-func (s *Service) runPatch(ctx context.Context, record *ImagePatchRecord, opts imagepatch.PatchOptions, reportData []byte, copaImageRef, activityID string) error {+func (s *Service) runPatch(ctx context.Context, record *ImagePatchRecord, opts imagepatch.PatchOptions, reportData []byte, copaImageRef, activityID string) (runErr error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "image.patch", trace.WithAttributes(+ attribute.String("arcane.environment.id", record.EnvironmentID),+ attribute.String("arcane.image.patch_id", record.ID),+ attribute.String("arcane.image.ref", record.OriginalRef),+ attribute.String("arcane.image.patched_ref", record.PatchedRef),+ attribute.String("arcane.image.patch_mode", record.Mode),+ attribute.String("arcane.activity.id", activityID),+ ))+ defer func() { tracing.End(span, runErr) }()fail := func(err error, durationMs int64) error {s.finishPatchRecord(ctx, record, imagepatch.PatchStatusFailed, err.Error(), nil, durationMs)s.completePatchActivity(ctx, activityID, false, err.Error())@@ -428,7 +444,7 @@ func platformPinnedRef(ctx context.Context, imageRef string, target v1.Platform)if err != nil {return ""}- desc, err := remote.Get(ref, remote.WithContext(ctx), remote.WithAuthFromKeychain(authn.DefaultKeychain))+ desc, err := remote.Get(ref, remote.WithContext(ctx), remote.WithTransport(registryTransport), remote.WithAuthFromKeychain(authn.DefaultKeychain))if err != nil || !desc.MediaType.IsIndex() {return ""}diff --git a/backend/internal/imageupdate/module.go b/backend/internal/imageupdate/module.goindex 9bb2f3037..c6698180d 100644--- a/backend/internal/imageupdate/module.go+++ b/backend/internal/imageupdate/module.go@@ -3,14 +3,20 @@ package imageupdateimport ("context"+ "errors"+ "log/slog""net/http""github.com/danielgtaylor/huma/v2""github.com/getarcaneapp/arcane/types/v2/image"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")type Module struct {@@ -18,7 +24,32 @@ type Module struct {getUpdateInfoByImageRefs func(context.Context, []string) (map[string]*image.UpdateInfo, error)}+// New wires image update routes and registers the arcane.image.updates gauges once.func New(service *ImageUpdateService, getUpdateInfo func(context.Context, []string) (map[string]*image.UpdateInfo, error)) *Module {+ if service != nil {+ meter := otel.Meter(tracing.InstrumentationName)+ updates, updatesErr := meter.Int64ObservableGauge("arcane.image.updates",+ metric.WithDescription("Local images with an available update by update type"), metric.WithUnit("{image}"))+ checkErrors, checkErrorsErr := meter.Int64ObservableGauge("arcane.image.update.check_errors",+ metric.WithDescription("Local images whose last update check failed"), metric.WithUnit("{image}"))+ err := errors.Join(updatesErr, checkErrorsErr)+ if err == nil {+ _, err = meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {+ summary, summaryErr := service.GetUpdateSummary(ctx)+ if summaryErr != nil {+ slog.DebugContext(ctx, "Skipping image update metrics", "error", summaryErr)+ return nil+ }+ o.ObserveInt64(updates, int64(summary.DigestUpdates), metric.WithAttributes(attribute.String("image.update.type", "digest")))+ o.ObserveInt64(updates, int64(summary.ImagesWithUpdates-summary.DigestUpdates), metric.WithAttributes(attribute.String("image.update.type", "tag")))+ o.ObserveInt64(checkErrors, int64(summary.ErrorsCount))+ return nil+ }, updates, checkErrors)+ }+ if err != nil {+ otel.Handle(err)+ }+ }return &Module{service: service, getUpdateInfoByImageRefs: getUpdateInfo}}@@ -41,13 +72,7 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContfunc RegisterImageUpdates(api huma.API,imageUpdateSvc *ImageUpdateService,- getUpdateInfoByImageRefs func(- context.Context,- []string,- ) (- map[string]*image.UpdateInfo,- error,- ),+ getUpdateInfoByImageRefs func(context.Context, []string) (map[string]*image.UpdateInfo, error),appCtx handlerutil.ActivityAppContext,) {h := &ImageUpdateHandler{diff --git a/backend/internal/job/service.go b/backend/internal/job/service.goindex c3620bde7..7f80a2634 100644--- a/backend/internal/job/service.go+++ b/backend/internal/job/service.go@@ -20,6 +20,10 @@ import ("github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/robfig/cron/v3""go.getarcane.app/kit/pkg"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric"+ "go.opentelemetry.io/otel/trace""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/config"@@ -35,6 +39,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")// JobService manages configuration for background job schedules.@@ -58,6 +63,9 @@ type JobService struct {scheduler scheduler.JobControllerlifecycleCtx context.Contextlocation *time.Location // Timezone for cron schedule calculations+ runCount metric.Int64Counter+ runDuration metric.Float64Histogram+ activeRuns metric.Int64UpDownCounter// environment-health is no longer a single scheduler job — it fans out to one// dynamic job per environment owned by environment.EnvironmentService. These bridge the Jobs@@ -88,6 +96,19 @@ func NewJobService(location: cfg.GetLocation(),}+ meter := otel.Meter(tracing.InstrumentationName)+ var errs [3]error+ service.runCount, errs[0] = meter.Int64Counter("arcane.job.runs",+ metric.WithDescription("Executed job runs by outcome"), metric.WithUnit("{run}"))+ service.runDuration, errs[1] = meter.Float64Histogram("arcane.job.duration",+ metric.WithDescription("Job run duration by outcome"), metric.WithUnit("s"),+ metric.WithExplicitBucketBoundaries(0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 120, 300, 600, 1800, 3600))+ service.activeRuns, errs[2] = meter.Int64UpDownCounter("arcane.job.active",+ metric.WithDescription("Job runs currently executing"), metric.WithUnit("{run}"))+ if err := errors.Join(errs[:]...); err != nil {+ otel.Handle(err)+ }+if coordinator != nil {coordinator.SetExecutor(service.executeRunInternal, service.reconcileRunInternal)if localActivity != nil {@@ -609,7 +630,33 @@ func (s *JobService) authorizeRunInternal(ctx context.Context, run scheduler.Runreturn nil}-func (s *JobService) executeRunInternal(ctx context.Context, run scheduler.Run) (scheduler.Outcome, error) {+func (s *JobService) executeRunInternal(ctx context.Context, run scheduler.Run) (result scheduler.Outcome, runErr error) {+ // Entity-scoped jobs ("gitops-sync:<id>") share one span name and metric series per job type.+ jobType, _, _ := strings.Cut(run.JobID, ":")+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "job "+jobType, trace.WithAttributes(+ attribute.String("arcane.job.id", run.JobID),+ attribute.String("arcane.job.run_id", run.ID),+ attribute.String("arcane.environment.id", run.EnvironmentID),+ ))+ jobAttr := attribute.String("arcane.job.id", jobType)+ started := time.Now()+ if s.activeRuns != nil {+ s.activeRuns.Add(ctx, 1, metric.WithAttributes(jobAttr))+ }+ defer func() {+ outcomeAttr := attribute.String("arcane.job.outcome", string(result.Status))+ span.SetAttributes(outcomeAttr)+ tracing.End(span, runErr)+ if s.activeRuns != nil {+ // Record against a non-canceled context so shutdown-interrupted runs still count.+ metricCtx := context.WithoutCancel(ctx)+ attrs := metric.WithAttributes(jobAttr, outcomeAttr)+ s.activeRuns.Add(metricCtx, -1, metric.WithAttributes(jobAttr))+ s.runCount.Add(metricCtx, 1, attrs)+ s.runDuration.Record(metricCtx, time.Since(started).Seconds(), attrs)+ }+ }()+if err := s.authorizeRunInternal(ctx, run); err != nil {return scheduler.Outcome{Status: scheduler.Failed}, err}diff --git a/backend/internal/middleware/environment.go b/backend/internal/middleware/environment.goindex d24466ae3..14dd1cabd 100644--- a/backend/internal/middleware/environment.go+++ b/backend/internal/middleware/environment.go@@ -2,6 +2,7 @@ package middlewareimport ("bytes"+ "cmp""context""encoding/json/v2""errors"@@ -25,8 +26,8 @@ import ("github.com/labstack/echo/v5""github.com/samber/mo""go.getarcane.app/kit/pkg"+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"- "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"@@ -37,8 +38,7 @@ const (apiEnvironmentsPrefix = "/api/environments/"environmentsPathMarker = "/environments/"- // proxyTimeout is intentionally generous because some proxied operations- // (e.g., image pulls with progress streaming) can take multiple minutes.+ // proxyTimeout is generous because proxied operations such as image pulls can stream for minutes.proxyTimeout = 30 * time.MinutemaxProxiedWorkspaceManifestBytes = 1024 * 1024@@ -63,14 +63,8 @@ var managementEndpointSet = map[string]struct{}{// Returns: apiURL, accessToken, enabled, errortype EnvResolver func(ctx context.Context, id string) (string, *string, bool, error)-// AuthValidator validates authentication for a request and resolves the-// caller's effective permission set. The boolean result reports whether the-// request is authenticated; the permission set is used to authorize proxied-// requests against the target environment. Sudo permission sets (internal-// agent proxies) bypass authorization. The resolved user is returned so-// per-user context (e.g. the icon catalog preference) can travel with the-// proxied request; it is nil for callers that are not a user (environment-// bootstrap keys).+// AuthValidator authenticates a request and returns the caller's permission set and user (nil for non-user callers).+// Sudo permission sets (internal agent proxies) bypass authorization.type AuthValidator func(ctx context.Context, c *echo.Context) (*authz.PermissionSet, *usertypes.Actor, bool)// EnvironmentMiddleware proxies requests for remote environments to their respective agents.@@ -82,10 +76,7 @@ type EnvironmentMiddleware struct {httpClient *http.Clientregistry *edge.TunnelRegistrymatcher *authz.PermissionMatcher- // checkOrigin is the same Origin validator the local WebSocket endpoints- // use. Proxied upgrades previously accepted any Origin, so a cross-origin- // page could ride the caller's session cookie into a remote environment's- // terminal or log stream.+ // checkOrigin is the local WebSocket Origin validator, so proxied upgrades cannot ride a cross-origin session.checkOrigin func(*http.Request) bool}@@ -103,12 +94,16 @@ func NewEnvProxyMiddlewareWithParamAndRegistry(registry = edge.NewTunnelRegistry()}+ // Traced so direct agents continue the manager's trace.+ httpClient := httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: proxyTimeout, TLSHandshakeTimeout: 10 * time.Second})+ httpClient.Transport = otelhttp.NewTransport(httpClient.Transport)+m := &EnvironmentMiddleware{localID: localID,paramName: paramName,resolver: resolver,authValidator: authValidator,- httpClient: httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: proxyTimeout, TLSHandshakeTimeout: 10 * time.Second}),+ httpClient: httpClient,registry: registry,matcher: matcher,checkOrigin: checkOrigin,@@ -122,13 +117,18 @@ func NewEnvProxyMiddlewareWithParamAndRegistry(// Handle is the main middleware handler.func (m *EnvironmentMiddleware) Handle(c *echo.Context, next echo.HandlerFunc) error {- envID := m.extractEnvironmentID(c)+ req := c.Request()+ ctx := req.Context()- if envID == "" || envID == m.localID {+ _, rest, hasMarker := strings.Cut(req.URL.Path, environmentsPathMarker)+ pathEnvID, _, _ := strings.Cut(rest, "/")+ envID := cmp.Or(c.Param(m.paramName), pathEnvID)+ if !hasMarker || envID == "" || envID == m.localID {return next(c)}- if !m.hasResourcePath(c, envID) {+ suffix, ok := strings.CutPrefix(req.URL.Path, apiEnvironmentsPrefix+envID)+ if !ok || len(suffix) <= 1 || suffix[0] != '/' || isManagementPath(req.Method, suffix) {return next(c)}@@ -136,168 +136,196 @@ func (m *EnvironmentMiddleware) Handle(c *echo.Context, next echo.HandlerFunc) evar perms *authz.PermissionSetvar user *usertypes.Actorif m.authValidator != nil {- ps, u, ok := m.authValidator(c.Request().Context(), c)- if !ok {- return c.JSON(http.StatusUnauthorized, map[string]any{- "success": false,- "data": map[string]any{"error": "Authentication required to access remote environments"},- })+ var authenticated bool+ perms, user, authenticated = m.authValidator(ctx, c)+ if !authenticated {+ return writeProxyError(c, http.StatusUnauthorized, "Authentication required to access remote environments")+ }+ if user != nil {+ RecordAuthenticatedRequest(ctx, req.Header, user)}- perms, user = ps, u}- m.setIconCatalogHeaderInternal(c, user)- m.setUpdateInitiatorHeadersInternal(c, user)+ // SECURITY: forwarded identity headers are always cleared first; only server-resolved values are sent.+ req.Header.Del(HeaderIconCatalog)+ if user != nil && user.Preferences.IconCatalog != nil && *user.Preferences.IconCatalog != "" {+ req.Header.Set(HeaderIconCatalog, *user.Preferences.IconCatalog)+ }+ req.Header.Del(HeaderUpdateInitiatorID)+ req.Header.Del(HeaderUpdateInitiatorName)+ req.Header.Del(HeaderUpdateInitiatorDisplayName)+ isContainerUpdate := req.Method == http.MethodPost && strings.Contains(req.URL.Path, "/containers/") && strings.HasSuffix(req.URL.Path, "/update")+ if isContainerUpdate && user != nil && user.ID != "" {+ req.Header.Set(HeaderUpdateInitiatorID, user.ID)+ req.Header.Set(HeaderUpdateInitiatorName, user.Username)+ if user.DisplayName != nil {+ req.Header.Set(HeaderUpdateInitiatorDisplayName, *user.DisplayName)+ }+ }- apiURL, accessToken, enabled, err := m.resolver(c.Request().Context(), envID)+ apiURL, accessToken, enabled, err := m.resolver(ctx, envID)if err != nil || apiURL == "" {- return c.JSON(http.StatusNotFound, map[string]any{- "success": false,- "data": map[string]any{"error": "Environment not found"},- })+ return writeProxyError(c, http.StatusNotFound, "Environment not found")}-if !enabled {- return c.JSON(http.StatusBadRequest, map[string]any{- "success": false,- "data": map[string]any{"error": errors.New("Environment is disabled").Error()}, //nolint:staticcheck // Preserve the existing error message.- })+ return writeProxyError(c, http.StatusBadRequest, "Environment is disabled")}- // SECURITY: Enforce the caller's per-environment permission BEFORE proxying.- // Remote agents run with a sudo permission set and perform no authorization- // of their own, so authorization for proxied requests must happen here,- // mirroring the per-operation RequirePermission checks used for the local- // environment.- if m.proxyPermissionDenied(c, perms, envID) {- return c.JSON(http.StatusForbidden, map[string]any{- "success": false,- "data": map[string]any{"error": "You don't have permission to perform this action on this environment"},- })+ // SECURITY: Remote agents trust the manager as sudo, so per-environment authorization happens here.+ if m.proxyPermissionDenied(c, perms, envID, suffix) {+ return writeProxyError(c, http.StatusForbidden, "You don't have permission to perform this action on this environment")}- isEdgeEnvironment := isEdgeEnvironmentURLInternal(apiURL)-- if handled, proxyActiveEdgeTunnelErr := m.proxyActiveEdgeTunnelInternal(c, envID, accessToken); handled {- return proxyActiveEdgeTunnelErr+ tunnel, hasTunnel := m.getActiveEdgeTunnel(envID).Get()+ if !hasTunnel && strings.HasPrefix(strings.ToLower(strings.TrimSpace(apiURL)), "edge://") {+ if tunnel, hasTunnel = m.waitForActiveEdgeTunnel(ctx, envID).Get(); !hasTunnel {+ slog.WarnContext(ctx, "No active edge tunnel for environment", "environmentId", envID)+ return writeProxyError(c, http.StatusBadGateway, "Edge agent is not connected")+ }+ slog.InfoContext(ctx, "Recovered edge tunnel during request", "environmentId", envID)}- if isEdgeEnvironment {- if handled, proxyRecoveredEdgeTunnelErr := m.proxyRecoveredEdgeTunnelInternal(c, envID, accessToken); handled {- return proxyRecoveredEdgeTunnelErr+ proxyPath := path.Join(apiEnvironmentsPrefix, m.localID) + suffix+ isWebSocket := httpx.IsWebSocketUpgradeRequest(req)+ if hasTunnel {+ slog.DebugContext(ctx, "Routing request through edge tunnel", "environmentId", envID, "path", req.URL.Path)+ edge.SetAgentToken(req, accessToken)+ if isWebSocket {+ return edge.ProxyWebSocketRequest(c, tunnel, proxyPath, m.checkOrigin)}+ return edge.ProxyHTTPRequest(c, tunnel, proxyPath)+ }- slog.WarnContext(c.Request().Context(), "No active edge tunnel for environment", "environmentId", envID)- return m.abortEdgeTunnelUnavailable(c)+ target := strings.TrimRight(apiURL, "/") + proxyPath+ if req.URL.RawQuery != "" {+ target += "?" + req.URL.RawQuery+ }+ if isWebSocket {+ if proxyErr := ws.ProxyHTTP(c.Response(), req, edge.HTTPToWebSocketURL(target), edge.BuildWebSocketHeaders(c, accessToken), m.checkOrigin); proxyErr != nil {+ slog.ErrorContext(ctx, "websocket proxy failed", "err", proxyErr)+ }+ return nil}- target := m.buildTargetURL(c, envID, apiURL)+ targetURL, err := httpx.ValidateOutboundHTTPURL(target)+ if err != nil {+ return writeProxyError(c, http.StatusInternalServerError, "Invalid proxy target URL: "+err.Error())+ }- if httpx.IsWebSocketUpgradeRequest(c.Request()) {- return m.proxyWebSocket(c, target, accessToken, envID)+ // Stream the body unbuffered (uploads reach gigabytes); ContentLength 0 means no body, -1 means chunked.+ var requestBody io.ReadCloser+ var contentLength int64+ switch {+ case req.Body == nil:+ case req.ContentLength != 0:+ requestBody, contentLength = req.Body, req.ContentLength+ default:+ _ = req.Body.Close()}- return m.proxyHTTP(c, target, accessToken)-}-// setIconCatalogHeaderInternal stamps the authenticated user's icon catalog-// preference onto the outgoing proxied request. Remote agents authenticate the-// proxy as a synthetic user with no preferences, so without this every remote-// environment resolves icons against the default catalog.-//-// SECURITY: the header is always cleared first, so a browser-supplied value-// never rides through; only the server-resolved preference is forwarded.-func (m *EnvironmentMiddleware) setIconCatalogHeaderInternal(c *echo.Context, user *usertypes.Actor) {- c.Request().Header.Del(HeaderIconCatalog)- if user == nil || user.Preferences.IconCatalog == nil || *user.Preferences.IconCatalog == "" {- return- }- c.Request().Header.Set(HeaderIconCatalog, *user.Preferences.IconCatalog)-}+ // The body is never logged: it can carry compose files and registry credentials.+ slog.DebugContext(ctx, "Creating proxy request", "method", req.Method, "target", target, "contentLength", req.ContentLength, "contentType", req.Header.Get("Content-Type"))++ proxyReq := (&http.Request{+ Method: req.Method,+ URL: targetURL,+ Host: targetURL.Host,+ Header: make(http.Header),+ Body: requestBody,+ ContentLength: contentLength,+ }).WithContext(ctx)++ edge.CopyRequestHeaders(req.Header, proxyReq.Header)+ edge.SetAuthHeader(proxyReq, c)+ edge.SetAgentToken(proxyReq, accessToken)+ edge.SetForwardedHeaders(proxyReq, c.RealIP(), req.Host)-func (m *EnvironmentMiddleware) setUpdateInitiatorHeadersInternal(c *echo.Context, user *usertypes.Actor) {- headers := c.Request().Header- headers.Del(HeaderUpdateInitiatorID)- headers.Del(HeaderUpdateInitiatorName)- headers.Del(HeaderUpdateInitiatorDisplayName)- updatePath := strings.Contains(c.Request().URL.Path, "/containers/") && strings.HasSuffix(c.Request().URL.Path, "/update")- if c.Request().Method != http.MethodPost || !updatePath {- return+ resp, err := m.httpClient.Do(proxyReq)+ if err != nil {+ return writeProxyError(c, http.StatusBadGateway, "Proxy request failed: "+err.Error())}- if user == nil || user.ID == "" {- return+ defer func() { _ = resp.Body.Close() }()++ // An upstream 401 means the agent rejected the manager's credentials; a 502 keeps browsers from dropping the session.+ if resp.StatusCode == http.StatusUnauthorized {+ slog.WarnContext(ctx, "Remote environment rejected the manager's credentials", "target", resp.Request.URL.Host)+ return writeProxyError(c, http.StatusBadGateway, "Remote environment rejected the manager's credentials")}- headers.Set(HeaderUpdateInitiatorID, user.ID)- headers.Set(HeaderUpdateInitiatorName, user.Username)- if user.DisplayName != nil {- headers.Set(HeaderUpdateInitiatorDisplayName, *user.DisplayName)+ w := c.Response()+ edge.CopyResponseHeaders(resp.Header, w.Header(), edge.BuildHopByHopHeaders(resp.Header))+ w.WriteHeader(resp.StatusCode)+ if req.Method != http.MethodHead {+ edge.CopyBodyWithFlush(w, resp.Body)}+ return nil}-// proxyPermissionDenied reports whether the caller lacks permission to perform-// the proxied request against environment envID. It mirrors the per-operation-// RequirePermission checks enforced for the local environment: the permission-// required for the (method, path) is looked up in the matcher and evaluated-// against the caller's permission set for the target environment.-//-// Sudo callers bypass the check. Requests whose (method, path) has no known-// permission mapping are denied (default-deny), so a newly added proxied route-// cannot silently bypass authorization before it is mapped.-func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authz.PermissionSet, envID string) bool {- if m.matcher == nil {- return false- }- if ps != nil && ps.Sudo {+// proxyPermissionDenied reports whether the caller lacks permission for the proxied request, mirroring local+// RequirePermission checks. Sudo bypasses it; routes without a permission mapping are denied by default.+func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authz.PermissionSet, envID, suffix string) bool {+ if m.matcher == nil || (ps != nil && ps.Sudo) {return false}- method := c.Request().Method- suffix := m.buildResourceSuffix(c.Request().URL.Path, envID)- if kind, ok := uploadSessionKindInternal(method, suffix); ok {- // Upload-session routes derive their permission from the {kind} path- // segment, so they carry no static mapping in the matcher; resolve the- // kind here and fail closed on unknown kinds.- perm, known := authz.UploadKindPermission(kind)- if !known || !ps.Allows(perm, envID) {- slog.DebugContext(c.Request().Context(), "Denying proxied upload session request: permission denied",- "method", method, "path", suffix, "kind", kind, "environmentId", envID)- return true+ req := c.Request()+ ctx := req.Context()+ method := req.Method+ segments := strings.Split(strings.Trim(suffix, "/"), "/")++ // Upload-session routes derive their permission from the {kind} segment; unknown kinds fail closed.+ if len(segments) >= 2 && segments[0] == "uploads" && segments[1] != "" {+ var isUploadSession bool+ switch len(segments) {+ case 2:+ isUploadSession = method == http.MethodPost+ case 3:+ isUploadSession = method == http.MethodGet || method == http.MethodDelete+ case 5:+ isUploadSession = method == http.MethodPut && segments[3] == "chunks"+ }+ if isUploadSession {+ perm, known := authz.UploadKindPermission(segments[1])+ denied := !known || !ps.Allows(perm, envID)+ if denied {+ slog.DebugContext(ctx, "Denying proxied upload session request: permission denied",+ "method", method, "path", suffix, "kind", segments[1], "environmentId", envID)+ }+ return denied}- return false}+perm, ok := m.matcher.Lookup(method, suffix).Get()if !ok {- slog.WarnContext(c.Request().Context(), "Denying proxied request with no known permission mapping",- "method", method, "path", suffix, "environmentId", envID)+ slog.WarnContext(ctx, "Denying proxied request with no known permission mapping", "method", method, "path", suffix, "environmentId", envID)return true}if perm == "" {- // Explicitly public proxied route (e.g. public settings): allowed for- // any authenticated caller, matching local enforcement.+ // Explicitly public route: allowed for any authenticated caller.return false}-- scopeEnvID := kit.Ternary(authz.IsEnvScoped(perm), envID, "")- if !ps.Allows(perm, scopeEnvID) {- slog.DebugContext(c.Request().Context(), "Denying proxied request: permission denied",- "method", method, "path", suffix, "permission", perm, "environmentId", envID)+ if !ps.Allows(perm, kit.Ternary(authz.IsEnvScoped(perm), envID, "")) {+ slog.DebugContext(ctx, "Denying proxied request: permission denied", "method", method, "path", suffix, "permission", perm, "environmentId", envID)return true}+var required []stringvar err error- if isVolumeWorkspaceUpdateRequestInternal(method, suffix) {- required, err = proxiedVolumeWorkspacePermissionsInternal(c.Request())+ if method == http.MethodPut && len(segments) == 3 && segments[0] == "volumes" && segments[1] != "" && segments[2] == "workspace" {+ required, err = proxiedVolumeWorkspacePermissions(req)} else {- required, err = proxiedGitOpsSyncPermissionsInternal(c.Request(), method, suffix)+ required, err = proxiedGitOpsSyncPermissions(req, method, segments)}if err != nil {- slog.DebugContext(c.Request().Context(), "Denying proxied request with invalid body",- "path", suffix, "environmentId", envID, "error", err)+ slog.DebugContext(ctx, "Denying proxied request with invalid body", "path", suffix, "environmentId", envID, "error", err)return true}- required = withContainerResourceSortPermissionsInternal(required, c.Request(), method, suffix)+ // Resource-sorted container lists collect per-container stats, so they also need containers:read.+ if method == http.MethodGet && len(segments) == 1 && segments[0] == "containers" && container.IsResourceSort(req.URL.Query().Get("sort")) {+ required = append(required, authz.PermContainersRead)+ }for _, operationPermission := range required {if !ps.Allows(operationPermission, envID) {- slog.DebugContext(c.Request().Context(), "Denying proxied request: body-derived permission denied",+ slog.DebugContext(ctx, "Denying proxied request: body-derived permission denied","path", suffix, "permission", operationPermission, "environmentId", envID)return true}@@ -305,35 +333,8 @@ func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authzreturn false}-// uploadSessionKindInternal extracts the upload kind from the exact-// upload-session route shapes: POST /uploads/{kind}, GET or DELETE-// /uploads/{kind}/{uploadId}, and PUT /uploads/{kind}/{uploadId}/chunks/{n}.-func uploadSessionKindInternal(method, suffix string) (string, bool) {- segments := strings.Split(strings.Trim(suffix, "/"), "/")- if len(segments) < 2 || segments[0] != "uploads" || segments[1] == "" {- return "", false- }- switch {- case len(segments) == 2 && method == http.MethodPost:- return segments[1], true- case len(segments) == 3 && (method == http.MethodGet || method == http.MethodDelete):- return segments[1], true- case len(segments) == 5 && method == http.MethodPut && segments[3] == "chunks":- return segments[1], true- }- return "", false-}--func isVolumeWorkspaceUpdateRequestInternal(method, suffix string) bool {- if method != http.MethodPut {- return false- }- segments := strings.Split(strings.Trim(suffix, "/"), "/")- return len(segments) == 3 && segments[0] == "volumes" && segments[1] != "" && segments[2] == "workspace"-}--func proxiedVolumeWorkspacePermissionsInternal(request *http.Request) ([]string, error) {- if request == nil || request.Body == nil {+func proxiedVolumeWorkspacePermissions(request *http.Request) ([]string, error) {+ if request.Body == nil {return nil, errors.New("missing multipart request body")}mediaType, params, err := mime.ParseMediaType(request.Header.Get("Content-Type"))@@ -349,12 +350,7 @@ func proxiedVolumeWorkspacePermissionsInternal(request *http.Request) ([]string,}defer func() {_, _ = captured.Seek(0, io.SeekStart)- request.Body = &proxiedReplayBodyInternal{- Reader: io.MultiReader(captured, originalBody),- captured: captured,- original: originalBody,- path: captured.Name(),- }+ request.Body = &proxiedReplayBody{Reader: io.MultiReader(captured, originalBody), captured: captured, original: originalBody}}()reader := multipart.NewReader(io.TeeReader(originalBody, captured), params["boundary"])@@ -392,17 +388,13 @@ func proxiedVolumeWorkspacePermissionsInternal(request *http.Request) ([]string,}}-// proxiedGitOpsSyncPermissionsInternal mirrors the handler-level lifecycle and-// backup gates for POST /gitops-syncs, POST /gitops-syncs/import, and PUT-// /gitops-syncs/{syncId}. The agent authenticates forwarded requests as sudo,-// so these body-dependent checks must run at the proxy. Returns nil for any-// other route.-func proxiedGitOpsSyncPermissionsInternal(request *http.Request, method, suffix string) ([]string, error) {- segments := strings.Split(strings.Trim(suffix, "/"), "/")+// proxiedGitOpsSyncPermissions mirrors the handler lifecycle and backup gates for gitops sync create, import,+// and update, since the agent trusts forwarded requests as sudo. Returns nil for any other route.+func proxiedGitOpsSyncPermissions(request *http.Request, method string, segments []string) ([]string, error) {isCreate := len(segments) == 1 && segments[0] == "gitops-syncs" && method == http.MethodPostisImport := len(segments) == 2 && segments[0] == "gitops-syncs" && segments[1] == "import" && method == http.MethodPostisUpdate := len(segments) == 2 && segments[0] == "gitops-syncs" && segments[1] != "" && method == http.MethodPut- if !isCreate && !isImport && !isUpdate || request == nil || request.Body == nil {+ if !isCreate && !isImport && !isUpdate || request.Body == nil {return nil, nil}@@ -416,228 +408,87 @@ func proxiedGitOpsSyncPermissionsInternal(request *http.Request, method, suffix}request.Body = io.NopCloser(bytes.NewReader(body))- type syncBodyInternal struct {+ type syncBody struct {gitops.PreDeployConfigRequestMode string `json:"mode"`}- var items []syncBodyInternal+ var items []syncBody+ var decodeErr errorif isImport {- if err := json.Unmarshal(body, &items); err != nil {- return nil, fmt.Errorf("decode gitops sync import body: %w", err)- }+ decodeErr = json.Unmarshal(body, &items)} else {- var single syncBodyInternal- if err := json.Unmarshal(body, &single); err != nil {- return nil, fmt.Errorf("decode gitops sync body: %w", err)- }- items = []syncBodyInternal{single}+ items = make([]syncBody, 1)+ decodeErr = json.Unmarshal(body, &items[0])}-- var required []string- for _, item := range items {- if item.HasPreDeployConfig() && !slices.Contains(required, authz.PermGitOpsLifecycle) {- required = append(required, authz.PermGitOpsLifecycle)- }- if isCreate && item.Mode == gitops.SyncModeBackup && !slices.Contains(required, authz.PermGitOpsBackup) {- required = append(required, authz.PermGitOpsBackup)- }+ if decodeErr != nil {+ return nil, fmt.Errorf("decode gitops sync body: %w", decodeErr)}- return required, nil-}-// withContainerResourceSortPermissionsInternal adds containers:read for-// resource-sorted container lists, which collect per-container stats.-func withContainerResourceSortPermissionsInternal(required []string, request *http.Request, method, suffix string) []string {- if request == nil || method != http.MethodGet || strings.Trim(suffix, "/") != "containers" {- return required+ var required []string+ if slices.ContainsFunc(items, func(item syncBody) bool { return item.HasPreDeployConfig() }) {+ required = append(required, authz.PermGitOpsLifecycle)}- if container.IsResourceSort(request.URL.Query().Get("sort")) {- return append(required, authz.PermContainersRead)+ if isCreate && slices.ContainsFunc(items, func(item syncBody) bool { return item.Mode == gitops.SyncModeBackup }) {+ required = append(required, authz.PermGitOpsBackup)}- return required+ return required, nil}-type proxiedReplayBodyInternal struct {+type proxiedReplayBody struct {io.Readercaptured *os.Fileoriginal io.ReadCloser- path string}-func (b *proxiedReplayBodyInternal) Close() error {+func (b *proxiedReplayBody) Close() error {capturedErr := b.captured.Close()originalErr := b.original.Close()// System temp scratch: no acfs root exists for it.- removeErr := os.Remove(b.path)+ removeErr := os.Remove(b.captured.Name())return errors.Join(capturedErr, originalErr, removeErr)}-func (m *EnvironmentMiddleware) proxyActiveEdgeTunnelInternal(c *echo.Context, envID string, accessToken *string) (bool, error) {- tunnel, ok := m.getActiveEdgeTunnelInternal(envID).Get()- if !ok {- return false, nil- }-- slog.DebugContext(c.Request().Context(), "Routing request through edge tunnel", "environmentId", envID, "path", c.Request().URL.Path)- m.setProxyContextHeadersInternal(c, accessToken)- return true, m.proxyThroughTunnelInternal(c, tunnel, envID)-}--func (m *EnvironmentMiddleware) proxyRecoveredEdgeTunnelInternal(c *echo.Context, envID string, accessToken *string) (bool, error) {- edge.TouchTunnelDemand(envID, edge.DefaultTunnelDemandTTL)-- tunnel, ok := m.waitForActiveEdgeTunnelInternal(c.Request().Context(), envID, edge.DefaultTunnelAcquireTimeout()).Get()- if !ok {- return false, nil- }-- slog.InfoContext(c.Request().Context(), "Recovered edge tunnel during request", "environmentId", envID)- m.setProxyContextHeadersInternal(c, accessToken)- return true, m.proxyThroughTunnelInternal(c, tunnel, envID)-}--func (m *EnvironmentMiddleware) setProxyContextHeadersInternal(c *echo.Context, accessToken *string) {- if accessToken != nil && *accessToken != "" {- c.Request().Header.Set(edge.HeaderAgentToken, *accessToken)- c.Request().Header.Set(edge.HeaderAPIKey, *accessToken)- }-}--func (m *EnvironmentMiddleware) proxyThroughTunnelInternal(c *echo.Context, tunnel *edge.AgentTunnel, envID string) error {- proxyPath := m.buildProxyPath(c, envID)- if httpx.IsWebSocketUpgradeRequest(c.Request()) {- return edge.ProxyWebSocketRequest(c, tunnel, proxyPath, m.checkOrigin)- }- return edge.ProxyHTTPRequest(c, tunnel, proxyPath)-}--// hasResourcePath reports whether the request targets a proxiable resource path.-func (m *EnvironmentMiddleware) hasResourcePath(c *echo.Context, envID string) bool {- suffix, ok := strings.CutPrefix(c.Request().URL.Path, apiEnvironmentsPrefix+envID)- if !ok || len(suffix) <= 1 || suffix[0] != '/' {- return false- }- return !isManagementPathInternal(c.Request().Method, suffix)-}--func isManagementPathInternal(method, suffix string) bool {- if suffix == "/jobs" || strings.HasPrefix(suffix, "/jobs/") {- return true- }- if isCentralSwarmManagementPathInternal(method, suffix) {- return true- }- if isCentralVolumeBackupPolicyPathInternal(method, suffix) {- return true- }- if suffix == "/activities" || strings.HasPrefix(suffix, "/activities/") {+// isManagementPath reports whether an environment-scoped path is served by the manager instead of being proxied.+func isManagementPath(method, suffix string) bool {+ if _, ok := managementEndpointSet[suffix]; ok {return true}-- if strings.HasPrefix(suffix, "/notifications") {- return true+ // Webhooks live in the manager DB and their public trigger endpoint resolves tokens there.+ for _, prefix := range []string{"/jobs", "/activities", "/webhooks"} {+ if suffix == prefix || strings.HasPrefix(suffix, prefix+"/") {+ return true+ }}-- // Webhooks are managed centrally: rows live in the manager DB (keyed by the- // real environment ID) and the public trigger endpoint resolves tokens there.- if suffix == "/webhooks" || strings.HasPrefix(suffix, "/webhooks/") {+ if strings.HasPrefix(suffix, "/notifications") || strings.HasPrefix(suffix, "/deployment/mtls/") {return true}- if strings.HasPrefix(suffix, "/deployment/mtls/") {+ switch method + " " + suffix {+ case "GET /swarm/join-candidates", "GET /swarm/nodes", "POST /swarm/join-environments", "POST /swarm/nodes/agents/reconcile":return true}- _, isManagement := managementEndpointSet[suffix]- return isManagement-}--func isCentralVolumeBackupPolicyPathInternal(method, suffix string) bool {- if method != http.MethodGet && method != http.MethodPut {- return false- }parts := strings.Split(strings.Trim(suffix, "/"), "/")- return len(parts) == 3 && parts[0] == "volumes" && parts[1] != "" && parts[2] == "backup-policy"-}--func isCentralSwarmManagementPathInternal(method, suffix string) bool {- if method == http.MethodGet && (suffix == "/swarm/join-candidates" || suffix == "/swarm/nodes") {- return true- }- if method == http.MethodPost && (suffix == "/swarm/join-environments" || suffix == "/swarm/nodes/agents/reconcile") {- return true+ if len(parts) == 3 && parts[0] == "volumes" && parts[1] != "" && parts[2] == "backup-policy" {+ return method == http.MethodGet || method == http.MethodPut}-- parts := strings.Split(strings.Trim(suffix, "/"), "/")- if len(parts) == 3 && method == http.MethodGet && parts[0] == "swarm" && parts[1] == "nodes" {- return true+ if len(parts) < 3 || parts[0] != "swarm" || parts[1] != "nodes" {+ return false}- if len(parts) == 5 && parts[0] == "swarm" && parts[1] == "nodes" && parts[3] == "agent" {- if parts[4] == "binding" {- return method == http.MethodPut || method == http.MethodDelete- }- if parts[4] == "deployment" {- return method == http.MethodPost || method == http.MethodDelete- }+ switch {+ case len(parts) == 3:+ return method == http.MethodGet+ case len(parts) == 5 && parts[3] == "agent" && parts[4] == "binding":+ return method == http.MethodPut || method == http.MethodDelete+ case len(parts) == 5 && parts[3] == "agent" && parts[4] == "deployment":+ return method == http.MethodPost || method == http.MethodDelete}-return false}-// extractEnvironmentID gets the environment ID from the request.-func (m *EnvironmentMiddleware) extractEnvironmentID(c *echo.Context) string {- requestPath := c.Request().URL.Path-- if !strings.Contains(requestPath, environmentsPathMarker) {- return ""- }-- if envID := c.Param(m.paramName); envID != "" {- return envID- }-- if _, rest, ok := strings.Cut(requestPath, environmentsPathMarker); ok {- if envID, _, _ := strings.Cut(rest, "/"); envID != "" {- return envID- }- }-- return ""-}--// buildResourceSuffix extracts the resource path after stripping the environment ID prefix.-func (m *EnvironmentMiddleware) buildResourceSuffix(requestPath, envID string) string {- suffix, _ := strings.CutPrefix(requestPath, apiEnvironmentsPrefix+envID)- if suffix != "" && suffix[0] != '/' {- suffix = "/" + suffix- }- return suffix-}--// buildTargetURL constructs the full proxy target URL for a remote environment.-func (m *EnvironmentMiddleware) buildTargetURL(c *echo.Context, envID, apiURL string) string {- req := c.Request()- suffix := m.buildResourceSuffix(req.URL.Path, envID)- target := strings.TrimRight(apiURL, "/") + path.Join(apiEnvironmentsPrefix, m.localID) + suffix- if qs := req.URL.RawQuery; qs != "" {- target += "?" + qs- }- return target-}--// buildProxyPath constructs the path sent through the edge tunnel.-func (m *EnvironmentMiddleware) buildProxyPath(c *echo.Context, envID string) string {- return path.Join(apiEnvironmentsPrefix, m.localID) + m.buildResourceSuffix(c.Request().URL.Path, envID)-}--func isEdgeEnvironmentURLInternal(apiURL string) bool {- normalized := strings.ToLower(strings.TrimSpace(apiURL))- return strings.HasPrefix(normalized, "edge://")-}--func (m *EnvironmentMiddleware) getActiveEdgeTunnelInternal(envID string) mo.Option[*edge.AgentTunnel] {+func (m *EnvironmentMiddleware) getActiveEdgeTunnel(envID string) mo.Option[*edge.AgentTunnel] {if m.registry == nil {return mo.None[*edge.AgentTunnel]()}@@ -649,158 +500,31 @@ func (m *EnvironmentMiddleware) getActiveEdgeTunnelInternal(envID string) mo.Optreturn mo.Some(tunnel)}-func (m *EnvironmentMiddleware) waitForActiveEdgeTunnelInternal(ctx context.Context, envID string, timeout time.Duration) mo.Option[*edge.AgentTunnel] {- if timeout <= 0 {- return m.getActiveEdgeTunnelInternal(envID)- }-- if tunnel, ok := m.getActiveEdgeTunnelInternal(envID).Get(); ok {- return mo.Some(tunnel)- }+// waitForActiveEdgeTunnel signals tunnel demand and polls until the edge agent connects or the acquire timeout elapses.+func (m *EnvironmentMiddleware) waitForActiveEdgeTunnel(ctx context.Context, envID string) mo.Option[*edge.AgentTunnel] {+ edge.TouchTunnelDemand(envID, edge.DefaultTunnelDemandTTL)- waitCtx, cancel := context.WithTimeout(ctx, timeout)+ waitCtx, cancel := context.WithTimeout(ctx, edge.DefaultTunnelAcquireTimeout())defer cancel()ticker := time.NewTicker(edge.DefaultTunnelAcquirePollEvery)defer ticker.Stop()for {+ if tunnel := m.getActiveEdgeTunnel(envID); tunnel.IsPresent() {+ return tunnel+ }select {case <-waitCtx.Done():return mo.None[*edge.AgentTunnel]()case <-ticker.C:- if tunnel, ok := m.getActiveEdgeTunnelInternal(envID).Get(); ok {- return mo.Some(tunnel)- }}}}-func (m *EnvironmentMiddleware) abortEdgeTunnelUnavailable(c *echo.Context) error {- return c.JSON(http.StatusBadGateway, map[string]any{+func writeProxyError(c *echo.Context, status int, message string) error {+ return c.JSON(status, map[string]any{"success": false,- "data": map[string]any{- "error": "Edge agent is not connected",- },+ "data": map[string]any{"error": message},})}--// proxyWebSocket handles WebSocket proxy requests.-func (m *EnvironmentMiddleware) proxyWebSocket(c *echo.Context, target string, accessToken *string, envID string) error {- if isEdgeEnvironmentURLInternal(target) {- slog.WarnContext(c.Request().Context(), "Refusing direct websocket proxy to edge environment without active tunnel", "environmentId", envID, "target", target)- return m.abortEdgeTunnelUnavailable(c)- }-- wsTarget := edge.HTTPToWebSocketURL(target)- headers := edge.BuildWebSocketHeaders(c, accessToken)-- if err := ws.ProxyHTTP(c.Response(), c.Request(), wsTarget, headers, m.checkOrigin); err != nil {- slog.ErrorContext(c.Request().Context(), "websocket proxy failed", "err", err)- }- return nil-}--// proxyHTTP handles standard HTTP proxy requests.-func (m *EnvironmentMiddleware) proxyHTTP(c *echo.Context, target string, accessToken *string) error {- if isEdgeEnvironmentURLInternal(target) {- slog.WarnContext(c.Request().Context(), "Refusing direct HTTP proxy to edge environment without active tunnel", "target", target)- return m.abortEdgeTunnelUnavailable(c)- }-- req, err := m.createProxyRequest(c, target, accessToken)- if err != nil {- errMessage := "Failed to create proxy request: " + err.Error()- if errors.Is(err, common.ErrEnvironmentInvalidProxyTarget) {- errMessage = err.Error()- }- return c.JSON(http.StatusInternalServerError, map[string]any{- "success": false,- "data": map[string]any{"error": errMessage},- })- }-- resp, err := m.httpClient.Do(req)- if err != nil {- return c.JSON(http.StatusBadGateway, map[string]any{- "success": false,- "data": map[string]any{"error": "Proxy request failed: " + err.Error()},- })- }- defer func() { _ = resp.Body.Close() }()-- return m.writeProxyResponse(c, resp)-}--// createProxyRequest builds the HTTP request to forward to the remote environment.-func (m *EnvironmentMiddleware) createProxyRequest(c *echo.Context, target string, accessToken *string) (*http.Request, error) {- srcReq := c.Request()- validatedTarget, err := httpx.ValidateOutboundHTTPURL(target)- if err != nil {- return nil, common.Classify(common.ErrEnvironmentInvalidProxyTarget, fmt.Errorf("Invalid proxy target URL: %w", err)) //nolint:staticcheck // Preserve the existing error message.- }-- // The body is streamed straight through rather than buffered: volume backup- // and image import uploads run to gigabytes, and reading them into memory- // cost roughly twice their size per in-flight request. This drops GetBody,- // so the transport can no longer replay the body across a redirect or an- // idle-connection retry; a proxied API call has no legitimate need for- // either, and a failure surfaces as a 502 rather than silent corruption.- //- // A server request's ContentLength is 0 only when there is genuinely no- // body (-1 means chunked/unknown), so it is the safe signal for whether to- // forward one at all.- var requestBody io.ReadCloser- var contentLength int64- switch {- case srcReq.Body == nil:- case srcReq.ContentLength != 0:- requestBody, contentLength = srcReq.Body, srcReq.ContentLength- default:- _ = srcReq.Body.Close()- }-- // The body is deliberately not logged: at debug level it would put compose- // files and registry credentials into the ring buffer served by- // /api/diagnostics/logs, and stringifying it allocated even when debug was off.- slog.DebugContext(srcReq.Context(), "Creating proxy request", "method", srcReq.Method, "target", target, "contentLength", srcReq.ContentLength, "contentType", srcReq.Header.Get("Content-Type"))-- requestURL := *validatedTarget- req := (&http.Request{- Method: srcReq.Method,- URL: &requestURL,- Host: requestURL.Host,- Header: make(http.Header),- Body: requestBody,- ContentLength: contentLength,- }).WithContext(srcReq.Context())-- edge.CopyRequestHeaders(srcReq.Header, req.Header)- edge.SetAuthHeader(req, c)- edge.SetAgentToken(req, accessToken)- edge.SetForwardedHeaders(req, c.RealIP(), srcReq.Host)-- return req, nil-}--// writeProxyResponse copies the proxy response back to the client. An upstream-// 401 means the agent rejected the manager's credentials; passing it through-// makes browser clients treat it as a lost session, so it becomes a 502.-func (m *EnvironmentMiddleware) writeProxyResponse(c *echo.Context, resp *http.Response) error {- if resp.StatusCode == http.StatusUnauthorized {- slog.WarnContext(c.Request().Context(), "Remote environment rejected the manager's credentials", "target", resp.Request.URL.Host)- return c.JSON(http.StatusBadGateway, map[string]any{- "success": false,- "data": map[string]any{"error": "Remote environment rejected the manager's credentials"},- })- }- w := c.Response()- hopByHop := edge.BuildHopByHopHeaders(resp.Header)- edge.CopyResponseHeaders(resp.Header, w.Header(), hopByHop)-- w.WriteHeader(resp.StatusCode)- if c.Request().Method != http.MethodHead {- edge.CopyBodyWithFlush(w, resp.Body)- }- return nil-}diff --git a/backend/internal/middleware/environment_test.go b/backend/internal/middleware/environment_test.goindex 36030b77b..d7148f89e 100644--- a/backend/internal/middleware/environment_test.go+++ b/backend/internal/middleware/environment_test.go@@ -277,42 +277,21 @@ func TestEnvironmentMiddleware_LocalEnvironmentSkipsProxyPermissionCheck(t *testassert.True(t, localHandlerHit, "local environment request must reach the local handler, not be proxy-authorized")}-func TestEnvironmentMiddleware_ProxyWebSocketRejectsEdgeTargetsWithoutTunnel(t *testing.T) {+func TestEnvironmentMiddleware_RejectsInvalidProxyTarget(t *testing.T) {middleware := newTestEnvironmentMiddleware()- e := echo.New()- recorder := httptest.NewRecorder()- req := httptest.NewRequest(http.MethodGet, "/api/environments/env-edge/ws/system/stats", http.NoBody)- c := e.NewContext(req, recorder)-- _ = middleware.proxyWebSocket(c, "edge://oracle-1/api/environments/0/ws/system/stats", nil, "env-edge")-- assert.Equal(t, http.StatusBadGateway, recorder.Code)- assert.Contains(t, recorder.Body.String(), "Edge agent is not connected")-}--func TestEnvironmentMiddleware_ProxyHTTPRejectsEdgeTargetsWithoutTunnel(t *testing.T) {- middleware := newTestEnvironmentMiddleware()- e := echo.New()- recorder := httptest.NewRecorder()- req := httptest.NewRequest(http.MethodGet, "/api/environments/env-edge/containers", http.NoBody)- c := e.NewContext(req, recorder)-- _ = middleware.proxyHTTP(c, "edge://oracle-1/api/environments/0/containers", nil)-- assert.Equal(t, http.StatusBadGateway, recorder.Code)- assert.Contains(t, recorder.Body.String(), "Edge agent is not connected")-}+ middleware.resolver = func(ctx context.Context, id string) (string, *string, bool, error) {+ _, _ = ctx, id+ return "ftp://example.com", nil, true, nil+ }+ router := echo.New()+ attachMiddleware(router, middleware)-func TestEnvironmentMiddleware_CreateProxyRequest_RejectsInvalidProxyTarget(t *testing.T) {- middleware := newTestEnvironmentMiddleware()- e := echo.New()+ req := httptest.NewRequest(http.MethodGet, "/api/environments/env-remote/containers", http.NoBody)recorder := httptest.NewRecorder()- req := httptest.NewRequest(http.MethodGet, "/api/environments/env-edge/containers", http.NoBody)- c := e.NewContext(req, recorder)+ router.ServeHTTP(recorder, req)- _, err := middleware.createProxyRequest(c, "ftp://example.com/containers", nil)- require.Error(t, err)- assert.Contains(t, err.Error(), "Invalid proxy target URL")+ assert.Equal(t, http.StatusInternalServerError, recorder.Code)+ assert.Contains(t, recorder.Body.String(), "Invalid proxy target URL")}func TestEnvironmentMiddleware_KeepsNodeAgentDeploymentCreationLocal(t *testing.T) {@@ -404,21 +383,21 @@ func TestEnvironmentMiddleware_ForwardsResolvedIconCatalogHeaderOnly(t *testing.}}-func TestIsCentralSwarmManagementPathInternal_IsMethodAware(t *testing.T) {- assert.True(t, isCentralSwarmManagementPathInternal(http.MethodGet, "/swarm/nodes"))- assert.True(t, isCentralSwarmManagementPathInternal(http.MethodGet, "/swarm/nodes/node-1"))- assert.False(t, isCentralSwarmManagementPathInternal(http.MethodPatch, "/swarm/nodes/node-1"))- assert.True(t, isCentralSwarmManagementPathInternal(http.MethodPost, "/swarm/nodes/node-1/agent/deployment"))- assert.True(t, isCentralSwarmManagementPathInternal(http.MethodDelete, "/swarm/nodes/node-1/agent/deployment"))- assert.True(t, isCentralSwarmManagementPathInternal(http.MethodPut, "/swarm/nodes/node-1/agent/binding"))- assert.True(t, isCentralSwarmManagementPathInternal(http.MethodDelete, "/swarm/nodes/node-1/agent/binding"))+func TestIsManagementPath_SwarmIsMethodAware(t *testing.T) {+ assert.True(t, isManagementPath(http.MethodGet, "/swarm/nodes"))+ assert.True(t, isManagementPath(http.MethodGet, "/swarm/nodes/node-1"))+ assert.False(t, isManagementPath(http.MethodPatch, "/swarm/nodes/node-1"))+ assert.True(t, isManagementPath(http.MethodPost, "/swarm/nodes/node-1/agent/deployment"))+ assert.True(t, isManagementPath(http.MethodDelete, "/swarm/nodes/node-1/agent/deployment"))+ assert.True(t, isManagementPath(http.MethodPut, "/swarm/nodes/node-1/agent/binding"))+ assert.True(t, isManagementPath(http.MethodDelete, "/swarm/nodes/node-1/agent/binding"))}-func TestIsCentralVolumeBackupPolicyPathInternal_IsMethodAware(t *testing.T) {- assert.True(t, isCentralVolumeBackupPolicyPathInternal(http.MethodGet, "/volumes/app-data/backup-policy"))- assert.True(t, isCentralVolumeBackupPolicyPathInternal(http.MethodPut, "/volumes/app-data/backup-policy"))- assert.False(t, isCentralVolumeBackupPolicyPathInternal(http.MethodPost, "/volumes/app-data/backup-policy"))- assert.False(t, isCentralVolumeBackupPolicyPathInternal(http.MethodGet, "/volumes/app-data/backups"))+func TestIsManagementPath_VolumeBackupPolicyIsMethodAware(t *testing.T) {+ assert.True(t, isManagementPath(http.MethodGet, "/volumes/app-data/backup-policy"))+ assert.True(t, isManagementPath(http.MethodPut, "/volumes/app-data/backup-policy"))+ assert.False(t, isManagementPath(http.MethodPost, "/volumes/app-data/backup-policy"))+ assert.False(t, isManagementPath(http.MethodGet, "/volumes/app-data/backups"))}const proxyTestEnvID = "remote-1"@@ -448,7 +427,7 @@ func TestProxyPermissionDeniedBlocksWriteForReadOnlyUser(t *testing.T) {c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),+ require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers/abc/restart"),"expected restart to be denied for a read-only user")}@@ -459,7 +438,7 @@ func TestProxyPermissionDeniedAllowsWriteForPermittedUser(t *testing.T) {c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),+ require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers/abc/restart"),"expected restart to be allowed for a user with containers:restart")}@@ -470,7 +449,7 @@ func TestProxyPermissionDeniedAllowsRead(t *testing.T) {c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers")- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),+ require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers"),"expected list to be allowed for a user with containers:list")}@@ -482,7 +461,7 @@ func TestProxyPermissionDeniedDeniesPermissionFromDifferentEnv(t *testing.T) {c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),+ require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers/abc/restart"),"expected denial: permission is scoped to a different environment")}@@ -490,7 +469,7 @@ func TestProxyPermissionDeniedSudoBypasses(t *testing.T) {m := newProxyAuthzMiddleware(containerMatcher())c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")- require.False(t, m.proxyPermissionDenied(c, authz.SudoPermissionSet(), proxyTestEnvID),+ require.False(t, m.proxyPermissionDenied(c, authz.SudoPermissionSet(), proxyTestEnvID, "/containers/abc/restart"),"expected sudo permission set to bypass the permission check")}@@ -501,7 +480,7 @@ func TestProxyPermissionDeniedDefaultDeniesUnmappedRoute(t *testing.T) {c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/unknown/resource")- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),+ require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/unknown/resource"),"expected an unmapped proxied route to be denied by default")}@@ -511,16 +490,10 @@ func TestProxyPermissionDeniedAllowsPublicRoute(t *testing.T) {c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/settings/public")- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),+ require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/settings/public"),"expected an explicitly public route to be allowed for any authenticated caller")}-func volumeWorkspaceMatcher() *authz.PermissionMatcher {- m := authz.NewPermissionMatcher()- m.Add(http.MethodPut, "/volumes/{volumeName}/workspace", authz.PermVolumesRead)- return m-}-func newProxyVolumeWorkspaceContext(t *testing.T, changes []volume.WorkspaceFileChange, fileFirst ...bool) (*echo.Context, []byte) {t.Helper()var body bytes.Buffer@@ -548,40 +521,36 @@ func newProxyVolumeWorkspaceContext(t *testing.T, changes []volume.WorkspaceFile}func TestProxyPermissionDeniedChecksVolumeWorkspaceManifestPermissions(t *testing.T) {- m := newProxyAuthzMiddleware(volumeWorkspaceMatcher())+ matcher := authz.NewPermissionMatcher()+ matcher.Add(http.MethodPut, "/volumes/{volumeName}/workspace", authz.PermVolumesRead)+ m := newProxyAuthzMiddleware(matcher)changes := []volume.WorkspaceFileChange{{Operation: volume.FileOpRename}}readOnly := authz.NewPermissionSet()readOnly.AddEnv(proxyTestEnvID, authz.PermVolumesRead)c, _ := newProxyVolumeWorkspaceContext(t, changes)- require.True(t, m.proxyPermissionDenied(c, readOnly, proxyTestEnvID))+ require.True(t, m.proxyPermissionDenied(c, readOnly, proxyTestEnvID, "/volumes/data/workspace"))uploadOnly := authz.NewPermissionSet()uploadOnly.AddEnv(proxyTestEnvID, authz.PermVolumesRead, authz.PermVolumesUpload)c, _ = newProxyVolumeWorkspaceContext(t, changes)- require.True(t, m.proxyPermissionDenied(c, uploadOnly, proxyTestEnvID))+ require.True(t, m.proxyPermissionDenied(c, uploadOnly, proxyTestEnvID, "/volumes/data/workspace"))permitted := authz.NewPermissionSet()permitted.AddEnv(proxyTestEnvID, authz.PermVolumesRead, authz.PermVolumesUpload, authz.PermVolumesDelete)c, rawBody := newProxyVolumeWorkspaceContext(t, changes)- require.False(t, m.proxyPermissionDenied(c, permitted, proxyTestEnvID))+ require.False(t, m.proxyPermissionDenied(c, permitted, proxyTestEnvID, "/volumes/data/workspace"))replayed, err := io.ReadAll(c.Request().Body)require.NoError(t, err)require.Equal(t, rawBody, replayed)c, rawBody = newProxyVolumeWorkspaceContext(t, changes, true)- require.False(t, m.proxyPermissionDenied(c, permitted, proxyTestEnvID))+ require.False(t, m.proxyPermissionDenied(c, permitted, proxyTestEnvID, "/volumes/data/workspace"))replayed, err = io.ReadAll(c.Request().Body)require.NoError(t, err)require.Equal(t, rawBody, replayed)}-func gitOpsSyncMatcher() *authz.PermissionMatcher {- m := authz.NewPermissionMatcher()- m.Add(http.MethodPost, "/gitops-syncs/import", authz.PermGitOpsCreate)- return m-}-func newProxyGitOpsImportContext(body string) *echo.Context {e := echo.New()req := httptest.NewRequest(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/gitops-syncs/import", bytes.NewReader([]byte(body)))@@ -590,35 +559,30 @@ func newProxyGitOpsImportContext(body string) *echo.Context {}func TestProxyPermissionDeniedChecksGitOpsImportLifecyclePermission(t *testing.T) {- m := newProxyAuthzMiddleware(gitOpsSyncMatcher())+ matcher := authz.NewPermissionMatcher()+ matcher.Add(http.MethodPost, "/gitops-syncs/import", authz.PermGitOpsCreate)+ m := newProxyAuthzMiddleware(matcher)hookBody := `[{"syncName":"a","gitRepo":"r","branch":"main","dockerComposePath":"compose.yaml","preDeployScriptPath":"scripts/run.sh"}]`plainBody := `[{"syncName":"a","gitRepo":"r","branch":"main","dockerComposePath":"compose.yaml"}]`createOnly := authz.NewPermissionSet()createOnly.AddEnv(proxyTestEnvID, authz.PermGitOpsCreate)- require.True(t, m.proxyPermissionDenied(newProxyGitOpsImportContext(hookBody), createOnly, proxyTestEnvID))- require.False(t, m.proxyPermissionDenied(newProxyGitOpsImportContext(plainBody), createOnly, proxyTestEnvID))+ require.True(t, m.proxyPermissionDenied(newProxyGitOpsImportContext(hookBody), createOnly, proxyTestEnvID, "/gitops-syncs/import"))+ require.False(t, m.proxyPermissionDenied(newProxyGitOpsImportContext(plainBody), createOnly, proxyTestEnvID, "/gitops-syncs/import"))withLifecycle := authz.NewPermissionSet()withLifecycle.AddEnv(proxyTestEnvID, authz.PermGitOpsCreate, authz.PermGitOpsLifecycle)c := newProxyGitOpsImportContext(hookBody)- require.False(t, m.proxyPermissionDenied(c, withLifecycle, proxyTestEnvID))+ require.False(t, m.proxyPermissionDenied(c, withLifecycle, proxyTestEnvID, "/gitops-syncs/import"))replayed, err := io.ReadAll(c.Request().Body)require.NoError(t, err)require.Equal(t, hookBody, string(replayed))}-// wsTerminalMatcher mirrors ws.AddProxiedPermissions for the container terminal-// stream: the proxy computes the suffix "/ws/containers/{id}/terminal" for a-// forwarded WebSocket request, and the matcher requires containers:exec for it.-func wsTerminalMatcher() *authz.PermissionMatcher {- m := authz.NewPermissionMatcher()- m.Add(http.MethodGet, "/ws/containers/{containerId}/terminal", authz.PermContainersExec)- return m-}-func TestProxyPermissionDeniedWSTerminalRequiresExec(t *testing.T) {- m := newProxyAuthzMiddleware(wsTerminalMatcher())+ matcher := authz.NewPermissionMatcher()+ matcher.Add(http.MethodGet, "/ws/containers/{containerId}/terminal", authz.PermContainersExec)+ m := newProxyAuthzMiddleware(matcher)// A caller who can read and list containers but lacks containers:exec must// not be able to open a terminal stream on the remote environment.@@ -627,13 +591,13 @@ func TestProxyPermissionDeniedWSTerminalRequiresExec(t *testing.T) {c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/ws/containers/abc/terminal")- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),+ require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/ws/containers/abc/terminal"),"expected WS terminal to be denied without containers:exec")// Granting containers:exec allows the same stream.ps.AddEnv(proxyTestEnvID, authz.PermContainersExec)- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID),+ require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/ws/containers/abc/terminal"),"expected WS terminal to be allowed with containers:exec")}@@ -645,21 +609,21 @@ func TestProxyPermissionDeniedResourceSortRequiresRead(t *testing.T) {for _, sort := range []string{"cpuUsage", "memoryUsage"} {c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers?sort="+sort)- require.True(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID),+ require.True(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID, "/containers"),"expected resource sort %q to be denied with only containers:list", sort)}c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers?sort=name")- require.False(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID),+ require.False(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID, "/containers"),"expected non-resource sorts to stay allowed with containers:list")c = newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers")- require.False(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID),+ require.False(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID, "/containers"),"expected the plain list to stay allowed with containers:list")withRead := authz.NewPermissionSet()withRead.AddEnv(proxyTestEnvID, authz.PermContainersList, authz.PermContainersRead)c = newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers?sort=memoryUsage&order=desc")- require.False(t, m.proxyPermissionDenied(c, withRead, proxyTestEnvID),+ require.False(t, m.proxyPermissionDenied(c, withRead, proxyTestEnvID, "/containers"),"expected resource sort to be allowed with containers:list and containers:read")}diff --git a/backend/internal/middleware/telemetry.go b/backend/internal/middleware/telemetry.gonew file mode 100644index 000000000..efd9e05ca--- /dev/null+++ b/backend/internal/middleware/telemetry.go@@ -0,0 +1,66 @@+package middleware++import (+ "context"+ "net/http"+ "strings"++ usertypes "github.com/getarcaneapp/arcane/types/v2/user"+ "github.com/labstack/echo/v5"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric"+ "go.opentelemetry.io/otel/trace"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"+)++var authenticatedRequests = func() metric.Int64Counter {+ counter, err := otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.auth.requests",+ metric.WithDescription("Authenticated API requests by credential type"),+ metric.WithUnit("{request}"),+ )+ if err != nil {+ otel.Handle(err)+ }+ return counter+}()++// RecordAuthenticatedRequest tags the request span with the acting user and+// counts the request by credential type.+func RecordAuthenticatedRequest(ctx context.Context, header http.Header, user *usertypes.Actor) {+ method := "session"+ switch {+ case header.Get(HeaderApiKey) != "":+ method = "api_key"+ case strings.HasPrefix(header.Get("Authorization"), "Bearer "):+ method = "bearer"+ }+ trace.SpanFromContext(ctx).SetAttributes(+ attribute.String("enduser.id", user.ID),+ attribute.String("user.name", user.Username),+ attribute.String("arcane.auth.method", method),+ )+ authenticatedRequests.Add(ctx, 1, metric.WithAttributes(attribute.String("arcane.auth.method", method)))+}++// SpanEnvironment tags the request span with the routed environment's ID, and with its name once the request+// succeeds, so unauthenticated or unknown-environment requests never trigger a lookup.+func SpanEnvironment(environmentName func(ctx context.Context, id string) string) echo.MiddlewareFunc {+ return func(next echo.HandlerFunc) echo.HandlerFunc {+ return func(c *echo.Context) error {+ ctx := c.Request().Context()+ span := trace.SpanFromContext(ctx)+ id := c.Param("id")+ if id == "" || !strings.HasPrefix(c.Path(), "/api/environments/:id") || !span.IsRecording() {+ return next(c)+ }+ span.SetAttributes(attribute.String("arcane.environment.id", id))+ err := next(c)+ if resp, unwrapErr := echo.UnwrapResponse(c.Response()); err == nil && unwrapErr == nil && resp.Status < http.StatusBadRequest {+ span.SetAttributes(attribute.String("arcane.environment.name", environmentName(ctx, id)))+ }+ return err+ }+ }+}diff --git a/backend/internal/notification/handler.go b/backend/internal/notification/handler.goindex 0ffacc4b2..8ed2fe903 100644--- a/backend/internal/notification/handler.go+++ b/backend/internal/notification/handler.go@@ -71,7 +71,7 @@ func normalizeNotificationTestType(testType string) string {}func isSupportedNotificationTestType(testType string) bool {- _, ok := supportedNotificationTestTypes[testType]+ _, ok := notificationTestEventTypes[testType]return ok}diff --git a/backend/internal/notification/service.go b/backend/internal/notification/service.goindex 4f137db25..ead660179 100644--- a/backend/internal/notification/service.go+++ b/backend/internal/notification/service.go@@ -20,6 +20,11 @@ import ("github.com/getarcaneapp/arcane/types/v2/system""go.getarcane.app/kit/pkg""go.getarcane.app/sys/crypto"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/codes"+ "go.opentelemetry.io/otel/metric"+ "go.opentelemetry.io/otel/trace""golang.org/x/sync/errgroup""github.com/getarcaneapp/arcane/backend/v2/internal/apns"@@ -34,10 +39,11 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation")-var notificationCredentialFieldsByProviderInternal = map[notifications.NotificationProvider][]string{+var notificationCredentialFieldsByProvider = map[notifications.NotificationProvider][]string{notifications.NotificationProviderDiscord: {"token"},notifications.NotificationProviderEmail: {"smtpPassword"},notifications.NotificationProviderTelegram: {"botToken"},@@ -47,12 +53,11 @@ var notificationCredentialFieldsByProviderInternal = map[notifications.Notificatnotifications.NotificationProviderPushover: {"token"},notifications.NotificationProviderGotify: {"token"},notifications.NotificationProviderMatrix: {"password"},- // The Google Chat incoming webhook URL embeds the key and token query- // parameters, so the whole URL is treated as a credential.+ // The Google Chat webhook URL embeds its key and token, so the whole URL is a credential.notifications.NotificationProviderGoogleChat: {"webhookUrl"},}-var notificationTargetFieldByProviderInternal = map[notifications.NotificationProvider]string{+var notificationTargetFieldByProvider = map[notifications.NotificationProvider]string{notifications.NotificationProviderEmail: "smtpHost",notifications.NotificationProviderSignal: "host",notifications.NotificationProviderNtfy: "host",@@ -61,7 +66,7 @@ var notificationTargetFieldByProviderInternal = map[notifications.NotificationPr}const (- notificationDispatchConcurrencyInternal = 4+ notificationDispatchConcurrency = 4notificationTestTypeSimple = "simple"notificationTestTypeImageUpdate = "image-update"@@ -71,11 +76,30 @@ const (notificationTestTypeAutoHeal = "auto-heal")+// notificationTestEventTypes lists supported test types and the event a real notification of that kind is gated on ("" = none).+var notificationTestEventTypes = map[string]notifications.NotificationEventType{+ notificationTestTypeSimple: "",+ notificationTestTypeImageUpdate: notifications.NotificationEventImageUpdate,+ notificationTestTypeBatchImageUpdate: notifications.NotificationEventImageUpdate,+ notificationTestTypeVulnerability: notifications.NotificationEventVulnerabilityFound,+ notificationTestTypePruneReport: notifications.NotificationEventPruneReport,+ notificationTestTypeAutoHeal: notifications.NotificationEventAutoHeal,+}+var (ErrUnauthorizedNotificationDispatch = errors.New("unauthorized notification dispatch")ErrUnsupportedDispatchKind = errors.New("unsupported notification dispatch kind"))+var notificationsSent = func() metric.Int64Counter {+ counter, err := otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.notifications.sent",+ metric.WithDescription("Notification provider sends by outcome"), metric.WithUnit("{notification}"))+ if err != nil {+ otel.Handle(err)+ }+ return counter+}()+type NotificationService struct {db *database.DBconfig *config.Config@@ -91,7 +115,7 @@ type NotificationTarget struct {EnvironmentName string}-func logManagerDispatchNotificationInternal(ctx context.Context, target NotificationTarget, kind notification.DispatchKind) {+func logManagerDispatchNotification(ctx context.Context, target NotificationTarget, kind notification.DispatchKind) {slog.InfoContext(ctx,"Manager dispatching notification on behalf of agent","environmentId", target.EnvironmentID,@@ -100,10 +124,6 @@ func logManagerDispatchNotificationInternal(ctx context.Context, target Notifica)}-func (s *NotificationService) ResolveNotificationTarget(ctx context.Context, environmentID string) (NotificationTarget, error) {- return s.resolveNotificationTargetInternal(ctx, environmentID)-}-func NewNotificationService(db *database.DB, cfg *config.Config, environmentSvc *environment.EnvironmentService, eventSvc *event.EventService, apnsSvc *apns.ApnsService) *NotificationService {return &NotificationService{db: db,@@ -116,19 +136,15 @@ func NewNotificationService(db *database.DB, cfg *config.Config, environmentSvc}}-func (s *NotificationService) resolveNotificationTargetInternal(ctx context.Context, environmentID string) (NotificationTarget, error) {+func (s *NotificationService) ResolveNotificationTarget(ctx context.Context, environmentID string) (NotificationTarget, error) {trimmedEnvironmentID := cmp.Or(strings.TrimSpace(environmentID), "0")if s.environmentSvc != nil {env, err := s.environmentSvc.GetEnvironmentByID(ctx, trimmedEnvironmentID)if err == nil && env != nil {- environmentName := strings.TrimSpace(env.Name)- if environmentName == "" && trimmedEnvironmentID == "0" {- environmentName = "Local Docker"- }return NotificationTarget{EnvironmentID: env.ID,- EnvironmentName: environmentName,+ EnvironmentName: cmp.Or(strings.TrimSpace(env.Name), kit.Ternary(env.ID == "0", "Local Docker", "")),}, nil}if trimmedEnvironmentID != "0" {@@ -148,31 +164,7 @@ func (s *NotificationService) resolveNotificationTargetInternal(ctx context.Cont}, nil}-func (s *NotificationService) resolveNotificationTargetForAccessTokenInternal(ctx context.Context, accessToken string) (NotificationTarget, error) {- if s.environmentSvc == nil {- return NotificationTarget{}, errors.New("environment service not initialized")- }-- env, err := s.environmentSvc.ResolveEnvironmentByAccessToken(ctx, accessToken)- if err != nil {- if errors.Is(err, environment.ErrEnvironmentAccessTokenRequired) || errors.Is(err, environment.ErrInvalidEnvironmentAccessToken) {- return NotificationTarget{}, ErrUnauthorizedNotificationDispatch- }- return NotificationTarget{}, err- }-- environmentName := strings.TrimSpace(env.Name)- if environmentName == "" && env.ID == "0" {- environmentName = "Local Docker"- }-- return NotificationTarget{- EnvironmentID: env.ID,- EnvironmentName: environmentName,- }, nil-}--func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.Context, payload notification.DispatchRequest) (notification.DispatchResponse, error) {+func (s *NotificationService) dispatchNotificationToManager(ctx context.Context, payload notification.DispatchRequest) (notification.DispatchResponse, error) {body, err := json.Marshal(payload)if err != nil {return notification.DispatchResponse{}, fmt.Errorf("failed to marshal notification dispatch payload: %w", err)@@ -186,9 +178,7 @@ func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.})}- // Tunnel-connected edge agents typically have neither MANAGER_API_URL nor- // AGENT_TOKEN configured for HTTP; ride the agent-to-manager event channel- // instead so their notifications are not silently lost (#3002).+ // Tunnel-connected edge agents often lack MANAGER_API_URL/AGENT_TOKEN, so they dispatch over the tunnel (#3002).if s.config == nil || strings.TrimSpace(httpx.ManagerBaseURL(s.config.ManagerApiUrl)) == "" || strings.TrimSpace(s.config.AgentToken) == "" {if publishViaTunnelErr := publishViaTunnel(); publishViaTunnelErr != nil {return notification.DispatchResponse{}, fmt.Errorf("notification dispatch needs either MANAGER_API_URL + AGENT_TOKEN or a connected edge tunnel: %w", publishViaTunnelErr)@@ -206,8 +196,7 @@ func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.resp, err := s.httpClient.Do(req)if err != nil {- // HTTP unreachable does not mean the manager is: an agent configured for- // HTTP may still hold a healthy edge tunnel, so ride that before giving up.+ // An agent configured for HTTP may still hold a healthy edge tunnel, so try it before giving up.if tunnelErr := publishViaTunnel(); tunnelErr == nil {return notification.DispatchResponse{Message: "notification dispatched via edge tunnel"}, nil}@@ -225,10 +214,7 @@ func (s *NotificationService) dispatchNotificationToManagerInternal(ctx context.return apiResponse.Data, nil}- // A non-2xx also means the notification was not dispatched, and the HTTP- // path can fail independently of the tunnel (stale AGENT_TOKEN, wrong- // MANAGER_API_URL, intermediate proxy), so the tunnel fallback applies here- // too before surfacing the status error.+ // A non-2xx (stale token, wrong URL, proxy) can fail independently of the tunnel, so fall back here too.responseBody, _ := io.ReadAll(resp.Body)if tunnelErr := publishViaTunnel(); tunnelErr == nil {return notification.DispatchResponse{Message: "notification dispatched via edge tunnel"}, nil@@ -240,32 +226,40 @@ func (s *NotificationService) DispatchNotification(ctx context.Context, accessToif s.config != nil && s.config.AgentMode {return notification.DispatchResponse{}, errors.New("notification dispatch is manager-only")}+ if s.environmentSvc == nil {+ return notification.DispatchResponse{}, errors.New("environment service not initialized")+ }- target, err := s.resolveNotificationTargetForAccessTokenInternal(ctx, accessToken)+ env, err := s.environmentSvc.ResolveEnvironmentByAccessToken(ctx, accessToken)if err != nil {+ if errors.Is(err, environment.ErrEnvironmentAccessTokenRequired) || errors.Is(err, environment.ErrInvalidEnvironmentAccessToken) {+ return notification.DispatchResponse{}, ErrUnauthorizedNotificationDispatch+ }return notification.DispatchResponse{}, err}- return s.dispatchForTargetInternal(ctx, target, payload)+ target := NotificationTarget{+ EnvironmentID: env.ID,+ EnvironmentName: cmp.Or(strings.TrimSpace(env.Name), kit.Ternary(env.ID == "0", "Local Docker", "")),+ }+ return s.dispatchForTarget(ctx, target, payload)}-// DispatchNotificationForEnvironment dispatches on behalf of an agent whose-// identity was already established by its edge tunnel session, so no access-// token is exchanged (#3002).+// DispatchNotificationForEnvironment dispatches for an agent already identified by its edge tunnel session (#3002).func (s *NotificationService) DispatchNotificationForEnvironment(ctx context.Context, environmentID string, payload notification.DispatchRequest) (notification.DispatchResponse, error) {if s.config != nil && s.config.AgentMode {return notification.DispatchResponse{}, errors.New("notification dispatch is manager-only")}- target, err := s.resolveNotificationTargetInternal(ctx, environmentID)+ target, err := s.ResolveNotificationTarget(ctx, environmentID)if err != nil {return notification.DispatchResponse{}, err}- return s.dispatchForTargetInternal(ctx, target, payload)+ return s.dispatchForTarget(ctx, target, payload)}-func (s *NotificationService) dispatchForTargetInternal(ctx context.Context, target NotificationTarget, payload notification.DispatchRequest) (notification.DispatchResponse, error) {+func (s *NotificationService) dispatchForTarget(ctx context.Context, target NotificationTarget, payload notification.DispatchRequest) (notification.DispatchResponse, error) {var err errordispatchResponse := notification.DispatchResponse{Message: "Notification dispatched successfully"}switch payload.Kind {@@ -273,40 +267,29 @@ func (s *NotificationService) dispatchForTargetInternal(ctx context.Context, tarif payload.ImageUpdate == nil {return notification.DispatchResponse{}, errors.New("image update payload is required")}- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)- dispatchResponse.Delivered, err = s.sendImageUpdateNotificationForTargetInternal(- ctx,- target,- payload.ImageUpdate.ImageRef,- &payload.ImageUpdate.UpdateInfo,- notifications.NotificationEventImageUpdate,- )+ logManagerDispatchNotification(ctx, target, payload.Kind)+ imageUpdate := payload.ImageUpdate+ dispatchResponse.Delivered, err = s.sendImageUpdateNotificationForTarget(ctx, target, imageUpdate.ImageRef, &imageUpdate.UpdateInfo, notifications.NotificationEventImageUpdate)return dispatchResponse, errcase notification.DispatchKindBatchImageUpdate:if payload.BatchImageUpdate == nil {return notification.DispatchResponse{}, errors.New("batch image update payload is required")}- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)- dispatchResponse.Delivered, err = s.sendBatchImageUpdateNotificationForTargetInternal(ctx, target, payload.BatchImageUpdate.Updates)+ logManagerDispatchNotification(ctx, target, payload.Kind)+ dispatchResponse.Delivered, err = s.sendBatchImageUpdateNotificationForTarget(ctx, target, payload.BatchImageUpdate.Updates)return dispatchResponse, errcase notification.DispatchKindContainerUpdate:if payload.ContainerUpdate == nil {return notification.DispatchResponse{}, errors.New("container update payload is required")}- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)- return dispatchResponse, s.sendContainerUpdateNotificationForTargetInternal(- ctx,- target,- payload.ContainerUpdate.ContainerName,- payload.ContainerUpdate.ImageRef,- payload.ContainerUpdate.OldDigest,- payload.ContainerUpdate.NewDigest,- )+ logManagerDispatchNotification(ctx, target, payload.Kind)+ update := payload.ContainerUpdate+ return dispatchResponse, s.sendContainerUpdateNotificationForTarget(ctx, target, update.ContainerName, update.ImageRef, update.OldDigest, update.NewDigest)case notification.DispatchKindBatchContainerUpdate:if payload.BatchContainerUpdate == nil {return notification.DispatchResponse{}, errors.New("batch container update payload is required")}- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)+ logManagerDispatchNotification(ctx, target, payload.Kind)entries := make([]notifications.ContainerUpdateBatchEntry, 0, len(payload.BatchContainerUpdate.Updates))for _, update := range payload.BatchContainerUpdate.Updates {entries = append(entries, notifications.ContainerUpdateBatchEntry{@@ -316,33 +299,25 @@ func (s *NotificationService) dispatchForTargetInternal(ctx context.Context, tarNewDigest: update.NewDigest,})}- return dispatchResponse, s.sendBatchContainerUpdateNotificationForTargetInternal(ctx, target, entries)+ return dispatchResponse, s.sendBatchContainerUpdateNotificationForTarget(ctx, target, entries)case notification.DispatchKindVulnerabilityFound:if payload.VulnerabilityFound == nil {return notification.DispatchResponse{}, errors.New("vulnerability payload is required")}- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)- return dispatchResponse, s.sendVulnerabilityNotificationForTargetInternal(ctx, target, VulnerabilityNotificationPayload{- CVEID: payload.VulnerabilityFound.CVEID,- CVELink: payload.VulnerabilityFound.CVELink,- Severity: payload.VulnerabilityFound.Severity,- ImageName: payload.VulnerabilityFound.ImageName,- FixedVersion: payload.VulnerabilityFound.FixedVersion,- PkgName: payload.VulnerabilityFound.PkgName,- InstalledVersion: payload.VulnerabilityFound.InstalledVersion,- })+ logManagerDispatchNotification(ctx, target, payload.Kind)+ return dispatchResponse, s.sendVulnerabilityNotificationForTarget(ctx, target, VulnerabilityNotificationPayload(*payload.VulnerabilityFound))case notification.DispatchKindPruneReport:if payload.PruneReport == nil {return notification.DispatchResponse{}, errors.New("prune report payload is required")}- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)- return dispatchResponse, s.sendPruneReportNotificationForTargetInternal(ctx, target, &payload.PruneReport.Result)+ logManagerDispatchNotification(ctx, target, payload.Kind)+ return dispatchResponse, s.sendPruneReportNotificationForTarget(ctx, target, &payload.PruneReport.Result)case notification.DispatchKindAutoHeal:if payload.AutoHeal == nil {return notification.DispatchResponse{}, errors.New("auto-heal payload is required")}- logManagerDispatchNotificationInternal(ctx, target, payload.Kind)- return dispatchResponse, s.sendAutoHealNotificationForTargetInternal(ctx, target, payload.AutoHeal.ContainerName, payload.AutoHeal.ContainerID)+ logManagerDispatchNotification(ctx, target, payload.Kind)+ return dispatchResponse, s.sendAutoHealNotificationForTarget(ctx, target, payload.AutoHeal.ContainerName, payload.AutoHeal.ContainerID)default:return notification.DispatchResponse{}, fmt.Errorf("%s: %w", payload.Kind, ErrUnsupportedDispatchKind)}@@ -383,7 +358,7 @@ func (s *NotificationService) CreateOrUpdateSettings(ctx context.Context, providerr := s.db.WithContext(ctx).Where("provider = ?", provider).First(&setting).ErrorexistingConfig := kit.Ternary(err == nil, setting.Config, database.JSON(nil))- encryptedConfig, encryptErr := encryptNotificationConfigCredentialsInternal(provider, localConfig, existingConfig)+ encryptedConfig, encryptErr := encryptNotificationConfigCredentials(provider, localConfig, existingConfig)if encryptErr != nil {return nil, encryptErr}@@ -416,8 +391,8 @@ func (s *NotificationService) CreateOrUpdateSettings(ctx context.Context, provid// RedactNotificationConfigCredentials returns a copy of config with provider credential fields blanked for API responses.func RedactNotificationConfigCredentials(provider notifications.NotificationProvider, localConfig database.JSON) database.JSON {- redacted := cloneNotificationConfigInternal(localConfig)- for _, field := range notificationCredentialFieldsByProviderInternal[provider] {+ redacted := cloneNotificationConfig(localConfig)+ for _, field := range notificationCredentialFieldsByProvider[provider] {value, ok := redacted[field]if !ok {continue@@ -431,16 +406,31 @@ func RedactNotificationConfigCredentials(provider notifications.NotificationProvreturn redacted}-func encryptNotificationConfigCredentialsInternal(provider notifications.NotificationProvider, localConfig, existingConfig database.JSON) (database.JSON, error) {- encryptedConfig := cloneNotificationConfigInternal(localConfig)- preserveConfig := existingConfig- if provider == notifications.NotificationProviderSignal {- preserveConfig = signalCredentialPreservationConfigInternal(localConfig, existingConfig)- }- if provider == notifications.NotificationProviderEmail {- preserveConfig = emailCredentialPreservationConfigInternal(localConfig, existingConfig)+// encryptNotificationConfigCredentials encrypts credential fields, preserving stored ones left blank.+// Kept separate from CreateOrUpdateSettings to stay under the gocognit limit.+func encryptNotificationConfigCredentials(provider notifications.NotificationProvider, localConfig, existingConfig database.JSON) (database.JSON, error) {+ encryptedConfig := cloneNotificationConfig(localConfig)++ // Switching auth mode drops the stored credential of the other mode.+ preserveConfig := cloneNotificationConfig(existingConfig)+ switch provider {+ case notifications.NotificationProviderSignal:+ user, _ := localConfig["user"].(string)+ password, _ := localConfig["password"].(string)+ token, _ := localConfig["token"].(string)+ if strings.TrimSpace(token) != "" {+ delete(preserveConfig, "password")+ }+ if strings.TrimSpace(user) != "" || strings.TrimSpace(password) != "" {+ delete(preserveConfig, "token")+ }+ case notifications.NotificationProviderEmail:+ if authMode, _ := localConfig["authMode"].(string); authMode == string(notifications.EmailAuthModeNone) {+ delete(preserveConfig, "smtpPassword")+ }}- if targetField := notificationTargetFieldByProviderInternal[provider]; targetField != "" {++ if targetField := notificationTargetFieldByProvider[provider]; targetField != "" {currentTarget, _ := existingConfig[targetField].(string)nextTarget, _ := encryptedConfig[targetField].(string)if strings.TrimSpace(nextTarget) == "" && strings.TrimSpace(currentTarget) != "" {@@ -448,9 +438,9 @@ func encryptNotificationConfigCredentialsInternal(provider notifications.NotificencryptedConfig[targetField] = currentTarget}- storedCredentials := make(map[string]bool, len(notificationCredentialFieldsByProviderInternal[provider]))- updatedCredentials := make(map[string]bool, len(notificationCredentialFieldsByProviderInternal[provider]))- for _, field := range notificationCredentialFieldsByProviderInternal[provider] {+ storedCredentials := make(map[string]bool, len(notificationCredentialFieldsByProvider[provider]))+ updatedCredentials := make(map[string]bool, len(notificationCredentialFieldsByProvider[provider]))+ for _, field := range notificationCredentialFieldsByProvider[provider] {preservedValue, _ := preserveConfig[field].(string)updatedValue, _ := encryptedConfig[field].(string)storedCredentials[field] = preservedValue != ""@@ -470,7 +460,8 @@ func encryptNotificationConfigCredentialsInternal(provider notifications.Notificreturn nil, err}}- for _, field := range notificationCredentialFieldsByProviderInternal[provider] {++ for _, field := range notificationCredentialFieldsByProvider[provider] {value, _ := encryptedConfig[field].(string)if value == "" {if existingValue, ok := preserveConfig[field].(string); ok && existingValue != "" {@@ -479,7 +470,11 @@ func encryptNotificationConfigCredentialsInternal(provider notifications.Notificcontinue}- encrypted, err := encryptNotificationCredentialInternal(value)+ // Already-encrypted values are kept as is.+ if _, decryptErr := crypto.Decrypt(value); decryptErr == nil {+ continue+ }+ encrypted, err := crypto.Encrypt(value)if err != nil {return nil, fmt.Errorf("failed to encrypt notification credential %q: %w", field, err)}@@ -488,41 +483,8 @@ func encryptNotificationConfigCredentialsInternal(provider notifications.Notificreturn encryptedConfig, nil}-func signalCredentialPreservationConfigInternal(localConfig, existingConfig database.JSON) database.JSON {- preserveConfig := cloneNotificationConfigInternal(existingConfig)- user, _ := localConfig["user"].(string)- password, _ := localConfig["password"].(string)- token, _ := localConfig["token"].(string)-- if strings.TrimSpace(token) != "" {- delete(preserveConfig, "password")- }- if strings.TrimSpace(user) != "" || strings.TrimSpace(password) != "" {- delete(preserveConfig, "token")- }-- return preserveConfig-}--func emailCredentialPreservationConfigInternal(localConfig, existingConfig database.JSON) database.JSON {- preserveConfig := cloneNotificationConfigInternal(existingConfig)- if authMode, _ := localConfig["authMode"].(string); authMode == string(notifications.EmailAuthModeNone) {- delete(preserveConfig, "smtpPassword")- }- return preserveConfig-}--func encryptNotificationCredentialInternal(value string) (string, error) {- if value == "" {- return "", nil- }- if _, err := crypto.Decrypt(value); err == nil {- return value, nil- }- return crypto.Encrypt(value)-}--func cloneNotificationConfigInternal(localConfig database.JSON) database.JSON {+// cloneNotificationConfig copies config, returning an empty non-nil map for nil.+func cloneNotificationConfig(localConfig database.JSON) database.JSON {if localConfig == nil {return database.JSON{}}@@ -538,92 +500,57 @@ func (s *NotificationService) DeleteSettings(ctx context.Context, provider notifreturn nil}+// isEventEnabled defaults to enabled when the config has no entry for eventType.func (s *NotificationService) isEventEnabled(localConfig database.JSON, eventType notifications.NotificationEventType) bool {events, ok := localConfig["events"].(map[string]any)if !ok {- return true // If no events config, default to enabled+ return true}-enabled, ok := events[string(eventType)].(bool)- if !ok {- return true // If event type not specified, default to enabled- }-- return enabled+ return !ok || enabled}-// logNotificationInternal records a delivery attempt in the event log so sends and-// failures are visible alongside every other Arcane event.-func (- s *NotificationService,-) logNotificationInternal(+// send delivers content to one provider inside a notification.send span and counts the outcome.+func (s *NotificationService) send(ctx context.Context,- environmentID string,- provider notifications.NotificationProvider,- subject, status string,- errMsg *string,- metadata database.JSON,-) {- if s.eventSvc == nil {- return- }-- severity := event.EventSeveritySuccess- title := fmt.Sprintf("Notification sent via %s", provider)- description := subject- if errMsg != nil {- severity = event.EventSeverityError- title = fmt.Sprintf("Notification failed via %s", provider)- description = fmt.Sprintf("%s: %s", subject, *errMsg)- }-- eventMetadata := cloneNotificationConfigInternal(metadata)- eventMetadata["provider"] = string(provider)- eventMetadata["status"] = status-- resourceType := "notification"- providerName := string(provider)- req := event.CreateEventRequest{- Type: event.EventTypeNotificationSend,- Severity: severity,- Title: title,- Description: description,- ResourceType: &resourceType,- ResourceName: &providerName,- EnvironmentID: &environmentID,- Metadata: eventMetadata,- }-- done := make(chan struct{})- go func() {- defer close(done)- defer utils.RecoverToError(nil, "notification event logging", "provider", providerName)- logCtx, cancelLog := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)- defer cancelLog()- if _, err := s.eventSvc.CreateEvent(logCtx, req); err != nil {- slog.WarnContext(logCtx, "Failed to log notification event", "provider", providerName, "error", err.Error())+ target NotificationTarget,+ eventType notifications.NotificationEventType,+ setting *NotificationSettings,+ content notifications.Content,+) (handled bool, err error) {+ providerAttr := attribute.String("arcane.notification.provider", string(setting.Provider))+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "notification.send", trace.WithAttributes(+ providerAttr,+ attribute.String("arcane.notification.kind", string(eventType)),+ attribute.String("arcane.environment.id", target.EnvironmentID),+ attribute.Int64("arcane.notification.setting_id", int64(setting.ID)),+ ))+ defer func() {+ // Delivery errors can embed webhook URLs whose paths hold secrets, so spans record only the error type.+ if err != nil {+ span.SetStatus(codes.Error, "notification delivery failed")+ span.SetAttributes(attribute.String("error.type", fmt.Sprintf("%T", err)))}+ span.End()}()- // Cancellation ends the caller's wait, not persistence of a completed attempt.- select {- case <-done:- case <-ctx.Done():+ handled, err = notifications.Deliver(ctx, setting.Provider, setting.Config, content)+ if handled {+ outcomeAttr := attribute.String("arcane.notification.outcome", kit.Ternary(err == nil, "success", "failure"))+ notificationsSent.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes(providerAttr, outcomeAttr))}+ return handled, err}-// notifyEnabledProvidersInternal is the single fan-out loop behind every-// notification event: it walks all provider settings, skips disabled providers-// and providers with the event unsubscribed, dispatches to the rest, logs each-// attempt, and aggregates send errors. It returns how many providers the-// notification was actually delivered to.-func (s *NotificationService) notifyEnabledProvidersInternal(+// notifyEnabledProviders sends content to every enabled provider subscribed to eventType, logs each attempt,+// and returns how many providers it was delivered to along with the aggregated send errors.+func (s *NotificationService) notifyEnabledProviders(ctx context.Context,target NotificationTarget,eventType notifications.NotificationEventType,logRef string,metadata database.JSON,- dispatch func(ctx context.Context, provider notifications.NotificationProvider, config database.JSON) (handled bool, err error),+ content notifications.Content,) (int, error) {settings, err := s.GetAllSettings(ctx)if err != nil {@@ -640,12 +567,12 @@ func (s *NotificationService) notifyEnabledProvidersInternal(handled := make([]bool, len(eligible))sendErrs := make([]error, len(eligible))var g errgroup.Group- g.SetLimit(notificationDispatchConcurrencyInternal)- for i, setting := range eligible {+ g.SetLimit(notificationDispatchConcurrency)+ for i := range eligible {g.Go(func() error {- defer utils.RecoverToError(&sendErrs[i], "notification dispatch", "provider", setting.Provider)+ defer utils.RecoverToError(&sendErrs[i], "notification dispatch", "provider", eligible[i].Provider)handled[i] = true- handled[i], sendErrs[i] = dispatch(ctx, setting.Provider, setting.Config)+ handled[i], sendErrs[i] = s.send(ctx, target, eventType, &eligible[i], content)return nil})}@@ -660,11 +587,49 @@ func (s *NotificationService) notifyEnabledProvidersInternal(slog.WarnContext(ctx, "Unknown notification provider", "provider", setting.Provider)continue}++ providerName := string(setting.Provider)+ req := event.CreateEventRequest{+ Type: event.EventTypeNotificationSend,+ Severity: event.EventSeveritySuccess,+ Title: "Notification sent via " + providerName,+ Description: logRef,+ ResourceType: new("notification"),+ ResourceName: &providerName,+ EnvironmentID: &target.EnvironmentID,+ Metadata: cloneNotificationConfig(metadata),+ }+ req.Metadata["provider"] = providerName+ req.Metadata["status"] = "success"if sendErrs[i] == nil {delivered+++ } else {+ msg := sendErrs[i].Error()+ errs = append(errs, fmt.Sprintf("%s: %s", providerName, msg))+ req.Severity = event.EventSeverityError+ req.Title = "Notification failed via " + providerName+ req.Description = fmt.Sprintf("%s: %s", logRef, msg)+ req.Metadata["status"] = "failed"+ }++ if s.eventSvc == nil {+ continue+ }+ done := make(chan struct{})+ go func() {+ defer close(done)+ defer utils.RecoverToError(nil, "notification event logging", "provider", providerName)+ logCtx, cancelLog := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)+ defer cancelLog()+ if _, createErr := s.eventSvc.CreateEvent(logCtx, req); createErr != nil {+ slog.WarnContext(logCtx, "Failed to log notification event", "provider", providerName, "error", createErr.Error())+ }+ }()+ // Cancellation ends the caller's wait, not persistence of a completed attempt.+ select {+ case <-done:+ case <-ctx.Done():}- status, errMsg := collectNotificationSendResultInternal(&errs, setting.Provider, sendErrs[i])- s.logNotificationInternal(ctx, target.EnvironmentID, setting.Provider, logRef, status, errMsg, metadata)}if s.apnsSvc != nil {@@ -679,25 +644,6 @@ func (s *NotificationService) notifyEnabledProvidersInternal(return delivered, nil}-func collectNotificationSendResultInternal(localErrors *[]string, provider notifications.NotificationProvider, sendErr error) (string, *string) {- if sendErr == nil {- return "success", nil- }-- msg := sendErr.Error()- *localErrors = append(*localErrors, fmt.Sprintf("%s: %s", provider, msg))- return "failed", &msg-}--var supportedNotificationTestTypes = map[string]struct{}{- notificationTestTypeSimple: {},- notificationTestTypeImageUpdate: {},- notificationTestTypeBatchImageUpdate: {},- notificationTestTypeVulnerability: {},- notificationTestTypePruneReport: {},- notificationTestTypeAutoHeal: {},-}-// VulnerabilityNotificationPayload is the data sent to all providers for vulnerability_found events.// Only vulnerabilities with a fixed version should trigger this notification.type VulnerabilityNotificationPayload struct {@@ -710,8 +656,9 @@ type VulnerabilityNotificationPayload struct {InstalledVersion string // optional}-// SendBatchContainerUpdateNotification dispatches a single batched-// notification covering all containers updated in one auto-update run.+// --- Event entry points ---++// SendBatchContainerUpdateNotification dispatches one notification covering all containers updated in an auto-update run.func (s *NotificationService) SendBatchContainerUpdateNotification(ctx context.Context, entries []notifications.ContainerUpdateBatchEntry) error {if len(entries) == 0 {return nil@@ -727,22 +674,22 @@ func (s *NotificationService) SendBatchContainerUpdateNotification(ctx context.CNewDigest: entry.NewDigest,})}- _, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{+ _, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{Kind: notification.DispatchKindBatchContainerUpdate,BatchContainerUpdate: ¬ification.DispatchBatchContainerUpdate{Updates: updates},})return err}- target, err := s.resolveNotificationTargetInternal(ctx, "")+ target, err := s.ResolveNotificationTarget(ctx, "")if err != nil {return err}- return s.sendBatchContainerUpdateNotificationForTargetInternal(ctx, target, entries)+ return s.sendBatchContainerUpdateNotificationForTarget(ctx, target, entries)}-func (s *NotificationService) sendBatchContainerUpdateNotificationForTargetInternal(ctx context.Context, target NotificationTarget, entries []notifications.ContainerUpdateBatchEntry) error {+func (s *NotificationService) sendBatchContainerUpdateNotificationForTarget(ctx context.Context, target NotificationTarget, entries []notifications.ContainerUpdateBatchEntry) error {containerNames := make([]string, 0, len(entries))for _, entry := range entries {containerNames = append(containerNames, entry.ContainerName)@@ -756,34 +703,20 @@ func (s *NotificationService) sendBatchContainerUpdateNotificationForTargetInter}content := s.templates.BatchContainerUpdate(target.EnvironmentName, entries)content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventContainerUpdate)- _, err := s.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventContainerUpdate, strings.Join(- containerNames,- ", ",- ), metadata, func(- ctx context.Context,- provider notifications.NotificationProvider,- config database.JSON,- ) (- bool,- error,- ) {- return notifications.Deliver(ctx, provider, config, content)- })+ logRef := strings.Join(containerNames, ", ")+ _, err := s.notifyEnabledProviders(ctx, target, notifications.NotificationEventContainerUpdate, logRef, metadata, content)return err}-// --- Event entry points -----// SendImageUpdateNotification dispatches a single-image update notification and-// returns the number of eligible providers it was delivered to (0 means no-// provider has this event enabled, so callers must not mark the update notified).+// SendImageUpdateNotification returns how many providers the notification was delivered to.+// Zero means no provider has this event enabled, so callers must not mark the update notified.func (s *NotificationService) SendImageUpdateNotification(ctx context.Context, imageRef string, updateInfo *imageupdate.Response, eventType notifications.NotificationEventType) (int, error) {if updateInfo == nil {return 0, errors.New("updateInfo is required")}if s.config != nil && s.config.AgentMode {- dispatchResponse, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{+ dispatchResponse, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{Kind: notification.DispatchKindImageUpdate,ImageUpdate: ¬ification.DispatchImageUpdate{ImageRef: imageRef,@@ -796,26 +729,21 @@ func (s *NotificationService) SendImageUpdateNotification(ctx context.Context, ireturn dispatchResponse.Delivered, nil}- target, err := s.resolveNotificationTargetInternal(ctx, "")+ target, err := s.ResolveNotificationTarget(ctx, "")if err != nil {return 0, err}- return s.sendImageUpdateNotificationForTargetInternal(ctx, target, imageRef, updateInfo, eventType)+ return s.sendImageUpdateNotificationForTarget(ctx, target, imageRef, updateInfo, eventType)}-func (- s *NotificationService,-) sendImageUpdateNotificationForTargetInternal(+func (s *NotificationService) sendImageUpdateNotificationForTarget(ctx context.Context,target NotificationTarget,imageRef string,updateInfo *imageupdate.Response,eventType notifications.NotificationEventType,-) (- int,- error,-) {+) (int, error) {metadata := database.JSON{"hasUpdate": updateInfo.HasUpdate,"currentDigest": updateInfo.CurrentDigest,@@ -825,14 +753,12 @@ func (}content := s.templates.ImageUpdate(target.EnvironmentName, imageRef, updateInfo)content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, eventType)- return s.notifyEnabledProvidersInternal(ctx, target, eventType, imageRef, metadata, func(ctx context.Context, provider notifications.NotificationProvider, config database.JSON) (bool, error) {- return notifications.Deliver(ctx, provider, config, content)- })+ return s.notifyEnabledProviders(ctx, target, eventType, imageRef, metadata, content)}func (s *NotificationService) SendContainerUpdateNotification(ctx context.Context, containerName, imageRef, oldDigest, newDigest string) error {if s.config != nil && s.config.AgentMode {- _, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{+ _, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{Kind: notification.DispatchKindContainerUpdate,ContainerUpdate: ¬ification.DispatchContainerUpdate{ContainerName: containerName,@@ -844,15 +770,15 @@ func (s *NotificationService) SendContainerUpdateNotification(ctx context.Contexreturn err}- target, err := s.resolveNotificationTargetInternal(ctx, "")+ target, err := s.ResolveNotificationTarget(ctx, "")if err != nil {return err}- return s.sendContainerUpdateNotificationForTargetInternal(ctx, target, containerName, imageRef, oldDigest, newDigest)+ return s.sendContainerUpdateNotificationForTarget(ctx, target, containerName, imageRef, oldDigest, newDigest)}-func (s *NotificationService) sendContainerUpdateNotificationForTargetInternal(ctx context.Context, target NotificationTarget, containerName, imageRef, oldDigest, newDigest string) error {+func (s *NotificationService) sendContainerUpdateNotificationForTarget(ctx context.Context, target NotificationTarget, containerName, imageRef, oldDigest, newDigest string) error {metadata := database.JSON{"containerName": containerName,"oldDigest": oldDigest,@@ -861,33 +787,20 @@ func (s *NotificationService) sendContainerUpdateNotificationForTargetInternal(c}content := s.templates.ContainerUpdate(target.EnvironmentName, containerName, imageRef, oldDigest, newDigest)content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventContainerUpdate)- _, err := s.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventContainerUpdate, imageRef, metadata, func(- ctx context.Context,- provider notifications.NotificationProvider,- config database.JSON,- ) (- bool,- error,- ) {- return notifications.Deliver(ctx, provider, config, content)- })+ _, err := s.notifyEnabledProviders(ctx, target, notifications.NotificationEventContainerUpdate, imageRef, metadata, content)return err}-func isVulnerabilitySummaryPayload(payload VulnerabilityNotificationPayload) bool {- return strings.HasPrefix(strings.ToUpper(strings.TrimSpace(payload.CVEID)), "DAILY SUMMARY")-}--// SendVulnerabilityNotification notifies all enabled providers that have vulnerability_found event enabled.+// SendVulnerabilityNotification notifies providers subscribed to vulnerability_found events.// Only daily summary payloads are sent; legacy per-CVE payloads are ignored.func (s *NotificationService) SendVulnerabilityNotification(ctx context.Context, payload VulnerabilityNotificationPayload) error {- if !isVulnerabilitySummaryPayload(payload) {+ if !strings.HasPrefix(strings.ToUpper(strings.TrimSpace(payload.CVEID)), "DAILY SUMMARY") {slog.InfoContext(ctx, "skipping legacy individual vulnerability notification payload", "cve", payload.CVEID)return nil}if s.config != nil && s.config.AgentMode {- _, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{+ _, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{Kind: notification.DispatchKindVulnerabilityFound,VulnerabilityFound: ¬ification.DispatchVulnerabilityFound{CVEID: payload.CVEID,@@ -902,15 +815,15 @@ func (s *NotificationService) SendVulnerabilityNotification(ctx context.Context,return err}- target, err := s.resolveNotificationTargetInternal(ctx, "")+ target, err := s.ResolveNotificationTarget(ctx, "")if err != nil {return err}- return s.sendVulnerabilityNotificationForTargetInternal(ctx, target, payload)+ return s.sendVulnerabilityNotificationForTarget(ctx, target, payload)}-func (s *NotificationService) sendVulnerabilityNotificationForTargetInternal(ctx context.Context, target NotificationTarget, payload VulnerabilityNotificationPayload) error {+func (s *NotificationService) sendVulnerabilityNotificationForTarget(ctx context.Context, target NotificationTarget, payload VulnerabilityNotificationPayload) error {metadata := database.JSON{"cveId": payload.CVEID,"severity": payload.Severity,@@ -919,30 +832,21 @@ func (s *NotificationService) sendVulnerabilityNotificationForTargetInternal(ctx}content := s.templates.Vulnerability(target.EnvironmentName, notification.DispatchVulnerabilityFound(payload))content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventVulnerabilityFound)- _, err := s.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventVulnerabilityFound, payload.ImageName, metadata, func(- ctx context.Context,- provider notifications.NotificationProvider,- config database.JSON,- ) (- bool,- error,- ) {- return notifications.Deliver(ctx, provider, config, content)- })+ _, err := s.notifyEnabledProviders(ctx, target, notifications.NotificationEventVulnerabilityFound, payload.ImageName, metadata, content)return err}-// SendBatchImageUpdateNotification dispatches a batched image-update notification-// and returns the number of eligible providers it was delivered to (0 means no-// provider has this event enabled, so callers must not mark the updates notified).+// SendBatchImageUpdateNotification returns how many providers the notification was delivered to.+// Zero means no provider has this event enabled, so callers must not mark the updates notified.func (s *NotificationService) SendBatchImageUpdateNotification(ctx context.Context, updates map[string]*imageupdate.Response) (int, error) {- updatesWithChanges := filterUpdatesWithChangesInternal(updates)+ updatesWithChanges := maps.Clone(updates)+ maps.DeleteFunc(updatesWithChanges, func(_ string, update *imageupdate.Response) bool { return update == nil || !update.HasUpdate })if len(updatesWithChanges) == 0 {return 0, nil}if s.config != nil && s.config.AgentMode {- dispatchResponse, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{+ dispatchResponse, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{Kind: notification.DispatchKindBatchImageUpdate,BatchImageUpdate: ¬ification.DispatchBatchImageUpdate{Updates: updatesWithChanges,@@ -954,33 +858,21 @@ func (s *NotificationService) SendBatchImageUpdateNotification(ctx context.Contereturn dispatchResponse.Delivered, nil}- target, err := s.resolveNotificationTargetInternal(ctx, "")+ target, err := s.ResolveNotificationTarget(ctx, "")if err != nil {return 0, err}- return s.sendBatchImageUpdateNotificationForTargetInternal(ctx, target, updatesWithChanges)+ return s.sendBatchImageUpdateNotificationForTarget(ctx, target, updatesWithChanges)}-func filterUpdatesWithChangesInternal(updates map[string]*imageupdate.Response) map[string]*imageupdate.Response {- updatesWithChanges := make(map[string]*imageupdate.Response, len(updates))- for imageRef, update := range updates {- if update != nil && update.HasUpdate {- updatesWithChanges[imageRef] = update- }- }- return updatesWithChanges-}--func (s *NotificationService) sendBatchImageUpdateNotificationForTargetInternal(ctx context.Context, target NotificationTarget, updates map[string]*imageupdate.Response) (int, error) {- updatesWithChanges := filterUpdatesWithChangesInternal(updates)-+func (s *NotificationService) sendBatchImageUpdateNotificationForTarget(ctx context.Context, target NotificationTarget, updates map[string]*imageupdate.Response) (int, error) {+ updatesWithChanges := maps.Clone(updates)+ maps.DeleteFunc(updatesWithChanges, func(_ string, update *imageupdate.Response) bool { return update == nil || !update.HasUpdate })if len(updatesWithChanges) == 0 {return 0, nil}- imageRefs := slices.Collect(maps.Keys(updatesWithChanges))-metadata := database.JSON{"updateCount": len(updatesWithChanges),"eventType": string(notifications.NotificationEventImageUpdate),@@ -988,19 +880,8 @@ func (s *NotificationService) sendBatchImageUpdateNotificationForTargetInternal(}content := s.templates.BatchImageUpdate(target.EnvironmentName, updatesWithChanges)content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventImageUpdate)- return s.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventImageUpdate, strings.Join(- imageRefs,- ", ",- ), metadata, func(- ctx context.Context,- provider notifications.NotificationProvider,- config database.JSON,- ) (- bool,- error,- ) {- return notifications.Deliver(ctx, provider, config, content)- })+ logRef := strings.Join(slices.Collect(maps.Keys(updatesWithChanges)), ", ")+ return s.notifyEnabledProviders(ctx, target, notifications.NotificationEventImageUpdate, logRef, metadata, content)}func (s *NotificationService) SendPruneReportNotification(ctx context.Context, result *system.PruneAllResult) error {@@ -1009,15 +890,22 @@ func (s *NotificationService) SendPruneReportNotification(ctx context.Context, rreturn nil}- hasChanges := pruneResultHasChangesInternal(result)+ hasChanges := result.SpaceReclaimed > 0 ||+ len(result.ContainersPruned) > 0 ||+ len(result.ImagesDeleted) > 0 ||+ len(result.VolumesDeleted) > 0 ||+ len(result.NetworksDeleted) > 0hasErrors := len(result.Errors) > 0if !hasChanges && !hasErrors {slog.InfoContext(ctx, "skipping prune report notification because no resources were pruned and no errors were reported")return nil}+ if !hasChanges {+ slog.WarnContext(ctx, "sending prune report notification with errors but no resources were pruned", "errorCount", len(result.Errors))+ }if s.config != nil && s.config.AgentMode {- _, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{+ _, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{Kind: notification.DispatchKindPruneReport,PruneReport: ¬ification.DispatchPruneReport{Result: *result,@@ -1026,63 +914,29 @@ func (s *NotificationService) SendPruneReportNotification(ctx context.Context, rreturn err}- target, err := s.resolveNotificationTargetInternal(ctx, "")+ target, err := s.ResolveNotificationTarget(ctx, "")if err != nil {return err}- return s.sendPruneReportNotificationForTargetInternal(ctx, target, result)+ return s.sendPruneReportNotificationForTarget(ctx, target, result)}-func (s *NotificationService) sendPruneReportNotificationForTargetInternal(ctx context.Context, target NotificationTarget, result *system.PruneAllResult) error {- hasChanges := pruneResultHasChangesInternal(result)- hasErrors := len(result.Errors) > 0-+func (s *NotificationService) sendPruneReportNotificationForTarget(ctx context.Context, target NotificationTarget, result *system.PruneAllResult) error {metadata := database.JSON{"spaceReclaimed": result.SpaceReclaimed,"eventType": string(notifications.NotificationEventPruneReport),}content := s.templates.PruneReport(target.EnvironmentName, result)content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventPruneReport)- _, err := s.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventPruneReport, "System Prune Report", metadata, func(- ctx context.Context,- provider notifications.NotificationProvider,- config database.JSON,- ) (- bool,- error,- ) {- return notifications.Deliver(ctx, provider, config, content)- })- if err != nil {- return err- }- if hasErrors && !hasChanges {- slog.WarnContext(ctx, "sending prune report notification with errors but no resources were pruned", "errorCount", len(result.Errors))- }-- return nil-}--func pruneResultHasChangesInternal(result *system.PruneAllResult) bool {- if result == nil {- return false- }-- if result.SpaceReclaimed > 0 {- return true- }-- return len(result.ContainersPruned) > 0 ||- len(result.ImagesDeleted) > 0 ||- len(result.VolumesDeleted) > 0 ||- len(result.NetworksDeleted) > 0+ _, err := s.notifyEnabledProviders(ctx, target, notifications.NotificationEventPruneReport, "System Prune Report", metadata, content)+ return err}// SendAutoHealNotification sends a notification when a container is auto-healed.func (s *NotificationService) SendAutoHealNotification(ctx context.Context, containerName, containerID string) error {if s.config != nil && s.config.AgentMode {- _, err := s.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{+ _, err := s.dispatchNotificationToManager(ctx, notification.DispatchRequest{Kind: notification.DispatchKindAutoHeal,AutoHeal: ¬ification.DispatchAutoHeal{ContainerName: containerName,@@ -1092,86 +946,73 @@ func (s *NotificationService) SendAutoHealNotification(ctx context.Context, contreturn err}- target, err := s.resolveNotificationTargetInternal(ctx, "")+ target, err := s.ResolveNotificationTarget(ctx, "")if err != nil {return err}- return s.sendAutoHealNotificationForTargetInternal(ctx, target, containerName, containerID)+ return s.sendAutoHealNotificationForTarget(ctx, target, containerName, containerID)}-func (s *NotificationService) sendAutoHealNotificationForTargetInternal(ctx context.Context, target NotificationTarget, containerName, containerID string) error {+func (s *NotificationService) sendAutoHealNotificationForTarget(ctx context.Context, target NotificationTarget, containerName, containerID string) error {metadata := database.JSON{"containerID": containerID,"eventType": string(notifications.NotificationEventAutoHeal),}content := s.templates.AutoHeal(target.EnvironmentName, containerName)content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, notifications.NotificationEventAutoHeal)- _, err := s.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventAutoHeal, containerName, metadata, func(- ctx context.Context,- provider notifications.NotificationProvider,- config database.JSON,- ) (- bool,- error,- ) {- return notifications.Deliver(ctx, provider, config, content)- })+ _, err := s.notifyEnabledProviders(ctx, target, notifications.NotificationEventAutoHeal, containerName, metadata, content)return err}// --- Test notifications ----// notificationEventTypeForTestTypeInternal maps a test type to the event type a-// real notification of that kind would be gated on ("" = no event gate).-func notificationEventTypeForTestTypeInternal(testType string) notifications.NotificationEventType {- switch testType {- case notificationTestTypeImageUpdate, notificationTestTypeBatchImageUpdate:- return notifications.NotificationEventImageUpdate- case notificationTestTypeVulnerability:- return notifications.NotificationEventVulnerabilityFound- case notificationTestTypePruneReport:- return notifications.NotificationEventPruneReport- case notificationTestTypeAutoHeal:- return notifications.NotificationEventAutoHeal- default:- return ""- }-}--// testNotificationWarningInternal reports why a real notification would not send-// even though the test did: the provider is disabled, or the tested event type is-// unsubscribed. Empty means real notifications would send.-func (s *NotificationService) testNotificationWarningInternal(setting *NotificationSettings, testType string) string {- if !setting.Enabled {- return fmt.Sprintf("%s is disabled, so real notifications will not send", setting.Provider)+// TestNotification sends a test message even to a disabled provider (testing before enabling is legitimate)+// and returns a warning when a real notification of the tested kind would not send.+func (s *NotificationService) TestNotification(ctx context.Context, environmentID string, provider notifications.NotificationProvider, testType string) (string, error) {+ setting, err := s.GetSettingsByProvider(ctx, provider)+ if err != nil {+ return "", fmt.Errorf("please save your %s settings before testing", provider)}- eventType := notificationEventTypeForTestTypeInternal(testType)- if eventType != "" && !s.isEventEnabled(setting.Config, eventType) {- return fmt.Sprintf("%s events are disabled for %s, so real notifications will not send", eventType, setting.Provider)+ testType = cmp.Or(strings.TrimSpace(testType), notificationTestTypeSimple)+ eventType, ok := notificationTestEventTypes[testType]+ if !ok {+ return "", fmt.Errorf("unsupported notification test type: %s", testType)}- if eventType == "" && !slices.ContainsFunc(notifications.AllNotificationEventTypes, func(candidate notifications.NotificationEventType) bool {++ var warning string+ switch {+ case !setting.Enabled:+ warning = fmt.Sprintf("%s is disabled, so real notifications will not send", setting.Provider)+ case eventType != "" && !s.isEventEnabled(setting.Config, eventType):+ warning = fmt.Sprintf("%s events are disabled for %s, so real notifications will not send", eventType, setting.Provider)+ case eventType == "" && !slices.ContainsFunc(notifications.AllNotificationEventTypes, func(candidate notifications.NotificationEventType) bool {return s.isEventEnabled(setting.Config, candidate)- }) {- return fmt.Sprintf("no events are subscribed for %s, so real notifications will not send", setting.Provider)+ }):+ warning = fmt.Sprintf("no events are subscribed for %s, so real notifications will not send", setting.Provider)}- return ""-}-func (s *NotificationService) testNotificationContentInternal(environmentName, testType string) notifications.Content {- switch testType {- case notificationTestTypeVulnerability:- return s.templates.Vulnerability(environmentName, notification.DispatchVulnerabilityFound{+ target, err := s.ResolveNotificationTarget(ctx, environmentID)+ if err != nil {+ return "", err+ }++ var content notifications.Content+ switch {+ case provider == notifications.NotificationProviderEmail && testType == notificationTestTypeSimple:+ content = s.templates.TestEmail(target.EnvironmentName)+ case testType == notificationTestTypeVulnerability:+ content = s.templates.Vulnerability(target.EnvironmentName, notification.DispatchVulnerabilityFound{CVEID: "Daily Summary - " + time.Now().UTC().Format("2006-01-02"),Severity: "Critical:1 High:3 Medium:2 Low:1 Unknown:0",ImageName: "5 image(s) scanned, 2 with fixable vulnerabilities",FixedVersion: "7 fixable vulnerability record(s)",PkgName: "CVE-2025-1234, CVE-2025-5678, CVE-2026-0001",})- case notificationTestTypeAutoHeal:- return s.templates.AutoHeal(environmentName, "test-container")- case notificationTestTypePruneReport:- return s.templates.PruneReport(environmentName, &system.PruneAllResult{+ case testType == notificationTestTypeAutoHeal:+ content = s.templates.AutoHeal(target.EnvironmentName, "test-container")+ case testType == notificationTestTypePruneReport:+ content = s.templates.PruneReport(target.EnvironmentName, &system.PruneAllResult{Success: true,ContainersPruned: []string{"a1b2c3d4e5f6", "f6e5d4c3b2a1"},ImagesDeleted: []string{"sha256:1111111111111111111111111111111111111111111111111111111111111111"},@@ -1184,8 +1025,8 @@ func (s *NotificationService) testNotificationContentInternal(environmentName, tBuildCacheSpaceReclaimed: 230162432,Errors: []string{},})- case notificationTestTypeBatchImageUpdate:- return s.templates.BatchImageUpdate(environmentName, map[string]*imageupdate.Response{+ case testType == notificationTestTypeBatchImageUpdate:+ content = s.templates.BatchImageUpdate(target.EnvironmentName, map[string]*imageupdate.Response{"nginx:latest": {HasUpdate: true,UpdateType: "digest",@@ -1213,7 +1054,7 @@ func (s *NotificationService) testNotificationContentInternal(environmentName, t})default: // simple and image-updateimageRef := kit.Ternary(testType == notificationTestTypeSimple, "test/image:latest", "nginx:latest")- return s.templates.ImageUpdate(environmentName, imageRef, &imageupdate.Response{+ content = s.templates.ImageUpdate(target.EnvironmentName, imageRef, &imageupdate.Response{HasUpdate: true,UpdateType: "digest",CurrentDigest: "sha256:abc123def456789012345678901234567890",@@ -1222,39 +1063,11 @@ func (s *NotificationService) testNotificationContentInternal(environmentName, tResponseTimeMs: 100,})}-}--// TestNotification sends a test message to the provider regardless of its enabled-// state (testing before enabling is legitimate) and returns a warning when a real-// notification of the tested kind would not send.-func (s *NotificationService) TestNotification(ctx context.Context, environmentID string, provider notifications.NotificationProvider, testType string) (string, error) {- setting, err := s.GetSettingsByProvider(ctx, provider)- if err != nil {- return "", fmt.Errorf("please save your %s settings before testing", provider)- }- testType = cmp.Or(strings.TrimSpace(testType), notificationTestTypeSimple)- if _, ok := supportedNotificationTestTypes[testType]; !ok {- return "", fmt.Errorf("unsupported notification test type: %s", testType)- }- warning := s.testNotificationWarningInternal(setting, testType)-- target, err := s.resolveNotificationTargetInternal(ctx, environmentID)- if err != nil {- return "", err- }-- if provider == notifications.NotificationProviderEmail && testType == notificationTestTypeSimple {- _, sendErr := notifications.Deliver(ctx, notifications.NotificationProviderEmail, setting.Config, s.templates.TestEmail(target.EnvironmentName))- return warning, sendErr- }- content := s.testNotificationContentInternal(target.EnvironmentName, testType)- // Stamp the event vars so the Test button exercises a configured generic- // payload template. The simple test renders image-update content, so it- // reports that event type.- testEventType := cmp.Or(notificationEventTypeForTestTypeInternal(testType), notifications.NotificationEventImageUpdate)+ // Event vars let the test exercise a generic payload template; the simple test renders image-update content.+ testEventType := cmp.Or(eventType, notifications.NotificationEventImageUpdate)content.Vars = notifications.EventVars(target.EnvironmentName, target.EnvironmentID, testEventType)- handled, sendErr := notifications.Deliver(ctx, provider, setting.Config, content)+ handled, sendErr := s.send(ctx, target, testEventType, setting, content)if !handled {return "", fmt.Errorf("unknown provider: %s", provider)}diff --git a/backend/internal/notification/service_test.go b/backend/internal/notification/service_test.goindex a6326c369..555d4ead7 100644--- a/backend/internal/notification/service_test.go+++ b/backend/internal/notification/service_test.go@@ -5,7 +5,6 @@ import ("context""encoding/base64""encoding/json"- "errors""log/slog""net/http""net/http/httptest"@@ -53,7 +52,7 @@ func setupNotificationTestDB(t *testing.T) *database.DB {return &database.DB{DB: db}}-func setupNotificationTestServiceInternal(t *testing.T) (*database.DB, *NotificationService) {+func setupNotificationTestService(t *testing.T) (*database.DB, *NotificationService) {t.Helper()db := setupNotificationTestDB(t)@@ -66,7 +65,7 @@ func setupNotificationTestServiceInternal(t *testing.T) (*database.DB, *Notificareturn db, NewNotificationService(db, cfg, envSvc, event.NewEventService(db, cfg, nil), nil)}-func newNotificationTestUpdateInfoInternal() *imageupdate.Response {+func newNotificationTestUpdateInfo() *imageupdate.Response {return &imageupdate.Response{HasUpdate: true,UpdateType: "digest",@@ -76,7 +75,7 @@ func newNotificationTestUpdateInfoInternal() *imageupdate.Response {}}-func captureNotificationServiceLogsInternal(t *testing.T) *bytes.Buffer {+func captureNotificationServiceLogs(t *testing.T) *bytes.Buffer {t.Helper()var buf bytes.Buffer@@ -90,11 +89,11 @@ func captureNotificationServiceLogsInternal(t *testing.T) *bytes.Buffer {return &buf}-func TestNotificationService_ResolveNotificationTargetInternal_UsesEnvironmentRecordAndFallback(t *testing.T) {+func TestNotificationService_ResolveNotificationTarget_UsesEnvironmentRecordAndFallback(t *testing.T) {ctx := t.Context()- db, svc := setupNotificationTestServiceInternal(t)+ db, svc := setupNotificationTestService(t)- target, err := svc.resolveNotificationTargetInternal(ctx, "")+ target, err := svc.ResolveNotificationTarget(ctx, "")require.NoError(t, err)require.Equal(t, "0", target.EnvironmentID)require.Equal(t, "Local Docker", target.EnvironmentName)@@ -108,42 +107,21 @@ func TestNotificationService_ResolveNotificationTargetInternal_UsesEnvironmentReStatus: string(environment.EnvironmentStatusOnline),}).Error)- target, err = svc.resolveNotificationTargetInternal(ctx, "env-remote")+ target, err = svc.ResolveNotificationTarget(ctx, "env-remote")require.NoError(t, err)require.Equal(t, "env-remote", target.EnvironmentID)require.Equal(t, "Remote Alpha", target.EnvironmentName)}-func TestNotificationService_ResolveNotificationTargetForAccessTokenInternal_UsesStoredEnvironmentName(t *testing.T) {- ctx := t.Context()- db, svc := setupNotificationTestServiceInternal(t)-- token := "remote-token"- now := time.Now()- require.NoError(t, db.WithContext(ctx).Create(&environment.Environment{- ID: "env-remote", CreatedAt: now, UpdatedAt: &now,- Name: "Remote Edge",- ApiUrl: "http://remote.example",- Enabled: true,- Status: string(environment.EnvironmentStatusOnline),- AccessToken: &token,- }).Error)-- target, err := svc.resolveNotificationTargetForAccessTokenInternal(ctx, token)- require.NoError(t, err)- require.Equal(t, "env-remote", target.EnvironmentID)- require.Equal(t, "Remote Edge", target.EnvironmentName)-}-func TestNotificationService_DispatchNotification_InvalidAccessTokenReturnsUnauthorizedSentinel(t *testing.T) {ctx := t.Context()- _, svc := setupNotificationTestServiceInternal(t)+ _, svc := setupNotificationTestService(t)_, err := svc.DispatchNotification(ctx, "missing-token", notification.DispatchRequest{Kind: notification.DispatchKindImageUpdate,ImageUpdate: ¬ification.DispatchImageUpdate{ImageRef: "nginx:latest",- UpdateInfo: *newNotificationTestUpdateInfoInternal(),+ UpdateInfo: *newNotificationTestUpdateInfo(),},})@@ -153,7 +131,7 @@ func TestNotificationService_DispatchNotification_InvalidAccessTokenReturnsUnautfunc TestNotificationService_DispatchNotification_UnsupportedKindReturnsSentinel(t *testing.T) {ctx := t.Context()- db, svc := setupNotificationTestServiceInternal(t)+ db, svc := setupNotificationTestService(t)token := "remote-token"now := time.Now()@@ -179,8 +157,8 @@ func TestNotificationService_DispatchNotification_UnsupportedKindReturnsSentinelfunc TestNotificationService_DispatchNotification_LogsManagerDispatchForAgent(t *testing.T) {ctx := t.Context()- db, svc := setupNotificationTestServiceInternal(t)- logBuffer := captureNotificationServiceLogsInternal(t)+ db, svc := setupNotificationTestService(t)+ logBuffer := captureNotificationServiceLogs(t)token := "remote-token"now := time.Now()@@ -197,7 +175,7 @@ func TestNotificationService_DispatchNotification_LogsManagerDispatchForAgent(tKind: notification.DispatchKindImageUpdate,ImageUpdate: ¬ification.DispatchImageUpdate{ImageRef: "nginx:latest",- UpdateInfo: *newNotificationTestUpdateInfoInternal(),+ UpdateInfo: *newNotificationTestUpdateInfo(),},})@@ -251,7 +229,7 @@ func TestNotificationService_SendImageUpdateNotification_AgentModeDispatchesToMaManagerApiUrl: server.URL,}, envSvc, nil, nil)- delivered, err := svc.SendImageUpdateNotification(ctx, "nginx:latest", newNotificationTestUpdateInfoInternal(), notifications.NotificationEventImageUpdate)+ delivered, err := svc.SendImageUpdateNotification(ctx, "nginx:latest", newNotificationTestUpdateInfo(), notifications.NotificationEventImageUpdate)require.NoError(t, err)require.Equal(t, 2, delivered)require.EqualValues(t, 1, calls.Load())@@ -260,7 +238,7 @@ func TestNotificationService_SendImageUpdateNotification_AgentModeDispatchesToMarequire.Equal(t, "nginx:latest", dispatched.ImageUpdate.ImageRef)}-func TestNotificationService_SendBatchImageUpdateNotification_AgentModeUsesManagerDeliveredCountInternal(t *testing.T) {+func TestNotificationService_SendBatchImageUpdateNotification_AgentModeUsesManagerDeliveredCount(t *testing.T) {ctx := t.Context()db := setupNotificationTestDB(t)envSvc := environment.NewEnvironmentService(db, nil, nil, nil, nil, nil)@@ -299,7 +277,7 @@ func TestNotificationService_SendBatchImageUpdateNotification_AgentModeUsesManag}, envSvc, nil, nil)delivered, err := svc.SendBatchImageUpdateNotification(ctx, map[string]*imageupdate.Response{- "nginx:latest": newNotificationTestUpdateInfoInternal(),+ "nginx:latest": newNotificationTestUpdateInfo(),})require.NoError(t, err)require.Equal(t, 0, delivered)@@ -323,7 +301,7 @@ func TestNotificationService_SendImageUpdateNotification_AgentModeRequiresUpdaterequire.Contains(t, err.Error(), "updateInfo is required")}-func TestNotificationService_SendBatchImageUpdateNotification_AgentModeSkipsNoOpDispatchInternal(t *testing.T) {+func TestNotificationService_SendBatchImageUpdateNotification_AgentModeSkipsNoOpDispatch(t *testing.T) {ctx := t.Context()db := setupNotificationTestDB(t)envSvc := environment.NewEnvironmentService(db, nil, nil, nil, nil, nil)@@ -364,8 +342,8 @@ func TestNotificationService_SendBatchImageUpdateNotification_AgentModeSkipsNoOp})}-func TestBuildImageUpdateNotificationMessageInternal_IncludesEnvironment(t *testing.T) {- updateInfo := newNotificationTestUpdateInfoInternal()+func TestBuildImageUpdateNotificationMessage_IncludesEnvironment(t *testing.T) {+ updateInfo := newNotificationTestUpdateInfo()message := notifications.BuildImageUpdateNotificationMessage(notifications.MessageFormatMarkdown, "Remote Alpha", "nginx:latest", updateInfo)require.Contains(t, message, "**Environment:** Remote Alpha")@@ -375,16 +353,16 @@ func TestBuildImageUpdateNotificationMessageInternal_IncludesEnvironment(t *testrequire.Contains(t, plainMessage, "Environment: Remote Alpha")}-func TestBuildContainerUpdateNotificationMessageInternal_IncludesEnvironment(t *testing.T) {+func TestBuildContainerUpdateNotificationMessage_IncludesEnvironment(t *testing.T) {message := notifications.BuildContainerUpdateNotificationMessage(notifications.MessageFormatMarkdown, "Local Lab", "nginx", "nginx:latest", "sha256:old", "sha256:new")require.Contains(t, message, "**Environment:** Local Lab")require.Equal(t, 1, strings.Count(message, "Environment"))}-func TestBuildBatchImageUpdateNotificationMessageInternal_EnvironmentAppearsOnce(t *testing.T) {+func TestBuildBatchImageUpdateNotificationMessage_EnvironmentAppearsOnce(t *testing.T) {updates := map[string]*imageupdate.Response{- "nginx:latest": newNotificationTestUpdateInfoInternal(),+ "nginx:latest": newNotificationTestUpdateInfo(),"redis:latest": {HasUpdate: true,UpdateType: "minor",@@ -399,7 +377,7 @@ func TestBuildBatchImageUpdateNotificationMessageInternal_EnvironmentAppearsOncerequire.Equal(t, 1, strings.Count(message, "Environment"))}-func TestBuildVulnerabilitySummaryNotificationMessageInternal_IncludesEnvironment(t *testing.T) {+func TestBuildVulnerabilitySummaryNotificationMessage_IncludesEnvironment(t *testing.T) {message := notifications.BuildVulnerabilitySummaryNotificationMessage(notifications.MessageFormatMarkdown,"Remote Alpha",@@ -414,7 +392,7 @@ func TestBuildVulnerabilitySummaryNotificationMessageInternal_IncludesEnvironmenrequire.Equal(t, 1, strings.Count(message, "Environment"))}-func TestBuildPruneReportNotificationMessageInternal_IncludesEnvironment(t *testing.T) {+func TestBuildPruneReportNotificationMessage_IncludesEnvironment(t *testing.T) {message := notifications.BuildPruneReportNotificationMessage(notifications.MessageFormatMarkdown, "Cluster One", &system.PruneAllResult{SpaceReclaimed: 3825205248,ContainerSpaceReclaimed: 503316480,@@ -427,14 +405,14 @@ func TestBuildPruneReportNotificationMessageInternal_IncludesEnvironment(t *testrequire.Equal(t, 1, strings.Count(message, "Environment"))}-func TestBuildAutoHealNotificationMessageInternal_IncludesEnvironment(t *testing.T) {+func TestBuildAutoHealNotificationMessage_IncludesEnvironment(t *testing.T) {message := notifications.BuildAutoHealNotificationMessage(notifications.MessageFormatMarkdown, "Cluster One", "nginx")require.Contains(t, message, "**Environment:** Cluster One")require.Equal(t, 1, strings.Count(message, "Environment"))}-func TestNotificationCredentialInternal_KeepsPlaintextLegacyValues(t *testing.T) {+func TestNotificationCredential_KeepsPlaintextLegacyValues(t *testing.T) {setupNotificationTestDB(t)value := "discord-webhook-token/plaintext"@@ -443,7 +421,7 @@ func TestNotificationCredentialInternal_KeepsPlaintextLegacyValues(t *testing.T)require.Equal(t, "discord-webhook-token/plaintext", value)}-func TestNotificationCredentialInternal_DecryptsEncryptedValues(t *testing.T) {+func TestNotificationCredential_DecryptsEncryptedValues(t *testing.T) {setupNotificationTestDB(t)encrypted, err := crypto.Encrypt("gotify-application-token")@@ -453,7 +431,7 @@ func TestNotificationCredentialInternal_DecryptsEncryptedValues(t *testing.T) {require.Equal(t, "gotify-application-token", encrypted)}-func TestNotificationCredentialInternal_ReturnsErrorForCorruptedCiphertext(t *testing.T) {+func TestNotificationCredential_ReturnsErrorForCorruptedCiphertext(t *testing.T) {setupNotificationTestDB(t)encrypted, err := crypto.Encrypt("gotify-application-token")@@ -465,7 +443,7 @@ func TestNotificationCredentialInternal_ReturnsErrorForCorruptedCiphertext(t *terequire.Error(t, notifications.DecryptStringCredential(new(base64.StdEncoding.EncodeToString(ciphertext))))}-func TestNotificationCredentialInternal_LeavesEmptyValuesEmpty(t *testing.T) {+func TestNotificationCredential_LeavesEmptyValuesEmpty(t *testing.T) {setupNotificationTestDB(t)value := ""@@ -474,7 +452,7 @@ func TestNotificationCredentialInternal_LeavesEmptyValuesEmpty(t *testing.T) {require.Empty(t, value)}-func TestNotificationService_CreateOrUpdateSettingsEncryptsCredentialFieldsInternal(t *testing.T) {+func TestNotificationService_CreateOrUpdateSettingsEncryptsCredentialFields(t *testing.T) {ctx := t.Context()db := setupNotificationTestDB(t)svc := NewNotificationService(db, &config.Config{}, nil, nil, nil)@@ -497,7 +475,7 @@ func TestNotificationService_CreateOrUpdateSettingsEncryptsCredentialFieldsInterrequire.Equal(t, "discord-secret-token", decrypted)}-func TestNotificationService_CreateOrUpdateSettingsPreservesStoredCredentialWhenEmptyInternal(t *testing.T) {+func TestNotificationService_CreateOrUpdateSettingsPreservesStoredCredentialWhenEmpty(t *testing.T) {ctx := t.Context()db := setupNotificationTestDB(t)svc := NewNotificationService(db, &config.Config{}, nil, nil, nil)@@ -525,7 +503,7 @@ func TestNotificationService_CreateOrUpdateSettingsPreservesStoredCredentialWhenrequire.Equal(t, "initial-gotify-token", decrypted)}-func TestNotificationService_CreateOrUpdateSettingsRejectsTargetChangeWithStoredCredentialInternal(t *testing.T) {+func TestNotificationService_CreateOrUpdateSettingsRejectsTargetChangeWithStoredCredential(t *testing.T) {ctx := t.Context()db := setupNotificationTestDB(t)svc := NewNotificationService(db, &config.Config{}, nil, nil, nil)@@ -560,7 +538,7 @@ func TestNotificationService_CreateOrUpdateSettingsRejectsTargetChangeWithStoredrequire.Equal(t, originalToken, updated.Config["token"])}-func TestNotificationService_CreateOrUpdateSettingsClearsEmailPasswordWhenAuthModeNoneInternal(t *testing.T) {+func TestNotificationService_CreateOrUpdateSettingsClearsEmailPasswordWhenAuthModeNone(t *testing.T) {ctx := t.Context()db := setupNotificationTestDB(t)svc := NewNotificationService(db, &config.Config{}, nil, nil, nil)@@ -584,7 +562,7 @@ func TestNotificationService_CreateOrUpdateSettingsClearsEmailPasswordWhenAuthMorequire.Empty(t, stored.Config["smtpPassword"])}-func TestNotificationService_CreateOrUpdateSettingsPreservesCredentialAcrossDisableInternal(t *testing.T) {+func TestNotificationService_CreateOrUpdateSettingsPreservesCredentialAcrossDisable(t *testing.T) {ctx := t.Context()db := setupNotificationTestDB(t)svc := NewNotificationService(db, &config.Config{}, nil, nil, nil)@@ -615,7 +593,7 @@ func TestNotificationService_CreateOrUpdateSettingsPreservesCredentialAcrossDisarequire.Equal(t, "initial-gotify-token", decrypted)}-func TestNotificationService_CreateOrUpdateSettingsKeepsConfigWhenDisabledInternal(t *testing.T) {+func TestNotificationService_CreateOrUpdateSettingsKeepsConfigWhenDisabled(t *testing.T) {ctx := t.Context()db := setupNotificationTestDB(t)svc := NewNotificationService(db, &config.Config{}, nil, nil, nil)@@ -651,53 +629,55 @@ func TestNotificationService_CreateOrUpdateSettingsKeepsConfigWhenDisabledInternrequire.Equal(t, false, events["prune_report"])}-func TestNotificationService_NotifyEnabledProvidersInternal_SkipsFiltersAndAggregatesInternal(t *testing.T) {+func TestNotificationService_NotifyEnabledProviders_SkipsFiltersAndAggregates(t *testing.T) {ctx := t.Context()db := setupNotificationTestDB(t)svc := NewNotificationService(db, &config.Config{}, nil, event.NewEventService(db, nil, nil), nil)+ var webhookCalls atomic.Int32+ var cancelOnWebhook atomic.Pointer[context.CancelFunc]+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ webhookCalls.Add(1)+ if cancelWebhook := cancelOnWebhook.Load(); cancelWebhook != nil {+ (*cancelWebhook)()+ }+ w.WriteHeader(http.StatusOK)+ }))+ defer server.Close()+rows := []NotificationSettings{{Provider: notifications.NotificationProviderDiscord, Enabled: false, Config: database.JSON{}},{Provider: notifications.NotificationProviderSlack, Enabled: true, Config: database.JSON{"events": map[string]any{string(notifications.NotificationEventPruneReport): false},}},- {Provider: notifications.NotificationProviderGotify, Enabled: true, Config: database.JSON{}},- {Provider: notifications.NotificationProviderNtfy, Enabled: true, Config: database.JSON{}},+ {Provider: notifications.NotificationProviderGeneric, Enabled: true, Config: database.JSON{"webhookUrl": server.URL}},+ // An unconfigured Telegram provider fails before any network call.+ {Provider: notifications.NotificationProviderTelegram, Enabled: true, Config: database.JSON{}},}for i := range rows {require.NoError(t, db.WithContext(ctx).Create(&rows[i]).Error)}- var dispatchedMu sync.Mutex- var dispatched []notifications.NotificationProvider+ content := notifications.Content{Text: notifications.TextByFormat(func(notifications.MessageFormat) string { return "loop-test" })}target := NotificationTarget{EnvironmentID: "0", EnvironmentName: "Local Docker"}- delivered, err := svc.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventPruneReport, "loop-test", database.JSON{"eventType": "prune_report"},- func(_ context.Context, provider notifications.NotificationProvider, _ database.JSON) (bool, error) {- dispatchedMu.Lock()- dispatched = append(dispatched, provider)- dispatchedMu.Unlock()- if provider == notifications.NotificationProviderNtfy {- return true, errors.New("boom")- }- return true, nil- })+ delivered, err := svc.notifyEnabledProviders(ctx, target, notifications.NotificationEventPruneReport, "loop-test", database.JSON{"eventType": "prune_report"}, content)// Disabled and event-disabled rows are never dispatched.- require.ElementsMatch(t, []notifications.NotificationProvider{notifications.NotificationProviderGotify, notifications.NotificationProviderNtfy}, dispatched)+ require.EqualValues(t, 1, webhookCalls.Load())require.Equal(t, 1, delivered)require.Error(t, err)- require.Contains(t, err.Error(), "notification errors: ntfy: boom")+ require.Contains(t, err.Error(), "notification errors: telegram: telegram bot token not configured")// Each dispatched attempt lands in the event log.var events []event.Eventrequire.NoError(t, db.WithContext(ctx).Where("type = ?", event.EventTypeNotificationSend).Order("created_at").Find(&events).Error)require.Len(t, events, 2)require.Equal(t, event.EventSeveritySuccess, events[0].Severity)- require.Equal(t, "Notification sent via gotify", events[0].Title)+ require.Equal(t, "Notification sent via generic", events[0].Title)require.Equal(t, "loop-test", events[0].Description)require.Equal(t, event.EventSeverityError, events[1].Severity)- require.Equal(t, "Notification failed via ntfy", events[1].Title)- require.Contains(t, events[1].Description, "boom")+ require.Equal(t, "Notification failed via telegram", events[1].Title)+ require.Contains(t, events[1].Description, "telegram bot token not configured")// Completed attempts persist even when cancellation ends the caller's wait.sqlDB, err := db.DB.DB()@@ -713,14 +693,11 @@ func TestNotificationService_NotifyEnabledProvidersInternal_SkipsFiltersAndAggre}))canceledCtx, cancel := context.WithCancel(ctx)defer cancel()+ cancelOnWebhook.Store(&cancel)returned := make(chan struct{})go func() {defer close(returned)- _, _ = svc.notifyEnabledProvidersInternal(canceledCtx, target, notifications.NotificationEventPruneReport, "canceled-loop-test", nil,- func(context.Context, notifications.NotificationProvider, database.JSON) (bool, error) {- cancel()- return true, nil- })+ _, _ = svc.notifyEnabledProviders(canceledCtx, target, notifications.NotificationEventPruneReport, "canceled-loop-test", nil, content)}()select {case <-returned:@@ -730,7 +707,7 @@ func TestNotificationService_NotifyEnabledProvidersInternal_SkipsFiltersAndAggrereleaseOnce.Do(func() { close(releaseLogging) })require.Eventually(t, func() bool {var count int64- return db.WithContext(ctx).Model(&event.Event{}).Where("description = ?", "canceled-loop-test").Count(&count).Error == nil && count == 2+ return db.WithContext(ctx).Model(&event.Event{}).Where("description LIKE ?", "canceled-loop-test%").Count(&count).Error == nil && count == 2}, time.Second, 10*time.Millisecond)}@@ -745,17 +722,17 @@ func TestSupportedNotificationTestTypes_IncludesAutoHeal(t *testing.T) {}for _, tt := range expected {- _, ok := supportedNotificationTestTypes[tt]- require.True(t, ok, "expected %q to be in supportedNotificationTestTypes", tt)+ _, ok := notificationTestEventTypes[tt]+ require.True(t, ok, "expected %q to be in notificationTestEventTypes", tt)}- require.Len(t, supportedNotificationTestTypes, len(expected),- "supportedNotificationTestTypes has unexpected entries")+ require.Len(t, notificationTestEventTypes, len(expected),+ "notificationTestEventTypes has unexpected entries")}func TestNotificationService_DispatchNotificationForEnvironment_ResolvesTunnelSessionEnvironment(t *testing.T) {ctx := t.Context()- db, svc := setupNotificationTestServiceInternal(t)+ db, svc := setupNotificationTestService(t)now := time.Now()require.NoError(t, db.WithContext(ctx).Create(&environment.Environment{@@ -770,7 +747,7 @@ func TestNotificationService_DispatchNotificationForEnvironment_ResolvesTunnelSeKind: notification.DispatchKindImageUpdate,ImageUpdate: ¬ification.DispatchImageUpdate{ImageRef: "nginx:latest",- UpdateInfo: *newNotificationTestUpdateInfoInternal(),+ UpdateInfo: *newNotificationTestUpdateInfo(),},})require.NoError(t, err)@@ -787,11 +764,11 @@ func TestNotificationService_AgentDispatchWithoutHTTPConfigFallsBackToTunnel(t *// No MANAGER_API_URL/AGENT_TOKEN and no active tunnel: the error must point// at both options instead of only the HTTP env vars (#3002).- _, err := svc.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{+ _, err := svc.dispatchNotificationToManager(ctx, notification.DispatchRequest{Kind: notification.DispatchKindImageUpdate,ImageUpdate: ¬ification.DispatchImageUpdate{ImageRef: "nginx:latest",- UpdateInfo: *newNotificationTestUpdateInfoInternal(),+ UpdateInfo: *newNotificationTestUpdateInfo(),},})require.Error(t, err)@@ -806,11 +783,11 @@ func TestNotificationService_AgentDispatchHTTPFailureFallsBackToTunnel(t *testin// HTTP dispatch fails and no tunnel is connected either: the fallback must// not mask the HTTP transport error.- _, err := svc.dispatchNotificationToManagerInternal(ctx, notification.DispatchRequest{+ _, err := svc.dispatchNotificationToManager(ctx, notification.DispatchRequest{Kind: notification.DispatchKindImageUpdate,ImageUpdate: ¬ification.DispatchImageUpdate{ImageRef: "nginx:latest",- UpdateInfo: *newNotificationTestUpdateInfoInternal(),+ UpdateInfo: *newNotificationTestUpdateInfo(),},})require.Error(t, err)diff --git a/backend/internal/oidc/service.go b/backend/internal/oidc/service.goindex bd147d5ab..aa32f7ac9 100644--- a/backend/internal/oidc/service.go+++ b/backend/internal/oidc/service.go@@ -9,17 +9,21 @@ import ("errors""fmt""log/slog"+ "maps""net/http""net/url""slices""strings"- "sync""time""github.com/coreos/go-oidc/v3/oidc"authtypes "github.com/getarcaneapp/arcane/types/v2/auth""github.com/samber/hot""go.getarcane.app/kit/pkg"+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/trace""golang.org/x/oauth2""github.com/getarcaneapp/arcane/backend/v2/internal/auth"@@ -27,6 +31,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/jwtclaims""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/oidcjwk"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")type OidcService struct {@@ -35,7 +40,6 @@ type OidcService struct {config *config.ConfighttpClient *http.ClientinsecureHttpClient *http.Client- providerMutex sync.RWMutexproviderCache *hot.HotCache[oidcProviderKey, *oidc.Provider]keySetManager *oidcjwk.KeySetManager}@@ -58,77 +62,18 @@ func NewOidcService(authService *auth.AuthService, settingsService *settings.SethttpClient = http.DefaultClient}- // Create a copy of the client to avoid modifying the shared one- // and remove the timeout so we can control it via context+ // Copy the shared client and drop its timeout so requests are bounded by context.oidcClient := *httpClientoidcClient.Timeout = 0+ insecureClient := oidcClient- service := &OidcService{- authService: authService,- settingsService: settingsService,- config: cfg,- httpClient: &oidcClient,- keySetManager: keySetManager,- }- service.providerCache = hot.NewHotCache[oidcProviderKey, *oidc.Provider](hot.LRU, 4).Build()- return service-}--func (s *OidcService) keySetInternal(ctx context.Context, jwksURL string, skipTLS bool) (oidc.KeySet, error) {- if s.keySetManager == nil {- return nil, errors.New("JWK set manager is not configured")- }- return s.keySetManager.KeySet(context.WithoutCancel(ctx), s.getHttpClientInternal(skipTLS), jwksURL)-}--func (s *OidcService) getEffectiveConfigInternal(ctx context.Context) (*settings.OidcConfig, error) {- oidcConfig, err := s.authService.GetOidcConfig(ctx)- if err != nil {- return nil, fmt.Errorf("failed to get OIDC config: %w", err)- }- if oidcConfig.IssuerURL == "" {- return nil, errors.New("issuer URL must be configured")- }- return oidcConfig, nil-}--func (s *OidcService) getHttpClientInternal(skipTlsVerify bool) *http.Client {- if skipTlsVerify {- return s.getInsecureHttpClientInternal()- }- return s.httpClient-}--func (s *OidcService) getInsecureHttpClientInternal() *http.Client {- s.providerMutex.RLock()- if s.insecureHttpClient != nil {- s.providerMutex.RUnlock()- return s.insecureHttpClient- }- s.providerMutex.RUnlock()-- s.providerMutex.Lock()- defer s.providerMutex.Unlock()-- if s.insecureHttpClient != nil {- return s.insecureHttpClient- }-- // Create insecure client- insecureClient := *s.httpClient-- var insecureTransport *http.Transport- if transport, ok := insecureClient.Transport.(*http.Transport); ok {+ // Derive the insecure transport from the caller's raw transport, before either client is instrumented.+ insecureTransport := &http.Transport{}+ if transport, ok := httpClient.Transport.(*http.Transport); ok {insecureTransport = transport.Clone()- } else {- // Transport is nil or not *http.Transport - create a new default transport- if defaultTransport, localOk := http.DefaultTransport.(*http.Transport); localOk {- insecureTransport = defaultTransport.Clone()- } else {- insecureTransport = &http.Transport{}- }+ } else if defaultTransport, isTransport := http.DefaultTransport.(*http.Transport); isTransport {+ insecureTransport = defaultTransport.Clone()}-if insecureTransport.TLSClientConfig == nil {// #nosec G402 - This is explicitly an insecure client for OIDC discovery when TLS verification is skippedinsecureTransport.TLSClientConfig = &tls.Config{@@ -144,38 +89,56 @@ func (s *OidcService) getInsecureHttpClientInternal() *http.Client {insecureTransport.Protocols.SetHTTP1(true)}insecureTransport.Protocols.SetHTTP2(true)- insecureClient.Transport = insecureTransport- s.insecureHttpClient = &insecureClient- return s.insecureHttpClient++ oidcClient.Transport = otelhttp.NewTransport(httpClient.Transport)+ insecureClient.Transport = otelhttp.NewTransport(insecureTransport)++ return &OidcService{+ authService: authService,+ settingsService: settingsService,+ config: cfg,+ httpClient: &oidcClient,+ insecureHttpClient: &insecureClient,+ keySetManager: keySetManager,+ providerCache: hot.NewHotCache[oidcProviderKey, *oidc.Provider](hot.LRU, 4).Build(),+ }}-func (s *OidcService) ensureOpenIDScopeInternal(scopes []string) []string {- hasOpenID := slices.Contains(scopes, oidc.ScopeOpenID)- if !hasOpenID {- scopes = append([]string{oidc.ScopeOpenID}, scopes...)+func (s *OidcService) effectiveConfig(ctx context.Context) (*settings.OidcConfig, error) {+ oidcConfig, err := s.authService.GetOidcConfig(ctx)+ if err != nil {+ return nil, fmt.Errorf("failed to get OIDC config: %w", err)}- return scopes+ if oidcConfig.IssuerURL == "" {+ return nil, errors.New("issuer URL must be configured")+ }+ return oidcConfig, nil+}++func (s *OidcService) httpClientFor(skipTLSVerify bool) *http.Client {+ return kit.Ternary(skipTLSVerify, s.insecureHttpClient, s.httpClient)}-func (s *OidcService) hasManualEndpointsInternal(cfg *settings.OidcConfig) bool {+func hasManualEndpoints(cfg *settings.OidcConfig) bool {return cfg.AuthorizationEndpoint != "" || cfg.TokenEndpoint != "" || cfg.UserinfoEndpoint != ""}-func (s *OidcService) getOauth2ConfigInternal(cfg *settings.OidcConfig, provider *oidc.Provider, origin, mobileRedirectURI string) (oauth2.Config, error) {+// requestedScopes returns the configured scopes (defaulting to email and profile), always including openid.+func requestedScopes(cfg *settings.OidcConfig) []string {scopes := strings.Fields(cfg.Scopes)if len(scopes) == 0 {scopes = []string{"email", "profile"}}- scopes = s.ensureOpenIDScopeInternal(scopes)+ if !slices.Contains(scopes, oidc.ScopeOpenID) {+ scopes = append([]string{oidc.ScopeOpenID}, scopes...)+ }+ return scopes+}- var endpoint oauth2.Endpoint+func (s *OidcService) buildOAuth2Config(cfg *settings.OidcConfig, provider *oidc.Provider, origin, mobileRedirectURI string) (oauth2.Config, error) {+ endpoint := oauth2.Endpoint{AuthURL: cfg.AuthorizationEndpoint, TokenURL: cfg.TokenEndpoint}if provider != nil {endpoint = provider.Endpoint()- } else {- endpoint = oauth2.Endpoint{- AuthURL: cfg.AuthorizationEndpoint,- TokenURL: cfg.TokenEndpoint,- }}if endpoint.AuthURL == "" || endpoint.TokenURL == "" {return oauth2.Config{}, errors.New("authorization and token endpoints must be configured")@@ -191,10 +154,18 @@ func (s *OidcService) getOauth2ConfigInternal(cfg *settings.OidcConfig, providerClientSecret: cfg.ClientSecret,Endpoint: endpoint,RedirectURL: redirectURL,- Scopes: scopes,+ Scopes: requestedScopes(cfg),}, nil}+func configAttributes(cfg *settings.OidcConfig) []attribute.KeyValue {+ return []attribute.KeyValue{+ attribute.String("arcane.oidc.issuer", cfg.IssuerURL),+ attribute.Bool("arcane.oidc.skip_tls_verify", cfg.SkipTlsVerify),+ attribute.Bool("arcane.oidc.manual_endpoints", hasManualEndpoints(cfg)),+ }+}+// GetMobileRedirectAllowlist returns the configured list of acceptable mobile OAuth redirect URIs.func (s *OidcService) GetMobileRedirectAllowlist(ctx context.Context) []string {raw := ""@@ -207,9 +178,7 @@ func (s *OidcService) GetMobileRedirectAllowlist(ctx context.Context) []string {return kit.TrimNonEmpty(strings.Split(raw, ","))}-// ValidateMobileRedirectURI returns nil if uri exactly matches one of the-// configured mobile redirect URIs. Full-string match is required — partial-// matches on scheme or host could be abused for open-redirect attacks.+// ValidateMobileRedirectURI requires an exact allowlist match; partial matches would allow open redirects.func (s *OidcService) ValidateMobileRedirectURI(ctx context.Context, uri string) error {if uri == "" {return errors.New("mobile redirect URI is empty")@@ -221,16 +190,16 @@ func (s *OidcService) ValidateMobileRedirectURI(ctx context.Context, uri string)}func (s *OidcService) GenerateAuthURL(ctx context.Context, redirectTo, origin, mobileRedirectURI string) (string, string, error) {- oidcConfig, err := s.getEffectiveConfigInternal(ctx)+ oidcConfig, err := s.effectiveConfig(ctx)if err != nil {slog.ErrorContext(ctx, "GenerateAuthURL: failed to get OIDC config", "error", err)return "", "", err}var provider *oidc.Provider- if !s.hasManualEndpointsInternal(oidcConfig) {+ if !hasManualEndpoints(oidcConfig) {var discoverProviderErr error- provider, discoverProviderErr = s.getOrDiscoverProviderInternal(ctx, oidcConfig)+ provider, discoverProviderErr = s.getOrDiscoverProvider(ctx, oidcConfig)if discoverProviderErr != nil {slog.ErrorContext(ctx, "GenerateAuthURL: provider discovery failed", "issuer", oidcConfig.IssuerURL, "error", discoverProviderErr)return "", "", fmt.Errorf("failed to discover provider: %w", discoverProviderErr)@@ -241,7 +210,7 @@ func (s *OidcService) GenerateAuthURL(ctx context.Context, redirectTo, origin, mnonce := kit.RandomString(32)codeVerifier := kit.RandomString(128)- oauth2Config, err := s.getOauth2ConfigInternal(oidcConfig, provider, origin, mobileRedirectURI)+ oauth2Config, err := s.buildOAuth2Config(oidcConfig, provider, origin, mobileRedirectURI)if err != nil {slog.ErrorContext(ctx, "GenerateAuthURL: invalid OIDC endpoints", "error", err)return "", "", err@@ -252,23 +221,20 @@ func (s *OidcService) GenerateAuthURL(ctx context.Context, redirectTo, origin, moauth2.S256ChallengeOption(codeVerifier),)- stateData := OidcState{+ stateJSON, err := json.Marshal(OidcState{State: state,Nonce: nonce,CodeVerifier: codeVerifier,RedirectTo: redirectTo,CreatedAt: time.Now(),- }-- stateJSON, err := json.Marshal(stateData)+ })if err != nil {slog.ErrorContext(ctx, "GenerateAuthURL: failed to marshal state", "error", err)return "", "", fmt.Errorf("failed to encode state: %w", err)}- encodedState := base64.URLEncoding.EncodeToString(stateJSON)slog.DebugContext(ctx, "GenerateAuthURL: generated authorization URL", "issuer", oidcConfig.IssuerURL, "scopes", oauth2Config.Scopes)- return authURL, encodedState, nil+ return authURL, base64.URLEncoding.EncodeToString(stateJSON), nil}func (s *OidcService) GetOidcRedirectURL(origin string) string {@@ -279,24 +245,17 @@ func (s *OidcService) GetOidcRedirectURL(origin string) string {return baseUrl + "/auth/oidc/callback"}-func (s *OidcService) getOrDiscoverProviderInternal(ctx context.Context, cfg *settings.OidcConfig) (*oidc.Provider, error) {- if s.providerCache == nil {- s.providerCache = hot.NewHotCache[oidcProviderKey, *oidc.Provider](hot.LRU, 4).Build()- }+func (s *OidcService) getOrDiscoverProvider(ctx context.Context, cfg *settings.OidcConfig) (*oidc.Provider, error) {key := oidcProviderKey{issuer: cfg.IssuerURL, skipTLS: cfg.SkipTlsVerify}provider, found, err := s.providerCache.GetWithLoaders(key, func(keys []oidcProviderKey) (map[oidcProviderKey]*oidc.Provider, error) {providers := make(map[oidcProviderKey]*oidc.Provider, len(keys))for _, providerKey := range keys {- discoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)- providerCtx := oidc.ClientContext(discoveryCtx, s.getHttpClientInternal(providerKey.skipTLS))- discovered, discoveredIssuer, discoverErr := s.discoverProviderInternal(providerCtx, providerKey.issuer)- cancel()+ discovered, discoverErr := s.discoverProvider(ctx, providerKey)if discoverErr != nil {- slog.ErrorContext(ctx, "getOrDiscoverProviderInternal: discovery failed", "issuer", providerKey.issuer, "skipTls", providerKey.skipTLS, "error", discoverErr)+ slog.ErrorContext(ctx, "getOrDiscoverProvider: discovery failed", "issuer", providerKey.issuer, "skipTls", providerKey.skipTLS, "error", discoverErr)return nil, fmt.Errorf("failed to discover provider at %s: %w", providerKey.issuer, discoverErr)}providers[providerKey] = discovered- slog.DebugContext(ctx, "getOrDiscoverProviderInternal: provider cached", "issuer", providerKey.issuer, "effectiveIssuer", discoveredIssuer, "skipTls", providerKey.skipTLS)}return providers, nil})@@ -309,213 +268,103 @@ func (s *OidcService) getOrDiscoverProviderInternal(ctx context.Context, cfg *sereturn provider, nil}-func (s *OidcService) discoverProviderInternal(ctx context.Context, issuer string) (*oidc.Provider, string, error) {- provider, err := oidc.NewProvider(ctx, issuer)+// discoverProvider fetches provider metadata, retrying once with the issuer's trailing slash toggled.+func (s *OidcService) discoverProvider(ctx context.Context, key oidcProviderKey) (_ *oidc.Provider, err error) {+ discoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)+ defer cancel()+ discoveryCtx, span := otel.Tracer(tracing.InstrumentationName).Start(discoveryCtx, "oidc.discover_provider", trace.WithAttributes(+ attribute.String("arcane.oidc.issuer", key.issuer),+ attribute.Bool("arcane.oidc.skip_tls_verify", key.skipTLS),+ ))+ defer func() { tracing.End(span, err) }()+ providerCtx := oidc.ClientContext(discoveryCtx, s.httpClientFor(key.skipTLS))++ provider, err := oidc.NewProvider(providerCtx, key.issuer)if err == nil {- return provider, issuer, nil- }-- altIssuer := strings.TrimRight(issuer, "/")- if altIssuer == issuer {- altIssuer = issuer + "/"+ slog.DebugContext(ctx, "getOrDiscoverProvider: provider cached", "issuer", key.issuer, "skipTls", key.skipTLS)+ return provider, nil}- slog.WarnContext(ctx, "getOrDiscoverProviderInternal: retrying discovery with alternate issuer", "configured", issuer, "alternate", altIssuer)- provider, altErr := oidc.NewProvider(ctx, altIssuer)- if altErr == nil {- return provider, altIssuer, nil+ altIssuer := strings.TrimRight(key.issuer, "/")+ if altIssuer == key.issuer {+ altIssuer = key.issuer + "/"}-- slog.ErrorContext(ctx, "getOrDiscoverProviderInternal: discovery failed with alternate issuer", "issuer", altIssuer, "error", altErr)- return nil, issuer, err-}--func (s *OidcService) exchangeTokenInternal(ctx context.Context, cfg *settings.OidcConfig, provider *oidc.Provider, code, verifier, origin, mobileRedirectURI string) (*oauth2.Token, error) {- oauth2Config, err := s.getOauth2ConfigInternal(cfg, provider, origin, mobileRedirectURI)- if err != nil {+ slog.WarnContext(ctx, "getOrDiscoverProvider: retrying discovery with alternate issuer", "configured", key.issuer, "alternate", altIssuer)+ provider, altErr := oidc.NewProvider(providerCtx, altIssuer)+ if altErr != nil {+ slog.ErrorContext(ctx, "getOrDiscoverProvider: discovery failed with alternate issuer", "issuer", altIssuer, "error", altErr)return nil, err}-- providerCtx := oidc.ClientContext(ctx, s.getHttpClientInternal(cfg.SkipTlsVerify))- token, err := oauth2Config.Exchange(providerCtx, code, oauth2.VerifierOption(verifier))- if err != nil {- slog.ErrorContext(ctx, "exchangeTokenInternal: token exchange failed", "tokenEndpoint", oauth2Config.Endpoint.TokenURL, "error", err)- return nil, fmt.Errorf("failed to exchange authorization code: %w", err)- }-- slog.DebugContext(ctx, "exchangeTokenInternal: token exchange successful", "hasAccessToken", token.AccessToken != "", "hasRefreshToken", token.RefreshToken != "")- return token, nil-}--func (s *OidcService) fetchClaimsInternal(ctx context.Context, cfg *settings.OidcConfig, provider *oidc.Provider, token *oauth2.Token, idToken *oidc.IDToken) (map[string]any, error) {- providerCtx := oidc.ClientContext(ctx, s.getHttpClientInternal(cfg.SkipTlsVerify))- var claims map[string]any-- if idToken != nil {- if err := idToken.Claims(&claims); err != nil {- slog.WarnContext(ctx, "fetchClaimsInternal: failed to extract claims from ID token", "error", err)- } else {- slog.DebugContext(ctx, "fetchClaimsInternal: extracted claims from ID token")- }- }-- var userInfoClaims map[string]any- switch {- case provider != nil:- userInfo, err := provider.UserInfo(providerCtx, oauth2.StaticTokenSource(token))- if err != nil {- slog.DebugContext(ctx, "fetchClaimsInternal: userinfo endpoint call failed", "error", err)- if claims != nil {- return claims, nil- }- return nil, fmt.Errorf("failed to fetch userinfo: %w", err)- }- if claimsErr := userInfo.Claims(&userInfoClaims); claimsErr != nil {- slog.WarnContext(ctx, "fetchClaimsInternal: failed to decode userinfo claims", "error", claimsErr)- if claims != nil {- return claims, nil- }- return nil, fmt.Errorf("failed to decode userinfo claims: %w", claimsErr)- }- slog.DebugContext(ctx, "fetchClaimsInternal: fetched userinfo claims successfully")- case cfg.UserinfoEndpoint != "":- manualClaims, err := s.fetchUserInfoClaimsInternal(providerCtx, cfg, token)- if err != nil {- slog.DebugContext(ctx, "fetchClaimsInternal: userinfo endpoint call failed", "error", err)- if claims != nil {- return claims, nil- }- return nil, fmt.Errorf("failed to fetch userinfo: %w", err)- }- userInfoClaims = manualClaims- slog.DebugContext(ctx, "fetchClaimsInternal: fetched userinfo claims successfully")- case claims != nil:- return claims, nil- default:- return nil, errors.New("userinfo endpoint not configured")- }-- if claims == nil {- claims = make(map[string]any)- }- for k, v := range userInfoClaims {- if _, exists := claims[k]; !exists {- claims[k] = v- }- }-- return claims, nil+ span.SetAttributes(attribute.String("arcane.oidc.effective_issuer", altIssuer))+ slog.DebugContext(ctx, "getOrDiscoverProvider: provider cached", "issuer", key.issuer, "effectiveIssuer", altIssuer, "skipTls", key.skipTLS)+ return provider, nil}-func (s *OidcService) fetchUserInfoClaimsInternal(ctx context.Context, cfg *settings.OidcConfig, token *oauth2.Token) (map[string]any, error) {- if cfg.UserinfoEndpoint == "" {- return nil, errors.New("userinfo endpoint not configured")- }- if token == nil || token.AccessToken == "" {- return nil, errors.New("missing access token for userinfo request")- }+func (s *OidcService) HandleCallback(ctx context.Context, code, state, storedState, origin, mobileRedirectURI string) (_ *authtypes.OidcUserInfo, _ *authtypes.OidcTokenResponse, err error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "oidc.callback")+ defer func() { tracing.End(span, err) }()+ slog.DebugContext(ctx, "HandleCallback: processing callback", "codePresent", code != "", "statePresent", state != "")- request, err := http.NewRequestWithContext(ctx, http.MethodGet, cfg.UserinfoEndpoint, http.NoBody)- if err != nil {- return nil, err+ var stateData OidcState+ stateJSON, err := base64.URLEncoding.DecodeString(storedState)+ if err == nil {+ err = json.Unmarshal(stateJSON, &stateData)}-- tokenType := cmp.Or(token.TokenType, "Bearer")- request.Header.Set("Authorization", fmt.Sprintf("%s %s", tokenType, token.AccessToken))- request.Header.Set("Accept", "application/json")-- client := s.getHttpClientInternal(cfg.SkipTlsVerify)- resp, err := client.Do(request)if err != nil {- return nil, err- }- defer func() { _ = resp.Body.Close() }()-- if resp.StatusCode < 200 || resp.StatusCode >= 300 {- return nil, fmt.Errorf("userinfo endpoint returned status %d", resp.StatusCode)+ slog.ErrorContext(ctx, "HandleCallback: failed to decode stored state", "error", err)+ return nil, nil, fmt.Errorf("invalid state parameter: %w", err)}-- var claims map[string]any- if unmarshalReadErr := json.UnmarshalRead(resp.Body, &claims); unmarshalReadErr != nil {- return nil, unmarshalReadErr+ if state != stateData.State {+ slog.ErrorContext(ctx, "HandleCallback: state mismatch", "receivedLen", len(state), "expectedLen", len(stateData.State))+ return nil, nil, errors.New("state parameter mismatch")}-- return claims, nil-}--func (s *OidcService) HandleCallback(ctx context.Context, code, state, storedState, origin, mobileRedirectURI string) (*authtypes.OidcUserInfo, *authtypes.OidcTokenResponse, error) {- slog.DebugContext(ctx, "HandleCallback: processing callback", "codePresent", code != "", "statePresent", state != "")-- stateData, err := s.validateStateInternal(ctx, state, storedState)- if err != nil {- return nil, nil, err+ if time.Since(stateData.CreatedAt) > 10*time.Minute {+ slog.ErrorContext(ctx, "HandleCallback: state expired", "age", time.Since(stateData.CreatedAt))+ return nil, nil, errors.New("authentication state has expired")}- cfg, err := s.getEffectiveConfigInternal(ctx)+ cfg, err := s.effectiveConfig(ctx)if err != nil {slog.ErrorContext(ctx, "HandleCallback: failed to get OIDC config", "error", err)return nil, nil, err}+ span.SetAttributes(configAttributes(cfg)...)var provider *oidc.Provider- if !s.hasManualEndpointsInternal(cfg) {- provider, err = s.getOrDiscoverProviderInternal(ctx, cfg)+ if !hasManualEndpoints(cfg) {+ provider, err = s.getOrDiscoverProvider(ctx, cfg)if err != nil {return nil, nil, err}}- token, err := s.exchangeTokenInternal(ctx, cfg, provider, code, stateData.CodeVerifier, origin, mobileRedirectURI)+ oauth2Config, err := s.buildOAuth2Config(cfg, provider, origin, mobileRedirectURI)if err != nil {return nil, nil, err}-- idToken, rawIDToken, err := s.verifyIDTokenInternal(ctx, provider, cfg, token, stateData.Nonce)+ token, err := oauth2Config.Exchange(oidc.ClientContext(ctx, s.httpClientFor(cfg.SkipTlsVerify)), code, oauth2.VerifierOption(stateData.CodeVerifier))if err != nil {- return nil, nil, err+ slog.ErrorContext(ctx, "HandleCallback: token exchange failed", "tokenEndpoint", oauth2Config.Endpoint.TokenURL, "error", err)+ return nil, nil, fmt.Errorf("failed to exchange authorization code: %w", err)}+ slog.DebugContext(ctx, "HandleCallback: token exchange successful", "hasAccessToken", token.AccessToken != "", "hasRefreshToken", token.RefreshToken != "")- return s.buildUserInfoInternal(ctx, provider, cfg, token, idToken, rawIDToken)-}--func (s *OidcService) validateStateInternal(ctx context.Context, state, storedState string) (*OidcState, error) {- stateData, err := s.decodeStateInternal(ctx, storedState)+ idToken, rawIDToken, err := s.verifyIDToken(ctx, provider, cfg, token, stateData.Nonce)if err != nil {- slog.ErrorContext(ctx, "HandleCallback: failed to decode stored state", "error", err)- return nil, fmt.Errorf("invalid state parameter: %w", err)- }-- if state != stateData.State {- slog.ErrorContext(ctx, "HandleCallback: state mismatch", "receivedLen", len(state), "expectedLen", len(stateData.State))- return nil, errors.New("state parameter mismatch")+ return nil, nil, err}- if time.Since(stateData.CreatedAt) > 10*time.Minute {- slog.ErrorContext(ctx, "HandleCallback: state expired", "age", time.Since(stateData.CreatedAt))- return nil, errors.New("authentication state has expired")- }- return stateData, nil+ return s.buildUserInfo(ctx, provider, cfg, token, idToken, rawIDToken)}-func (s *OidcService) verifyIDTokenInternal(ctx context.Context, provider *oidc.Provider, cfg *settings.OidcConfig, token *oauth2.Token, nonce string) (*oidc.IDToken, string, error) {- var rawIDToken string- if idTokenValue := token.Extra("id_token"); idTokenValue != nil {- if idTokenStr, ok := idTokenValue.(string); ok {- rawIDToken = idTokenStr- }- }-+func (s *OidcService) verifyIDToken(ctx context.Context, provider *oidc.Provider, cfg *settings.OidcConfig, token *oauth2.Token, nonce string) (*oidc.IDToken, string, error) {+ rawIDToken, _ := token.Extra("id_token").(string)if rawIDToken == "" {slog.WarnContext(ctx, "HandleCallback: no ID token in response (non-compliant OIDC response)")return nil, "", nil}- verifierConfig := &oidc.Config{- ClientID: cfg.ClientID,- SupportedSigningAlgs: oidcjwk.SupportedSigningAlgs(),- }-- var issuer, jwksURL string+ issuer, jwksURL := cfg.IssuerURL, cfg.JwksURIif provider != nil {var meta struct {Issuer string `json:"issuer"`@@ -525,21 +374,24 @@ func (s *OidcService) verifyIDTokenInternal(ctx context.Context, provider *oidc.return nil, "", fmt.Errorf("failed to read provider metadata: %w", err)}issuer, jwksURL = meta.Issuer, meta.JWKSURL- } else {- issuer, jwksURL = cfg.IssuerURL, cfg.JwksURI}if jwksURL == "" {return nil, "", errors.New("jwks URI must be configured when using manual OIDC endpoints")}+ if s.keySetManager == nil {+ return nil, "", errors.New("failed to configure provider JWK set: JWK set manager is not configured")+ }- keySet, err := s.keySetInternal(ctx, jwksURL, cfg.SkipTlsVerify)+ client := s.httpClientFor(cfg.SkipTlsVerify)+ keySet, err := s.keySetManager.KeySet(context.WithoutCancel(ctx), client, jwksURL)if err != nil {return nil, "", fmt.Errorf("failed to configure provider JWK set: %w", err)}- providerCtx := oidc.ClientContext(ctx, s.getHttpClientInternal(cfg.SkipTlsVerify))- verifier := oidc.NewVerifier(issuer, keySet, verifierConfig)-- idToken, err := verifier.Verify(providerCtx, rawIDToken)+ verifier := oidc.NewVerifier(issuer, keySet, &oidc.Config{+ ClientID: cfg.ClientID,+ SupportedSigningAlgs: oidcjwk.SupportedSigningAlgs(),+ })+ idToken, err := verifier.Verify(oidc.ClientContext(ctx, client), rawIDToken)if err != nil {slog.ErrorContext(ctx, "HandleCallback: ID token verification failed", "error", err)return nil, "", fmt.Errorf("failed to verify ID token: %w", err)@@ -563,24 +415,63 @@ func (s *OidcService) verifyIDTokenInternal(ctx context.Context, provider *oidc.return idToken, rawIDToken, nil}-func (- s *OidcService,-) buildUserInfoInternal(- ctx context.Context,- provider *oidc.Provider,- cfg *settings.OidcConfig,- token *oauth2.Token,- idToken *oidc.IDToken,- rawIDToken string,-) (- *authtypes.OidcUserInfo,- *authtypes.OidcTokenResponse,- error,-) {- claims, err := s.fetchClaimsInternal(ctx, cfg, provider, token, idToken)- if err != nil {- slog.ErrorContext(ctx, "HandleCallback: failed to fetch claims", "error", err)- return nil, nil, fmt.Errorf("failed to fetch user claims: %w", err)+func (s *OidcService) buildUserInfo(+ ctx context.Context, provider *oidc.Provider, cfg *settings.OidcConfig, token *oauth2.Token, idToken *oidc.IDToken, rawIDToken string,+) (*authtypes.OidcUserInfo, *authtypes.OidcTokenResponse, error) {+ var claims map[string]any+ if idToken != nil {+ if claimsErr := idToken.Claims(&claims); claimsErr != nil {+ slog.WarnContext(ctx, "HandleCallback: failed to extract claims from ID token", "error", claimsErr)+ }+ }++ // Userinfo claims fill gaps; ID token claims win on conflict.+ var userInfoClaims map[string]any+ var userInfoErr error+ client := s.httpClientFor(cfg.SkipTlsVerify)+ switch {+ case provider != nil:+ userInfo, fetchErr := provider.UserInfo(oidc.ClientContext(ctx, client), oauth2.StaticTokenSource(token))+ if fetchErr != nil {+ userInfoErr = fmt.Errorf("failed to fetch userinfo: %w", fetchErr)+ } else if decodeErr := userInfo.Claims(&userInfoClaims); decodeErr != nil {+ userInfoErr = fmt.Errorf("failed to decode userinfo claims: %w", decodeErr)+ }+ case cfg.UserinfoEndpoint != "" && token.AccessToken == "":+ userInfoErr = errors.New("failed to fetch userinfo: missing access token for userinfo request")+ case cfg.UserinfoEndpoint != "":+ request, requestErr := http.NewRequestWithContext(ctx, http.MethodGet, cfg.UserinfoEndpoint, http.NoBody)+ if requestErr != nil {+ userInfoErr = fmt.Errorf("failed to fetch userinfo: %w", requestErr)+ break+ }+ request.Header.Set("Authorization", cmp.Or(token.TokenType, "Bearer")+" "+token.AccessToken)+ request.Header.Set("Accept", "application/json")+ resp, doErr := client.Do(request)+ if doErr != nil {+ userInfoErr = fmt.Errorf("failed to fetch userinfo: %w", doErr)+ break+ }+ defer func() { _ = resp.Body.Close() }()+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {+ userInfoErr = fmt.Errorf("failed to fetch userinfo: userinfo endpoint returned status %d", resp.StatusCode)+ } else if decodeErr := json.UnmarshalRead(resp.Body, &userInfoClaims); decodeErr != nil {+ userInfoErr = fmt.Errorf("failed to fetch userinfo: %w", decodeErr)+ }+ case claims == nil:+ userInfoErr = errors.New("userinfo endpoint not configured")+ }+ if userInfoErr != nil {+ slog.DebugContext(ctx, "HandleCallback: userinfo request failed", "error", userInfoErr)+ if claims == nil {+ slog.ErrorContext(ctx, "HandleCallback: failed to fetch claims", "error", userInfoErr)+ return nil, nil, fmt.Errorf("failed to fetch user claims: %w", userInfoErr)+ }+ userInfoClaims = nil+ }+ if merged := maps.Clone(userInfoClaims); merged != nil {+ maps.Copy(merged, claims)+ claims = merged}subject := jwtclaims.GetStringClaim(claims, "sub")@@ -603,81 +494,102 @@ func (Extra: claims,}- tokenType := cmp.Or(token.TokenType, "Bearer")-tokenResp := &authtypes.OidcTokenResponse{AccessToken: token.AccessToken,- TokenType: tokenType,+ TokenType: cmp.Or(token.TokenType, "Bearer"),RefreshToken: token.RefreshToken,IDToken: rawIDToken,}if !token.Expiry.IsZero() {- expiresIn := max(int(time.Until(token.Expiry).Seconds()), 0)- tokenResp.ExpiresIn = expiresIn+ tokenResp.ExpiresIn = max(int(time.Until(token.Expiry).Seconds()), 0)}+ trace.SpanFromContext(ctx).SetAttributes(attribute.Int("arcane.oidc.groups", len(userInfoDto.Groups)), attribute.Int("arcane.oidc.roles", len(userInfoDto.Roles)))slog.InfoContext(ctx, "HandleCallback: authentication successful", "subject", userInfoDto.Subject, "email", userInfoDto.Email)return &userInfoDto, tokenResp, nil}-func (s *OidcService) decodeStateInternal(ctx context.Context, encodedState string) (*OidcState, error) {- stateJSON, err := base64.URLEncoding.DecodeString(encodedState)+// postForm sends an OAuth form POST and decodes the JSON response body, along with the HTTP status code.+func postForm(ctx context.Context, client *http.Client, endpoint string, values url.Values) (map[string]any, int, error) {+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(values.Encode()))if err != nil {- slog.ErrorContext(ctx, "decodeStateInternal: failed to decode base64 state", "error", err)- return nil, err+ return nil, 0, err}+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")+ req.Header.Set("Accept", "application/json")- var stateData OidcState- if unmarshalErr := json.Unmarshal(stateJSON, &stateData); unmarshalErr != nil {- slog.ErrorContext(ctx, "decodeStateInternal: failed to unmarshal state JSON", "error", unmarshalErr)- return nil, unmarshalErr+ resp, err := client.Do(req)+ if err != nil {+ return nil, 0, err}+ defer func() { _ = resp.Body.Close() }()- return &stateData, nil+ var body map[string]any+ decodeErr := json.UnmarshalRead(resp.Body, &body)+ return body, resp.StatusCode, decodeErr}// InitiateDeviceAuth initiates the OIDC device authorization flow.-func (s *OidcService) InitiateDeviceAuth(ctx context.Context) (*authtypes.OidcDeviceAuthResponse, error) {- cfg, err := s.getEffectiveConfigInternal(ctx)- if err != nil {- slog.ErrorContext(ctx, "InitiateDeviceAuth: failed to get OIDC config", "error", err)- return nil, err- }+func (s *OidcService) InitiateDeviceAuth(ctx context.Context) (_ *authtypes.OidcDeviceAuthResponse, err error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "oidc.device_authorize")+ defer func() { tracing.End(span, err) }()- deviceEndpoint, err := s.getDeviceAuthorizationEndpointInternal(ctx, cfg)+ cfg, err := s.effectiveConfig(ctx)if err != nil {- slog.ErrorContext(ctx, "InitiateDeviceAuth: failed to get device endpoint", "error", err)+ slog.ErrorContext(ctx, "InitiateDeviceAuth: failed to get OIDC config", "error", err)return nil, err}+ span.SetAttributes(configAttributes(cfg)...)- scopes := strings.Fields(cfg.Scopes)- if len(scopes) == 0 {- scopes = []string{"email", "profile"}+ deviceEndpoint := cfg.DeviceAuthorizationEndpoint+ if deviceEndpoint == "" {+ provider, discoverErr := s.getOrDiscoverProvider(ctx, cfg)+ if discoverErr != nil {+ return nil, fmt.Errorf("failed to discover provider: %w", discoverErr)+ }+ var claims struct {+ DeviceAuthorizationEndpoint string `json:"device_authorization_endpoint"`+ }+ if claimsErr := provider.Claims(&claims); claimsErr != nil {+ return nil, fmt.Errorf("failed to get device authorization endpoint from provider: %w", claimsErr)+ }+ if claims.DeviceAuthorizationEndpoint == "" {+ return nil, errors.New("device authorization endpoint not found in provider configuration")+ }+ deviceEndpoint = claims.DeviceAuthorizationEndpoint}- scopes = s.ensureOpenIDScopeInternal(scopes)values := url.Values{}values.Set("client_id", cfg.ClientID)- values.Set("scope", strings.Join(scopes, " "))+ values.Set("scope", strings.Join(requestedScopes(cfg), " "))if cfg.ClientSecret != "" {values.Set("client_secret", cfg.ClientSecret)}- respData, err := s.makeDeviceAuthRequestInternal(ctx, deviceEndpoint, values, cfg.SkipTlsVerify)- if err != nil {- return nil, err+ respData, status, err := postForm(ctx, s.httpClientFor(cfg.SkipTlsVerify), deviceEndpoint, values)+ switch {+ case status == 0:+ slog.ErrorContext(ctx, "InitiateDeviceAuth: request failed", "error", err)+ return nil, fmt.Errorf("device authorization request failed: %w", err)+ case status < 200 || status >= 300:+ if errMsg, ok := respData["error"].(string); ok && err == nil {+ return nil, fmt.Errorf("device authorization failed: %s", errMsg)+ }+ return nil, fmt.Errorf("device authorization endpoint returned status %d", status)+ case err != nil:+ return nil, fmt.Errorf("failed to decode device authorization response: %w", err)}- deviceCode, ok := respData["device_code"].(string)- if !ok || deviceCode == "" {+ deviceCode, _ := respData["device_code"].(string)+ if deviceCode == "" {return nil, errors.New("invalid device_code in response")}- userCode, ok := respData["user_code"].(string)- if !ok || userCode == "" {+ userCode, _ := respData["user_code"].(string)+ if userCode == "" {return nil, errors.New("invalid user_code in response")}- verificationUri, ok := respData["verification_uri"].(string)- if !ok || verificationUri == "" {+ verificationUri, _ := respData["verification_uri"].(string)+ if verificationUri == "" {return nil, errors.New("invalid verification_uri in response")}expiresIn, ok := respData["expires_in"].(float64)@@ -686,200 +598,100 @@ func (s *OidcService) InitiateDeviceAuth(ctx context.Context) (*authtypes.OidcDe}response := &authtypes.OidcDeviceAuthResponse{- DeviceCode: deviceCode,- UserCode: userCode,- VerificationUri: verificationUri,- ExpiresIn: int(expiresIn),- }-- if uri, localOk := respData["verification_uri_complete"].(string); localOk {- response.VerificationUriComplete = uri- }- if interval, localOk2 := respData["interval"].(float64); localOk2 {- response.Interval = int(interval)- } else {- response.Interval = 5+ DeviceCode: deviceCode,+ UserCode: userCode,+ VerificationUri: verificationUri,+ VerificationUriComplete: kit.As(respData["verification_uri_complete"], ""),+ ExpiresIn: int(expiresIn),+ Interval: int(kit.As(respData["interval"], float64(5))),}slog.DebugContext(ctx, "InitiateDeviceAuth: device authorization initiated", "userCode", response.UserCode, "expiresIn", response.ExpiresIn)return response, nil}-// getDeviceAuthorizationEndpointInternal discovers or returns the configured device authorization endpoint.-func (s *OidcService) getDeviceAuthorizationEndpointInternal(ctx context.Context, cfg *settings.OidcConfig) (string, error) {- if cfg.DeviceAuthorizationEndpoint != "" {- return cfg.DeviceAuthorizationEndpoint, nil- }-- provider, err := s.getOrDiscoverProviderInternal(ctx, cfg)- if err != nil {- return "", fmt.Errorf("failed to discover provider: %w", err)- }-- var claims struct {- DeviceAuthorizationEndpoint string `json:"device_authorization_endpoint"`- }- if claimsErr := provider.Claims(&claims); claimsErr != nil {- return "", fmt.Errorf("failed to get device authorization endpoint from provider: %w", claimsErr)- }-- if claims.DeviceAuthorizationEndpoint == "" {- return "", errors.New("device authorization endpoint not found in provider configuration")- }-- return claims.DeviceAuthorizationEndpoint, nil-}--// makeDeviceAuthRequestInternal makes a device authorization request.-func (s *OidcService) makeDeviceAuthRequestInternal(ctx context.Context, endpoint string, params url.Values, skipTls bool) (map[string]any, error) {- body := params.Encode()-- req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(body))- if err != nil {- return nil, err- }- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")- req.Header.Set("Accept", "application/json")-- client := s.getHttpClientInternal(skipTls)- resp, err := client.Do(req)- if err != nil {- slog.ErrorContext(ctx, "makeDeviceAuthRequestInternal: request failed", "error", err)- return nil, fmt.Errorf("device authorization request failed: %w", err)- }- defer func() { _ = resp.Body.Close() }()-- if resp.StatusCode < 200 || resp.StatusCode >= 300 {- var errorResp map[string]any- if unmarshalReadErr := json.UnmarshalRead(resp.Body, &errorResp); unmarshalReadErr == nil {- if errMsg, ok := errorResp["error"].(string); ok {- return nil, fmt.Errorf("device authorization failed: %s", errMsg)- }- }- return nil, fmt.Errorf("device authorization endpoint returned status %d", resp.StatusCode)- }-- var respData map[string]any- if decodeDeviceAuthorizationErr := json.UnmarshalRead(resp.Body, &respData); decodeDeviceAuthorizationErr != nil {- return nil, fmt.Errorf("failed to decode device authorization response: %w", decodeDeviceAuthorizationErr)- }-- return respData, nil-}-// ExchangeDeviceToken exchanges a device code for tokens.-func (s *OidcService) ExchangeDeviceToken(ctx context.Context, deviceCode string) (*authtypes.OidcUserInfo, *authtypes.OidcTokenResponse, error) {- cfg, err := s.getEffectiveConfigInternal(ctx)+func (s *OidcService) ExchangeDeviceToken(ctx context.Context, deviceCode string) (_ *authtypes.OidcUserInfo, _ *authtypes.OidcTokenResponse, err error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "oidc.device_token")+ defer func() {+ // Pending and slow_down are normal polling outcomes, not failures.+ pending := err != nil && (err.Error() == "authorization_pending" || err.Error() == "slow_down")+ tracing.End(span, kit.Ternary(pending, nil, err))+ }()++ cfg, err := s.effectiveConfig(ctx)if err != nil {slog.ErrorContext(ctx, "ExchangeDeviceToken: failed to get OIDC config", "error", err)return nil, nil, err}+ span.SetAttributes(configAttributes(cfg)...)- var tokenEndpoint string- if cfg.TokenEndpoint != "" {- tokenEndpoint = cfg.TokenEndpoint- } else {- provider, getOrDiscoverProviderErr := s.getOrDiscoverProviderInternal(ctx, cfg)- if getOrDiscoverProviderErr != nil {- return nil, nil, getOrDiscoverProviderErr+ var provider *oidc.Provider+ if !hasManualEndpoints(cfg) {+ if provider, err = s.getOrDiscoverProvider(ctx, cfg); err != nil {+ return nil, nil, err}- tokenEndpoint = provider.Endpoint().TokenURL}-- params := map[string]string{- "grant_type": "urn:ietf:params:oauth:grant-type:device_code",- "device_code": deviceCode,- "client_id": cfg.ClientID,+ tokenEndpoint := cfg.TokenEndpoint+ if tokenEndpoint == "" {+ endpointProvider := provider+ if endpointProvider == nil {+ if endpointProvider, err = s.getOrDiscoverProvider(ctx, cfg); err != nil {+ return nil, nil, err+ }+ }+ tokenEndpoint = endpointProvider.Endpoint().TokenURL}++ values := url.Values{}+ values.Set("grant_type", "urn:ietf:params:oauth:grant-type:device_code")+ values.Set("device_code", deviceCode)+ values.Set("client_id", cfg.ClientID)if cfg.ClientSecret != "" {- params["client_secret"] = cfg.ClientSecret+ values.Set("client_secret", cfg.ClientSecret)}- tokenResp, err := s.makeTokenRequestInternal(ctx, tokenEndpoint, params, cfg.SkipTlsVerify)- if err != nil {- return nil, nil, err+ tokenResp, status, err := postForm(ctx, s.httpClientFor(cfg.SkipTlsVerify), tokenEndpoint, values)+ if status == 0 {+ slog.ErrorContext(ctx, "ExchangeDeviceToken: token request failed", "error", err)+ return nil, nil, fmt.Errorf("token request failed: %w", err)}-- var provider *oidc.Provider- if !s.hasManualEndpointsInternal(cfg) {- provider, err = s.getOrDiscoverProviderInternal(ctx, cfg)- if err != nil {- return nil, nil, err+ if err != nil {+ return nil, nil, fmt.Errorf("failed to decode token response: %w", err)+ }+ if status < 200 || status >= 300 {+ errMsg, ok := tokenResp["error"].(string)+ switch {+ case !ok:+ return nil, nil, fmt.Errorf("token endpoint returned status %d", status)+ case errMsg == "authorization_pending", errMsg == "slow_down", errMsg == "expired_token", errMsg == "access_denied":+ return nil, nil, errors.New(errMsg)+ default:+ return nil, nil, fmt.Errorf("token exchange failed: %s", errMsg)}}- accessToken, ok := tokenResp["access_token"].(string)- if !ok || accessToken == "" {+ accessToken, _ := tokenResp["access_token"].(string)+ if accessToken == "" {return nil, nil, errors.New("invalid access_token in response")}token := &oauth2.Token{- AccessToken: accessToken,- TokenType: kit.As(tokenResp["token_type"], "Bearer"),- }- if refreshToken, localOk := tokenResp["refresh_token"].(string); localOk {- token.RefreshToken = refreshToken+ AccessToken: accessToken,+ TokenType: kit.As(tokenResp["token_type"], "Bearer"),+ RefreshToken: kit.As(tokenResp["refresh_token"], ""),}- if expiresIn, localOk2 := tokenResp["expires_in"].(float64); localOk2 {+ if expiresIn, ok := tokenResp["expires_in"].(float64); ok {token.Expiry = time.Now().Add(time.Duration(expiresIn) * time.Second)}- if idToken, localOk3 := tokenResp["id_token"].(string); localOk3 {+ if idToken, ok := tokenResp["id_token"].(string); ok {token = token.WithExtra(map[string]any{"id_token": idToken})}- idToken, rawIDToken, err := s.verifyIDTokenInternal(ctx, provider, cfg, token, "")+ idToken, rawIDToken, err := s.verifyIDToken(ctx, provider, cfg, token, "")if err != nil {return nil, nil, err}- return s.buildUserInfoInternal(ctx, provider, cfg, token, idToken, rawIDToken)-}--// makeTokenRequestInternal makes a token exchange request.-func (s *OidcService) makeTokenRequestInternal(ctx context.Context, endpoint string, params map[string]string, skipTls bool) (map[string]any, error) {- values := url.Values{}- for k, v := range params {- values.Set(k, v)- }- body := values.Encode()-- req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(body))- if err != nil {- return nil, err- }- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")- req.Header.Set("Accept", "application/json")-- client := s.getHttpClientInternal(skipTls)- resp, err := client.Do(req)- if err != nil {- slog.ErrorContext(ctx, "makeTokenRequestInternal: request failed", "error", err)- return nil, fmt.Errorf("token request failed: %w", err)- }- defer func() { _ = resp.Body.Close() }()-- var tokenResp map[string]any- if unmarshalReadErr := json.UnmarshalRead(resp.Body, &tokenResp); unmarshalReadErr != nil {- return nil, fmt.Errorf("failed to decode token response: %w", unmarshalReadErr)- }-- if resp.StatusCode < 200 || resp.StatusCode >= 300 {- if errMsg, ok := tokenResp["error"].(string); ok {- switch errMsg {- case "authorization_pending":- return nil, errors.New("authorization_pending")- case "slow_down":- return nil, errors.New("slow_down")- case "expired_token":- return nil, errors.New("expired_token")- case "access_denied":- return nil, errors.New("access_denied")- default:- return nil, fmt.Errorf("token exchange failed: %s", errMsg)- }- }- return nil, fmt.Errorf("token endpoint returned status %d", resp.StatusCode)- }-- return tokenResp, nil+ return s.buildUserInfo(ctx, provider, cfg, token, idToken, rawIDToken)}diff --git a/backend/internal/oidc/service_test.go b/backend/internal/oidc/service_test.goindex 0024dcce3..8289428c4 100644--- a/backend/internal/oidc/service_test.go+++ b/backend/internal/oidc/service_test.go@@ -67,7 +67,7 @@ func TestGetMobileRedirectAllowlistTrimsWhitespace(t *testing.T) {func TestGetMobileRedirectAllowlistUsesSettings(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsService, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err,"settings.NewSettingsService: %v", err)@@ -108,7 +108,7 @@ func setupSettingsTestDB(t *testing.T) *database.DB {return &database.DB{DB: db}}-func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {+func newSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {t.Helper()svc, err := settings.NewSettingsService(ctx, db)if err == nil {diff --git a/backend/internal/registry/children/browse/service.go b/backend/internal/registry/children/browse/service.goindex 46476ed72..d759a256a 100644--- a/backend/internal/registry/children/browse/service.go+++ b/backend/internal/registry/children/browse/service.go@@ -17,11 +17,15 @@ import ("github.com/google/go-containerregistry/pkg/v1""github.com/google/go-containerregistry/pkg/v1/remote""github.com/google/go-containerregistry/pkg/v1/remote/transport"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/trace""golang.org/x/sync/errgroup""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/registryauth""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")const tagDetailsConcurrency = 4@@ -52,7 +56,13 @@ func NewService(// ListRepositories lists the repositories of a configured registry through the// catalog API, limited to the namespace of the registry URL.-func (s *Service) ListRepositories(ctx context.Context, id string, params pagination.QueryParams) ([]containerregistry.Repository, pagination.Response, error) {+func (s *Service) ListRepositories(ctx context.Context, id string, params pagination.QueryParams) (listed []containerregistry.Repository, paging pagination.Response, listErr error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "registry.list_repositories", trace.WithAttributes(attribute.String("arcane.registry.id", id)))+ defer func() {+ span.SetAttributes(attribute.Int("arcane.result.count", len(listed)))+ tracing.End(span, listErr)+ }()+lookupCtx, cancel := s.lookupContext(ctx)defer cancel()@@ -82,7 +92,16 @@ func (s *Service) ListRepositories(ctx context.Context, id string, params pagina// ListRepositoryTags lists the tags of a repository. Manifest details are only// fetched for the requested page, and a failure is recorded on the tag itself// so one unreadable manifest does not hide the rest of the page.-func (s *Service) ListRepositoryTags(ctx context.Context, id, repository string, params pagination.QueryParams) ([]containerregistry.RepositoryTag, pagination.Response, error) {+func (s *Service) ListRepositoryTags(ctx context.Context, id, repository string, params pagination.QueryParams) (listed []containerregistry.RepositoryTag, paging pagination.Response, listErr error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "registry.list_tags", trace.WithAttributes(+ attribute.String("arcane.registry.id", id),+ attribute.String("arcane.registry.repository", repository),+ ))+ defer func() {+ span.SetAttributes(attribute.Int("arcane.result.count", len(listed)))+ tracing.End(span, listErr)+ }()+lookupCtx, cancel := s.lookupContext(ctx)defer cancel()@@ -121,7 +140,17 @@ func (s *Service) ListRepositoryTags(ctx context.Context, id, repository string,// DeleteRepositoryTag deletes the manifest a tag points to. Registries delete// manifests by digest, so every tag sharing that digest is removed too.-func (s *Service) DeleteRepositoryTag(ctx context.Context, id, repository, tag string) (string, error) {+func (s *Service) DeleteRepositoryTag(ctx context.Context, id, repository, tag string) (deleted string, opErr error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "registry.delete_tag", trace.WithAttributes(+ attribute.String("arcane.registry.id", id),+ attribute.String("arcane.registry.repository", repository),+ attribute.String("arcane.image.tag", tag),+ ))+ defer func() {+ span.SetAttributes(attribute.String("arcane.image.digest", deleted))+ tracing.End(span, opErr)+ }()+lookupCtx, cancel := s.lookupContext(ctx)defer cancel()diff --git a/backend/internal/registry/service.go b/backend/internal/registry/service.goindex 65bfef964..608720e39 100644--- a/backend/internal/registry/service.go+++ b/backend/internal/registry/service.go@@ -36,6 +36,10 @@ import ("go.getarcane.app/updater/digest""go.getarcane.app/updater/refs""go.getarcane.app/updater/registry"+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/trace""golang.org/x/sync/singleflight""gorm.io/gorm"@@ -48,6 +52,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation")@@ -110,6 +115,10 @@ func NewContainerRegistryService(if len(distributionHTTPClients) > 0 && distributionHTTPClients[0] != nil {distributionHTTPClient = distributionHTTPClients[0]}+ // Trace a copy so the injected client keeps its raw transport.+ traced := *distributionHTTPClient+ traced.Transport = otelhttp.NewTransport(distributionHTTPClient.Transport)+ distributionHTTPClient = &tracedservice := &ContainerRegistryService{db: db,dockerClient: dockerClient,@@ -795,7 +804,11 @@ func registryDisplayNameInternal(registryHost, registryType string) string {}}-func (s *ContainerRegistryService) TestRegistry(ctx context.Context, registryURL, username, token string) error {+func (s *ContainerRegistryService) TestRegistry(ctx context.Context, registryURL, username, token string) (testErr error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "registry.test_credentials", trace.WithAttributes(+ attribute.String("arcane.registry.host", registryauth.NormalizeRegistryForComparison(registryURL)),+ ))+ defer func() { tracing.End(span, testErr) }()if strings.TrimSpace(username) == "" && strings.TrimSpace(token) == "" {// No credentials configured — skip the credential test.return nil@@ -870,7 +883,14 @@ func (s *ContainerRegistryService) ImageDigest(ctx context.Context, imageRef strreturn digestValue, nil}-func (s *ContainerRegistryService) InspectImageDigest(ctx context.Context, imageRef string, externalCreds []containerregistry.Credential) (*containerregistry.DigestResult, error) {+func (s *ContainerRegistryService) InspectImageDigest(ctx context.Context, imageRef string, externalCreds []containerregistry.Credential) (inspected *containerregistry.DigestResult, digestErr error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "registry.inspect_digest", trace.WithAttributes(attribute.String("arcane.image.ref", imageRef)))+ defer func() {+ if inspected != nil && inspected.Digest != "" {+ span.SetAttributes(attribute.String("arcane.image.digest", inspected.Digest))+ }+ tracing.End(span, digestErr)+ }()parts, err := refs.NormalizeReference(imageRef)if err != nil {return nil, common.Classify(common.ErrValidation, err)@@ -1556,7 +1576,12 @@ func (s *ContainerRegistryService) fetchDigestFromRegistryInternal(ctx context.C// ListImageTags discovers repository tags with the same credential precedence as// digest checks. External credentials replace local credentials when supplied.-func (s *ContainerRegistryService) ListImageTags(ctx context.Context, imageRef string, externalCreds []containerregistry.Credential) ([]string, error) {+func (s *ContainerRegistryService) ListImageTags(ctx context.Context, imageRef string, externalCreds []containerregistry.Credential) (tagList []string, tagsErr error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "registry.list_tags", trace.WithAttributes(attribute.String("arcane.image.ref", imageRef)))+ defer func() {+ span.SetAttributes(attribute.Int("arcane.result.count", len(tagList)))+ tracing.End(span, tagsErr)+ }()if refs.IsDigestPinnedReference(imageRef) || refs.IsImageIDLikeReference(imageRef) {return nil, errors.New("cannot discover tags for an immutable image reference")}diff --git a/backend/internal/settings/handler.go b/backend/internal/settings/handler.goindex a16d49612..c978b53a6 100644--- a/backend/internal/settings/handler.go+++ b/backend/internal/settings/handler.go@@ -1,11 +1,13 @@package settingsimport (+ "cmp""context"- "errors""fmt"+ "maps""net/http""os"+ "slices""strconv""strings"@@ -22,6 +24,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/search"+ "github.com/getarcaneapp/arcane/backend/v2/internal/telemetry""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/getarcaneapp/arcane/backend/v2/pkg/projects"@@ -75,110 +78,18 @@ type GetCategoriesOutput struct {Body []category.Category}-// validateProjectsDirectoryValueInternal validates a projects directory value allowing:-// - Unix absolute paths (/...)-// - Windows drive paths (C:/..., C:\...)-// - Mapping format "container:host" where container is absolute Unix or Windows path-func validateProjectsDirectoryValueInternal(path string) error {- switch {- case projects.IsWindowsDrivePath(path):- return nil- case strings.Contains(path, ":"):- parts := strings.SplitN(path, ":", 2)- if len(parts) != 2 {- return errors.New("projectsDirectory must be an absolute path or valid mapping format")- }- container := parts[0]- if !strings.HasPrefix(container, "/") && !projects.IsWindowsDrivePath(container) {- return errors.New("projectsDirectory mapping format: container path must be absolute")- }- return nil- default:- if !strings.HasPrefix(path, "/") {- return errors.New("projectsDirectory must be an absolute path starting with '/'")- }- return nil- }-}--// validateAbsoluteDirectoryPathInternal validates a plain absolute directory path allowing:-// - Unix absolute paths (/...)-// - Windows drive paths (C:/..., C:\...)-func validateAbsoluteDirectoryPathInternal(path string) error {- switch {- case projects.IsWindowsDrivePath(path):- return nil- case strings.HasPrefix(path, "/"):- return nil- default:- return errors.New("must be an absolute path")- }-}--func filterSettingsCategoriesInternal(ps *authz.PermissionSet, categories []category.Category) []category.Category {- if ps == nil {- return []category.Category{}- }- filtered := make([]category.Category, 0, len(categories))- for _, cat := range categories {- if canAccessSettingsCategoryAtAnyScopeInternal(ps, cat.ID) {- filtered = append(filtered, cat)- }- }- return filtered-}--func canAccessSettingsCategoryAtAnyScopeInternal(ps *authz.PermissionSet, categoryID string) bool {- if ps == nil {- return false- }- if authz.CanAccessSettingsCategory(ps, categoryID, "") {- return true- }- for envID := range ps.PerEnv {- if authz.CanAccessSettingsCategory(ps, categoryID, envID) {- return true- }- }- return false-}--func (h *SettingsHandler) appendRuntimeSettingsInternal(settingsDto []settings.PublicSetting, includeAuthenticatedOnly bool) []settings.PublicSetting {- if !includeAuthenticatedOnly {- return settingsDto- }-- uiConfigDisabled := false- if h.cfg != nil {- uiConfigDisabled = h.cfg.UIConfigurationDisabled- }- settingsDto = append(settingsDto, settings.PublicSetting{- Key: "uiConfigDisabled",- Value: strconv.FormatBool(uiConfigDisabled),- Type: "boolean",- })-- projectWorkspaceMaxFileSizeMB := 10- volumeWorkspaceMaxFileSizeMB := 10- if h.cfg != nil {- projectWorkspaceMaxFileSizeMB = workspace.EffectiveMaxFileSizeMB(h.cfg.ProjectWorkspaceMaxFileSizeMB)- volumeWorkspaceMaxFileSizeMB = workspace.EffectiveMaxFileSizeMB(h.cfg.VolumeWorkspaceMaxFileSizeMB)- }+func (h *SettingsHandler) appendRuntimeSettings(settingsDto []settings.PublicSetting, includeAuthenticatedOnly bool) []settings.PublicSetting {+ relay := h.cfg != nil && !h.cfg.AgentModesettingsDto = append(settingsDto,- settings.PublicSetting{Key: projectWorkspaceMaxFileSizeSettingKey, Value: strconv.Itoa(projectWorkspaceMaxFileSizeMB), Type: "number"},- settings.PublicSetting{Key: volumeWorkspaceMaxFileSizeSettingKey, Value: strconv.Itoa(volumeWorkspaceMaxFileSizeMB), Type: "number"},+ settings.PublicSetting{Key: "frontendTracingEnabled", Value: strconv.FormatBool(relay && telemetry.Endpoint("traces") != ""), Type: "boolean"},+ settings.PublicSetting{Key: "frontendMetricsEnabled", Value: strconv.FormatBool(relay && telemetry.Endpoint("metrics") != ""), Type: "boolean"},+ settings.PublicSetting{Key: "frontendLogsEnabled", Value: strconv.FormatBool(relay && telemetry.Endpoint("logs") != ""), Type: "boolean"},)-- backupVolumeName := "arcane-backups"- if h.cfg != nil && strings.TrimSpace(h.cfg.BackupVolumeName) != "" {- backupVolumeName = h.cfg.BackupVolumeName+ if !includeAuthenticatedOnly {+ return settingsDto}- settingsDto = append(settingsDto, settings.PublicSetting{- Key: "backupVolumeName",- Value: backupVolumeName,- Type: "string",- })+ cfg := cmp.Or(h.cfg, &config.Config{})var edgeCfg *edge.Configif h.cfg != nil {edgeCfg = &edge.Config{@@ -188,15 +99,23 @@ func (h *SettingsHandler) appendRuntimeSettingsInternal(settingsDto []settings.P}}_, edgeMTLSCAErr := edge.AvailableManagerMTLSCAPath(edgeCfg)- settingsDto = append(settingsDto, settings.PublicSetting{- Key: "edgeMTLSManagerCAAvailable",- Value: strconv.FormatBool(edgeMTLSCAErr == nil),- Type: "boolean",- })+ projectWorkspaceMaxFileSizeMB := workspace.EffectiveMaxFileSizeMB(cfg.ProjectWorkspaceMaxFileSizeMB)+ volumeWorkspaceMaxFileSizeMB := workspace.EffectiveMaxFileSizeMB(cfg.VolumeWorkspaceMaxFileSizeMB)+ settingsDto = append(settingsDto,+ settings.PublicSetting{Key: "uiConfigDisabled", Value: strconv.FormatBool(cfg.UIConfigurationDisabled), Type: "boolean"},+ settings.PublicSetting{Key: projectWorkspaceMaxFileSizeSettingKey, Value: strconv.Itoa(projectWorkspaceMaxFileSizeMB), Type: "number"},+ settings.PublicSetting{Key: volumeWorkspaceMaxFileSizeSettingKey, Value: strconv.Itoa(volumeWorkspaceMaxFileSizeMB), Type: "number"},+ settings.PublicSetting{+ Key: "backupVolumeName",+ Value: kit.Ternary(strings.TrimSpace(cfg.BackupVolumeName) != "", cfg.BackupVolumeName, "arcane-backups"),+ Type: "string",+ },+ settings.PublicSetting{Key: "edgeMTLSManagerCAAvailable", Value: strconv.FormatBool(edgeMTLSCAErr == nil), Type: "boolean"},+ )if h.settingsService != nil {- cfg := h.settingsService.GetSettingsConfig()- depotConfigured := strings.TrimSpace(cfg.DepotProjectId.Value) != "" && strings.TrimSpace(cfg.DepotToken.Value) != ""+ settingsCfg := h.settingsService.GetSettingsConfig()+ depotConfigured := strings.TrimSpace(settingsCfg.DepotProjectId.Value) != "" && strings.TrimSpace(settingsCfg.DepotToken.Value) != ""settingsDto = append(settingsDto,settings.PublicSetting{Key: "depotConfigured", Value: strconv.FormatBool(depotConfigured), Type: "boolean"},settings.PublicSetting{Key: "enableGravatarEnvForced", Value: strconv.FormatBool(h.settingsService.IsEnvOverrideActive("enableGravatar")), Type: "boolean"},@@ -223,7 +142,7 @@ func (h *SettingsHandler) GetPublicSettings(ctx context.Context, input *GetPublireturn nil, huma.Error500InternalServerError("Failed to map settings")}- return &GetPublicSettingsOutput{Body: h.appendRuntimeSettingsInternal(settingsDto, false)}, nil+ return &GetPublicSettingsOutput{Body: h.appendRuntimeSettings(settingsDto, false)}, nil}// GetSettings returns all settings for an environment.@@ -244,13 +163,10 @@ func (h *SettingsHandler) GetSettings(ctx context.Context, input *GetSettingsInp}allowedKeys[projectWorkspaceMaxFileSizeSettingKey] = struct{}{}allowedKeys[volumeWorkspaceMaxFileSizeSettingKey] = struct{}{}- filtered := make([]settings.PublicSetting, 0, len(*settingsDto))- for _, setting := range *settingsDto {- if _, ok := allowedKeys[setting.Key]; ok {- filtered = append(filtered, setting)- }- }- settingsDto = &filtered+ *settingsDto = slices.DeleteFunc(*settingsDto, func(setting settings.PublicSetting) bool {+ _, ok := allowedKeys[setting.Key]+ return !ok+ })}return &GetSettingsOutput{Body: *settingsDto}, nil}@@ -262,99 +178,79 @@ func (h *SettingsHandler) GetSettings(ctx context.Context, input *GetSettingsInpreturn nil, huma.Error500InternalServerError("Failed to map settings")}- return &GetSettingsOutput{Body: h.appendRuntimeSettingsInternal(settingsDto, true)}, nil+ return &GetSettingsOutput{Body: h.appendRuntimeSettings(settingsDto, true)}, nil}// UpdateSettings updates settings for an environment.func (h *SettingsHandler) UpdateSettings(ctx context.Context, input *UpdateSettingsInput) (*handlerutil.Out[[]settings.SettingDto], error) {- if err := h.validateSettingsUpdateInput(input.Body); err != nil {- return nil, err+ update := input.Body+ // Only validate changed directories so env-provided values do not block unrelated saves.+ currentCfg := h.settingsService.GetSettingsConfig()+ projectsDir := ""+ if update.ProjectsDirectory != nil && *update.ProjectsDirectory != currentCfg.ProjectsDirectory.Value {+ projectsDir = *update.ProjectsDirectory}-- if input.EnvironmentID != "0" {- return h.updateSettingsForRemoteEnvironment(ctx, input)- }-- return h.updateSettingsForLocalEnvironment(ctx, input.Body)-}--func (h *SettingsHandler) validateSettingsUpdateInput(input settings.Update) error {- // Validate projects directory if provided and changed from current value.- // Skip validation when the value matches the current (possibly env-overridden) setting- // so that saving unrelated settings doesn't fail due to env-provided directory formats.- if input.ProjectsDirectory != nil && *input.ProjectsDirectory != "" {- currentDir := h.settingsService.GetSettingsConfig().ProjectsDirectory.Value- if *input.ProjectsDirectory != currentDir {- if err := validateProjectsDirectoryValueInternal(*input.ProjectsDirectory); err != nil {- return huma.Error400BadRequest(err.Error())- }+ if projectsDir != "" && !projects.IsWindowsDrivePath(projectsDir) {+ // Mapping format is "container:host"; the container path must be absolute.+ if container, _, isMapping := strings.Cut(projectsDir, ":"); !strings.HasPrefix(container, "/") {+ return nil, huma.Error400BadRequest(kit.Ternary(isMapping,+ "projectsDirectory mapping format: container path must be absolute",+ "projectsDirectory must be an absolute path starting with '/'"))}}- if input.SwarmStackSourcesDirectory != nil && *input.SwarmStackSourcesDirectory != "" {- currentDir := h.settingsService.GetSettingsConfig().SwarmStackSourcesDirectory.Value- if *input.SwarmStackSourcesDirectory != currentDir {- if err := validateAbsoluteDirectoryPathInternal(*input.SwarmStackSourcesDirectory); err != nil {- return huma.Error400BadRequest("swarmStackSourcesDirectory " + err.Error())- }- }+ if dir := update.SwarmStackSourcesDirectory; dir != nil && *dir != "" && *dir != currentCfg.SwarmStackSourcesDirectory.Value &&+ !projects.IsWindowsDrivePath(*dir) && !strings.HasPrefix(*dir, "/") {+ return nil, huma.Error400BadRequest("swarmStackSourcesDirectory must be an absolute path")}- if input.AvatarMaxUploadSizeMb != nil && strings.TrimSpace(*input.AvatarMaxUploadSizeMb) != "" {- value, err := strconv.Atoi(strings.TrimSpace(*input.AvatarMaxUploadSizeMb))+ if update.AvatarMaxUploadSizeMb != nil && strings.TrimSpace(*update.AvatarMaxUploadSizeMb) != "" {+ value, err := strconv.Atoi(strings.TrimSpace(*update.AvatarMaxUploadSizeMb))if err != nil || value < 1 || value > 50 {- return huma.Error400BadRequest("avatarMaxUploadSizeMb must be between 1 and 50")+ return nil, huma.Error400BadRequest("avatarMaxUploadSizeMb must be between 1 and 50")}}- if input.TrivyConfig != nil && strings.TrimSpace(*input.TrivyConfig) != "" {+ if update.TrivyConfig != nil && strings.TrimSpace(*update.TrivyConfig) != "" {var doc map[string]any- if err := yaml.Unmarshal([]byte(*input.TrivyConfig), &doc); err != nil {- return huma.Error400BadRequest("trivyConfig must be a YAML mapping: " + err.Error())+ if err := yaml.Unmarshal([]byte(*update.TrivyConfig), &doc); err != nil {+ return nil, huma.Error400BadRequest("trivyConfig must be a YAML mapping: " + err.Error())}if doc == nil {- return huma.Error400BadRequest("trivyConfig must be a YAML mapping")+ return nil, huma.Error400BadRequest("trivyConfig must be a YAML mapping")}}- return nil-}--func (h *SettingsHandler) updateSettingsForRemoteEnvironment(ctx context.Context, input *UpdateSettingsInput) (*handlerutil.Out[[]settings.SettingDto], error) {- // Check if trying to update auth settings on non-local environment.- if hasAuthSettingsUpdateInternal(input.Body) {- return nil, huma.Error403Forbidden("Authentication settings can only be updated from the main environment")- }-- apiResp, err := h.proxyRemoteJSON.JSON[base.ApiResponse[[]settings.SettingDto]](ctx, input.EnvironmentID, http.MethodPut, "/api/environments/0/settings", input.Body)- if err != nil {- return nil, err+ if input.EnvironmentID != "0" {+ if update.AuthLocalEnabled != nil || update.OidcEnabled != nil || update.AuthSessionTimeout != nil || update.AuthPasswordPolicy != nil ||+ update.OidcClientId != nil || update.OidcClientSecret != nil || update.OidcIssuerUrl != nil || update.OidcScopes != nil ||+ update.OidcMergeAccounts != nil || update.OidcSkipTlsVerify != nil || update.OidcAutoRedirectToProvider != nil ||+ update.OidcProviderName != nil || update.OidcProviderLogoUrl != nil || update.OidcGroupsClaim != nil {+ return nil, huma.Error403Forbidden("Authentication settings can only be updated from the main environment")+ }+ apiResp, err := h.proxyRemoteJSON.JSON[base.ApiResponse[[]settings.SettingDto]](ctx, input.EnvironmentID, http.MethodPut, "/api/environments/0/settings", update)+ if err != nil {+ return nil, err+ }+ return &handlerutil.Out[[]settings.SettingDto]{Body: *apiResp}, nil}- return &handlerutil.Out[[]settings.SettingDto]{Body: *apiResp}, nil-}--func (h *SettingsHandler) updateSettingsForLocalEnvironment(ctx context.Context, input settings.Update) (*handlerutil.Out[[]settings.SettingDto], error) {- if input.ProjectsDirectory != nil && *input.ProjectsDirectory != "" {- currentDir := h.settingsService.GetSettingsConfig().ProjectsDirectory.Value- if *input.ProjectsDirectory != currentDir {- resolved, err := projects.GetProjectsDirectory(ctx, strings.TrimSpace(*input.ProjectsDirectory))- if err != nil {- return nil, huma.Error400BadRequest(fmt.Sprintf("cannot use projects directory %q: %v", *input.ProjectsDirectory, err))- }- // os.* rather than acfs: this validates a user-typed directory before- // it becomes a confinement root, so no root exists for it yet.- f, err := os.Open(resolved)- if err != nil {- return nil, huma.Error400BadRequest(fmt.Sprintf("cannot read projects directory %q: %v", resolved, err))- }- if closeErr := f.Close(); closeErr != nil {- return nil, huma.Error400BadRequest(fmt.Sprintf("cannot read projects directory %q: %v", resolved, closeErr))- }+ if projectsDir != "" {+ resolved, err := projects.GetProjectsDirectory(ctx, strings.TrimSpace(projectsDir))+ if err != nil {+ return nil, huma.Error400BadRequest(fmt.Sprintf("cannot use projects directory %q: %v", projectsDir, err))+ }+ // os rather than acfs: the directory is validated before it becomes a confinement root.+ f, err := os.Open(resolved)+ if err == nil {+ err = f.Close()+ }+ if err != nil {+ return nil, huma.Error400BadRequest(fmt.Sprintf("cannot read projects directory %q: %v", resolved, err))}}- updatedSettings, err := h.settingsService.UpdateSettings(ctx, input)+ updatedSettings, err := h.settingsService.UpdateSettings(ctx, update)if err != nil {apiErr := common.ToAPIError(err)if apiErr.HTTPStatus() == http.StatusInternalServerError {@@ -365,49 +261,43 @@ func (h *SettingsHandler) updateSettingsForLocalEnvironment(ctx context.Context,settingDtos := make([]settings.SettingDto, 0, len(updatedSettings))for _, setting := range updatedSettings {- settingDtos = append(settingDtos, settings.SettingDto{- Key: setting.Key,- Type: "string",- Value: setting.Value,- })+ settingDtos = append(settingDtos, settings.SettingDto{Key: setting.Key, Type: "string", Value: setting.Value})}return &handlerutil.Out[[]settings.SettingDto]{- Body: base.ApiResponse[[]settings.SettingDto]{- Success: true,- Data: settingDtos,- },+ Body: base.ApiResponse[[]settings.SettingDto]{Success: true, Data: settingDtos},}, nil}-func hasAuthSettingsUpdateInternal(req settings.Update) bool {- return req.AuthLocalEnabled != nil || req.OidcEnabled != nil ||- req.AuthSessionTimeout != nil || req.AuthPasswordPolicy != nil ||- req.OidcClientId != nil ||- req.OidcClientSecret != nil || req.OidcIssuerUrl != nil ||- req.OidcScopes != nil ||- req.OidcMergeAccounts != nil ||- req.OidcSkipTlsVerify != nil || req.OidcAutoRedirectToProvider != nil ||- req.OidcProviderName != nil || req.OidcProviderLogoUrl != nil ||- req.OidcGroupsClaim != nil-}-// Search searches settings by query.func (h *SettingsHandler) Search(ctx context.Context, input *SearchSettingsInput) (*SearchSettingsOutput, error) {if strings.TrimSpace(input.Body.Query) == "" {return nil, huma.Error400BadRequest("Query parameter is required")}- ps, _ := middleware.PermissionsFromContext(ctx)- results := search.Search(h.settingsSearchService.GetSettingsCategories(), input.Body.Query, searchtypes.SettingsProfile)- results.Results = filterSettingsCategoriesInternal(ps, results.Results)- results.Count = len(results.Results)+ categories, err := h.GetCategories(ctx, nil)+ if err != nil {+ return nil, err+ }+ results := search.Search(categories.Body, input.Body.Query, searchtypes.SettingsProfile)+ if results.Results == nil {+ results.Results = []category.Category{}+ }return &SearchSettingsOutput{Body: results}, nil}-// GetCategories returns all available settings categories.+// GetCategories returns the settings categories reachable at any scope.func (h *SettingsHandler) GetCategories(ctx context.Context, input *struct{}) (*GetCategoriesOutput, error) {+ categories := []category.Category{}ps, _ := middleware.PermissionsFromContext(ctx)- categories := filterSettingsCategoriesInternal(ps, h.settingsSearchService.GetSettingsCategories())+ if ps == nil {+ return &GetCategoriesOutput{Body: categories}, nil+ }+ scopes := append([]string{""}, slices.Collect(maps.Keys(ps.PerEnv))...)+ for _, cat := range h.settingsSearchService.GetSettingsCategories() {+ if slices.ContainsFunc(scopes, func(envID string) bool { return authz.CanAccessSettingsCategory(ps, cat.ID, envID) }) {+ categories = append(categories, cat)+ }+ }return &GetCategoriesOutput{Body: categories}, nil}diff --git a/backend/internal/settings/handler_test.go b/backend/internal/settings/handler_test.goindex 312e4420c..562d9f61e 100644--- a/backend/internal/settings/handler_test.go+++ b/backend/internal/settings/handler_test.go@@ -19,7 +19,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge")-func TestSettingsHandlerAppendRuntimeSettingsInternal(t *testing.T) {+func TestSettingsHandlerAppendRuntimeSettings(t *testing.T) {handler := &SettingsHandler{cfg: &config.Config{UIConfigurationDisabled: true,BackupVolumeName: "custom-backups",@@ -27,13 +27,13 @@ func TestSettingsHandlerAppendRuntimeSettingsInternal(t *testing.T) {VolumeWorkspaceMaxFileSizeMB: 18,}}- publicKeys := runtimeSettingKeysInternal(handler.appendRuntimeSettingsInternal(nil, false))+ publicKeys := runtimeSettingKeys(handler.appendRuntimeSettings(nil, false))require.NotContains(t, publicKeys, "uiConfigDisabled")require.NotContains(t, publicKeys, "backupVolumeName")require.NotContains(t, publicKeys, "depotConfigured")require.NotContains(t, publicKeys, "edgeMTLSManagerCAAvailable")- authenticatedKeys := runtimeSettingKeysInternal(handler.appendRuntimeSettingsInternal(nil, true))+ authenticatedKeys := runtimeSettingKeys(handler.appendRuntimeSettings(nil, true))require.Equal(t, "12", authenticatedKeys[projectWorkspaceMaxFileSizeSettingKey])require.Equal(t, "18", authenticatedKeys[volumeWorkspaceMaxFileSizeSettingKey])require.Equal(t, "true", authenticatedKeys["uiConfigDisabled"])@@ -41,20 +41,20 @@ func TestSettingsHandlerAppendRuntimeSettingsInternal(t *testing.T) {require.Equal(t, "false", authenticatedKeys["edgeMTLSManagerCAAvailable"])}-func TestSettingsHandlerAppendRuntimeSettingsDoesNotGenerateEdgeMTLSCAInternal(t *testing.T) {+func TestSettingsHandlerAppendRuntimeSettingsDoesNotGenerateEdgeMTLSCA(t *testing.T) {assetsDir := t.TempDir()handler := &SettingsHandler{cfg: &config.Config{EdgeMTLSMode: edge.EdgeMTLSModeRequired,EdgeMTLSAssetsDir: assetsDir,}}- authenticatedKeys := runtimeSettingKeysInternal(handler.appendRuntimeSettingsInternal(nil, true))+ authenticatedKeys := runtimeSettingKeys(handler.appendRuntimeSettings(nil, true))require.Equal(t, "false", authenticatedKeys["edgeMTLSManagerCAAvailable"])require.NoFileExists(t, filepath.Join(assetsDir, "ca.crt"))require.NoFileExists(t, filepath.Join(assetsDir, "ca.key"))}-func TestSettingsHandlerUpdateLocalEnvironmentRejectsUnreadableProjectsDirectoryInternal(t *testing.T) {+func TestSettingsHandlerUpdateLocalEnvironmentRejectsUnreadableProjectsDirectory(t *testing.T) {if runtime.GOOS == "windows" {t.Skip("POSIX permission-denied behavior is not portable to Windows")}@@ -63,7 +63,7 @@ func TestSettingsHandlerUpdateLocalEnvironmentRejectsUnreadableProjectsDirectory}ctx := t.Context()- settingsService, err := newSettingsServiceForTestInternal(t, ctx, setupSettingsTestDB(t))+ settingsService, err := newSettingsServiceForTest(t, ctx, setupSettingsTestDB(t))require.NoError(t, err)originalDir := settingsService.GetSettingsConfig().ProjectsDirectory.Value@@ -72,7 +72,7 @@ func TestSettingsHandlerUpdateLocalEnvironmentRejectsUnreadableProjectsDirectoryt.Cleanup(func() { _ = os.Chmod(unreadable, 0o700) })handler := &SettingsHandler{settingsService: settingsService, cfg: &config.Config{}}- _, err = handler.updateSettingsForLocalEnvironment(ctx, settings.Update{ProjectsDirectory: new(unreadable)})+ _, err = handler.UpdateSettings(ctx, &UpdateSettingsInput{EnvironmentID: "0", Body: settings.Update{ProjectsDirectory: new(unreadable)}})require.Error(t, err)var statusErr huma.StatusError@@ -82,7 +82,7 @@ func TestSettingsHandlerUpdateLocalEnvironmentRejectsUnreadableProjectsDirectoryrequire.Equal(t, originalDir, settingsService.GetSettingsConfig().ProjectsDirectory.Value)}-func TestSettingsHandlerRemoteWorkspaceSettingsVisibilityInternal(t *testing.T) {+func TestSettingsHandlerRemoteWorkspaceSettingsVisibility(t *testing.T) {remoteSettings := []settings.PublicSetting{{Key: "dockerHost", Type: "string", Value: "unix:///var/run/docker.sock"},{Key: "baseServerUrl", Type: "string", Value: "https://manager.example"},@@ -99,7 +99,7 @@ func TestSettingsHandlerRemoteWorkspaceSettingsVisibilityInternal(t *testing.T)return json.Unmarshal(payload, output)}- settingsService, err := newSettingsServiceForTestInternal(t, t.Context(), setupSettingsTestDB(t))+ settingsService, err := newSettingsServiceForTest(t, t.Context(), setupSettingsTestDB(t))require.NoError(t, err)handler := &SettingsHandler{settingsService: settingsService, proxyRemoteJSON: proxy}@@ -116,7 +116,7 @@ func TestSettingsHandlerRemoteWorkspaceSettingsVisibilityInternal(t *testing.T)require.Equal(t, remoteSettings, output.Body)}-func runtimeSettingKeysInternal(publicSettings []settings.PublicSetting) map[string]string {+func runtimeSettingKeys(publicSettings []settings.PublicSetting) map[string]string {keys := make(map[string]string, len(publicSettings))for _, setting := range publicSettings {keys[setting.Key] = setting.Valuediff --git a/backend/internal/settings/helpers.go b/backend/internal/settings/helpers.gonew file mode 100644index 000000000..9aa2c7798--- /dev/null+++ b/backend/internal/settings/helpers.go@@ -0,0 +1,47 @@+package settings++import (+ "context"+ "fmt"+ "strings"+ "uuid"++ "go.getarcane.app/kit/pkg"+ "gorm.io/gorm"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+)++// EnsureInstanceID returns this installation's persisted instance ID, creating it on first start.+func EnsureInstanceID(ctx context.Context, db *database.DB) (string, error) {+ return ensureSettingValue(ctx, db, "instanceId", "instance ID", func() (string, error) { return uuid.New().String(), nil })+}++func ensureSettingValue(ctx context.Context, db *database.DB, key, label string, generate func() (string, error)) (string, error) {+ var value string+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ var setting SettingVariable+ if err := tx.Where("key = ?", key).Limit(1).Find(&setting).Error; err != nil {+ return fmt.Errorf("failed to load %s: %w", label, err)+ }+ if setting.Value != "" {+ value = setting.Value+ return nil+ }++ generated, err := generate()+ if err != nil {+ return err+ }+ if saveErr := tx.Save(&SettingVariable{Key: key, Value: generated}).Error; saveErr != nil {+ return fmt.Errorf("failed to persist %s: %w", label, saveErr)+ }+ value = generated+ return nil+ })+ return value, err+}++func settingEnvName(key string) string {+ return strings.ToUpper(kit.SnakeCase(key))+}diff --git a/backend/internal/settings/service.go b/backend/internal/settings/service.goindex dc28a73bd..0e2e0fd2c 100644--- a/backend/internal/settings/service.go+++ b/backend/internal/settings/service.go@@ -9,7 +9,6 @@ import ("errors""fmt""log/slog"- "maps""os""path/filepath""reflect"@@ -51,12 +50,8 @@ const (type SettingsService struct {db *database.DBconfig atomic.Pointer[Settings]- // effectiveConfig is the env-override-applied snapshot, rebuilt whenever- // config is stored. Overrides are resolved once at startup and never- // change, so materializing them here turns every read (GetSettings and- // the typed getters — ~100 call sites, some in per-project loops) into a- // pointer load instead of a full struct Clone plus a reflection pass.- // The snapshot is shared: readers must never mutate it.+ // effectiveConfig is the shared, read-only env-override-applied snapshot,+ // rebuilt whenever config is stored so reads are a pointer load.effectiveConfig atomic.Pointer[Settings]envOverrides []settingsEnvOverridewrites sync.Mutex@@ -69,15 +64,9 @@ type SettingsService struct {changes *concurrency.Signal[[]libarcane.SettingUpdate]}-type settingsUpdateResultInternal struct {- settings []SettingVariable- changes []libarcane.SettingUpdate-}-type settingsEnvOverride struct {fieldIndex intkey string- envVarName stringvalue string}@@ -92,28 +81,57 @@ func NewSettingsService(ctx context.Context, db *database.DB) (*SettingsService,lifecycleCtx: ctx,changes: concurrency.NewSignal[[]libarcane.SettingUpdate](),}- svc.envOverrides = resolveSettingsEnvOverridesInternal()+ fields, _ := getSettingsFieldCacheInternal()+ for _, field := range fields {+ if !slices.Contains(splitSettingAttrsInternal(field.attrs), "envOverride") {+ continue+ }+ if val, ok, _ := utils.LookupEnvOrFile(settingEnvName(field.key)); ok && val != "" {+ svc.envOverrides = append(svc.envOverrides, settingsEnvOverride{fieldIndex: field.index, key: field.key, value: kit.TrimQuotes(val)})+ }+ }if len(svc.envOverrides) > 0 {slog.InfoContext(ctx, "Loaded Environment Settings Overrides", "count", len(svc.envOverrides))}- err := svc.LoadDatabaseSettings(ctx)- if err != nil {- return nil, fmt.Errorf("failed to load settings: %w", err)- }-- err = svc.setupInstanceID(ctx)- if err != nil {+ if _, err := EnsureInstanceID(ctx, db); err != nil {return nil, fmt.Errorf("failed to setup instance ID: %w", err)}+ if err := svc.LoadDatabaseSettings(ctx); err != nil {+ return nil, fmt.Errorf("failed to load settings: %w", err)+ }- go svc.runEffectsInternal()+ go func() {+ defer close(svc.effectsDone)+ for {+ svc.effectsMu.Lock()+ if len(svc.effects) > 0 {+ effect := svc.effects[0]+ svc.effects[0] = nil+ svc.effects = svc.effects[1:]+ svc.effectsMu.Unlock()+ func() { defer utils.RecoverToError(nil, "settings effect"); effect() }()+ continue+ }+ closed := svc.effectsClosed+ svc.effectsMu.Unlock()+ if closed {+ return+ }+ select {+ case <-svc.effectsWake:+ case <-svc.lifecycleCtx.Done():+ svc.effectsMu.Lock()+ svc.effectsClosed = true+ svc.effectsMu.Unlock()+ }+ }+ }()return svc, nil}-// SubscribeSettingsChanges invokes callback once when any subscribed key is-// present in a successful update. The callback receives only matching values.-// Callbacks run serially on the ordered effects worker, outside the write mutex.+// SubscribeSettingsChanges invokes callback once with the matching values when any subscribed key+// is present in a successful update. Callbacks run serially on the effects worker, outside the write mutex.func (s *SettingsService) SubscribeSettingsChanges(keys []string, callback func([]libarcane.SettingUpdate)) func() {if callback == nil || len(keys) == 0 {return func() {}@@ -130,7 +148,7 @@ func (s *SettingsService) SubscribeSettingsChanges(keys []string, callback func(}}if len(matching) > 0 {- if err := s.enqueueEffectInternal(func() { callback(matching) }); err != nil && s.lifecycleCtx.Err() == nil {+ if err := s.enqueueEffect(func() { callback(matching) }); err != nil && s.lifecycleCtx.Err() == nil {slog.ErrorContext(s.lifecycleCtx, "Failed to queue settings change effects", "error", err)}}@@ -154,7 +172,7 @@ func (s *SettingsService) NotifySettingsChanges(ctx context.Context, keys ...str}updates = append(updates, libarcane.SettingUpdate{Key: key, Value: value})}- s.publishSettingsChangesInternal(updates)+ s.publishSettingsChanges(updates)return nil}@@ -167,33 +185,75 @@ func (s *SettingsService) GetSettingsConfig() *Settings {return v}-func (s *SettingsService) LoadDatabaseSettings(ctx context.Context) (err error) {- dst, err := s.loadDatabaseSettingsInternal(ctx, s.db)- if err != nil {- return err+func (s *SettingsService) LoadDatabaseSettings(ctx context.Context) error {+ var stored []SettingVariable+ queryCtx, queryCancel := context.WithTimeout(ctx, 10*time.Second)+ defer queryCancel()+ if err := s.db.WithContext(queryCtx).Find(&stored).Error; err != nil {+ return fmt.Errorf("failed to load configuration from the database: %w", err)+ }++ dst := DefaultSettingsConfig()+ appCfg := config.Load()+ fields, _ := getSettingsFieldCacheInternal()+ rv := reflect.ValueOf(dst).Elem()+ if appCfg.UIConfigurationDisabled || appCfg.AgentMode {+ slog.DebugContext(ctx, "LoadDatabaseSettings: using env path",+ "uiConfigurationDisabled", appCfg.UIConfigurationDisabled, "agentMode", appCfg.AgentMode, "environment", appCfg.Environment)++ storedValues := make(map[string]string, len(stored))+ for _, setting := range stored {+ storedValues[setting.Key] = setting.Value+ }+ for _, field := range fields {+ value := rv.Field(field.index).FieldByName("Value")+ if slices.Contains(splitSettingAttrsInternal(field.attrs), "internal") {+ if val, ok := storedValues[field.key]; ok {+ value.SetString(val)+ }+ continue+ }++ envVarName := settingEnvName(field.key)+ if val, ok, _ := utils.LookupEnvOrFile(envVarName); ok {+ mask := kit.Ternary(val == "", "<empty>", fmt.Sprintf("%d chars", len(val)))+ slog.DebugContext(ctx, "LoadDatabaseSettings: env override found", "key", field.key, "env", envVarName, "valueMasked", mask)+ value.SetString(kit.TrimQuotes(val))+ continue+ }+ if val, ok := storedValues[field.key]; ok {+ slog.DebugContext(ctx, "LoadDatabaseSettings: using database fallback", "key", field.key)+ value.SetString(val)+ continue+ }+ slog.DebugContext(ctx, "LoadDatabaseSettings: env not set and no database value", "key", field.key, "env", envVarName)+ }++ count := 0+ for _, field := range fields {+ if rv.Field(field.index).FieldByName("Value").String() != "" {+ count+++ }+ }+ slog.DebugContext(ctx, "LoadDatabaseSettings: completed env load", "loadedFields", count)+ } else {+ for _, setting := range stored {+ if err := dst.UpdateField(setting.Key, setting.Value, false); err != nil && !errors.Is(err, SettingKeyNotFoundError{}) {+ return fmt.Errorf("failed to process settings for key '%s': %w", setting.Key, err)+ }+ }+ s.applyEnvOverrides(dst)}s.config.Store(dst)effective := dst.Clone()- s.applyEnvOverrides(ctx, effective)+ s.applyEnvOverrides(effective)s.effectiveConfig.Store(effective)return nil}-func (s *SettingsService) refreshSettingsCacheInternal(ctx context.Context) error {- if err := s.LoadDatabaseSettings(ctx); err != nil {- return fmt.Errorf("failed to refresh settings cache: %w", err)- }-- return nil-}--func (s *SettingsService) getDefaultSettings() *Settings {- return DefaultSettingsConfig()-}-// DefaultSettingsConfig returns the canonical default settings model used by Arcane.func DefaultSettingsConfig() *Settings {return &Settings{@@ -317,111 +377,7 @@ func DefaultSettingsConfig() *Settings {}}-func (s *SettingsService) loadDatabaseSettingsInternal(ctx context.Context, db *database.DB) (*Settings, error) {- if config.Load().UIConfigurationDisabled || config.Load().AgentMode {- slog.DebugContext(- ctx,- "loadDatabaseSettingsInternal: using env path",- "uiConfigurationDisabled",- config.Load().UIConfigurationDisabled,- "agentMode",- config.Load().AgentMode,- "environment",- config.Load().Environment,- )- return s.loadDatabaseConfigFromEnv(ctx, db)- }-- dest := s.getDefaultSettings()-- var loaded []SettingVariable- queryCtx, queryCancel := context.WithTimeout(ctx, 10*time.Second)- defer queryCancel()- err := db.WithContext(queryCtx).- Find(&loaded).Error- if err != nil {- return nil, fmt.Errorf("failed to load configuration from the database: %w", err)- }-- for _, v := range loaded {- err = dest.UpdateField(v.Key, v.Value, false)-- if err != nil && !errors.Is(err, SettingKeyNotFoundError{}) {- return nil, fmt.Errorf("failed to process settings for key '%s': %w", v.Key, err)- }- }-- // Apply environment variable overrides for fields tagged with "envOverride"- s.applyEnvOverrides(ctx, dest)-- return dest, nil-}--func (s *SettingsService) loadDatabaseConfigFromEnv(ctx context.Context, db *database.DB) (*Settings, error) {- dest := s.getDefaultSettings()-- // Fetch all settings once to avoid N+1 queries for internal keys- var allSettings []SettingVariable- if err := db.WithContext(ctx).Find(&allSettings).Error; err != nil {- return nil, fmt.Errorf("failed to load settings for env config: %w", err)- }- settingsMap := make(map[string]string, len(allSettings))- for _, s := range allSettings {- settingsMap[s.Key] = s.Value- }-- rt := reflect.ValueOf(dest).Elem().Type()- rv := reflect.ValueOf(dest).Elem()- for i := range rt.NumField() {- field := rt.Field(i)-- tagParts := strings.Split(field.Tag.Get("key"), ",")- key := tagParts[0]- isInternal := slices.Contains(tagParts[1:], "internal")-- if isInternal {- if val, ok := settingsMap[key]; ok {- rv.Field(i).FieldByName("Value").SetString(val)- }- continue- }-- envVarName := strings.ToUpper(kit.SnakeCase(key))-- // debug: log each env name checked and whether a value exists- if val, ok, _ := utils.LookupEnvOrFile(envVarName); ok {- mask := "<empty>"- if val != "" {- mask = fmt.Sprintf("%d chars", len(val))- }- slog.DebugContext(ctx, "loadDatabaseConfigFromEnv: env override found", "key", key, "env", envVarName, "valueMasked", mask)- rv.Field(i).FieldByName("Value").SetString(kit.TrimQuotes(val))- continue- }- if val, ok := settingsMap[key]; ok {- // Fallback to database if environment variable is not set- slog.DebugContext(ctx, "loadDatabaseConfigFromEnv: using database fallback", "key", key)- rv.Field(i).FieldByName("Value").SetString(val)- continue- }- slog.DebugContext(ctx, "loadDatabaseConfigFromEnv: env not set and no database value", "key", key, "env", envVarName)- }-- // debug: final snapshot (only show which fields are non-empty)- count := 0- for i := range rt.NumField() {- v := rv.Field(i).FieldByName("Value").String()- if v != "" {- count++- }- }- slog.DebugContext(ctx, "loadDatabaseConfigFromEnv: completed env load", "loadedFields", count)-- return dest, nil-}--func (s *SettingsService) applyEnvOverrides(ctx context.Context, dest *Settings) {- _ = ctx+func (s *SettingsService) applyEnvOverrides(dest *Settings) {rv := reflect.ValueOf(dest).Elem()for _, override := range s.envOverrides {@@ -429,64 +385,27 @@ func (s *SettingsService) applyEnvOverrides(ctx context.Context, dest *Settings)}}-func resolveSettingsEnvOverridesInternal() []settingsEnvOverride {- rt := reflect.TypeFor[Settings]()- overrides := make([]settingsEnvOverride, 0)-- for i := range rt.NumField() {- field := rt.Field(i)- tagValue := field.Tag.Get("key")- if tagValue == "" {- continue- }-- // Parse tag attributes (e.g., "dockerHost,public,envOverride")- parts := strings.Split(tagValue, ",")- key := parts[0]- hasEnvOverride := slices.Contains(parts[1:], "envOverride")-- if !hasEnvOverride {- continue- }-- envVarName := strings.ToUpper(kit.SnakeCase(key))- if val, ok, _ := utils.LookupEnvOrFile(envVarName); ok && val != "" {- overrides = append(overrides, settingsEnvOverride{- fieldIndex: i,- key: key,- envVarName: envVarName,- value: kit.TrimQuotes(val),- })- }- }-- return overrides+// IsEnvOverrideActive reports whether a non-empty environment override controls the setting.+func (s *SettingsService) IsEnvOverrideActive(key string) bool {+ return slices.ContainsFunc(s.envOverrides, func(override settingsEnvOverride) bool { return override.key == key })}-// IsEnvOverrideActive reports whether an environment override currently controls the setting.-// and its corresponding environment variable is currently set to a non-empty value.-func (s *SettingsService) IsEnvOverrideActive(key string) bool {- for _, override := range s.envOverrides {- if override.key == key {- return true- }+// GetSettings returns the shared, read-only effective (env-override-applied) settings snapshot.+// Mutation flows go through UpdateSettings, which works on an explicit clone.+func (s *SettingsService) GetSettings(_ context.Context) (*Settings, error) {+ if effective := s.effectiveConfig.Load(); effective != nil {+ return effective, nil}- return false-}--// GetSettings returns the effective (env-override-applied) settings snapshot.-// The snapshot is shared across callers and must be treated as read-only;-// mutation flows go through UpdateSettings, which works on an explicit clone.-func (s *SettingsService) GetSettings(ctx context.Context) (*Settings, error) {- settingsCfg := s.getEffectiveSettingsConfigInternal(ctx)+ // Only reachable before LoadDatabaseSettings has completed in the+ // constructor; fall back to building the snapshot per call.+ settingsCfg := s.GetSettingsConfig().Clone()+ s.applyEnvOverrides(settingsCfg)return settingsCfg, nil}-// GetSettingsOrDefaults is a convenience for hot paths that need a snapshot but cannot-// meaningfully recover from a settings load failure. It logs any error and guarantees a-// non-nil *Settings (defaults: a zero-valued struct, which the SettingVariable helpers-// like kit.ParseOrDefault treat as "use the caller's default").+// GetSettingsOrDefaults logs any settings load failure and always returns a non-nil *Settings;+// a zero-valued struct makes SettingVariable helpers fall back to the caller's default.func (s *SettingsService) GetSettingsOrDefaults(ctx context.Context) *Settings {cfg, err := s.GetSettings(ctx)if err != nil {@@ -495,19 +414,7 @@ func (s *SettingsService) GetSettingsOrDefaults(ctx context.Context) *Settings {return kit.Ternary(cfg == nil, &Settings{}, cfg)}-func (s *SettingsService) getEffectiveSettingsConfigInternal(ctx context.Context) *Settings {- if effective := s.effectiveConfig.Load(); effective != nil {- return effective- }-- // Only reachable before LoadDatabaseSettings has completed in the- // constructor; fall back to building the snapshot per call.- settingsCfg := s.GetSettingsConfig().Clone()- s.applyEnvOverrides(ctx, settingsCfg)- return settingsCfg-}--func validateSettingValueInternal(key, value string) error {+func validateSettingValue(key, value string) error {if key == "upgradeLogRetentionDays" && value != "" {days, err := strconv.Atoi(value)if err != nil || days < 0 || days > 3650 {@@ -523,7 +430,7 @@ func validateSettingValueInternal(key, value string) error {}func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string) error {- if err := validateSettingValueInternal(key, value); err != nil {+ if err := validateSettingValue(key, value); err != nil {return err}if err := libarcane.ValidateCronSetting(key, value); err != nil {@@ -535,10 +442,7 @@ func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string)return err}- if err := s.updateSettingValueNoRefreshInternal(ctx, key, value); err != nil {- return err- }- return s.refreshSettingsCacheInternal(context.WithoutCancel(ctx))+ return s.persistSettings(ctx, []SettingVariable{{Key: key, Value: value}})}// UpdateSettingValues persists a group of internal setting values atomically@@ -546,7 +450,7 @@ func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string)func (s *SettingsService) UpdateSettingValues(ctx context.Context, updates []libarcane.SettingUpdate) error {values := make([]SettingVariable, 0, len(updates))for _, update := range updates {- if err := validateSettingValueInternal(update.Key, update.Value); err != nil {+ if err := validateSettingValue(update.Key, update.Value); err != nil {return err}values = append(values, SettingVariable{Key: update.Key, Value: update.Value})@@ -557,23 +461,7 @@ func (s *SettingsService) UpdateSettingValues(ctx context.Context, updates []libreturn err}- if err := s.persistSettings(ctx, values); err != nil {- return err- }- return s.refreshSettingsCacheInternal(context.WithoutCancel(ctx))-}--func (s *SettingsService) updateSettingValueNoRefreshInternal(ctx context.Context, key, value string) error {- if key == "oidcProviderName" {- value = normalization.Text(value, true, true)- }- return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- settingVar := &SettingVariable{- Key: key,- Value: value,- }- return tx.Save(settingVar).Error- })+ return s.persistSettings(ctx, values)}// UpdateSettings publishes the refreshed snapshot before returning. Each@@ -587,33 +475,7 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.Uif err := ctx.Err(); err != nil {return nil, err}- result, err := s.updateSettingsInternal(ctx, updates)- if err == nil {- s.publishSettingsChangesInternal(result.changes)- }- return result.settings, err-}--func (s *SettingsService) publishSettingsChangesInternal(updates []libarcane.SettingUpdate) {- safe := make([]libarcane.SettingUpdate, 0, len(updates))- cfg := s.GetSettingsConfig()- for _, update := range updates {- _, _, sensitive, err := cfg.FieldByKey(update.Key)- if err != nil {- slog.WarnContext(s.lifecycleCtx, "Skipping settings change notification for unknown key", "key", update.Key, "error", err)- continue- }- if !sensitive {- safe = append(safe, update)- }- }- if len(safe) > 0 {- s.changes.Publish(safe)- }-}-func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates settings.Update) (settingsUpdateResultInternal, error) {- defaultCfg := s.getDefaultSettings()cfg := s.GetSettingsConfig().Clone()oidcClientSecretUpdated := updates.OidcClientSecret != nil && !s.IsEnvOverrideActive("oidcClientSecret")trivyServerTokenUpdated := updates.TrivyServerToken != nil && *updates.TrivyServerToken != "" && !s.IsEnvOverrideActive("trivyServerToken")@@ -633,7 +495,7 @@ func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates semap[string]bool{"oidcClientSecret": cfg.OidcClientSecret.Value != ""},map[string]bool{"oidcClientSecret": updates.OidcClientSecret != nil},); err != nil {- return settingsUpdateResultInternal{}, err+ return nil, err}effectiveValue := func(current SettingVariable, next *string) string {@@ -642,10 +504,7 @@ func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates se}return current.Value}- oidcClientSecret := cfg.OidcClientSecret.Value- if oidcClientSecretUpdated {- oidcClientSecret = *updates.OidcClientSecret- }+ oidcClientSecret := kit.Ternary(oidcClientSecretUpdated, kit.FromPtr(updates.OidcClientSecret), cfg.OidcClientSecret.Value)if effectiveValue(cfg.OidcEnabled, updates.OidcEnabled) == "true" {required := []struct{ key, value string }{{"oidcClientId", effectiveValue(cfg.OidcClientId, updates.OidcClientId)},@@ -654,16 +513,7 @@ func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates se}for _, field := range required {if strings.TrimSpace(field.value) == "" {- return settingsUpdateResultInternal{}, common.Classify(- common.ErrValidation,- &base.FieldError{- Field: field.key,- Err: fmt.Errorf(- "enabling OIDC requires %s",- field.key,- ),- },- )+ return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: field.key, Err: fmt.Errorf("enabling OIDC requires %s", field.key)})}}}@@ -676,12 +526,12 @@ func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates semap[string]bool{"trivyServerToken": cfg.TrivyServerToken.Value != ""},map[string]bool{"trivyServerToken": trivyServerTokenUpdated},); err != nil {- return settingsUpdateResultInternal{}, err+ return nil, err}- valuesToUpdate, err := s.prepareUpdateValues(updates, cfg, defaultCfg)+ valuesToUpdate, err := s.prepareUpdateValues(updates, cfg, DefaultSettingsConfig())if err != nil {- return settingsUpdateResultInternal{}, err+ return nil, err}if oidcClientSecretUpdated {valuesToUpdate = append(valuesToUpdate, SettingVariable{Key: "oidcClientSecret", Value: *updates.OidcClientSecret})@@ -691,21 +541,34 @@ func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates se}if persistSettingsErr := s.persistSettings(ctx, valuesToUpdate); persistSettingsErr != nil {- return settingsUpdateResultInternal{}, persistSettingsErr+ return nil, persistSettingsErr}- if refreshSettingsCacheErr := s.refreshSettingsCacheInternal(context.WithoutCancel(ctx)); refreshSettingsCacheErr != nil {- return settingsUpdateResultInternal{}, refreshSettingsCacheErr- }- settingsCfg := s.GetSettingsConfig()changes := make([]libarcane.SettingUpdate, 0, len(valuesToUpdate))for _, value := range valuesToUpdate {changes = append(changes, libarcane.SettingUpdate{Key: value.Key, Value: value.Value})}- return settingsUpdateResultInternal{- settings: settingsCfg.ToSettingVariableSlice(SettingVisibilityNonAdmin, false),- changes: changes,- }, nil+ result := s.GetSettingsConfig().ToSettingVariableSlice(SettingVisibilityNonAdmin, false)+ s.publishSettingsChanges(changes)+ return result, nil+}++func (s *SettingsService) publishSettingsChanges(updates []libarcane.SettingUpdate) {+ safe := make([]libarcane.SettingUpdate, 0, len(updates))+ cfg := s.GetSettingsConfig()+ for _, update := range updates {+ _, _, sensitive, err := cfg.FieldByKey(update.Key)+ if err != nil {+ slog.WarnContext(s.lifecycleCtx, "Skipping settings change notification for unknown key", "key", update.Key, "error", err)+ continue+ }+ if !sensitive {+ safe = append(safe, update)+ }+ }+ if len(safe) > 0 {+ s.changes.Publish(safe)+ }}func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defaultCfg *Settings) ([]SettingVariable, error) {@@ -714,15 +577,20 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defavaluesToUpdate := make([]SettingVariable, 0)for i := range rt.NumField() {- field := rt.Field(i)fieldValue := rv.Field(i)-- key, value, ok := extractUpdateValue(field, fieldValue)- if !ok {+ if fieldValue.Kind() == reflect.Pointer && fieldValue.IsNil() {continue}+ key, _, _ := strings.Cut(rt.Field(i).Tag.Get("json"), ",")+ if key == "" {+ continue+ }+ var value string+ if fieldValue.Kind() == reflect.Pointer {+ value = fieldValue.Elem().String()+ }- if err := validateSettingValueInternal(key, value); err != nil {+ if err := validateSettingValue(key, value); err != nil {return nil, err}@@ -746,18 +614,11 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defareturn nil, fmt.Errorf("invalid cron expression for %s: %w", key, err)}- var valueToSave string- var err error-- if value == "" {- defaultValue, _, _, _ := defaultCfg.FieldByKey(key)- valueToSave = defaultValue- err = cfg.UpdateField(key, defaultValue, true)- } else {- valueToSave = value- err = cfg.UpdateField(key, value, true)+ valueToSave := value+ if valueToSave == "" {+ valueToSave, _, _, _ = defaultCfg.FieldByKey(key)}-+ err := cfg.UpdateField(key, valueToSave, true)if errors.Is(err, SettingSensitiveForbiddenError{}) {continue}@@ -771,26 +632,8 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defareturn valuesToUpdate, nil}-func extractUpdateValue(field reflect.StructField, fieldValue reflect.Value) (string, string, bool) {- if fieldValue.Kind() == reflect.Pointer && fieldValue.IsNil() {- return "", "", false- }-- key, _, _ := strings.Cut(field.Tag.Get("json"), ",")- if key == "" {- return "", "", false- }-- var value string- if fieldValue.Kind() == reflect.Pointer {- value = fieldValue.Elem().String()- }-- return key, value, true-}-func (s *SettingsService) persistSettings(ctx context.Context, values []SettingVariable) error {- return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {for _, setting := range values {if setting.Key == "oidcProviderName" {setting.Value = normalization.Text(setting.Value, true, true)@@ -800,12 +643,19 @@ func (s *SettingsService) persistSettings(ctx context.Context, values []SettingV}}return nil- })+ }); err != nil {+ return err+ }++ if err := s.LoadDatabaseSettings(context.WithoutCancel(ctx)); err != nil {+ return fmt.Errorf("failed to refresh settings cache: %w", err)+ }+ return nil}-// retiredSettingValuesInternal lists former defaults that startup replaces+// retiredSettingValues lists former defaults that startup replaces// with the current default when an install still holds them verbatim.-var retiredSettingValuesInternal = map[string][]string{+var retiredSettingValues = map[string][]string{"dockerClientRefreshInterval": {"*/30 * * * * *"},"autoHealInterval": {"*/30 * * * * *"},}@@ -817,10 +667,9 @@ func (s *SettingsService) EnsureDefaultSettings(ctx context.Context) error {return err}- defaultSettings := s.getDefaultSettings()- defaultSettingVars := defaultSettings.ToSettingVariableSlice(SettingVisibilityAll, false)+ defaultSettingVars := DefaultSettingsConfig().ToSettingVariableSlice(SettingVisibilityAll, false)- if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {for _, defaultSetting := range defaultSettingVars {var existing SettingVariableerr := tx.Where("key = ?", defaultSetting.Key).First(&existing).Error@@ -832,17 +681,15 @@ func (s *SettingsService) EnsureDefaultSettings(ctx context.Context) error {}case err != nil:return fmt.Errorf("failed to check for existing setting %s: %w", defaultSetting.Key, err)- case slices.Contains(retiredSettingValuesInternal[defaultSetting.Key], existing.Value):- if replaceDefaultSettingErr := tx.Model(&SettingVariable{}).Where("key = ?", defaultSetting.Key).Update("value", defaultSetting.Value).Error; replaceDefaultSettingErr != nil {+ case slices.Contains(retiredSettingValues[defaultSetting.Key], existing.Value):+ replaceDefaultSettingErr := tx.Model(&SettingVariable{}).Where("key = ?", defaultSetting.Key).Update("value", defaultSetting.Value).Error+ if replaceDefaultSettingErr != nil {return fmt.Errorf("failed to replace retired default for setting %s: %w", defaultSetting.Key, replaceDefaultSettingErr)}}}return nil- }); err != nil {- return err- }- return nil+ })}func (s *SettingsService) PruneUnknownSettings(ctx context.Context) error {@@ -852,12 +699,12 @@ func (s *SettingsService) PruneUnknownSettings(ctx context.Context) error {return err}- allowedKeys := allowedSettingKeys()- if len(allowedKeys) == 0 {- return nil+ fields, _ := getSettingsFieldCacheInternal()+ keys := make([]string, 0, len(fields)+1)+ for _, field := range fields {+ keys = append(keys, field.key)}-- keys := slices.Collect(maps.Keys(allowedKeys))+ keys = append(keys, "encryptionKey")result := s.db.WithContext(ctx).Where("key NOT IN ?", keys).Delete(&SettingVariable{})if result.Error != nil {@@ -876,14 +723,38 @@ func (s *SettingsService) PersistEnvSettingsIfMissing(ctx context.Context) errorreturn err}- rt := reflect.TypeFor[Settings]()appCfg := config.Load()isEnvOnlyMode := appCfg.AgentMode || appCfg.UIConfigurationDisabled+ fields, _ := getSettingsFieldCacheInternal()if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- for field := range rt.Fields() {- if err := s.processEnvField(ctx, tx, field, isEnvOnlyMode); err != nil {- return err+ for _, field := range fields {+ attrs := splitSettingAttrsInternal(field.attrs)+ // Outside env-only mode, only persist fields explicitly marked as envOverride.+ if slices.Contains(attrs, "internal") || (!isEnvOnlyMode && !slices.Contains(attrs, "envOverride")) {+ continue+ }+ envVal, ok, _ := utils.LookupEnvOrFile(settingEnvName(field.key))+ if !ok {+ continue+ }+ envVal = kit.TrimQuotes(envVal)++ var existing SettingVariable+ err := tx.Where("key = ?", field.key).First(&existing).Error+ switch {+ case errors.Is(err, gorm.ErrRecordNotFound):+ if createEnvSettingErr := tx.Create(&SettingVariable{Key: field.key, Value: envVal}).Error; createEnvSettingErr != nil {+ return fmt.Errorf("persist env setting %s: %w", field.key, createEnvSettingErr)+ }+ slog.DebugContext(ctx, "Created setting from environment", "key", field.key)+ case err != nil:+ return fmt.Errorf("check setting %s: %w", field.key, err)+ case existing.Value != envVal:+ if updateEnvSettingErr := tx.Model(&existing).Update("value", envVal).Error; updateEnvSettingErr != nil {+ return fmt.Errorf("update env setting %s: %w", field.key, updateEnvSettingErr)+ }+ slog.DebugContext(ctx, "Updated setting from environment", "key", field.key)}}return nil@@ -893,115 +764,26 @@ func (s *SettingsService) PersistEnvSettingsIfMissing(ctx context.Context) errorreturn s.LoadDatabaseSettings(context.WithoutCancel(ctx))}-func allowedSettingKeys() map[string]struct{} {- allowed := make(map[string]struct{})-- settingsType := reflect.TypeFor[Settings]()- for field := range settingsType.Fields() {- key, _, _ := strings.Cut(field.Tag.Get("key"), ",")- if key == "" {- continue- }- allowed[key] = struct{}{}- }-- allowed["encryptionKey"] = struct{}{}-- return allowed-}--func (s *SettingsService) processEnvField(ctx context.Context, tx *gorm.DB, field reflect.StructField, isEnvOnlyMode bool) error {- tag := field.Tag.Get("key")- key, attrs, _ := strings.Cut(tag, ",")-- if !s.shouldProcessField(key, attrs, isEnvOnlyMode) {- return nil- }-- envVarName := strings.ToUpper(kit.SnakeCase(key))- envVal, ok, _ := utils.LookupEnvOrFile(envVarName)- if !ok {- return nil- }- envVal = kit.TrimQuotes(envVal)-- return s.upsertEnvSetting(ctx, tx, key, envVal)-}--func (s *SettingsService) shouldProcessField(key, attrs string, isEnvOnlyMode bool) bool {- if key == "" || strings.Contains(attrs, "internal") {- return false- }-- // If not in env-only mode, only persist if it's explicitly marked as envOverride- return isEnvOnlyMode || strings.Contains(attrs, "envOverride")-}--func (s *SettingsService) upsertEnvSetting(ctx context.Context, tx *gorm.DB, key, envVal string) error {- var existing SettingVariable- err := tx.Where("key = ?", key).First(&existing).Error-- switch {- case errors.Is(err, gorm.ErrRecordNotFound):- newVar := SettingVariable{Key: key, Value: envVal}- if createEnvSettingErr := tx.Create(&newVar).Error; createEnvSettingErr != nil {- return fmt.Errorf("persist env setting %s: %w", key, createEnvSettingErr)- }- slog.DebugContext(ctx, "Created setting from environment", "key", key)- case err != nil:- return fmt.Errorf("check setting %s: %w", key, err)- default:- if existing.Value != envVal {- if updateEnvSettingErr := tx.Model(&existing).Update("value", envVal).Error; updateEnvSettingErr != nil {- return fmt.Errorf("update env setting %s: %w", key, updateEnvSettingErr)- }- slog.DebugContext(ctx, "Updated setting from environment", "key", key)- }- }-- return nil-}-func (s *SettingsService) ListSettings(visibility SettingVisibility) []SettingVariable {return s.GetSettingsConfig().ToSettingVariableSlice(visibility, true)}// GetSettingType returns the type from the setting metadatafunc (s *SettingsService) GetSettingType(key string) string {- rt := reflect.TypeFor[Settings]()- for field := range rt.Fields() {- keyTag := field.Tag.Get("key")- fieldKey, _, _ := strings.Cut(keyTag, ",")- if fieldKey == key {- metaTag := field.Tag.Get("meta")- parts := strings.SplitSeq(metaTag, ";")- for part := range parts {- if after, ok := strings.CutPrefix(part, "type="); ok {- return after- }+ _, byKey := getSettingsFieldCacheInternal()+ if field, ok := byKey[key]; ok {+ metaTag := reflect.TypeFor[Settings]().Field(field.index).Tag.Get("meta")+ for part := range strings.SplitSeq(metaTag, ";") {+ if after, found := strings.CutPrefix(part, "type="); found {+ return after}- return "text" // default type}}- return "text" // default if not found-}--func (s *SettingsService) setupInstanceID(ctx context.Context) error {- instanceID := s.GetSettingsConfig().InstanceID.Value- if instanceID != "" {- return nil- }-- err := s.UpdateSetting(ctx, "instanceId", uuid.New().String())- if err != nil {- return fmt.Errorf("failed to set instance ID in database: %w", err)- }-- return nil+ return "text"}func (s *SettingsService) settingValue[T any](ctx context.Context, key string, parse func(string) (T, error)) mo.Option[T] {- cfg := s.getEffectiveSettingsConfigInternal(ctx)+ cfg := s.GetSettingsOrDefaults(ctx)value, _, _, err := cfg.FieldByKey(key)if err != nil || value == "" {return mo.None[T]()@@ -1038,9 +820,8 @@ func (s *SettingsService) SetStringSetting(ctx context.Context, key, value strinreturn s.UpdateSetting(ctx, key, value)}-// SetContainerAutoUpdateExclusionInternal adds or removes a container name from-// the autoUpdateExcludedContainers setting. When excluded is true the container-// is added to the list; when false it is removed.+// SetContainerAutoUpdateExclusionInternal adds (excluded) or removes a container name+// in the autoUpdateExcludedContainers setting.func (s *SettingsService) SetContainerAutoUpdateExclusionInternal(ctx context.Context, containerName string, excluded bool) error {s.writes.Lock()defer s.writes.Unlock()@@ -1049,23 +830,13 @@ func (s *SettingsService) SetContainerAutoUpdateExclusionInternal(ctx context.Co}ordered := kit.Unique(kit.TrimNonEmpty(strings.Split(s.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""), ",")))-if excluded {ordered = kit.Unique(append(ordered, containerName))} else {- filtered := ordered[:0]- for _, name := range ordered {- if name != containerName {- filtered = append(filtered, name)- }- }- ordered = filtered+ ordered = slices.DeleteFunc(ordered, func(name string) bool { return name == containerName })}- if err := s.updateSettingValueNoRefreshInternal(ctx, "autoUpdateExcludedContainers", strings.Join(ordered, ",")); err != nil {- return err- }- return s.refreshSettingsCacheInternal(context.WithoutCancel(ctx))+ return s.persistSettings(ctx, []SettingVariable{{Key: "autoUpdateExcludedContainers", Value: strings.Join(ordered, ",")}})}func (s *SettingsService) EnsureEncryptionKey(ctx context.Context) (string, error) {@@ -1075,50 +846,14 @@ func (s *SettingsService) EnsureEncryptionKey(ctx context.Context) (string, erroreturn "", err}- const keyName = "encryptionKey"- var key string-- err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- var sv SettingVariable- err := tx.Where("key = ?", keyName).First(&sv).Error-- if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {- return fmt.Errorf("failed to load encryption key: %w", err)- }-- if sv.Value != "" {- key = sv.Value- return nil- }-- notFound := errors.Is(err, gorm.ErrRecordNotFound)+ return ensureSettingValue(ctx, s.db, "encryptionKey", "encryption key", func() (string, error) {sum := sha256.Sum256([]byte(uuid.New().String()))- generatedKey := base64.StdEncoding.EncodeToString(sum[:])- key = generatedKey-- if notFound {- if createErr := tx.Create(&SettingVariable{Key: keyName, Value: generatedKey}).Error; createErr != nil {- return fmt.Errorf("failed to persist encryption key: %w", createErr)- }- return nil- }-- if updErr := tx.Model(&SettingVariable{}).- Where("key = ?", keyName).- Update("value", generatedKey).Error; updErr != nil {- return fmt.Errorf("failed to update encryption key: %w", updErr)- }- return nil+ return base64.StdEncoding.EncodeToString(sum[:]), nil})- if err != nil {- return "", err- }-- return key, nil}func (s *SettingsService) EnsureJwtSigningKey(ctx context.Context) (*mldsa.PrivateKey, error) {- seed, err := s.ensureEncryptedKeyInternal(ctx, "jwtSigningKeySeed", mldsa.PrivateKeySize)+ seed, err := s.ensureEncryptedKey(ctx, "jwtSigningKeySeed", mldsa.PrivateKeySize)if err != nil {return nil, err}@@ -1130,66 +865,46 @@ func (s *SettingsService) EnsureJwtSigningKey(ctx context.Context) (*mldsa.Priva}func (s *SettingsService) EnsureBrowserSessionSigningKey(ctx context.Context) ([]byte, error) {- return s.ensureEncryptedKeyInternal(ctx, "browserSessionSigningKey", browserSessionSigningKeySize)+ return s.ensureEncryptedKey(ctx, "browserSessionSigningKey", browserSessionSigningKeySize)}-func (s *SettingsService) ensureEncryptedKeyInternal(ctx context.Context, keyName string, size int) ([]byte, error) {+func (s *SettingsService) ensureEncryptedKey(ctx context.Context, keyName string, size int) ([]byte, error) {s.writes.Lock()defer s.writes.Unlock()if err := ctx.Err(); err != nil {return nil, err}- var key []byte-- err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- var sv SettingVariable- err := tx.Where("key = ?", keyName).First(&sv).Error- if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {- return fmt.Errorf("failed to load signing key: %w", err)- }-- if sv.Value != "" {- decoded, decErr := crypto.Decrypt(sv.Value)- if decErr != nil {- return fmt.Errorf("failed to decrypt signing key: %w", decErr)- }- key, decErr = base64.StdEncoding.DecodeString(decoded)- if decErr != nil {- return fmt.Errorf("failed to decode signing key: %w", decErr)- }- if len(key) != size {- return fmt.Errorf("invalid signing key length: got %d, want %d", len(key), size)- }- return nil- }-- key = make([]byte, size)- if _, genErr := rand.Read(key); genErr != nil {- return fmt.Errorf("failed to generate signing key: %w", genErr)+ var generated []byte+ stored, err := ensureSettingValue(ctx, s.db, keyName, "signing key", func() (string, error) {+ generated = make([]byte, size)+ if _, genErr := rand.Read(generated); genErr != nil {+ return "", fmt.Errorf("failed to generate signing key: %w", genErr)}- encrypted, encErr := crypto.Encrypt(base64.StdEncoding.EncodeToString(key))+ encrypted, encErr := crypto.Encrypt(base64.StdEncoding.EncodeToString(generated))if encErr != nil {- return fmt.Errorf("failed to encrypt signing key: %w", encErr)- }-- if errors.Is(err, gorm.ErrRecordNotFound) {- if createSigningKeyErr := tx.Create(&SettingVariable{Key: keyName, Value: encrypted}).Error; createSigningKeyErr != nil {- return fmt.Errorf("failed to persist signing key: %w", createSigningKeyErr)- }- return nil- }- if updateSigningKeyErr := tx.Model(&SettingVariable{}).- Where("key = ?", keyName).- Update("value", encrypted).Error; updateSigningKeyErr != nil {- return fmt.Errorf("failed to update signing key: %w", updateSigningKeyErr)+ return "", fmt.Errorf("failed to encrypt signing key: %w", encErr)}- return nil+ return encrypted, nil})if err != nil {return nil, err}+ if generated != nil {+ return generated, nil+ }+ decoded, err := crypto.Decrypt(stored)+ if err != nil {+ return nil, fmt.Errorf("failed to decrypt signing key: %w", err)+ }+ key, err := base64.StdEncoding.DecodeString(decoded)+ if err != nil {+ return nil, fmt.Errorf("failed to decode signing key: %w", err)+ }+ if len(key) != size {+ return nil, fmt.Errorf("invalid signing key length: got %d, want %d", len(key), size)+ }return key, nil}@@ -1198,95 +913,56 @@ func (s *SettingsService) NormalizeProjectsDirectory(ctx context.Context, projecslog.DebugContext(ctx, "PROJECTS_DIRECTORY environment variable is set, skipping normalization", "value", projectsDirEnv)return nil}-- s.writes.Lock()- defer s.writes.Unlock()- if err := ctx.Err(); err != nil {- return err- }-- var projectsDirSetting SettingVariable- err := s.db.WithContext(ctx).Where("key = ?", "projectsDirectory").First(&projectsDirSetting).Error- if errors.Is(err, gorm.ErrRecordNotFound) {- slog.DebugContext(ctx, "No projectsDirectory setting found, skipping normalization")- return nil- }- if err != nil {- return fmt.Errorf("failed to load projectsDirectory setting: %w", err)- }-- value := strings.TrimSpace(projectsDirSetting.Value)- isMapping := strings.Contains(value, ":") && (strings.HasPrefix(value, "/") || projects.IsWindowsDrivePath(value))- if filepath.IsAbs(value) || isMapping {- slog.DebugContext(ctx, "Projects directory already normalized or custom, skipping", "value", projectsDirSetting.Value)- return nil- }-- cwd, _ := os.Getwd()- absPath, absErr := filepath.Abs(value)- if absErr != nil {- return fmt.Errorf("failed to resolve relative path to absolute: %w", absErr)- }- slog.InfoContext(ctx, "Normalizing projects directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)- if updateSettingValueNoRefreshErr := s.updateSettingValueNoRefreshInternal(ctx, "projectsDirectory", absPath); updateSettingValueNoRefreshErr != nil {- return fmt.Errorf("failed to update projectsDirectory: %w", updateSettingValueNoRefreshErr)- }- if refreshSettingsCacheErr := s.refreshSettingsCacheInternal(context.WithoutCancel(ctx)); refreshSettingsCacheErr != nil {- return refreshSettingsCacheErr- }- slog.InfoContext(ctx, "Successfully normalized projects directory")- s.publishSettingsChangesInternal([]libarcane.SettingUpdate{{Key: "projectsDirectory", Value: absPath}})- return nil+ return s.normalizeDirectorySetting(ctx, "projectsDirectory")}func (s *SettingsService) NormalizeBuildsDirectory(ctx context.Context) error {- const buildsKey = "buildsDirectory"- envVarName := strings.ToUpper(kit.SnakeCase(buildsKey))- if envVal, ok, _ := utils.LookupEnvOrFile(envVarName); ok && strings.TrimSpace(envVal) != "" {+ if envVal, ok, _ := utils.LookupEnvOrFile(settingEnvName("buildsDirectory")); ok && strings.TrimSpace(envVal) != "" {slog.DebugContext(ctx, "BUILDS_DIRECTORY environment variable is set, skipping normalization", "value", envVal)return nil}+ return s.normalizeDirectorySetting(ctx, "buildsDirectory")+}+func (s *SettingsService) normalizeDirectorySetting(ctx context.Context, key string) error {s.writes.Lock()defer s.writes.Unlock()if err := ctx.Err(); err != nil {return err}- var buildsDirSetting SettingVariable- err := s.db.WithContext(ctx).Where("key = ?", buildsKey).First(&buildsDirSetting).Error+ var setting SettingVariable+ err := s.db.WithContext(ctx).Where("key = ?", key).First(&setting).Errorif errors.Is(err, gorm.ErrRecordNotFound) {- slog.DebugContext(ctx, "No buildsDirectory setting found, skipping normalization")+ slog.DebugContext(ctx, "No directory setting found, skipping normalization", "key", key)return nil}if err != nil {- return fmt.Errorf("failed to load buildsDirectory setting: %w", err)+ return fmt.Errorf("failed to load %s setting: %w", key, err)}- value := strings.TrimSpace(buildsDirSetting.Value)- if value == "" || filepath.IsAbs(value) {- slog.DebugContext(ctx, "Builds directory already normalized or empty, skipping", "value", buildsDirSetting.Value)+ value := strings.TrimSpace(setting.Value)+ isMapping := strings.Contains(value, ":") && (strings.HasPrefix(value, "/") || projects.IsWindowsDrivePath(value))+ if value == "" || filepath.IsAbs(value) || isMapping {+ slog.DebugContext(ctx, "Directory setting already normalized, custom, or empty, skipping", "key", key, "value", setting.Value)return nil}cwd, _ := os.Getwd()- absPath, absErr := filepath.Abs(value)- if absErr != nil {- return fmt.Errorf("failed to resolve relative path to absolute: %w", absErr)- }- slog.InfoContext(ctx, "Normalizing builds directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)- if updateSettingValueNoRefreshErr := s.updateSettingValueNoRefreshInternal(ctx, buildsKey, absPath); updateSettingValueNoRefreshErr != nil {- return fmt.Errorf("failed to update buildsDirectory: %w", updateSettingValueNoRefreshErr)+ absPath, err := filepath.Abs(value)+ if err != nil {+ return fmt.Errorf("failed to resolve relative path to absolute: %w", err)}- if refreshSettingsCacheErr := s.refreshSettingsCacheInternal(context.WithoutCancel(ctx)); refreshSettingsCacheErr != nil {- return refreshSettingsCacheErr+ slog.InfoContext(ctx, "Normalizing directory setting from relative to absolute path", "key", key, "from", value, "to", absPath, "base", cwd)+ if persistSettingsErr := s.persistSettings(ctx, []SettingVariable{{Key: key, Value: absPath}}); persistSettingsErr != nil {+ return persistSettingsErr}- slog.InfoContext(ctx, "Successfully normalized builds directory")- s.publishSettingsChangesInternal([]libarcane.SettingUpdate{{Key: buildsKey, Value: absPath}})+ slog.InfoContext(ctx, "Successfully normalized directory setting", "key", key)+ s.publishSettingsChanges([]libarcane.SettingUpdate{{Key: key, Value: absPath}})return nil}-func (s *SettingsService) enqueueEffectInternal(effect func()) error {+func (s *SettingsService) enqueueEffect(effect func()) error {s.effectsMu.Lock()defer s.effectsMu.Unlock()if s.effectsClosed {@@ -1300,33 +976,6 @@ func (s *SettingsService) enqueueEffectInternal(effect func()) error {return nil}-func (s *SettingsService) runEffectsInternal() {- defer close(s.effectsDone)- for {- s.effectsMu.Lock()- if len(s.effects) > 0 {- effect := s.effects[0]- s.effects[0] = nil- s.effects = s.effects[1:]- s.effectsMu.Unlock()- func() { var err error; defer utils.RecoverToError(&err, "settings effect"); effect() }()- continue- }- closed := s.effectsClosed- s.effectsMu.Unlock()- if closed {- return- }- select {- case <-s.effectsWake:- case <-s.lifecycleCtx.Done():- s.effectsMu.Lock()- s.effectsClosed = true- s.effectsMu.Unlock()- }- }-}-// Stop drains accepted effects and joins the settings worker.func (s *SettingsService) Stop(ctx context.Context) error {s.writes.Lock()diff --git a/backend/internal/settings/service_test.go b/backend/internal/settings/service_test.goindex 59ef71d37..e135751a1 100644--- a/backend/internal/settings/service_test.go+++ b/backend/internal/settings/service_test.go@@ -32,7 +32,7 @@ func setupSettingsTestDB(t *testing.T) *database.DB {return &database.DB{DB: db}}-func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*SettingsService, error) {+func newSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*SettingsService, error) {t.Helper()svc, err := NewSettingsService(ctx, db)if err == nil {@@ -41,11 +41,11 @@ func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *dareturn svc, err}-func waitForSettingsNotificationsInternal(t *testing.T, svc *SettingsService) {+func waitForSettingsNotifications(t *testing.T, svc *SettingsService) {t.Helper()done := make(chan struct{})svc.writes.Lock()- err := svc.enqueueEffectInternal(func() { close(done) })+ err := svc.enqueueEffect(func() { close(done) })svc.writes.Unlock()require.NoError(t, err)select {@@ -58,7 +58,7 @@ func waitForSettingsNotificationsInternal(t *testing.T, svc *SettingsService) {func TestSettingsService_EnsureDefaultSettings_ReplacesRetiredDefaults(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, db.Create(&SettingVariable{Key: "dockerClientRefreshInterval", Value: "*/30 * * * * *"}).Error)require.NoError(t, db.Create(&SettingVariable{Key: "autoHealInterval", Value: "*/10 * * * * *"}).Error)@@ -75,7 +75,7 @@ func TestSettingsService_EnsureDefaultSettings_ReplacesRetiredDefaults(t *testinfunc TestSettingsService_EnsureDefaultSettings_Idempotent(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -164,7 +164,7 @@ func TestSettingsService_EnsureDefaultSettings_Idempotent(t *testing.T) {func TestSettingsService_ImageEventWatcherSettingPersists(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))require.False(t, svc.GetBoolSetting(ctx, "imageEventWatcherEnabled", true))@@ -181,7 +181,7 @@ func TestSettingsService_ImageEventWatcherSettingPersists(t *testing.T) {func TestSettingsService_GetSettings_UnknownKeysIgnored(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.db.WithContext(ctx).@@ -194,7 +194,7 @@ func TestSettingsService_GetSettings_UnknownKeysIgnored(t *testing.T) {func TestSettingsService_GetSettings_UsesCachedSnapshotWithoutDatabase(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.SetStringSetting(ctx, "baseServerUrl", "http://cached"))@@ -210,7 +210,7 @@ func TestSettingsService_GetSettings_UsesCachedSnapshotWithoutDatabase(t *testinfunc TestSettingsService_AvatarMaxUploadSizeDefaultAndUpdate(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)current, err := svc.GetSettings(ctx)@@ -228,10 +228,10 @@ func TestSettingsService_AvatarMaxUploadSizeDefaultAndUpdate(t *testing.T) {require.Equal(t, "8", current.AvatarMaxUploadSizeMb.Value)}-func TestSettingsServiceUpdateSettingsRejectsOIDCIssuerChangeWithStoredSecretInternal(t *testing.T) {+func TestSettingsServiceUpdateSettingsRejectsOIDCIssuerChangeWithStoredSecret(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "oidcIssuerUrl", "https://issuer.example.com"))require.NoError(t, svc.UpdateSetting(ctx, "oidcClientSecret", "old-client-secret"))@@ -247,10 +247,10 @@ func TestSettingsServiceUpdateSettingsRejectsOIDCIssuerChangeWithStoredSecretIntrequire.Equal(t, "old-client-secret", current.OidcClientSecret.Value)}-func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWhenClearingSecretInternal(t *testing.T) {+func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWhenClearingSecret(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "oidcIssuerUrl", "https://issuer.example.com"))require.NoError(t, svc.UpdateSetting(ctx, "oidcClientSecret", "old-client-secret"))@@ -267,10 +267,10 @@ func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWhenClearingSecretInrequire.Empty(t, current.OidcClientSecret.Value)}-func TestSettingsServiceUpdateSettingsRejectsEnablingOIDCWithoutSecretInternal(t *testing.T) {+func TestSettingsServiceUpdateSettingsRejectsEnablingOIDCWithoutSecret(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)_, err = svc.UpdateSettings(ctx, settings.Update{@@ -285,10 +285,10 @@ func TestSettingsServiceUpdateSettingsRejectsEnablingOIDCWithoutSecretInternal(trequire.Equal(t, "false", current.OidcEnabled.Value)}-func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWithReplacementSecretInternal(t *testing.T) {+func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWithReplacementSecret(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "oidcIssuerUrl", "https://issuer.example.com"))require.NoError(t, svc.UpdateSetting(ctx, "oidcClientSecret", "old-client-secret"))@@ -312,10 +312,10 @@ func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWithReplacementSecrerequire.Equal(t, "Café Login", current.OidcProviderName.Value)}-func TestSettingsServiceUpdateSettingsRejectsTrivyServerChangeWithStoredTokenInternal(t *testing.T) {+func TestSettingsServiceUpdateSettingsRejectsTrivyServerChangeWithStoredToken(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))require.NoError(t, svc.UpdateSetting(ctx, "trivyServerToken", "old-trivy-token"))@@ -331,10 +331,10 @@ func TestSettingsServiceUpdateSettingsRejectsTrivyServerChangeWithStoredTokenIntrequire.Equal(t, "old-trivy-token", current.TrivyServerToken.Value)}-func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithReplacementTokenInternal(t *testing.T) {+func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithReplacementToken(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))require.NoError(t, svc.UpdateSetting(ctx, "trivyServerToken", "old-trivy-token"))@@ -351,10 +351,10 @@ func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithReplacementTokerequire.Equal(t, "new-trivy-token", current.TrivyServerToken.Value)}-func TestSettingsServiceUpdateSettingsAllowsClearingTrivyServerWithStoredTokenInternal(t *testing.T) {+func TestSettingsServiceUpdateSettingsAllowsClearingTrivyServerWithStoredToken(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))require.NoError(t, svc.UpdateSetting(ctx, "trivyServerToken", "old-trivy-token"))@@ -370,10 +370,10 @@ func TestSettingsServiceUpdateSettingsAllowsClearingTrivyServerWithStoredTokenInrequire.Equal(t, "old-trivy-token", current.TrivyServerToken.Value)}-func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithoutStoredTokenInternal(t *testing.T) {+func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithoutStoredToken(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))@@ -390,7 +390,7 @@ func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithoutStoredTokenIfunc TestSettingsService_PruneUnknownSettings_RemovesStaleKeys(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "projectsDirectory", "/tmp/projects"))@@ -417,7 +417,7 @@ func TestSettingsService_GetSettings_EnvOverride_OidcMergeAccounts(t *testing.T)db := setupSettingsTestDB(t)t.Setenv("OIDC_MERGE_ACCOUNTS", "true")- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -431,7 +431,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyScanTimeout(t *testing.T)db := setupSettingsTestDB(t)t.Setenv("TRIVY_SCAN_TIMEOUT", "1800")- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -447,7 +447,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyResourceLimits(t *testing.t.Setenv("TRIVY_CPU_LIMIT", "2.5")t.Setenv("TRIVY_MEMORY_LIMIT_MB", "2048")- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -463,7 +463,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyNetwork(t *testing.T) {db := setupSettingsTestDB(t)t.Setenv("TRIVY_NETWORK", "arcane-external")- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -477,7 +477,7 @@ func TestSettingsService_GetSettings_EnvOverride_FollowProjectSymlinks(t *testindb := setupSettingsTestDB(t)t.Setenv("FOLLOW_PROJECT_SYMLINKS", "true")- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -492,7 +492,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyRuntimeSecurity(t *testingt.Setenv("TRIVY_SECURITY_OPTS", "label=disable,\nlabel=type:container_runtime_t")t.Setenv("TRIVY_PRIVILEGED", "true")- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -507,7 +507,7 @@ func TestSettingsService_GetStringSetting_EnvOverride_SwarmStackSourcesDirectorydb := setupSettingsTestDB(t)t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", "/mnt/swarm-from-env")- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "swarmStackSourcesDirectory", "/tmp/swarm-from-db"))@@ -515,21 +515,21 @@ func TestSettingsService_GetStringSetting_EnvOverride_SwarmStackSourcesDirectoryrequire.Equal(t, "/mnt/swarm-from-env", svc.GetSettingsConfig().SwarmStackSourcesDirectory.Value)}-func TestSettingsService_isEnvOverrideActiveInternal(t *testing.T) {+func TestSettingsService_isEnvOverrideActive(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.False(t, svc.IsEnvOverrideActive("oidcEnabled"))t.Setenv("OIDC_ENABLED", "false")- svcWithOverride, err := newSettingsServiceForTestInternal(t, ctx, db)+ svcWithOverride, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.True(t, svcWithOverride.IsEnvOverrideActive("oidcEnabled"))t.Setenv("AUTH_SESSION_TIMEOUT", "120")- svcWithNonOverrideEnv, err := newSettingsServiceForTestInternal(t, ctx, db)+ svcWithNonOverrideEnv, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.False(t, svcWithNonOverrideEnv.IsEnvOverrideActive("authSessionTimeout"))}@@ -537,7 +537,7 @@ func TestSettingsService_isEnvOverrideActiveInternal(t *testing.T) {func TestSettingsService_GetSetHelpers(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)// Defaults for missing keys@@ -559,7 +559,7 @@ func TestSettingsService_GetSetHelpers(t *testing.T) {func TestSettingsService_UpdateSetting(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "pruneImageMode", "all"))@@ -572,7 +572,7 @@ func TestSettingsService_UpdateSetting(t *testing.T) {func TestSettingsService_UpdateSettingRejectsInvalidCronBeforePersistence(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "pollingInterval", "*/5 * * * * *"))@@ -586,7 +586,7 @@ func TestSettingsService_UpdateSettingRejectsInvalidCronBeforePersistence(t *tesfunc TestSettingsService_UpdateSetting_RefreshesCachedSnapshot(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.Equal(t, "http://localhost", svc.GetSettingsConfig().BaseServerURL.Value)@@ -602,7 +602,7 @@ func TestSettingsService_UpdateSetting_RefreshesCachedSnapshot(t *testing.T) {func TestSettingsService_UpdateSettings_PruneModesDoNotTriggerScheduledPruneCallback(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)callbackCalls := 0@@ -615,14 +615,14 @@ func TestSettingsService_UpdateSettings_PruneModesDoNotTriggerScheduledPruneCallPruneContainerUntil: new("24h"),})require.NoError(t, err)- waitForSettingsNotificationsInternal(t, svc)+ waitForSettingsNotifications(t, svc)require.Equal(t, 0, callbackCalls)}func TestSettingsService_UpdateSettings_ScheduledPruneScheduleTriggersCallback(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)callbackCalls := 0@@ -634,14 +634,14 @@ func TestSettingsService_UpdateSettings_ScheduledPruneScheduleTriggersCallback(tScheduledPruneEnabled: new("true"),})require.NoError(t, err)- waitForSettingsNotificationsInternal(t, svc)+ waitForSettingsNotifications(t, svc)require.Equal(t, 1, callbackCalls)}-func TestSettingsServiceActorSerializesConcurrentUpdatesInternal(t *testing.T) {+func TestSettingsServiceActorSerializesConcurrentUpdates(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -666,10 +666,10 @@ func TestSettingsServiceActorSerializesConcurrentUpdatesInternal(t *testing.T) {require.Equal(t, "/data/templates-b", current.TemplatesDirectory.Value)}-func TestSettingsServiceActorPublishesSnapshotBeforeAsynchronousNotificationInternal(t *testing.T) {+func TestSettingsServiceActorPublishesSnapshotBeforeAsynchronousNotification(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -708,10 +708,10 @@ func TestSettingsServiceActorPublishesSnapshotBeforeAsynchronousNotificationInteclose(releaseCallback)}-func TestSettingsServiceActorNotifiesSubscriberOnceForMultipleMatchingKeysInternal(t *testing.T) {+func TestSettingsServiceActorNotifiesSubscriberOnceForMultipleMatchingKeys(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -731,10 +731,10 @@ func TestSettingsServiceActorNotifiesSubscriberOnceForMultipleMatchingKeysInternrequire.Equal(t, int32(1), calls.Load())}-func TestSettingsServiceActorDoesNotPublishSensitiveValuesInternal(t *testing.T) {+func TestSettingsServiceActorDoesNotPublishSensitiveValues(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -744,14 +744,14 @@ func TestSettingsServiceActorDoesNotPublishSensitiveValuesInternal(t *testing.T)})_, err = svc.UpdateSettings(ctx, settings.Update{OidcClientSecret: new("should-not-leave-settings-service")})require.NoError(t, err)- waitForSettingsNotificationsInternal(t, svc)+ waitForSettingsNotifications(t, svc)require.Zero(t, calls.Load())}-func TestSettingsServiceActorSerializesContainerExclusionUpdatesInternal(t *testing.T) {+func TestSettingsServiceActorSerializesContainerExclusionUpdates(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -780,7 +780,7 @@ func BenchmarkSettingsService_GetSettings(b *testing.B) {require.FailNowf(b, "benchmark database migration failed", "%v", autoMigrateErr)}settingsDB := &database.DB{DB: db}- svc, err := newSettingsServiceForTestInternal(b, ctx, settingsDB)+ svc, err := newSettingsServiceForTest(b, ctx, settingsDB)if err != nil {require.FailNowf(b, "benchmark service setup failed", "%v", err)}@@ -802,7 +802,7 @@ func BenchmarkSettingsService_GetSettings(b *testing.B) {func TestSettingsService_EnsureEncryptionKey(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)k1, err := svc.EnsureEncryptionKey(ctx)@@ -834,7 +834,7 @@ func TestSettingsService_EnsureEncryptionKey(t *testing.T) {func TestSettingsService_LoadDatabaseSettings_ReloadsChanges(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)// Initially empty DB -> defaults (not persisted yet)@@ -864,7 +864,7 @@ func TestSettingsService_LoadDatabaseSettings_UIConfigurationDisabled_Env(t *tesctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)// Reload explicitly (NewSettingsService already did, but explicit for clarity)@@ -878,7 +878,7 @@ func TestSettingsService_LoadDatabaseSettings_UIConfigurationDisabled_Env(t *tesfunc TestSettingsService_UpdateSettings_RefreshesCache(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -907,7 +907,7 @@ func TestSettingsService_UpdateSettings_ReturnsEnvOverriddenValues(t *testing.T)ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -929,7 +929,7 @@ func TestSettingsService_UpdateSettings_ReturnsEnvOverriddenValues(t *testing.T)func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyScanTimeout(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -940,7 +940,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyScanTimeout(_, err = svc.UpdateSettings(ctx, settings.Update{TrivyScanTimeout: new("1200")})require.NoError(t, err)- waitForSettingsNotificationsInternal(t, svc)+ waitForSettingsNotifications(t, svc)require.NotNil(t, callbackPayload)require.Contains(t, callbackPayload, libarcane.SettingUpdate{Key: "trivyScanTimeout", Value: "1200"})@@ -949,7 +949,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyScanTimeout(func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyResourceLimits(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -964,7 +964,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyResourceLimiTrivyMemoryLimitMb: new("3072"),})require.NoError(t, err)- waitForSettingsNotificationsInternal(t, svc)+ waitForSettingsNotifications(t, svc)require.NotNil(t, callbackPayload)require.Contains(t, callbackPayload, libarcane.SettingUpdate{Key: "trivyResourceLimitsEnabled", Value: "false"})@@ -975,7 +975,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyResourceLimifunc TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyConcurrentScanContainers(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -986,7 +986,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyConcurrentSc_, err = svc.UpdateSettings(ctx, settings.Update{TrivyConcurrentScanContainers: new("4")})require.NoError(t, err)- waitForSettingsNotificationsInternal(t, svc)+ waitForSettingsNotifications(t, svc)require.NotNil(t, callbackPayload)require.Contains(t, callbackPayload, libarcane.SettingUpdate{Key: "trivyConcurrentScanContainers", Value: "4"})@@ -995,7 +995,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyConcurrentScfunc TestSettingsService_UpdateSettings_TrivyNetworkTriggersVulnerabilityCallback(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -1006,14 +1006,14 @@ func TestSettingsService_UpdateSettings_TrivyNetworkTriggersVulnerabilityCallbac_, err = svc.UpdateSettings(ctx, settings.Update{TrivyNetwork: new("arcane-external")})require.NoError(t, err)- waitForSettingsNotificationsInternal(t, svc)+ waitForSettingsNotifications(t, svc)require.True(t, callbackCalled)}func TestSettingsService_UpdateSettings_TrivyNetworkDoesNotTriggerTimeoutCallback(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -1024,14 +1024,14 @@ func TestSettingsService_UpdateSettings_TrivyNetworkDoesNotTriggerTimeoutCallbac_, err = svc.UpdateSettings(ctx, settings.Update{TrivyNetwork: new("arcane-external")})require.NoError(t, err)- waitForSettingsNotificationsInternal(t, svc)+ waitForSettingsNotifications(t, svc)require.Nil(t, callbackPayload)}func TestSettingsService_UpdateSettings_TrivyRuntimeSecurityTriggersVulnerabilityCallback(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -1045,14 +1045,14 @@ func TestSettingsService_UpdateSettings_TrivyRuntimeSecurityTriggersVulnerabilitTrivyPrivileged: new("true"),})require.NoError(t, err)- waitForSettingsNotificationsInternal(t, svc)+ waitForSettingsNotifications(t, svc)require.True(t, callbackCalled)}func TestSettingsService_UpdateSettings_TrivyRuntimeSecurityDoesNotTriggerTimeoutCallback(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -1066,7 +1066,7 @@ func TestSettingsService_UpdateSettings_TrivyRuntimeSecurityDoesNotTriggerTimeouTrivyPrivileged: new("true"),})require.NoError(t, err)- waitForSettingsNotificationsInternal(t, svc)+ waitForSettingsNotifications(t, svc)require.Nil(t, callbackPayload)}@@ -1082,7 +1082,7 @@ func TestSettingsService_LoadDatabaseSettings_InternalKeys_EnvMode(t *testing.T)internalVal := "test-instance-id"require.NoError(t, db.DB.Create(&SettingVariable{Key: internalKey, Value: internalVal}).Error)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)// Reload explicitly to trigger the env loading path@@ -1096,7 +1096,7 @@ func TestSettingsService_LoadDatabaseSettings_InternalKeys_EnvMode(t *testing.T)func TestSettingsService_NormalizeProjectsDirectory_ConvertsRelativeToAbsolute(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)// Seed with relative path@@ -1119,7 +1119,7 @@ func TestSettingsService_NormalizeProjectsDirectory_ConvertsRelativeToAbsolute(tfunc TestSettingsService_NormalizeProjectsDirectory_SkipsWhenEnvSet(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)// Seed with relative path@@ -1138,7 +1138,7 @@ func TestSettingsService_NormalizeProjectsDirectory_SkipsWhenEnvSet(t *testing.Tfunc TestSettingsService_NormalizeProjectsDirectory_LeavesOtherPathsUnchanged(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)customPath := "/custom/projects/path"@@ -1157,7 +1157,7 @@ func TestSettingsService_NormalizeProjectsDirectory_LeavesOtherPathsUnchanged(tfunc TestSettingsService_NormalizeProjectsDirectory_HandlesNotFound(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)// Don't create the setting at all@@ -1170,7 +1170,7 @@ func TestSettingsService_NormalizeProjectsDirectory_HandlesNotFound(t *testing.Tfunc TestSettingsService_NormalizeProjectsDirectory_UpdatesCacheAfterNormalization(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -1193,10 +1193,10 @@ func TestSettingsService_NormalizeProjectsDirectory_UpdatesCacheAfterNormalizatirequire.True(t, filepath.IsAbs(cfg2.ProjectsDirectory.Value), "path should be absolute")}-func TestSettingsService_NormalizeProjectsDirectoryPublishesChangeInternal(t *testing.T) {+func TestSettingsService_NormalizeProjectsDirectoryPublishesChange(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))require.NoError(t, svc.UpdateSetting(ctx, "projectsDirectory", "data/projects"))@@ -1208,18 +1208,16 @@ func TestSettingsService_NormalizeProjectsDirectoryPublishesChangeInternal(t *tedefer unsubscribe()require.NoError(t, svc.NormalizeProjectsDirectory(ctx, ""))- waitForSettingsNotificationsInternal(t, svc)+ waitForSettingsNotifications(t, svc)updates := <-notifiedrequire.Len(t, updates, 1)require.Equal(t, "projectsDirectory", updates[0].Key)require.True(t, filepath.IsAbs(updates[0].Value))}-// TestSettingsServiceEffectiveSnapshotMaterializedInternal verifies the-// env-override-applied snapshot is built once per refresh: reads share one-// pointer (no per-call clone), env overrides win over database values, and-// an update rebuilds the snapshot while keeping the override applied.-func TestSettingsServiceEffectiveSnapshotMaterializedInternal(t *testing.T) {+// TestSettingsServiceEffectiveSnapshotMaterialized verifies reads share one env-override-applied+// snapshot per refresh, and an update rebuilds it with the override still applied.+func TestSettingsServiceEffectiveSnapshotMaterialized(t *testing.T) {ctx := t.Context()t.Setenv("PROJECTS_DIRECTORY", "/env/projects")@@ -1227,7 +1225,7 @@ func TestSettingsServiceEffectiveSnapshotMaterializedInternal(t *testing.T) {require.NoError(t, db.Create(&SettingVariable{Key: "projectsDirectory", Value: "/db/projects"}).Error)require.NoError(t, db.Create(&SettingVariable{Key: "baseServerUrl", Value: "http://before.example"}).Error)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)// Env override beats the database value on the effective snapshot.@@ -1257,7 +1255,7 @@ func TestSettingsService_FeatureDefaultsAndUnknownFeature(t *testing.T) {require.NoError(t, svc.RequireFeature(t.Context(), features.VulnerabilityManagement))require.False(t, svc.IsFeatureEnabled(t.Context(), features.ID("unknown")))- svc, err := newSettingsServiceForTestInternal(t, t.Context(), setupSettingsTestDB(t))+ svc, err := newSettingsServiceForTest(t, t.Context(), setupSettingsTestDB(t))require.NoError(t, err)require.True(t, svc.IsFeatureEnabled(t.Context(), features.VulnerabilityManagement))require.NoError(t, svc.EnsureDefaultSettings(t.Context()))@@ -1270,9 +1268,9 @@ func TestSettingsService_FeatureDefaultsAndUnknownFeature(t *testing.T) {func TestSettingsService_FeaturePersistenceAndEnvironmentIsolation(t *testing.T) {ctx := t.Context()db := setupSettingsTestDB(t)- svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ svc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)- other, err := newSettingsServiceForTestInternal(t, ctx, setupSettingsTestDB(t))+ other, err := newSettingsServiceForTest(t, ctx, setupSettingsTestDB(t))require.NoError(t, err)var notifications int@@ -1286,7 +1284,7 @@ func TestSettingsService_FeaturePersistenceAndEnvironmentIsolation(t *testing.T)disabled := "false"_, err = svc.UpdateSettings(ctx, settings.Update{FeatureVulnerabilityManagementEnabled: &disabled})require.NoError(t, err)- waitForSettingsNotificationsInternal(t, svc)+ waitForSettingsNotifications(t, svc)require.Equal(t, 1, notifications)require.False(t, svc.IsFeatureEnabled(ctx, features.VulnerabilityManagement))require.True(t, other.IsFeatureEnabled(ctx, features.VulnerabilityManagement))@@ -1298,7 +1296,7 @@ func TestSettingsService_FeaturePersistenceAndEnvironmentIsolation(t *testing.T)require.Equal(t, http.StatusForbidden, apiErr.HTTPStatus())require.Equal(t, common.APIErrorCodeFeatureDisabled, apiErr.Code)- reloaded, err := newSettingsServiceForTestInternal(t, ctx, db)+ reloaded, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.False(t, reloaded.IsFeatureEnabled(ctx, features.VulnerabilityManagement))enabled := "true"@@ -1313,7 +1311,7 @@ func TestSettingsService_FeaturePersistenceAndEnvironmentIsolation(t *testing.T)func TestSettingsService_FeatureEnvironmentOverride(t *testing.T) {t.Setenv("FEATURE_VULNERABILITY_MANAGEMENT_ENABLED", "false")- svc, err := newSettingsServiceForTestInternal(t, t.Context(), setupSettingsTestDB(t))+ svc, err := newSettingsServiceForTest(t, t.Context(), setupSettingsTestDB(t))require.NoError(t, err)enabled := "true"_, err = svc.UpdateSettings(t.Context(), settings.Update{FeatureVulnerabilityManagementEnabled: &enabled})@@ -1326,7 +1324,7 @@ func TestSettingsService_FeatureEnvironmentOverride(t *testing.T) {func TestSettingsService_RejectInvalidFeatureBoolean(t *testing.T) {for _, invalid := range []string{"", "yes", "0", "TRUE"} {t.Run(invalid, func(t *testing.T) {- svc, err := newSettingsServiceForTestInternal(t, t.Context(), setupSettingsTestDB(t))+ svc, err := newSettingsServiceForTest(t, t.Context(), setupSettingsTestDB(t))require.NoError(t, err)_, err = svc.UpdateSettings(t.Context(), settings.Update{FeatureVulnerabilityManagementEnabled: &invalid})require.ErrorIs(t, err, common.ErrValidation)@@ -1343,7 +1341,7 @@ func TestSettingsService_SwarmFeatureDefaultsOffAndPersists(t *testing.T) {var nilService *SettingsServicerequire.False(t, nilService.IsFeatureEnabled(t.Context(), features.Swarm))- svc, err := newSettingsServiceForTestInternal(t, t.Context(), setupSettingsTestDB(t))+ svc, err := newSettingsServiceForTest(t, t.Context(), setupSettingsTestDB(t))require.NoError(t, err)require.False(t, svc.IsFeatureEnabled(t.Context(), features.Swarm))require.Contains(t, svc.ListSettings(SettingVisibilityPublic), SettingVariable{Key: features.SwarmSettingKey, Value: "false"})diff --git a/backend/internal/system/children/backup/service.go b/backend/internal/system/children/backup/service.goindex 797219455..62ba31dab 100644--- a/backend/internal/system/children/backup/service.go+++ b/backend/internal/system/children/backup/service.go@@ -31,6 +31,10 @@ import ("go.getarcane.app/docker/compat"kit "go.getarcane.app/kit/pkg""go.getarcane.app/sys/cgroup"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric"+ "go.opentelemetry.io/otel/trace""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/activity"@@ -54,6 +58,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/backupbrowser"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")const (@@ -139,6 +144,7 @@ type Service struct {// backupWorkflow runs backups StartBackup accepted; policyWorkflow runs scheduled policies.backupWorkflow *flow.WorkflowpolicyWorkflow *flow.Workflow+ operations metric.Int64Counter}// manualBackupInput is a manual backup's workflow payload; the run row is created before it starts.@@ -167,6 +173,12 @@ func NewService(deps Dependencies) *Service {recoveryKeys: deps.RecoveryKeys, resolveRuntimeOptions: deps.ResolveRuntimeOptions,jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),}+ var counterErr error+ service.operations, counterErr = otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.backups",+ metric.WithDescription("Backup operations by outcome"), metric.WithUnit("{backup}"))+ if counterErr != nil {+ otel.Handle(counterErr)+ }service.snapshots = snapshots.NewService(deps.Engine,deps.SQLDB,@@ -230,6 +242,22 @@ func (s *Service) recoveryEnvironment(ctx context.Context) map[string]string {return result}+// startOperation opens a backup span; the returned func ends it and counts the outcome.+func (s *Service) startOperation(ctx context.Context, operation string, attrs ...attribute.KeyValue) (context.Context, func(err error, results ...attribute.KeyValue)) {+ attrs = append(attrs, attribute.String("arcane.environment.id", "0"), attribute.String("arcane.backup.kind", "system"))+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "backup."+operation, trace.WithAttributes(attrs...))+ return ctx, func(err error, results ...attribute.KeyValue) {+ span.SetAttributes(results...)+ tracing.End(span, err)+ if s.operations != nil {+ s.operations.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes(+ attribute.String("arcane.backup.operation", operation),+ attribute.String("arcane.backup.outcome", kit.Ternary(err == nil, "success", "failure")),+ ))+ }+ }+}+// acquireRun takes the shared system backup admission lease.func (s *Service) acquireRun(ctx context.Context) (*runs.Lease, error) {lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID)@@ -431,6 +459,11 @@ func (s *Service) prepareBackup(ctx context.Context, trigger string, request bac// executeBackup stages one snapshot, keeps it locally and/or replicates it to S3, then records the run's result.func (s *Service) executeBackup(ctx context.Context, prepared *preparedSystemBackup) (_ *backuptypes.SystemBackupRun, err error) {run, key, checkpoint := prepared.run, prepared.recoveryKey, prepared.checkpoint+ ctx, finish := s.startOperation(ctx, "create", attribute.String("arcane.backup.id", run.ID), attribute.String("arcane.backup.trigger", run.Trigger))+ defer func() {+ finish(err, attribute.String("arcane.backup.target", string(run.Destination)), attribute.Int64("arcane.backup.size_bytes", run.Size),+ attribute.String("arcane.backup.s3_destination_id", run.S3DestinationID))+ }()defer func() {if err = cmp.Or(err, ctx.Err()); err != nil {run.Status, run.Error = backuptypes.SystemBackupStatusFailed, err.Error()@@ -560,7 +593,10 @@ func (s *Service) BrowseBackupFiles(ctx context.Context, id, recoveryKey, reques// RestoreBackupFiles restores selected project files into the current projects// directory after creating a safety snapshot.-func (s *Service) RestoreBackupFiles(ctx context.Context, id string, request backuptypes.RestoreSystemBackupFilesRequest, user usertypes.Actor) error {+func (s *Service) RestoreBackupFiles(ctx context.Context, id string, request backuptypes.RestoreSystemBackupFilesRequest, user usertypes.Actor) (opErr error) {+ ctx, finish := s.startOperation(ctx, "restore_files", attribute.String("arcane.backup.id", id),+ attribute.Int("arcane.result.count", len(request.Paths)), attribute.Bool("arcane.backup.select_all", request.SelectAll))+ defer func() { finish(opErr) }()lease, err := s.acquireRun(ctx)if err != nil {return err@@ -591,7 +627,9 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, id string, request bacreturn s.snapshots.RestoreFiles(ctx, dockerClient, key, *run, request.RestoreSelection, safetyBackup)}-func (s *Service) DeleteBackup(ctx context.Context, id, recoveryKey string) error {+func (s *Service) DeleteBackup(ctx context.Context, id, recoveryKey string) (opErr error) {+ ctx, finish := s.startOperation(ctx, "delete", attribute.String("arcane.backup.id", id))+ defer func() { finish(opErr) }()// Deletes contend with create/restore/upload on the system run lease so a// slow operation cannot resurrect or double-forget snapshots.lease, err := s.acquireRun(ctx)@@ -805,7 +843,9 @@ func (s *Service) UploadBackup(ctx context.Context, id string, request backuptypreturn run, nil}-func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, user usertypes.Actor) error {+func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, user usertypes.Actor) (opErr error) {+ ctx, finish := s.startOperation(ctx, "restore", attribute.String("arcane.backup.id", id))+ defer func() { finish(opErr) }()run, err := s.store.Run(ctx, id)if err != nil {return errdiff --git a/backend/internal/telemetry/module.go b/backend/internal/telemetry/module.gonew file mode 100644index 000000000..7b2250c10--- /dev/null+++ b/backend/internal/telemetry/module.go@@ -0,0 +1,64 @@+// Package telemetry relays browser OpenTelemetry data to the collector configured+// through the standard OTEL_* exporter, endpoint, protocol, and header variables.+package telemetry++import (+ "context"+ "errors"+ "log/slog"+ "mime"+ "net/http"+ "os"+ "time"++ "github.com/labstack/echo/v5"+ "golang.org/x/time/rate"++ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"+)++// RegisterRoutes adds an unauthenticated relay route per enabled signal; disabled+// signals return 404. Routes share a per-IP limit, a global limit, and a cap on concurrent requests.+func RegisterRoutes(g *echo.Group, service *Service) {+ if service == nil {+ return+ }+ perIP := middleware.PerIPRateLimit(120, 30)+ global := rate.NewLimiter(rate.Every(time.Minute/600), 60)+ inFlight := make(chan struct{}, 32)+ for signal := range service.destinations {+ g.POST("/telemetry/"+signal, func(c *echo.Context) error {+ req := c.Request()+ if !global.Allow() {+ c.Response().Header().Set("Retry-After", "60")+ return c.NoContent(http.StatusTooManyRequests)+ }+ select {+ case inFlight <- struct{}{}:+ defer func() { <-inFlight }()+ default:+ return c.NoContent(http.StatusTooManyRequests)+ }+ // Slow uploads must not hold a relay slot indefinitely.+ if err := http.NewResponseController(c.Response()).SetReadDeadline(time.Now().Add(10 * time.Second)); err != nil {+ slog.DebugContext(req.Context(), "Failed to set relay read deadline", "error", err)+ }+ if mediaType, _, _ := mime.ParseMediaType(req.Header.Get("Content-Type")); mediaType != "application/json" {+ return c.NoContent(http.StatusUnsupportedMediaType)+ }+ status, err := service.Forward(context.WithoutCancel(req.Context()), signal, req.Body)+ switch {+ case errors.Is(err, ErrPayloadTooLarge):+ return c.NoContent(http.StatusRequestEntityTooLarge)+ case errors.Is(err, ErrInvalidPayload):+ return c.NoContent(http.StatusBadRequest)+ case errors.Is(err, os.ErrDeadlineExceeded):+ return c.NoContent(http.StatusRequestTimeout)+ case err != nil:+ slog.DebugContext(req.Context(), "Failed to relay browser telemetry", "signal", signal, "error", err)+ return c.NoContent(http.StatusBadGateway)+ }+ return c.NoContent(status)+ }, perIP)+ }+}diff --git a/backend/internal/telemetry/service.go b/backend/internal/telemetry/service.gonew file mode 100644index 000000000..ebec31eb5--- /dev/null+++ b/backend/internal/telemetry/service.go@@ -0,0 +1,161 @@+package telemetry++import (+ "bytes"+ "cmp"+ "context"+ "encoding/json/v2"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "os"+ "slices"+ "strings"+ "time"+)++const maxPayloadBytes = 512 << 10++var (+ ErrPayloadTooLarge = errors.New("telemetry payload exceeds 512 KiB")+ ErrInvalidPayload = errors.New("telemetry payload is not OTLP/JSON from arcane-frontend")+ // resourceKeys names each signal's top-level OTLP/JSON resource list.+ resourceKeys = map[string]string{"traces": "resourceSpans", "metrics": "resourceMetrics", "logs": "resourceLogs"}+)++type otlpAttribute struct {+ Key string `json:"key"`+ Value struct {+ StringValue string `json:"stringValue"`+ } `json:"value"`+}++// otlpEnvelope decodes only the resource identities of an OTLP/JSON export.+type otlpEnvelope map[string][]struct {+ Resource struct {+ Attributes []otlpAttribute `json:"attributes"`+ } `json:"resource"`+}++// Service relays browser OTLP/JSON payloads to the configured collector so the+// collector and its credentials stay server-side.+type Service struct {+ destinations map[string]destination+ client *http.Client+}++type destination struct {+ endpoint string+ headers http.Header+}++// SDKDisabled reports whether OTEL_SDK_DISABLED turns all telemetry off.+func SDKDisabled() bool {+ return strings.EqualFold(strings.TrimSpace(os.Getenv("OTEL_SDK_DISABLED")), "true")+}++// Endpoint returns the OTLP/HTTP URL for signal, or "" when its exporter is+// off, uses gRPC, or has no endpoint configured.+func Endpoint(signal string) string {+ upper := strings.ToUpper(signal)+ if SDKDisabled() || strings.TrimSpace(os.Getenv("OTEL_"+upper+"_EXPORTER")) != "otlp" {+ return ""+ }+ protocol := cmp.Or(os.Getenv("OTEL_EXPORTER_OTLP_"+upper+"_PROTOCOL"), os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL"), "http/protobuf")+ if !strings.HasPrefix(protocol, "http/") {+ return ""+ }+ endpoint := os.Getenv("OTEL_EXPORTER_OTLP_" + upper + "_ENDPOINT")+ if endpoint == "" {+ base := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")+ if base == "" {+ return ""+ }+ endpoint = strings.TrimRight(base, "/") + "/v1/" + signal+ }+ parsed, err := url.Parse(endpoint)+ if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {+ return ""+ }+ return parsed.String()+}++// NewService returns nil when no signal can be relayed.+func NewService() *Service {+ destinations := make(map[string]destination, 3)+ for _, signal := range []string{"traces", "metrics", "logs"} {+ endpoint := Endpoint(signal)+ if endpoint == "" {+ continue+ }+ headers := http.Header{}+ rawHeaders := cmp.Or(os.Getenv("OTEL_EXPORTER_OTLP_"+strings.ToUpper(signal)+"_HEADERS"), os.Getenv("OTEL_EXPORTER_OTLP_HEADERS"))+ for pair := range strings.SplitSeq(rawHeaders, ",") {+ key, value, ok := strings.Cut(pair, "=")+ key = strings.TrimSpace(key)+ if !ok || key == "" {+ continue+ }+ if decoded, err := url.PathUnescape(strings.TrimSpace(value)); err == nil {+ value = decoded+ }+ headers.Set(key, value)+ }+ destinations[signal] = destination{endpoint: endpoint, headers: headers}+ }+ if len(destinations) == 0 {+ return nil+ }++ return &Service{+ destinations: destinations,+ client: &http.Client{+ Timeout: 10 * time.Second,+ CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },+ },+ }+}++// Forward posts an OTLP/JSON payload of at most 512 KiB for signal to the collector+// and returns its status code. Every resource must identify as arcane-frontend.+func (s *Service) Forward(ctx context.Context, signal string, body io.Reader) (int, error) {+ dest, ok := s.destinations[signal]+ if !ok {+ return 0, fmt.Errorf("telemetry signal %q is not relayed", signal)+ }+ payload, err := io.ReadAll(io.LimitReader(body, maxPayloadBytes+1))+ if err != nil {+ return 0, fmt.Errorf("read %s payload: %w", signal, err)+ }+ if len(payload) > maxPayloadBytes {+ return 0, ErrPayloadTooLarge+ }+ var envelope otlpEnvelope+ if json.Unmarshal(payload, &envelope) != nil || len(envelope) != 1 || len(envelope[resourceKeys[signal]]) == 0 {+ return 0, ErrInvalidPayload+ }+ for _, entry := range envelope[resourceKeys[signal]] {+ if !slices.ContainsFunc(entry.Resource.Attributes, func(attr otlpAttribute) bool {+ return attr.Key == "service.name" && attr.Value.StringValue == "arcane-frontend"+ }) {+ return 0, ErrInvalidPayload+ }+ }++ req, err := http.NewRequestWithContext(ctx, http.MethodPost, dest.endpoint, bytes.NewReader(payload))+ if err != nil {+ return 0, fmt.Errorf("create %s forward request: %w", signal, err)+ }+ req.Header = dest.headers.Clone()+ req.Header.Set("Content-Type", "application/json")++ resp, err := s.client.Do(req)+ if err != nil {+ return 0, fmt.Errorf("forward %s: %w", signal, err)+ }+ defer func() { _ = resp.Body.Close() }()+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxPayloadBytes))+ return resp.StatusCode, nil+}diff --git a/backend/internal/template/service.go b/backend/internal/template/service.goindex 0e82ebca5..169fac4a9 100644--- a/backend/internal/template/service.go+++ b/backend/internal/template/service.go@@ -10,8 +10,10 @@ import ("log/slog""maps""net/http"+ "net/url""os""path/filepath"+ "slices""strings""sync""sync/atomic"@@ -26,6 +28,9 @@ import ("go.getarcane.app/kit/normalization""go.getarcane.app/kit/pkg""go.getarcane.app/kit/pkg/mapping"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/trace""golang.org/x/sync/errgroup""gorm.io/gorm"@@ -36,6 +41,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/projects""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")type registryFetchMeta struct {@@ -68,7 +74,7 @@ const (remoteIDPrefix = "remote")-var errRemoteCacheInvalidatedInternal = errors.New("remote template cache invalidated")+var errRemoteCacheInvalidated = errors.New("remote template cache invalidated")func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.Client, settingsService *settings.SettingsService) *TemplateService {if httpClient == nil {@@ -83,14 +89,13 @@ func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.CregistryFetchMeta: make(map[string]*registryFetchMeta),registryErrors: make(map[string]string),}- service.safeHTTPClient = service.newSafeHTTPClientInternal(ctx)revalidationCtx := context.WithoutCancel(ctx)loader := func(generations []uint64) (map[uint64][]ComposeTemplate, error) {loadCtx, cancel := context.WithTimeout(revalidationCtx, 2*time.Minute)defer cancel()catalogs := make(map[uint64][]ComposeTemplate, len(generations))for _, generation := range generations {- templates, err := service.loadRemoteTemplatesInternal(loadCtx, generation)+ templates, err := service.loadRemoteTemplates(loadCtx, generation)if err != nil {return nil, err}@@ -105,31 +110,27 @@ func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.CWithRevalidationErrorPolicy(hot.KeepOnError).Build()- if err := projects.EnsureDefaultTemplates(ctx, service.configuredTemplatesDirSettingInternal(ctx)); err != nil {+ if err := projects.EnsureDefaultTemplates(ctx, service.configuredTemplatesDirSetting(ctx)); err != nil {slog.WarnContext(ctx, "failed to ensure default templates", "error", err)}return service}-// configuredTemplatesDirSettingInternal returns the raw user-configured templates-// directory string (from the settingsService), without resolving it to an absolute-// path. Use this when calling helpers like projects.GetTemplatesDirectory that-// perform their own resolution.-func (s *TemplateService) configuredTemplatesDirSettingInternal(ctx context.Context) string {+// configuredTemplatesDirSetting returns the raw templates directory setting; callers resolve it.+func (s *TemplateService) configuredTemplatesDirSetting(ctx context.Context) string {if s.settingsService == nil {return ""}return s.settingsService.GetStringSetting(ctx, "templatesDirectory", "/app/data/templates")}-// getTemplatesDirectoryInternal resolves the effective templates directory by-// applying the same resolution rules used elsewhere for the projects directory.-func (s *TemplateService) getTemplatesDirectoryInternal(ctx context.Context) (string, error) {- return projects.GetTemplatesDirectory(ctx, strings.TrimSpace(s.configuredTemplatesDirSettingInternal(ctx)))+// getTemplatesDirectory resolves the effective templates directory.+func (s *TemplateService) getTemplatesDirectory(ctx context.Context) (string, error) {+ return projects.GetTemplatesDirectory(ctx, strings.TrimSpace(s.configuredTemplatesDirSetting(ctx)))}-func (s *TemplateService) remoteTemplatesInternal(ctx context.Context, refresh bool) ([]ComposeTemplate, error) {+func (s *TemplateService) remoteTemplates(ctx context.Context, refresh bool) ([]ComposeTemplate, error) {if s.remoteCache == nil {return nil, errors.New("remote template cache is not initialized")}@@ -141,7 +142,7 @@ func (s *TemplateService) remoteTemplatesInternal(ctx context.Context, refresh bvar templates []ComposeTemplatevar err errorif refresh {- templates, err = s.loadRemoteTemplatesInternal(ctx, generation)+ templates, err = s.loadRemoteTemplates(ctx, generation)s.registryMu.Lock()if generation == s.remoteGeneration.Load() && err == nil {s.remoteCache.Set(generation, templates)@@ -165,11 +166,28 @@ func (s *TemplateService) remoteTemplatesInternal(ctx context.Context, refresh b}func (s *TemplateService) GetAllTemplates(ctx context.Context) ([]ComposeTemplate, error) {- return s.getMergedTemplatesInternal(ctx)+ if err := s.SyncLocalTemplatesFromFilesystem(ctx); err != nil {+ slog.WarnContext(ctx, "failed to sync filesystem templates", "error", err)+ }++ var templates []ComposeTemplate+ // Omit heavy content fields which are not needed for listing.+ if err := s.db.WithContext(ctx).Omit("Content", "EnvContent").Preload("Registry").Find(&templates).Error; err != nil {+ return nil, fmt.Errorf("failed to get local templates: %w", err)+ }++ remoteTemplates, err := s.remoteTemplates(ctx, false)+ if err != nil {+ slog.WarnContext(ctx, "failed to load remote templates", "error", err)+ } else {+ templates = append(templates, cloneRemoteTemplates(remoteTemplates)...)+ }++ return templates, nil}func (s *TemplateService) GetAllTemplatesPaginated(ctx context.Context, params pagination.QueryParams) ([]tmpl.Template, pagination.Response, error) {- templates, err := s.getMergedTemplatesInternal(ctx)+ templates, err := s.GetAllTemplates(ctx)if err != nil {return nil, pagination.Response{}, err}@@ -236,7 +254,7 @@ func (s *TemplateService) GetAllTemplatesPaginated(ctx context.Context, params p}func (s *TemplateService) GetTemplate(ctx context.Context, id string) (*ComposeTemplate, error) {- if err := s.syncFilesystemTemplatesInternal(ctx); err != nil {+ if err := s.SyncLocalTemplatesFromFilesystem(ctx); err != nil {slog.WarnContext(ctx, "failed to sync filesystem templates", "error", err)}@@ -247,55 +265,33 @@ func (s *TemplateService) GetTemplate(ctx context.Context, id string) (*ComposeTreturn nil, fmt.Errorf("failed to query local template: %w", err)}- templates, err := s.remoteTemplatesInternal(ctx, false)+ templates, err := s.remoteTemplates(ctx, false)if err != nil {return nil, fmt.Errorf("template %q lookup failed: registry refresh error: %w", id, err)}- if found := s.lookupRemoteTemplateInternal(templates, id); found != nil {- return found, nil+ matchesID := func(t ComposeTemplate) bool { return t.ID == id }+ if i := slices.IndexFunc(templates, matchesID); i >= 0 {+ return &cloneRemoteTemplates(templates[i : i+1])[0], nil}- // Remote IDs (remote:registry:slug) deserve a synchronous force-refresh on miss- // before we return "not found" — the cache may be stale or the previous refresh- // silently returned empty.+ // Force-refresh remote IDs (remote:registry:slug) on a miss; the cache may be stale.if strings.HasPrefix(id, remoteIDPrefix+":") {slog.InfoContext(ctx, "remote template not in cache, forcing registry refresh", "templateId", id, "cacheSize", len(templates))- localTemplates, refreshErr := s.remoteTemplatesInternal(ctx, true)+ refreshed, refreshErr := s.remoteTemplates(ctx, true)if refreshErr != nil {return nil, fmt.Errorf("template %q not found and registry refresh failed: %w", id, refreshErr)}- if found := s.lookupRemoteTemplateInternal(localTemplates, id); found != nil {- return found, nil+ if i := slices.IndexFunc(refreshed, matchesID); i >= 0 {+ return &cloneRemoteTemplates(refreshed[i : i+1])[0], nil}- return nil, common.Classify(- common.ErrTemplateNotFound,- fmt.Errorf(- "template not found: %w",- fmt.Errorf(- "template %q not found in any registered registry (cache size=%d after refresh)",- id,- len(- localTemplates,- ),- ),- ),- )+ notFoundErr := fmt.Errorf("template %q not found in any registered registry (cache size=%d after refresh)", id, len(refreshed))+ return nil, common.Classify(common.ErrTemplateNotFound, fmt.Errorf("template not found: %w", notFoundErr))}return nil, common.Classify(common.ErrTemplateNotFound, errors.New("template not found"))}-func (s *TemplateService) lookupRemoteTemplateInternal(templates []ComposeTemplate, id string) *ComposeTemplate {- for i := range templates {- if templates[i].ID == id {- cloned := cloneRemoteTemplates(templates[i : i+1])- return &cloned[0]- }- }- return nil-}-func (s *TemplateService) CreateTemplate(ctx context.Context, template *ComposeTemplate) error {if err := normalization.Normalize(template); err != nil {return err@@ -357,7 +353,7 @@ func (s *TemplateService) DeleteTemplate(ctx context.Context, id string) error {return errors.New("cannot delete remote template directly")}- baseDir, err := s.getTemplatesDirectoryInternal(ctx)+ baseDir, err := s.getTemplatesDirectory(ctx)if err != nil {return fmt.Errorf("failed to get templates directory: %w", err)}@@ -377,12 +373,9 @@ func (s *TemplateService) DeleteTemplate(ctx context.Context, id string) error {})}-// readDefaultTemplateInternal reads one of the default template files from the-// configured templates directory. These used to be read from a CWD-relative-// "data/templates", which only resolved when the process happened to run from-// the repository root.-func (s *TemplateService) readDefaultTemplateInternal(ctx context.Context, fileName string) (string, error) {- baseDir, err := s.getTemplatesDirectoryInternal(ctx)+// readDefaultTemplate reads one of the default template files from the configured templates directory.+func (s *TemplateService) readDefaultTemplate(ctx context.Context, fileName string) (string, error) {+ baseDir, err := s.getTemplatesDirectory(ctx)if err != nil {return "", fmt.Errorf("failed to get templates directory: %w", err)}@@ -394,7 +387,7 @@ func (s *TemplateService) readDefaultTemplateInternal(ctx context.Context, fileN}func (s *TemplateService) GetComposeTemplate(ctx context.Context) string {- content, err := s.readDefaultTemplateInternal(ctx, ".compose.template")+ content, err := s.readDefaultTemplate(ctx, ".compose.template")if err != nil {slog.WarnContext(ctx, "failed to read compose template", "error", err)return ""@@ -403,7 +396,7 @@ func (s *TemplateService) GetComposeTemplate(ctx context.Context) string {}func (s *TemplateService) GetSwarmStackTemplate(ctx context.Context) string {- content, err := s.readDefaultTemplateInternal(ctx, ".swarm-stack.template")+ content, err := s.readDefaultTemplate(ctx, ".swarm-stack.template")if err != nil {slog.WarnContext(ctx, "failed to read swarm stack template", "error", err)return projects.DefaultSwarmStackTemplate()@@ -412,7 +405,7 @@ func (s *TemplateService) GetSwarmStackTemplate(ctx context.Context) string {}func (s *TemplateService) GetSwarmStackEnvTemplate(ctx context.Context) string {- content, err := s.readDefaultTemplateInternal(ctx, ".swarm-stack.env.template")+ content, err := s.readDefaultTemplate(ctx, ".swarm-stack.env.template")if err != nil {slog.WarnContext(ctx, "failed to read swarm stack env template", "error", err)return projects.DefaultSwarmStackEnvTemplate()@@ -421,7 +414,7 @@ func (s *TemplateService) GetSwarmStackEnvTemplate(ctx context.Context) string {}func (s *TemplateService) SaveComposeTemplate(ctx context.Context, content string) error {- baseDir, err := s.getTemplatesDirectoryInternal(ctx)+ baseDir, err := s.getTemplatesDirectory(ctx)if err != nil {return fmt.Errorf("failed to get templates directory: %w", err)}@@ -429,7 +422,7 @@ func (s *TemplateService) SaveComposeTemplate(ctx context.Context, content strin}func (s *TemplateService) GetEnvTemplate(ctx context.Context) string {- content, err := s.readDefaultTemplateInternal(ctx, ".env.template")+ content, err := s.readDefaultTemplate(ctx, ".env.template")if err != nil {slog.WarnContext(ctx, "failed to read env template", "error", err)return ""@@ -438,7 +431,7 @@ func (s *TemplateService) GetEnvTemplate(ctx context.Context) string {}func (s *TemplateService) SaveEnvTemplate(ctx context.Context, content string) error {- baseDir, err := s.getTemplatesDirectoryInternal(ctx)+ baseDir, err := s.getTemplatesDirectory(ctx)if err != nil {return fmt.Errorf("failed to get templates directory: %w", err)}@@ -474,12 +467,8 @@ func (s *TemplateService) CreateRegistry(ctx context.Context, registry *Templatereturn errors.New("registry URL is required")}if manifest, err := s.fetchRegistryManifest(ctx, registry.URL); err == nil {- if registry.Name == "" {- registry.Name = manifest.Name- }- if registry.Description == "" {- registry.Description = manifest.Description- }+ registry.Name = cmp.Or(registry.Name, manifest.Name)+ registry.Description = cmp.Or(registry.Description, manifest.Description)} else if registry.Name == "" || registry.Description == "" {return fmt.Errorf("failed to fetch registry manifest: %w", err)}@@ -502,7 +491,7 @@ func (s *TemplateService) CreateRegistry(ctx context.Context, registry *Templatereturn err}- s.invalidateRemoteCacheInternal()+ s.invalidateRemoteCache()return nil}@@ -519,8 +508,16 @@ func (s *TemplateService) UpdateRegistry(ctx context.Context, id string, updatesreturn fmt.Errorf("failed to find registry: %w", err)}- if err := s.hydrateRegistryUpdates(ctx, updates, &existing); err != nil {- return err+ // Hydrate blank fields from the manifest; a changed URL must yield them.+ urlChanged := updates.URL != "" && updates.URL != existing.URL+ manifestURL := cmp.Or(updates.URL, existing.URL)+ if (urlChanged || updates.Name == "" || updates.Description == "") && manifestURL != "" {+ if manifest, err := s.fetchRegistryManifest(ctx, manifestURL); err == nil {+ updates.Name = cmp.Or(updates.Name, manifest.Name)+ updates.Description = cmp.Or(updates.Description, manifest.Description)+ } else if urlChanged && (updates.Name == "" || updates.Description == "") {+ return fmt.Errorf("failed to fetch registry manifest: %w", err)+ }}if err := normalization.Normalize(updates); err != nil {@@ -537,30 +534,7 @@ func (s *TemplateService) UpdateRegistry(ctx context.Context, id string, updatesreturn err}- s.invalidateRemoteCacheInternal()- return nil-}--func (s *TemplateService) hydrateRegistryUpdates(ctx context.Context, updates, existing *TemplateRegistry) error {- urlChanged := updates.URL != "" && updates.URL != existing.URL- needsHydration := updates.Name == "" || updates.Description == ""-- if (urlChanged || needsHydration) && (updates.URL != "" || existing.URL != "") {- manifestURL := updates.URL- if manifestURL == "" {- manifestURL = existing.URL- }- if manifest, err := s.fetchRegistryManifest(ctx, manifestURL); err == nil {- if updates.Name == "" {- updates.Name = manifest.Name- }- if updates.Description == "" {- updates.Description = manifest.Description- }- } else if urlChanged && (updates.Name == "" || updates.Description == "") {- return fmt.Errorf("failed to fetch registry manifest: %w", err)- }- }+ s.invalidateRemoteCache()return nil}@@ -579,11 +553,17 @@ func (s *TemplateService) DeleteRegistry(ctx context.Context, id string) error {return err}- s.invalidateRemoteCacheInternal()+ s.invalidateRemoteCache()return nil}-func (s *TemplateService) loadRemoteTemplatesInternal(ctx context.Context, generation uint64) ([]ComposeTemplate, error) {+func (s *TemplateService) loadRemoteTemplates(ctx context.Context, generation uint64) (templates []ComposeTemplate, err error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "template.sync_remote")+ defer func() {+ span.SetAttributes(attribute.Int("arcane.result.count", len(templates)))+ tracing.End(span, err)+ }()+registries, err := s.GetRegistries(ctx)if err != nil {return nil, err@@ -591,8 +571,8 @@ func (s *TemplateService) loadRemoteTemplatesInternal(ctx context.Context, genervar (mu sync.Mutex- templates []ComposeTemplatefetchErrors []error+ enabledRegistries intsuccessfulRegistries int)@@ -603,11 +583,12 @@ func (s *TemplateService) loadRemoteTemplatesInternal(ctx context.Context, generif !reg.Enabled {continue}+ enabledRegistries++g.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "template worker")- remoteTemplates, fetchRegistryTemplatesErr := s.fetchRegistryTemplatesInternal(groupCtx, ®, generation)+ remoteTemplates, fetchRegistryTemplatesErr := s.fetchRegistryTemplates(groupCtx, ®, generation)if fetchRegistryTemplatesErr != nil {slog.WarnContext(groupCtx, "failed to fetch templates from registry", "registry", reg.Name, "url", reg.URL, "error", fetchRegistryTemplatesErr)s.registryMu.Lock()@@ -639,11 +620,16 @@ func (s *TemplateService) loadRemoteTemplatesInternal(ctx context.Context, gener})}- if waitErr := g.Wait(); waitErr != nil {+ waitErr := g.Wait()+ span.SetAttributes(+ attribute.Int("arcane.template.registry_count", enabledRegistries),+ attribute.Int("arcane.template.failed_registry_count", len(fetchErrors)),+ )+ if waitErr != nil {return nil, waitErr}if generation != s.remoteGeneration.Load() {- return nil, errRemoteCacheInvalidatedInternal+ return nil, errRemoteCacheInvalidated}if successfulRegistries == 0 && len(fetchErrors) > 0 {return nil, errors.Join(fetchErrors...)@@ -652,44 +638,52 @@ func (s *TemplateService) loadRemoteTemplatesInternal(ctx context.Context, generreturn templates, nil}-func (s *TemplateService) FetchRaw(ctx context.Context, url string) ([]byte, error) {- return s.doGET(ctx, url)-}--func (s *TemplateService) doGET(ctx context.Context, url string) ([]byte, error) {- client, req, err := s.newSafeRequestInternal(ctx, http.MethodGet, url)+func (s *TemplateService) FetchRaw(ctx context.Context, rawURL string) ([]byte, error) {+ client, req, err := s.newSafeRequest(ctx, http.MethodGet, rawURL)if err != nil {return nil, err}resp, err := client.Do(req)if err != nil {- return nil, fmt.Errorf("failed to fetch %s: %w", url, err)+ return nil, fmt.Errorf("failed to fetch %s: %w", rawURL, err)}defer func() { _ = resp.Body.Close() }()if resp.StatusCode != http.StatusOK {- return nil, fmt.Errorf("HTTP status %d for URL %s", resp.StatusCode, url)+ return nil, fmt.Errorf("HTTP status %d for URL %s", resp.StatusCode, rawURL)}body, err := io.ReadAll(resp.Body)if err != nil {- return nil, fmt.Errorf("failed to read response body from %s: %w", url, err)+ return nil, fmt.Errorf("failed to read response body from %s: %w", rawURL, err)}return body, nil}-// fetchRegistryTemplatesInternal performs a conditional GET using If-Modified-Since.-// If the server replies 304 Not Modified, cached templates for the registry are reused.-func (s *TemplateService) fetchRegistryTemplatesInternal(ctx context.Context, reg *TemplateRegistry, generation uint64) ([]ComposeTemplate, error) {+// fetchRegistryTemplates performs a conditional GET; a 304 reuses the registry's cached templates.+func (s *TemplateService) fetchRegistryTemplates(ctx context.Context, reg *TemplateRegistry, generation uint64) (templates []ComposeTemplate, err error) {+ registryHost := ""+ if parsedURL, parseErr := url.Parse(reg.URL); parseErr == nil {+ registryHost = parsedURL.Host+ }+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "template.fetch_registry", trace.WithAttributes(+ attribute.String("arcane.template.registry_id", reg.ID),+ attribute.String("arcane.template.registry_host", registryHost),+ ))+ defer func() {+ span.SetAttributes(attribute.Int("arcane.result.count", len(templates)))+ tracing.End(span, err)+ }()+s.registryMu.RLock()if generation != s.remoteGeneration.Load() {s.registryMu.RUnlock()- return nil, errRemoteCacheInvalidatedInternal+ return nil, errRemoteCacheInvalidated}fetchMeta := s.registryFetchMeta[reg.ID]s.registryMu.RUnlock()- client, req, err := s.newSafeRequestInternal(ctx, http.MethodGet, reg.URL)+ client, req, err := s.newSafeRequest(ctx, http.MethodGet, reg.URL)if err != nil {return nil, fmt.Errorf("create request: %w", err)}@@ -703,6 +697,7 @@ func (s *TemplateService) fetchRegistryTemplatesInternal(ctx context.Context, re}defer func() { _ = resp.Body.Close() }()+ span.SetAttributes(attribute.Bool("arcane.template.not_modified", resp.StatusCode == http.StatusNotModified))if resp.StatusCode == http.StatusNotModified {if fetchMeta != nil {return cloneRemoteTemplates(fetchMeta.Templates), nil@@ -723,20 +718,36 @@ func (s *TemplateService) fetchRegistryTemplatesInternal(ctx context.Context, rereturn nil, fmt.Errorf("parse registry JSON: %w", unmarshalErr)}- templates := make([]ComposeTemplate, 0, len(regDTO.Templates))- for _, remoteTemplate := range regDTO.Templates {- templates = append(templates, s.convertRemoteToLocal(remoteTemplate, reg))+ templates = make([]ComposeTemplate, 0, len(regDTO.Templates))+ for _, remote := range regDTO.Templates {+ templates = append(templates, ComposeTemplate{+ ID: fmt.Sprintf("%s:%s:%s", remoteIDPrefix, reg.ID, remote.ID),+ Name: remote.Name,+ Description: remote.Description,+ IsRemote: true,+ RegistryID: mo.EmptyableToOption(strings.TrimSpace(reg.ID)).ToPointer(),+ Registry: cloneRegistry(reg),+ Metadata: &ComposeTemplateMetadata{+ Version: mo.EmptyableToOption(strings.TrimSpace(remote.Version)).ToPointer(),+ Author: mo.EmptyableToOption(strings.TrimSpace(remote.Author)).ToPointer(),+ Tags: remote.Tags,+ RemoteURL: mo.EmptyableToOption(strings.TrimSpace(remote.ComposeURL)).ToPointer(),+ EnvURL: mo.EmptyableToOption(strings.TrimSpace(remote.EnvURL)).ToPointer(),+ DocumentationURL: mo.EmptyableToOption(strings.TrimSpace(remote.DocumentationURL)).ToPointer(),+ IconURL: mo.EmptyableToOption(strings.TrimSpace(remote.IconURL)).ToPointer(),+ RegistryIconURL: mo.EmptyableToOption(strings.TrimSpace(remote.IconURL)).ToPointer(),+ },+ })}- lm := resp.Header.Get("Last-Modified")newMeta := ®istryFetchMeta{- LastModified: lm,+ LastModified: resp.Header.Get("Last-Modified"),Templates: cloneRemoteTemplates(templates),}s.registryMu.Lock()if generation != s.remoteGeneration.Load() {s.registryMu.Unlock()- return nil, errRemoteCacheInvalidatedInternal+ return nil, errRemoteCacheInvalidated}s.registryFetchMeta[reg.ID] = newMetas.registryMu.Unlock()@@ -744,8 +755,8 @@ func (s *TemplateService) fetchRegistryTemplatesInternal(ctx context.Context, rereturn templates, nil}-func (s *TemplateService) fetchRegistryManifest(ctx context.Context, url string) (*tmpl.RemoteRegistry, error) {- body, err := s.doGET(ctx, url)+func (s *TemplateService) fetchRegistryManifest(ctx context.Context, rawURL string) (*tmpl.RemoteRegistry, error) {+ body, err := s.FetchRaw(ctx, rawURL)if err != nil {return nil, err}@@ -759,95 +770,43 @@ func (s *TemplateService) fetchRegistryManifest(ctx context.Context, url string)return ®, nil}-func (s *TemplateService) convertRemoteToLocal(remote tmpl.RemoteTemplate, registry *TemplateRegistry) ComposeTemplate {- publicID := fmt.Sprintf("%s:%s:%s", remoteIDPrefix, registry.ID, remote.ID)-- return ComposeTemplate{- ID: publicID,- Name: remote.Name,- Description: remote.Description,- Content: "",- EnvContent: nil,- IsCustom: false,- IsRemote: true,- RegistryID: mo.EmptyableToOption(strings.TrimSpace(registry.ID)).ToPointer(),- Registry: cloneRegistry(registry),- Metadata: &ComposeTemplateMetadata{- Version: mo.EmptyableToOption(strings.TrimSpace(remote.Version)).ToPointer(),- Author: mo.EmptyableToOption(strings.TrimSpace(remote.Author)).ToPointer(),- Tags: remote.Tags,- RemoteURL: mo.EmptyableToOption(strings.TrimSpace(remote.ComposeURL)).ToPointer(),- EnvURL: mo.EmptyableToOption(strings.TrimSpace(remote.EnvURL)).ToPointer(),- DocumentationURL: mo.EmptyableToOption(strings.TrimSpace(remote.DocumentationURL)).ToPointer(),- IconURL: mo.EmptyableToOption(strings.TrimSpace(remote.IconURL)).ToPointer(),- RegistryIconURL: mo.EmptyableToOption(strings.TrimSpace(remote.IconURL)).ToPointer(),- },- }-}--func (s *TemplateService) FetchTemplateContent(ctx context.Context, template *ComposeTemplate) (string, string, error) {+func (s *TemplateService) FetchTemplateContent(ctx context.Context, template *ComposeTemplate) (composeContent, envContent string, err error) {if !template.IsRemote {- envContent := ""- if template.EnvContent != nil {- envContent = *template.EnvContent- }- return template.Content, envContent, nil+ return template.Content, mo.PointerToOption(template.EnvContent).OrEmpty(), nil}if template.Metadata == nil || template.Metadata.RemoteURL == nil || strings.TrimSpace(*template.Metadata.RemoteURL) == "" {return "", "", fmt.Errorf("remote template %q is missing compose_url in registry metadata", template.ID)}- return s.fetchRemoteTemplateFiles(ctx, template)-}+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "template.fetch", trace.WithAttributes(+ attribute.String("arcane.template.id", template.ID),+ attribute.String("arcane.template.registry_id", mo.PointerToOption(template.RegistryID).OrEmpty()),+ ))+ defer func() { tracing.End(span, err) }()-func (s *TemplateService) fetchRemoteTemplateFiles(ctx context.Context, template *ComposeTemplate) (string, string, error) {- if template == nil || template.Metadata == nil || template.Metadata.RemoteURL == nil {- return "", "", errors.New("not a remote template or missing remote URL")- }-- composeContent, err := s.fetchURL(ctx, *template.Metadata.RemoteURL)+ composeBody, err := s.FetchRaw(ctx, *template.Metadata.RemoteURL)if err != nil {return "", "", fmt.Errorf("failed to fetch compose content from %s: %w", *template.Metadata.RemoteURL, err)}- var envContent string- if template.Metadata.EnvURL != nil && *template.Metadata.EnvURL != "" {- envContent, err = s.fetchURL(ctx, *template.Metadata.EnvURL)- if err != nil {- slog.WarnContext(ctx, "failed to fetch env content", "url", *template.Metadata.EnvURL, "error", err)- envContent = ""+ if envURL := mo.PointerToOption(template.Metadata.EnvURL).OrEmpty(); envURL != "" {+ envBody, envErr := s.FetchRaw(ctx, envURL)+ if envErr != nil {+ slog.WarnContext(ctx, "failed to fetch env content", "url", envURL, "error", envErr)}+ envContent = string(envBody)}- return composeContent, envContent, nil+ return string(composeBody), envContent, nil}-func (s *TemplateService) fetchURL(ctx context.Context, url string) (string, error) {- body, err := s.doGET(ctx, url)- if err != nil {- return "", err- }- return string(body), nil-}--func (s *TemplateService) newSafeHTTPClientInternal(ctx context.Context) *http.Client {- client, err := httpx.NewSafeOutboundHTTPClient(s.httpClient, s.lookupIP)- if err != nil {- slog.WarnContext(ctx, "failed to configure safe HTTP client", "error", err)- return nil- }- return client-}--func (s *TemplateService) newSafeRequestInternal(ctx context.Context, method, rawURL string) (*http.Client, *http.Request, error) {+func (s *TemplateService) newSafeRequest(ctx context.Context, method, rawURL string) (*http.Client, *http.Request, error) {parsedURL, err := httpx.ValidateSafeRemoteURL(ctx, rawURL, s.lookupIP)if err != nil {return nil, nil, err}- // Double-checked under registryMu: the lazy init ran unsynchronized, so- // concurrent template downloads raced on s.safeHTTPClient — a data race, and- // each loser silently built and leaked its own client.+ // Lazily built once under registryMu (double-checked) so concurrent requests share one client.s.registryMu.RLock()client := s.safeHTTPClients.registryMu.RUnlock()@@ -855,7 +814,11 @@ func (s *TemplateService) newSafeRequestInternal(ctx context.Context, method, raif client == nil {s.registryMu.Lock()if s.safeHTTPClient == nil {- s.safeHTTPClient = s.newSafeHTTPClientInternal(ctx)+ safeClient, clientErr := httpx.NewSafeOutboundHTTPClient(s.httpClient, s.lookupIP)+ if clientErr != nil {+ slog.WarnContext(ctx, "failed to configure safe HTTP client", "error", clientErr)+ }+ s.safeHTTPClient = safeClient}client = s.safeHTTPClients.registryMu.Unlock()@@ -878,59 +841,45 @@ func (s *TemplateService) DownloadTemplate(ctx context.Context, remoteTemplate *return nil, errors.New("template is not remote")}- base := s.templateBaseFromRemote(remoteTemplate)+ base := projects.Slugify(remoteTemplate.Name)+ if base == "" {+ base = cmp.Or(projects.Slugify(remoteTemplate.ID[strings.LastIndex(remoteTemplate.ID, ":")+1:]), "template-"+uuid.New().String())+ }- dir, composePath, envPath, err := projects.EnsureTemplateDir(ctx, s.configuredTemplatesDirSettingInternal(ctx), base)+ dir, composePath, envPath, err := projects.EnsureTemplateDir(ctx, s.configuredTemplatesDirSetting(ctx), base)if err != nil {return nil, err}srcDesc := fmt.Sprintf("Imported from %s/compose.yaml", dir)- return s.downloadTemplateTransaction(ctx, remoteTemplate, base, composePath, envPath, srcDesc)-}--func (s *TemplateService) downloadTemplateTransaction(ctx context.Context, remoteTemplate *ComposeTemplate, base, composePath, envPath, srcDesc string) (*ComposeTemplate, error) {var resultTemplate *ComposeTemplate-- err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ err = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {var existing ComposeTemplate- if err := tx.Where("is_remote = ? AND registry_id IS NULL AND (description = ? OR name = ?)", false, srcDesc, base).- First(&existing).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {- return fmt.Errorf("failed to check existing template: %w", err)- } else if err == nil {- // Existing template found- composeContent, envContent, fetchTemplateContentErr := s.FetchTemplateContent(ctx, remoteTemplate)- if fetchTemplateContentErr != nil {- return fmt.Errorf("failed to fetch template content for existing local template: %w", fetchTemplateContentErr)- }+ findErr := tx.Where("is_remote = ? AND registry_id IS NULL AND (description = ? OR name = ?)", false, srcDesc, base).First(&existing).Error+ if findErr != nil && !errors.Is(findErr, gorm.ErrRecordNotFound) {+ return fmt.Errorf("failed to check existing template: %w", findErr)+ }- envPtr, werr := projects.WriteTemplateFiles(composePath, envPath, composeContent, envContent)- if werr != nil {- return werr- }+ composeContent, envContent, fetchErr := s.FetchTemplateContent(ctx, remoteTemplate)+ if fetchErr != nil {+ return fmt.Errorf("failed to fetch template content for download: %w", fetchErr)+ }+ envPtr, writeErr := projects.WriteTemplateFiles(composePath, envPath, composeContent, envContent)+ if writeErr != nil {+ return writeErr+ }+ if findErr == nil {existing.Content = composeContentexisting.EnvContent = envPtrexisting.Metadata = cloneTemplateMetadata(remoteTemplate.Metadata)-- if updateLocalTemplateErr := tx.Save(&existing).Error; updateLocalTemplateErr != nil {- return fmt.Errorf("failed to update existing local template: %w", updateLocalTemplateErr)+ if saveErr := tx.Save(&existing).Error; saveErr != nil {+ return fmt.Errorf("failed to update existing local template: %w", saveErr)}resultTemplate = &existingreturn nil}- // New template- composeContent, envContent, err := s.FetchTemplateContent(ctx, remoteTemplate)- if err != nil {- return fmt.Errorf("failed to fetch template content for download: %w", err)- }-- envPtr, werr := projects.WriteTemplateFiles(composePath, envPath, composeContent, envContent)- if werr != nil {- return werr- }-localTemplate := &ComposeTemplate{ID: uuid.New().String(),Name: base,@@ -938,14 +887,10 @@ func (s *TemplateService) downloadTemplateTransaction(ctx context.Context, remotContent: composeContent,EnvContent: envPtr,IsCustom: true,- IsRemote: false,- RegistryID: nil,- Registry: nil,Metadata: cloneTemplateMetadata(remoteTemplate.Metadata),}-- if createLocalTemplateErr := tx.Create(localTemplate).Error; createLocalTemplateErr != nil {- return fmt.Errorf("failed to save local template: %w", createLocalTemplateErr)+ if createErr := tx.Create(localTemplate).Error; createErr != nil {+ return fmt.Errorf("failed to save local template: %w", createErr)}resultTemplate = localTemplatereturn nil@@ -956,21 +901,6 @@ func (s *TemplateService) downloadTemplateTransaction(ctx context.Context, remotreturn resultTemplate, nil}-func (s *TemplateService) templateBaseFromRemote(remoteTemplate *ComposeTemplate) string {- base := projects.Slugify(remoteTemplate.Name)- if base != "" {- return base- }- parts := strings.Split(remoteTemplate.ID, ":")- if len(parts) > 0 {- base = projects.Slugify(parts[len(parts)-1])- }- if base == "" {- base = "template-" + uuid.New().String()- }- return base-}-func cloneTemplateMetadata(meta *ComposeTemplateMetadata) *ComposeTemplateMetadata {if meta == nil {return nil@@ -1010,7 +940,7 @@ func cloneRegistry(registry *TemplateRegistry) *TemplateRegistry {return new(*registry)}-func (s *TemplateService) invalidateRemoteCacheInternal() {+func (s *TemplateService) invalidateRemoteCache() {s.registryMu.Lock()defer s.registryMu.Unlock()s.remoteGeneration.Add(1)@@ -1021,71 +951,8 @@ func (s *TemplateService) invalidateRemoteCacheInternal() {}}-func (s *TemplateService) SyncLocalTemplatesFromFilesystem(ctx context.Context) error {- return s.syncFilesystemTemplatesInternal(ctx)-}--func (s *TemplateService) upsertFilesystemTemplate(ctx context.Context, name, desc, compose string, envPtr *string) error {- iconURL := projects.ResolveTemplateIconURL(ctx, compose, mo.PointerToOption(envPtr).OrEmpty())-- return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- var existing ComposeTemplate- // Match on description (which encodes the absolute compose path) OR name (folder name).- // Matching on name as a fallback keeps a single row across templates-directory- // reconfigurations or compose-filename renames within the same folder, but- // only for templates previously imported from disk.- q := tx.Where("is_remote = ? AND registry_id IS NULL AND (description = ? OR (name = ? AND description LIKE ?))", false, desc, name, "Imported from %").- First(&existing)-- if q.Error == nil {- existing.Name = name- existing.Description = desc- existing.Content = compose- existing.EnvContent = envPtr- existing.IsCustom = true- existing.IsRemote = false- setTemplateIconURL(&existing, iconURL)- if err := tx.Save(&existing).Error; err != nil {- return fmt.Errorf("update template %s: %w", existing.ID, err)- }- return nil- }- if !errors.Is(q.Error, gorm.ErrRecordNotFound) {- if err := q.Error; err != nil {- return fmt.Errorf("query existing template: %w", err)- }- return nil- }-- tpl := &ComposeTemplate{- ID: uuid.New().String(),- Name: name,- Description: desc,- Content: compose,- EnvContent: envPtr,- IsCustom: true,- IsRemote: false,- RegistryID: nil,- Registry: nil,- Metadata: nil,- }- setTemplateIconURL(tpl, iconURL)- if err := tx.Create(tpl).Error; err != nil {- return fmt.Errorf("insert template %s: %w", name, err)- }- return nil- })-}--func (s *TemplateService) processFolderEntry(ctx context.Context, baseDir, folder string) error {- compose, envPtr, desc, found, err := projects.ReadFolderComposeTemplate(ctx, baseDir, folder)- if err != nil || !found {- return err- }- return s.upsertFilesystemTemplate(ctx, folder, desc, compose, envPtr)-}--func (s *TemplateService) syncFilesystemTemplatesInternal(ctx context.Context) error {+// SyncLocalTemplatesFromFilesystem imports template folders from the templates directory, at most once per fsSyncInterval.+func (s *TemplateService) SyncLocalTemplatesFromFilesystem(ctx context.Context) (err error) {s.fsSyncMu.Lock()defer s.fsSyncMu.Unlock()@@ -1093,7 +960,14 @@ func (s *TemplateService) syncFilesystemTemplatesInternal(ctx context.Context) ereturn nil}- dir, err := s.getTemplatesDirectoryInternal(ctx)+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "template.sync_filesystem")+ synced, failed := 0, 0+ defer func() {+ span.SetAttributes(attribute.Int("arcane.result.count", synced), attribute.Int("arcane.template.failed_count", failed))+ tracing.End(span, err)+ }()++ dir, err := s.getTemplatesDirectory(ctx)if err != nil {return fmt.Errorf("ensure templates dir: %w", err)}@@ -1111,9 +985,52 @@ func (s *TemplateService) syncFilesystemTemplatesInternal(ctx context.Context) eif !ent.IsDirectory {continue}- if processFolderEntryErr := s.processFolderEntry(ctx, dir, ent.Name); processFolderEntryErr != nil {- slog.WarnContext(ctx, "failed to read folder template", "folder", ent.Name, "error", processFolderEntryErr)+ compose, envPtr, desc, found, folderErr := projects.ReadFolderComposeTemplate(ctx, dir, ent.Name)+ if folderErr == nil && !found {+ continue+ }+ if folderErr == nil {+ iconURL := projects.ResolveTemplateIconURL(ctx, compose, mo.PointerToOption(envPtr).OrEmpty())+ folderErr = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ // Match on description (the absolute compose path) or, for disk imports, the folder name so+ // directory reconfigurations and compose renames keep a single row.+ var existing ComposeTemplate+ findErr := tx.Where("is_remote = ? AND registry_id IS NULL AND (description = ? OR (name = ? AND description LIKE ?))", false, desc, ent.Name, "Imported from %").+ First(&existing).Error+ if findErr != nil && !errors.Is(findErr, gorm.ErrRecordNotFound) {+ return fmt.Errorf("query existing template: %w", findErr)+ }++ tpl := &existing+ if findErr != nil {+ tpl = &ComposeTemplate{ID: uuid.New().String()}+ }+ tpl.Name = ent.Name+ tpl.Description = desc+ tpl.Content = compose+ tpl.EnvContent = envPtr+ tpl.IsCustom = true+ tpl.IsRemote = false+ setTemplateIconURL(tpl, iconURL)++ if findErr == nil {+ if saveErr := tx.Save(tpl).Error; saveErr != nil {+ return fmt.Errorf("update template %s: %w", tpl.ID, saveErr)+ }+ return nil+ }+ if createErr := tx.Create(tpl).Error; createErr != nil {+ return fmt.Errorf("insert template %s: %w", ent.Name, createErr)+ }+ return nil+ })+ }+ if folderErr != nil {+ failed+++ slog.WarnContext(ctx, "failed to read folder template", "folder", ent.Name, "error", folderErr)+ continue}+ synced++}s.lastFsSync = time.Now()@@ -1152,17 +1069,11 @@ func (s *TemplateService) GetTemplateContentWithParsedData(ctx context.Context,return nil, err}- var composeContent, envContent string- if composeTemplate.IsRemote {- composeContent, envContent, err = s.FetchTemplateContent(ctx, composeTemplate)- if err != nil {- return nil, fmt.Errorf("failed to fetch template content: %w", err)- }- } else {- composeContent = composeTemplate.Content- if composeTemplate.EnvContent != nil {- envContent = *composeTemplate.EnvContent- }+ composeContent, envContent, err := s.FetchTemplateContent(ctx, composeTemplate)+ if err != nil {+ return nil, fmt.Errorf("failed to fetch template content: %w", err)+ }+ if !composeTemplate.IsRemote {setTemplateIconURL(composeTemplate, projects.ResolveTemplateIconURL(ctx, composeContent, envContent))}@@ -1189,28 +1100,3 @@ func (s *TemplateService) GetTemplateContentWithParsedData(ctx context.Context,EnvVariables: envVars,}, nil}--func (s *TemplateService) getMergedTemplatesInternal(ctx context.Context) ([]ComposeTemplate, error) {- if err := s.syncFilesystemTemplatesInternal(ctx); err != nil {- slog.WarnContext(ctx, "failed to sync filesystem templates", "error", err)- }-- var templates []ComposeTemplate- // Use Omit to avoid fetching heavy content fields which are not needed for listing- if err := s.db.WithContext(ctx).Omit("Content", "EnvContent").Preload("Registry").Find(&templates).Error; err != nil {- return nil, fmt.Errorf("failed to get local templates: %w", err)- }-- remoteTemplates, err := s.remoteTemplatesInternal(ctx, false)- if err != nil {- slog.WarnContext(ctx, "failed to load remote templates", "error", err)- } else {- copied := cloneRemoteTemplates(remoteTemplates)-- if len(copied) > 0 {- templates = append(templates, copied...)- }- }-- return templates, nil-}diff --git a/backend/internal/template/service_test.go b/backend/internal/template/service_test.goindex 60de58db9..91e9cf45f 100644--- a/backend/internal/template/service_test.go+++ b/backend/internal/template/service_test.go@@ -148,7 +148,7 @@ services:Enabled: true,}- templates, err := service.fetchRegistryTemplatesInternal(t.Context(), registry, service.remoteGeneration.Load())+ templates, err := service.fetchRegistryTemplates(t.Context(), registry, service.remoteGeneration.Load())require.NoError(t, err)require.Len(t, templates, 2)require.NotNil(t, templates[0].Metadata)@@ -156,7 +156,7 @@ services:require.Equal(t, "https://cdn.example/good.png", *templates[0].Metadata.IconURL)require.Nil(t, templates[1].Metadata.IconURL)- cachedTemplates, err := service.fetchRegistryTemplatesInternal(t.Context(), registry, service.remoteGeneration.Load())+ cachedTemplates, err := service.fetchRegistryTemplates(t.Context(), registry, service.remoteGeneration.Load())require.NoError(t, err)require.Len(t, cachedTemplates, 2)require.NotNil(t, cachedTemplates[0].Metadata)@@ -301,12 +301,12 @@ func TestGetAllTemplatesPaginated_FiltersByType(t *testing.T) {Filters: map[string]string{"type": tt.typeFilter},})require.NoError(t, err)- require.ElementsMatch(t, tt.wantIDs, templateIDsInternal(templates))+ require.ElementsMatch(t, tt.wantIDs, templateIDs(templates))})}}-func templateIDsInternal(templates []tmpl.Template) []string {+func templateIDs(templates []tmpl.Template) []string {ids := make([]string, 0, len(templates))for _, template := range templates {ids = append(ids, template.ID)@@ -326,7 +326,7 @@ func TestFetchRaw_BlocksUnsafeRemoteURL(t *testing.T) {require.ErrorIs(t, err, common.ErrUnsafeRemoteURL)}-func TestSyncFilesystemTemplatesInternal_PopulatesIconURL(t *testing.T) {+func TestSyncLocalTemplatesFromFilesystem_PopulatesIconURL(t *testing.T) {tempDir := t.TempDir()templatesRoot := filepath.Join(tempDir, "templates")@@ -349,7 +349,7 @@ services:registryFetchMeta: make(map[string]*registryFetchMeta),}- require.NoError(t, service.syncFilesystemTemplatesInternal(t.Context()))+ require.NoError(t, service.SyncLocalTemplatesFromFilesystem(t.Context()))var stored ComposeTemplaterequire.NoError(t, service.db.WithContext(t.Context()).First(&stored, "name = ?", "example").Error)@@ -410,7 +410,6 @@ func TestGetTemplate_ForceRefreshesRemoteCacheOnMiss(t *testing.T) {service := NewTemplateService(t.Context(), db, client, settingsSvc)service.lookupIP = lookupIP- service.safeHTTPClient = service.newSafeHTTPClientInternal(t.Context())// Cache starts empty; GetTemplate for a remote ID should force a refresh and find the template.got, err := service.GetTemplate(t.Context(), "remote:reg-1:affine")@@ -431,12 +430,12 @@ func minimalSettingsServiceForTest(t *testing.T) *settings.SettingsService {db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)require.NoError(t, db.AutoMigrate(&settings.SettingVariable{}))- svc, err := newSettingsServiceForTestInternal(t, t.Context(), &database.DB{DB: db})+ svc, err := newSettingsServiceForTest(t, t.Context(), &database.DB{DB: db})require.NoError(t, err)return svc}-func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {+func newSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {t.Helper()svc, err := settings.NewSettingsService(ctx, db)if err == nil {diff --git a/backend/internal/vulnerability/children/scan/children/trivy/service.go b/backend/internal/vulnerability/children/scan/children/trivy/service.goindex 8c169bfd2..25864f123 100644--- a/backend/internal/vulnerability/children/scan/children/trivy/service.go+++ b/backend/internal/vulnerability/children/scan/children/trivy/service.go@@ -23,6 +23,9 @@ import ("go.getarcane.app/docker""go.getarcane.app/kit/pkg""go.getarcane.app/sys/cgroup"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/trace""github.com/getarcaneapp/arcane/backend/v2/internal/config"dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"@@ -34,6 +37,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/vuln""github.com/getarcaneapp/arcane/backend/v2/pkg/projects""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")const (@@ -55,11 +59,9 @@ type Service struct {registryAuthConfigs func(ctx context.Context) (map[string]registry.AuthConfig, error)saveReport func(ctx context.Context, imageID, data string) errorsetPhase func(imageID string, phase vulnerability.ScanPhase)- // scanLocks allows concurrent scans of different images while preventing- // duplicate scans of the same image.+ // scanLocks allows concurrent scans of different images but not duplicate scans of one image.scanLocks utils.KeyedMutex- // trivyScanSlots limits concurrent Trivy scans (manual + scheduled) and- // provides deterministic slot IDs for shared-cache sharding.+ // trivyScanSlots limits concurrent manual and scheduled scans and provides stable slot IDs for cache sharding.trivyScanSlots chan inttrivyScanMu sync.Mutex}@@ -84,9 +86,12 @@ func NewService(}}-type trivyRuntimeSecurityInternal struct {- securityOpts []string- privileged bool+// settingsConfig returns the current settings, or nil without a settings service.+func (s *Service) settingsConfig() *settings.Settings {+ if s.settingsService == nil {+ return nil+ }+ return s.settingsService.GetSettingsConfig()}// A nil release without an error means the scan must wait in the queue.@@ -111,7 +116,18 @@ func (s *Service) AcquireSlot(ctx context.Context, imageID string, wait bool) (s}}()- slotCh := s.getTrivyScanSlotChannelInternal(s.ConcurrentScanContainers())+ limit := s.ConcurrentScanContainers()+ s.trivyScanMu.Lock()+ // Resize only when idle so tokens never split across channels.+ if s.trivyScanSlots == nil || (cap(s.trivyScanSlots) != limit && len(s.trivyScanSlots) == cap(s.trivyScanSlots)) {+ s.trivyScanSlots = make(chan int, limit)+ for i := range limit {+ s.trivyScanSlots <- i+ }+ }+ slotCh := s.trivyScanSlots+ s.trivyScanMu.Unlock()+if wait {select {case slotID = <-slotCh:@@ -142,72 +158,14 @@ func (s *Service) AcquireSlot(ctx context.Context, imageID string, wait bool) (s}func (s *Service) ConcurrentScanContainers() int {- if s.settingsService == nil {- return 1- }-- cfg := s.settingsService.GetSettingsConfig()- if cfg == nil {- return 1- }-- limit := cfg.TrivyConcurrentScanContainers.AsInt()- return kit.Ternary(limit < 1, 1, limit)-}--func (s *Service) getTrivyScanSlotChannelInternal(limit int) chan int {- if limit < 1 {- limit = 1- }-- createSlots := func(size int) chan int {- slots := make(chan int, size)- for i := range size {- slots <- i- }- return slots- }-- s.trivyScanMu.Lock()- defer s.trivyScanMu.Unlock()-- if s.trivyScanSlots == nil {- s.trivyScanSlots = createSlots(limit)- return s.trivyScanSlots- }-- if cap(s.trivyScanSlots) == limit {- return s.trivyScanSlots+ if cfg := s.settingsConfig(); cfg != nil {+ return max(cfg.TrivyConcurrentScanContainers.AsInt(), 1)}-- // Apply capacity changes only when idle to avoid splitting tokens across channels.- if len(s.trivyScanSlots) == cap(s.trivyScanSlots) {- s.trivyScanSlots = createSlots(limit)- }-- return s.trivyScanSlots-}--func (s *Service) buildTrivyRegistryConfigInternal(ctx context.Context) []byte {- if s.registryAuthConfigs == nil {- return nil- }-- authConfigs, err := s.registryAuthConfigs(ctx)- if err != nil {- slog.WarnContext(ctx, "failed to load registry credentials for trivy scan; continuing anonymously", "error", err)- return nil- }-- configJSON, err := vuln.BuildDockerConfigJSON(authConfigs)- if err != nil {- slog.WarnContext(ctx, "failed to build trivy registry config; continuing anonymously", "error", err)- return nil- }-- return configJSON+ return 1}+// SetupBatchContainers starts long-running scanner containers that scheduled scans exec into,+// avoiding a container lifecycle per image.func (s *Service) SetupBatchContainers(ctx context.Context, count int) ([]string, map[string][]byte, func(), error) {if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {return nil, nil, nil, err@@ -225,53 +183,22 @@ func (s *Service) SetupBatchContainers(ctx context.Context, count int) ([]stringserverURL, _ := s.ServerConfig()var cacheVolume stringif serverURL == "" {- cacheVolume, err = s.ensureTrivyCacheVolumeInternal(ctx)+ cacheVolume, err = s.ensureTrivyCacheVolume(ctx)if err != nil {return nil, nil, nil, fmt.Errorf("failed to ensure trivy cache volume: %w", err)}}- files := s.trivyContainerFilesInternal(ctx)- containerIDs := make([]string, 0, max(count, 1))- cleanup := func() {- for _, containerID := range containerIDs {- vuln.RemoveContainer(ctx, dockerClient, containerID, "batch trivy container")- }- }-- for range max(count, 1) {- containerID, createErr := s.createBatchTrivyContainer(ctx, scannerImage, cacheVolume, files)- if createErr != nil {- cleanup()- return nil, nil, nil, fmt.Errorf("failed to create batch trivy container: %w", createErr)- }- containerIDs = append(containerIDs, containerID)- }-- return containerIDs, files, cleanup, nil-}--// createBatchTrivyContainer creates a long-running Trivy container that stays-// alive so we can exec individual scan commands into it. This avoids the-// overhead of creating/starting/removing a container for every image.-func (s *Service) createBatchTrivyContainer(ctx context.Context, scannerImage, cacheVolume string, files map[string][]byte) (string, error) {- dockerClient, err := s.dockerService.GetClient(ctx)+ files := s.trivyContainerFiles(ctx)+ runtimeOptions, err := s.resolveTrivyRuntimeOptions(ctx, dockerClient)if err != nil {- return "", fmt.Errorf("failed to connect to Docker: %w", err)- }-- runtimeOptions, err := s.resolveTrivyRuntimeOptionsInternal(ctx, dockerClient)- if err != nil {- return "", err+ return nil, nil, nil, fmt.Errorf("failed to create batch trivy container: %w", err)}-env := append([]string(nil), runtimeOptions.ContainerEnv...)if files[vuln.RegistryConfigFile] != nil {+ // ECR tokens refresh only at batch-container creation; a multi-hour batch can outlive them.env = append(env, "DOCKER_CONFIG="+vuln.RegistryConfigDir)- // ECR tokens are refreshed when the batch container is created; a multi-hour- // batch could outlive the token and fall back to anonymous pulls.}-containerConfig := &container.Config{Image: scannerImage,Entrypoint: []string{"sh", "-c", "trap 'exit 0' TERM; while :; do sleep 1; done"},@@ -280,40 +207,46 @@ func (s *Service) createBatchTrivyContainer(ctx context.Context, scannerImage, clibarcane.InternalResourceLabel: "true",},}-- runtimeSecurity := s.getTrivyRuntimeSecurityInternal().OrEmpty()- securityOpts, privileged := runtimeSecurity.securityOpts, runtimeSecurity.privileged+ var securityOpts []string+ privileged := false+ if cfg := s.settingsConfig(); cfg != nil {+ securityOpts, privileged = vuln.ParseSecurityOpts(cfg.TrivySecurityOpts.Value), cfg.TrivyPrivileged.IsTrue()+ }hostConfig := vuln.BuildBatchHostConfig(cacheVolume, runtimeOptions, securityOpts, privileged)-- resources, cpuSet, applyLimits, err := s.getTrivyContainerResourceOptionsInternal(ctx, dockerClient)+ resources, cpuSet, applyLimits, err := s.getTrivyContainerResourceOptions(ctx, dockerClient)if err != nil {- return "", err+ return nil, nil, nil, fmt.Errorf("failed to create batch trivy container: %w", err)}vuln.ApplyContainerResources(hostConfig, resources, cpuSet, applyLimits)- containerID, err := s.createAndStartTrivyContainerInternal(ctx, dockerClient, "batch", containerConfig, hostConfig, files)- if err != nil {- return "", err+ containerIDs := make([]string, 0, max(count, 1))+ cleanup := func() {+ for _, containerID := range containerIDs {+ vuln.RemoveContainer(ctx, dockerClient, containerID, "batch trivy container")+ }+ }+ for range max(count, 1) {+ containerID, createErr := s.createAndStartTrivyContainer(ctx, dockerClient, "batch", containerConfig, hostConfig, files)+ if createErr != nil {+ cleanup()+ return nil, nil, nil, fmt.Errorf("failed to create batch trivy container: %w", createErr)+ }+ slog.InfoContext(ctx, "batch trivy container started", "containerId", containerID)+ containerIDs = append(containerIDs, containerID)}- slog.InfoContext(ctx, "batch trivy container started", "containerId", containerID)-- return containerID, nil+ return containerIDs, files, cleanup, nil}-// ExecScan runs a trivy scan for a single image inside an-// already-running container via docker exec.-func (- s *Service,-) ExecScan(- ctx context.Context,- containerID, imageName, imageID, scannerVersion string,- slotID int,- files map[string][]byte,-) (- *vulnerability.ScanResult,- error,-) {+// ExecScan runs a trivy scan for one image inside an already-running batch container via docker exec.+func (s *Service) ExecScan(ctx context.Context, containerID, imageName, imageID, scannerVersion string, slotID int, files map[string][]byte) (result *vulnerability.ScanResult, err error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "trivy.scan", trace.WithAttributes(+ attribute.String("container.image.name", imageName),+ attribute.String("container.image.id", imageID),+ attribute.String("arcane.trivy.mode", "exec"),+ ))+ defer func() { tracing.End(span, err) }()+dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {return nil, fmt.Errorf("failed to connect to Docker: %w", err)@@ -337,10 +270,11 @@ func (serverURL, serverToken := s.ServerConfig()var cacheDir stringif serverURL == "" {- cacheDir = s.getTrivySingleScanCacheDirForSlotInternal(slotID)+ cacheDir = fmt.Sprintf("%s/slot-%d", trivyCacheSlotRootDir, max(slotID, 0))}execCmd := append([]string{"trivy"}, s.buildTrivyCommandArgs(imageName, cacheDir, outputPath, files)...)+ startTime := time.Now()exitCode, err := docker.ExecInContainer(ctx, dockerClient, containerID, client.ExecCreateOptions{Cmd: execCmd,Env: vuln.ServerTokenEnv(serverToken),@@ -351,115 +285,86 @@ func (return nil, err}- if exitCode != 0 {- errMsg := vuln.ReadTempFileExcerpt(stderrFile)- if errMsg == "" {- errMsg = vuln.ReadTempFileExcerpt(stdoutFile)- }- if errMsg == "" {- errMsg = fmt.Sprintf("exit status %d", exitCode)- }- slog.WarnContext(ctx,- "trivy scan diagnostics",- "image", imageName,- "imageId", imageID,- "reason", "exec non-zero exit",- "exitCode", int64(exitCode),- "stdoutBytes", vuln.TempFileSize(stdoutFile),- "stderrBytes", vuln.TempFileSize(stderrFile),- )- return nil, vuln.FormatExitError(int64(exitCode), errMsg, imageName)+ run := trivyContainerResult{+ stdoutFile: stdoutFile, stderrFile: stderrFile, duration: time.Since(startTime).Milliseconds(),+ statusCode: int64(exitCode), containerID: containerID,}-- trivyOutput, err := vuln.ReadOutputFromContainerFile(ctx, dockerClient, containerID, outputPath)+ result, err = s.scanResult(ctx, dockerClient, run, outputPath, imageName, imageID, "exec")if err != nil {- slog.WarnContext(ctx,- "failed to read trivy output file from batch container; falling back to stdout parsing",- "containerId", containerID,- "outputPath", outputPath,- "error", err,- )- trivyOutput = nil+ return nil, err}+ result.ScannerVersion = scannerVersion+ return result, nil+}- var trivyReport *vulnerability.TrivyReport- if len(trivyOutput) > 0 {- trivyReport, err = vuln.DecodeReportFromBytes(trivyOutput)- } else {- trivyReport, err = vuln.DecodeReportFromFile(stdoutFile)+// scanResult decodes a finished scan's report (output file first, then stdout), persists the raw report for+// report-based patching, and records the result counts on the active span.+func (s *Service) scanResult(ctx context.Context, dockerClient *client.Client, run trivyContainerResult, outputPath, imageName, imageID, mode string) (*vulnerability.ScanResult, error) {+ diagnose := func(reason string) {+ slog.WarnContext(ctx, "trivy scan diagnostics", "image", imageName, "imageId", imageID, "reason", mode+" "+reason, "exitCode", run.statusCode,+ "stdoutBytes", vuln.TempFileSize(run.stdoutFile), "stderrBytes", vuln.TempFileSize(run.stderrFile))+ }+ if run.statusCode != 0 {+ errMsg := cmp.Or(vuln.ReadTempFileExcerpt(run.stderrFile), vuln.ReadTempFileExcerpt(run.stdoutFile), fmt.Sprintf("exit status %d", run.statusCode))+ diagnose("non-zero exit")+ return nil, vuln.FormatExitError(run.statusCode, errMsg, imageName)}- if err != nil {- if errors.Is(err, io.EOF) {- errMsg := cmp.Or(vuln.ReadTempFileExcerpt(stderrFile), "trivy scan produced no output")- slog.WarnContext(ctx,- "trivy scan diagnostics",- "image", imageName,- "imageId", imageID,- "reason", "exec empty output",- "exitCode", int64(exitCode),- "stdoutBytes", vuln.TempFileSize(stdoutFile),- "stderrBytes", vuln.TempFileSize(stderrFile),- )- return nil, fmt.Errorf("trivy scan failed: %s", errMsg)- }- slog.WarnContext(ctx,- "trivy scan diagnostics",- "image", imageName,- "imageId", imageID,- "reason", "exec parse error",- "exitCode", int64(exitCode),- "stdoutBytes", vuln.TempFileSize(stdoutFile),- "stderrBytes", vuln.TempFileSize(stderrFile),- )- return nil, fmt.Errorf("failed to parse trivy output: %w", err)+ raw, err := vuln.ReadOutputFromContainerFile(ctx, dockerClient, run.containerID, outputPath)+ if err != nil {+ slog.WarnContext(ctx, "failed to read trivy output file; falling back to stdout parsing",+ "mode", mode, "containerId", run.containerID, "outputPath", outputPath, "error", err)+ raw = nil}- if trivyReport == nil {- errMsg := cmp.Or(vuln.ReadTempFileExcerpt(stderrFile), "trivy scan produced no output")- slog.WarnContext(ctx,- "trivy scan diagnostics",- "image", imageName,- "imageId", imageID,- "reason", "exec nil report",- "exitCode", int64(exitCode),- "stdoutBytes", vuln.TempFileSize(stdoutFile),- "stderrBytes", vuln.TempFileSize(stderrFile),- )- return nil, fmt.Errorf("trivy scan failed: %s", errMsg)+ var report *vulnerability.TrivyReport+ if len(raw) > 0 {+ report, err = vuln.DecodeReportFromBytes(raw)+ } else {+ report, err = vuln.DecodeReportFromFile(run.stdoutFile)+ }+ switch {+ case errors.Is(err, io.EOF):+ diagnose("empty output")+ return nil, fmt.Errorf("trivy scan failed: %s", cmp.Or(vuln.ReadTempFileExcerpt(run.stderrFile), "trivy scan produced no output"))+ case err != nil:+ diagnose("parse error")+ return nil, fmt.Errorf("failed to parse trivy output: %w", err)+ case report == nil:+ diagnose("nil report")+ return nil, fmt.Errorf("trivy scan failed: %s", cmp.Or(vuln.ReadTempFileExcerpt(run.stderrFile), "trivy scan produced no output"))}// The name can move to another image after it was selected; never store that image's findings under imageID.- if scanned := trivyReport.Metadata.ImageID; scanned != "" && strings.TrimPrefix(scanned, "sha256:") != strings.TrimPrefix(imageID, "sha256:") {+ if scanned := report.Metadata.ImageID; mode == "exec" && scanned != "" && strings.TrimPrefix(scanned, "sha256:") != strings.TrimPrefix(imageID, "sha256:") {return nil, fmt.Errorf("%s now refers to image %s, not the selected image %s", imageName, scanned, imageID)}- result := vulnerability.ConvertTrivyReportToScanResult(trivyReport, imageID, time.Now(), 0)- result.ScannerVersion = scannerVersion- result.HasReport = s.persistRawTrivyReportInternal(ctx, imageID, trivyOutput, stdoutFile)-- return result, nil-}--// persistRawTrivyReportInternal stores the raw Trivy JSON report in the-// database so it can later drive report-based image patching. Reports are-// keyed by image ID, so a re-scan replaces the previous report. When the-// output-file read produced nothing, the stdout temp file the report was-// decoded from is used instead.-func (s *Service) persistRawTrivyReportInternal(ctx context.Context, imageID string, raw []byte, stdoutFallback *os.File) bool {- if len(raw) == 0 && stdoutFallback != nil {- if _, err := stdoutFallback.Seek(0, io.SeekStart); err == nil {- raw, _ = io.ReadAll(stdoutFallback)+ result := vulnerability.ConvertTrivyReportToScanResult(report, imageID, time.Now(), run.duration)+ if len(raw) == 0 {+ if _, seekErr := run.stdoutFile.Seek(0, io.SeekStart); seekErr == nil {+ raw, _ = io.ReadAll(run.stdoutFile)}}- if len(raw) == 0 || s.saveReport == nil {- return false+ if len(raw) > 0 && s.saveReport != nil {+ if saveErr := s.saveReport(ctx, imageID, string(raw)); saveErr != nil {+ slog.WarnContext(ctx, "failed to persist raw trivy report", "imageId", imageID, "error", saveErr)+ } else {+ result.HasReport = true+ }}- if err := s.saveReport(ctx, imageID, string(raw)); err != nil {- slog.WarnContext(ctx, "failed to persist raw trivy report", "imageId", imageID, "error", err)- return false- }- return true+ summary := cmp.Or(result.Summary, &vulnerability.SeveritySummary{})+ trace.SpanFromContext(ctx).SetAttributes(+ attribute.String("arcane.scan.status", string(result.Status)),+ attribute.Int("vulnerability.critical", summary.Critical),+ attribute.Int("vulnerability.high", summary.High),+ attribute.Int("vulnerability.medium", summary.Medium),+ attribute.Int("vulnerability.low", summary.Low),+ attribute.Int("vulnerability.unknown", summary.Unknown),+ attribute.Int("vulnerability.total", summary.Total),+ )+ slog.DebugContext(ctx, "trivy scan finished", "imageId", imageID, "imageName", imageName, "durationMs", run.duration, "status", result.Status, "vulnerabilities", summary.Total)+ return result, nil}// Version returns the Trivy version from the configured scanner image.@@ -556,72 +461,23 @@ func (s *Service) Version(ctx context.Context) string {return vuln.ParseVersion(vuln.ReadTempFileExcerpt(stdoutFile))}-func (s *Service) getTrivyImageRef() string {- toolsRegistry := ""- if s.settingsService != nil {- toolsRegistry = s.settingsService.GetSettingsConfig().ToolsImageRegistry.Value+func (s *Service) resolveTrivyRuntimeOptions(ctx context.Context, dockerClient *client.Client) (vuln.RuntimeOptions, error) {+ var configuredNetwork, configuredDockerHost string+ if cfg := s.settingsConfig(); cfg != nil {+ configuredNetwork, configuredDockerHost = strings.TrimSpace(cfg.TrivyNetwork.Value), strings.TrimSpace(cfg.DockerHost.Value)}- return volumehelper.ToolsImage(toolsRegistry)-}-func (s *Service) getTrivyDbRegistryInternal() string {- if s.settingsService == nil {- return ""- }- return s.settingsService.GetSettingsConfig().TrivyDbRegistry.Value-}--func (s *Service) getConfiguredTrivyNetworkModeInternal() string {- if s.settingsService == nil {- return ""- }-- cfg := s.settingsService.GetSettingsConfig()- if cfg == nil {- return ""- }-- return strings.TrimSpace(cfg.TrivyNetwork.Value)-}--func (s *Service) getEffectiveDockerHostInternal() string {- if s.dockerService != nil {- if host := strings.TrimSpace(s.dockerService.DockerHost()); host != "" {- return host+ options := vuln.RuntimeOptions{NetworkMode: configuredNetwork}+ if options.NetworkMode == "" {+ if inspect, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient); inspectErr == nil {+ options.NetworkMode = docker.SelectAutoNetworkMode(inspect)+ } else {+ slog.DebugContext(ctx, "failed to auto-detect Trivy network; falling back to bridge", "error", inspectErr)+ options.NetworkMode = vuln.DefaultNetworkMode}}- if s.settingsService != nil {- if cfg := s.settingsService.GetSettingsConfig(); cfg != nil {- if host := strings.TrimSpace(cfg.DockerHost.Value); host != "" {- return host- }- }- }-- return vuln.DefaultDockerHostURI-}--func (s *Service) resolveTrivyNetworkModeInternal(ctx context.Context, dockerClient *client.Client) string {- if configuredNetwork := s.getConfiguredTrivyNetworkModeInternal(); configuredNetwork != "" {- return configuredNetwork- }-- inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)- if err == nil {- return docker.SelectAutoNetworkMode(inspect)- }-- slog.DebugContext(ctx, "failed to auto-detect Trivy network; falling back to bridge", "error", err)- return vuln.DefaultNetworkMode-}--func (s *Service) resolveTrivyRuntimeOptionsInternal(ctx context.Context, dockerClient *client.Client) (vuln.RuntimeOptions, error) {- options := vuln.RuntimeOptions{- NetworkMode: s.resolveTrivyNetworkModeInternal(ctx, dockerClient),- }-- dockerHost := s.getEffectiveDockerHostInternal()+ dockerHost := cmp.Or(strings.TrimSpace(s.dockerService.DockerHost()), configuredDockerHost, vuln.DefaultDockerHostURI)scheme, socketPath, err := vuln.ParseDockerHost(dockerHost)if err != nil {return vuln.RuntimeOptions{}, fmt.Errorf("failed to resolve trivy docker host: %w", err)@@ -659,58 +515,29 @@ func (s *Service) resolveTrivyRuntimeOptionsInternal(ctx context.Context, dockerreturn options, nil}-func (s *Service) getTrivyRuntimeSecurityInternal() mo.Option[trivyRuntimeSecurityInternal] {- if s.settingsService == nil {- return mo.None[trivyRuntimeSecurityInternal]()- }-- cfg := s.settingsService.GetSettingsConfig()- if cfg == nil {- return mo.None[trivyRuntimeSecurityInternal]()- }-- return mo.Some(trivyRuntimeSecurityInternal{- securityOpts: vuln.ParseSecurityOpts(cfg.TrivySecurityOpts.Value),- privileged: cfg.TrivyPrivileged.IsTrue(),- })-}--func (s *Service) getTrivyScanTimeoutArg() string {+func (s *Service) scanTimeoutArg() string {timeoutSeconds := 0- if s.settingsService != nil {- timeoutSeconds = s.settingsService.GetSettingsConfig().TrivyScanTimeout.AsInt()+ if cfg := s.settingsConfig(); cfg != nil {+ timeoutSeconds = cfg.TrivyScanTimeout.AsInt()}-return timeouts.GetDuration(timeoutSeconds, timeouts.DefaultTrivyScan).String()}-func (s *Service) getTrivySingleScanCacheDirForSlotInternal(slotID int) string {- if slotID < 0 {- slotID = 0- }-- // Use a fixed cache directory per concurrency slot on the shared cache- // volume. This avoids Trivy fs-cache lock contention while bounding cache- // growth to the configured slot count.- return fmt.Sprintf("%s/slot-%d", trivyCacheSlotRootDir, slotID)-}-func (s *Service) EnsureImage(ctx context.Context) (string, error) {dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {return "", fmt.Errorf("failed to connect to Docker: %w", err)}- scannerImage := s.getTrivyImageRef()+ toolsRegistry, pullTimeoutSeconds := "", 0+ if cfg := s.settingsConfig(); cfg != nil {+ toolsRegistry, pullTimeoutSeconds = cfg.ToolsImageRegistry.Value, cfg.DockerImagePullTimeout.AsInt()+ }+ scannerImage := volumehelper.ToolsImage(toolsRegistry)if _, imageInspectErr := dockerClient.ImageInspect(ctx, scannerImage); imageInspectErr == nil {return scannerImage, nil}- pullTimeoutSeconds := 0- if s.settingsService != nil && s.settingsService.GetSettingsConfig() != nil {- pullTimeoutSeconds = s.settingsService.GetSettingsConfig().DockerImagePullTimeout.AsInt()- }-pullCtx, pullCancel := context.WithTimeout(ctx, timeouts.GetDuration(pullTimeoutSeconds, timeouts.DefaultDockerImagePull))defer pullCancel()@@ -730,7 +557,7 @@ func (s *Service) EnsureImage(ctx context.Context) (string, error) {return scannerImage, nil}-func (s *Service) ensureTrivyCacheVolumeInternal(ctx context.Context) (string, error) {+func (s *Service) ensureTrivyCacheVolume(ctx context.Context) (string, error) {dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {return "", err@@ -753,27 +580,10 @@ func (s *Service) ensureTrivyCacheVolumeInternal(ctx context.Context) (string, ereturn trivyCacheVolumeName, nil}-func (s *Service) getTrivyConfigFiles() (configContent, ignoreContent string) {- if s.settingsService == nil {- return "", ""- }-- cfg := s.settingsService.GetSettingsConfig()- if cfg == nil {- return "", ""- }-- return cfg.TrivyConfig.Value, cfg.TrivyIgnore.Value-}-// ServerConfig returns the remote Trivy server URL and optional token when// client/server mode is enabled, or empty strings for standalone (local DB) scanning.func (s *Service) ServerConfig() (serverURL, token string) {- if s.settingsService == nil {- return "", ""- }-- cfg := s.settingsService.GetSettingsConfig()+ cfg := s.settingsConfig()if cfg == nil || !cfg.TrivyServerEnabled.IsTrue() {return "", ""}@@ -784,25 +594,28 @@ func (s *Service) ServerConfig() (serverURL, token string) {// IgnoreUnfixed reports whether scans should only return vulnerabilities// that have a known fix (--ignore-unfixed).func (s *Service) IgnoreUnfixed() bool {- if s.settingsService == nil {- return false- }-- cfg := s.settingsService.GetSettingsConfig()+ cfg := s.settingsConfig()return cfg != nil && cfg.TrivyIgnoreUnfixed.IsTrue()}-func (s *Service) trivyContainerFilesInternal(ctx context.Context) map[string][]byte {+func (s *Service) trivyContainerFiles(ctx context.Context) map[string][]byte {files := map[string][]byte{}- if registryConfigJSON := s.buildTrivyRegistryConfigInternal(ctx); len(registryConfigJSON) > 0 {- files[vuln.RegistryConfigFile] = registryConfigJSON- }- configContent, ignoreContent := s.getTrivyConfigFiles()- if strings.TrimSpace(configContent) != "" {- files[vuln.TrivyConfigFile] = []byte(configContent)+ if s.registryAuthConfigs != nil {+ if authConfigs, err := s.registryAuthConfigs(ctx); err != nil {+ slog.WarnContext(ctx, "failed to load registry credentials for trivy scan; continuing anonymously", "error", err)+ } else if configJSON, buildErr := vuln.BuildDockerConfigJSON(authConfigs); buildErr != nil {+ slog.WarnContext(ctx, "failed to build trivy registry config; continuing anonymously", "error", buildErr)+ } else if len(configJSON) > 0 {+ files[vuln.RegistryConfigFile] = configJSON+ }}- if strings.TrimSpace(ignoreContent) != "" {- files[vuln.TrivyIgnoreFile] = []byte(ignoreContent)+ if cfg := s.settingsConfig(); cfg != nil {+ if strings.TrimSpace(cfg.TrivyConfig.Value) != "" {+ files[vuln.TrivyConfigFile] = []byte(cfg.TrivyConfig.Value)+ }+ if strings.TrimSpace(cfg.TrivyIgnore.Value) != "" {+ files[vuln.TrivyIgnoreFile] = []byte(cfg.TrivyIgnore.Value)+ }}return files}@@ -810,8 +623,12 @@ func (s *Service) trivyContainerFilesInternal(ctx context.Context) map[string][]// Flags passed here take precedence over the user-supplied trivy.yaml.func (s *Service) buildTrivyCommandArgs(imageName, cacheDir, outputPath string, files map[string][]byte) []string {serverURL, _ := s.ServerConfig()- cmdArgs := []string{"image", "--format", "json", "--quiet", "--exit-code", "0", "--exit-on-eol", "0", "--timeout", s.getTrivyScanTimeoutArg()}- cmdArgs = append(cmdArgs, vuln.ScanSourceArgs(serverURL, s.getTrivyDbRegistryInternal())...)+ dbRegistry := ""+ if cfg := s.settingsConfig(); cfg != nil {+ dbRegistry = cfg.TrivyDbRegistry.Value+ }+ cmdArgs := []string{"image", "--format", "json", "--quiet", "--exit-code", "0", "--exit-on-eol", "0", "--timeout", s.scanTimeoutArg()}+ cmdArgs = append(cmdArgs, vuln.ScanSourceArgs(serverURL, dbRegistry)...)cmdArgs = append(cmdArgs, vuln.IgnoreUnfixedArgs(s.IgnoreUnfixed())...)if files[vuln.TrivyConfigFile] != nil {@@ -832,7 +649,7 @@ func (s *Service) buildTrivyCommandArgs(imageName, cacheDir, outputPath string,return append(cmdArgs, imageName)}-func (s *Service) createAndStartTrivyContainerInternal(+func (s *Service) createAndStartTrivyContainer(ctx context.Context,dockerClient *client.Client,scope string,@@ -859,8 +676,7 @@ func (s *Service) createAndStartTrivyContainerInternal(vuln.RemoveContainer(ctx, dockerClient, resp.ID, "trivy container after file copy failure")return "", fmt.Errorf("failed to copy scan files into container: %w", copyFilesToContainerErr)}- // DOCKER_CONFIG may still point at the target directory. When the- // config file is absent, Trivy falls back to anonymous registry access.+ // DOCKER_CONFIG may still point here; without the file Trivy falls back to anonymous registry access.slog.WarnContext(ctx,"failed to inject registry credentials into trivy container; continuing anonymously","scope", scope,@@ -890,17 +706,8 @@ func (s *Service) createAndStartTrivyContainerInternal(return resp.ID, nil}-func (s *Service) getTrivyContainerResourcesInternal() mo.Option[container.Resources] {- var cfg *settings.Settings- if s.settingsService != nil {- cfg = s.settingsService.GetSettingsConfig()- }-- return buildTrivyContainerResourcesInternal(cfg)-}--func (s *Service) getTrivyContainerResourceOptionsInternal(ctx context.Context, dockerClient *client.Client) (container.Resources, string, bool, error) {- resources, applyLimits := s.getTrivyContainerResourcesInternal().Get()+func (s *Service) getTrivyContainerResourceOptions(ctx context.Context, dockerClient *client.Client) (container.Resources, string, bool, error) {+ resources, applyLimits := buildTrivyContainerResources(s.settingsConfig()).Get()if !applyLimits || resources.NanoCPUs == 0 {return resources, "", applyLimits, nil}@@ -911,13 +718,53 @@ func (s *Service) getTrivyContainerResourceOptionsInternal(ctx context.Context,return container.Resources{}, "", false, err}- capabilities, err := s.getTrivyCPUCapabilitiesInternal(ctx, dockerClient)- if err != nil {- if ctx.Err() != nil {+ // Capabilities, not limits, are cached per daemon so a new service instance skips the Info call.+ key := "trivy:cpu-capabilities:v1:" + kit.SHA256Hex(dockerClient.DaemonHost()+"\x00"+dockerClient.ClientVersion())+ var capabilities trivyCPUCapabilities+ fresh := false+ if s.kvService != nil {+ raw, found, err := s.kvService.Get(ctx, key)+ if err != nil {+ slog.WarnContext(ctx, "failed to read cached Trivy CPU capabilities", "error", err)+ } else if found {+ if unmarshalErr := json.Unmarshal([]byte(raw), &capabilities); unmarshalErr != nil || capabilities.FetchedAt.IsZero() {+ slog.WarnContext(ctx, "ignoring invalid cached Trivy CPU capabilities", "error", unmarshalErr)+ capabilities = trivyCPUCapabilities{}+ } else if age := time.Since(capabilities.FetchedAt); age >= 0 && age < trivyCPUCapabilitiesCacheTTL {+ fresh = true+ }+ }+ }++ if !fresh {+ infoCtx, cancel := context.WithTimeout(ctx, trivyCPUCapabilitiesLookupTimeout)+ info, err := dockerClient.Info(infoCtx, client.InfoOptions{})+ cancel()+ switch {+ case err != nil && ctx.Err() != nil:return container.Resources{}, "", false, ctx.Err()+ case err != nil && capabilities.FetchedAt.IsZero():+ slog.WarnContext(ctx, "unable to determine Trivy CPU capabilities; using configured CPU limits", "error", err)+ return resources, "", applyLimits, nil+ case err != nil:+ slog.WarnContext(ctx, "failed to refresh Trivy CPU capabilities; using cached capabilities", "error", err)+ default:+ capabilities = trivyCPUCapabilities{+ Quota: info.Info.CPUCfsQuota && !vuln.IsSynologyDockerHost(info.Info.OperatingSystem),+ CPUSet: info.Info.CPUSet,+ CPUs: info.Info.NCPU,+ FetchedAt: time.Now().UTC(),+ }+ if s.kvService != nil {+ payload, marshalErr := json.Marshal(capabilities)+ if marshalErr == nil {+ marshalErr = s.kvService.Set(ctx, key, string(payload))+ }+ if marshalErr != nil {+ slog.WarnContext(ctx, "failed to cache Trivy CPU capabilities", "error", marshalErr)+ }+ }}- slog.WarnContext(ctx, "unable to determine Trivy CPU capabilities; using configured CPU limits", "error", err)- return resources, "", applyLimits, nil}if capabilities.Quota {@@ -931,20 +778,15 @@ func (s *Service) getTrivyContainerResourceOptionsInternal(ctx context.Context,resources.NanoCPUs = 0if cpuSet != "" {- slog.InfoContext(ctx,- "docker host does not support CPU CFS quota; applying Trivy CPU limit via cpuset",- "cpuset", cpuSet,- )+ slog.InfoContext(ctx, "docker host does not support CPU CFS quota; applying Trivy CPU limit via cpuset", "cpuset", cpuSet)} else {- slog.InfoContext(ctx,- "docker host does not support CPU CFS quota or cpuset; skipping Trivy CPU limit",- )+ slog.InfoContext(ctx, "docker host does not support CPU CFS quota or cpuset; skipping Trivy CPU limit")}return resources, cpuSet, applyLimits, nil}-func buildTrivyContainerResourcesInternal(cfg *settings.Settings) mo.Option[container.Resources] {+func buildTrivyContainerResources(cfg *settings.Settings) mo.Option[container.Resources] {limitsEnabled := truecpuLimitCores := defaultTrivyCPULimitCoresmemoryLimitMB := defaultTrivyMemoryLimitMB@@ -973,7 +815,7 @@ func buildTrivyContainerResourcesInternal(cfg *settings.Settings) mo.Option[contreturn mo.Some(resources)}-type trivyContainerResultInternal struct {+type trivyContainerResult struct {stdoutFile *os.FilestderrFile *os.Fileduration int64@@ -981,38 +823,80 @@ type trivyContainerResultInternal struct {containerID string}-func (s *Service) runTrivyContainerInternal(- ctx context.Context,- dockerClient *client.Client,- imageID string,- containerConfig *container.Config,- hostConfig *container.HostConfig,- files map[string][]byte,-) (trivyContainerResultInternal, error) {+func (s *Service) Scan(ctx context.Context, scannerImage, imageName, imageID string, slotID int) (result *vulnerability.ScanResult, err error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "trivy.scan", trace.WithAttributes(+ attribute.String("container.image.name", imageName),+ attribute.String("container.image.id", imageID),+ attribute.String("arcane.trivy.mode", "container"),+ ))+ defer func() { tracing.End(span, err) }()++ slog.DebugContext(ctx, "running trivy scan", "imageId", imageID, "imageName", imageName, "scannerImage", scannerImage)++ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return nil, fmt.Errorf("failed to connect to Docker: %w", err)+ }++ serverURL, serverToken := s.ServerConfig()+ var cacheVolume, cacheDir string+ if serverURL == "" {+ if cacheVolume, err = s.ensureTrivyCacheVolume(ctx); err != nil {+ return nil, err+ }+ // A fixed cache dir per slot avoids fs-cache lock contention while bounding growth to the slot count.+ cacheDir = fmt.Sprintf("%s/slot-%d", trivyCacheSlotRootDir, max(slotID, 0))+ slog.DebugContext(ctx, "using slot-based trivy cache dir for scan", "slotId", slotID, "cacheDir", cacheDir, "concurrentScanContainers", s.ConcurrentScanContainers())+ }++ outputPath := vuln.NewOutputPath()+ files := s.trivyContainerFiles(ctx)+ cmdArgs := s.buildTrivyCommandArgs(imageName, cacheDir, outputPath, files)++ runtimeOptions, err := s.resolveTrivyRuntimeOptions(ctx, dockerClient)+ if err != nil {+ return nil, err+ }++ env := append([]string(nil), runtimeOptions.ContainerEnv...)+ if files[vuln.RegistryConfigFile] != nil {+ env = append(env, "DOCKER_CONFIG="+vuln.RegistryConfigDir)+ }+ if serverToken != "" {+ env = append(env, vuln.ServerTokenEnv(serverToken)...)+ }++ containerConfig := vuln.BuildContainerConfig(scannerImage, cmdArgs, env)+ resources, cpuSet, applyLimits, err := s.getTrivyContainerResourceOptions(ctx, dockerClient)+ if err != nil {+ return nil, err+ }+ var securityOpts []string+ privileged := false+ if cfg := s.settingsConfig(); cfg != nil {+ securityOpts, privileged = vuln.ParseSecurityOpts(cfg.TrivySecurityOpts.Value), cfg.TrivyPrivileged.IsTrue()+ }+ hostConfig := vuln.BuildHostConfig(cacheVolume, resources, cpuSet, applyLimits, runtimeOptions, securityOpts, privileged)+ s.setPhase(imageID, vulnerability.ScanPhaseCreatingContainer)+stdoutFile, err := vuln.CreateLogTempFile("trivy-stdout-*")if err != nil {- return trivyContainerResultInternal{}, fmt.Errorf("failed to create trivy stdout temp file: %w", err)+ return nil, fmt.Errorf("failed to create trivy stdout temp file: %w", err)}stderrFile, err := vuln.CreateLogTempFile("trivy-stderr-*")if err != nil {vuln.CleanupLogTempFiles(ctx, stdoutFile)- return trivyContainerResultInternal{}, fmt.Errorf("failed to create trivy stderr temp file: %w", err)+ return nil, fmt.Errorf("failed to create trivy stderr temp file: %w", err)}- success := false- containerID := ""- defer func() {- if !success {- vuln.RemoveContainer(ctx, dockerClient, containerID, "trivy scan container")- vuln.CleanupLogTempFiles(ctx, stdoutFile, stderrFile)- }- }()+ defer vuln.CleanupLogTempFiles(ctx, stdoutFile, stderrFile)- containerID, err = s.createAndStartTrivyContainerInternal(ctx, dockerClient, "single-scan", containerConfig, hostConfig, files)+ containerID, err := s.createAndStartTrivyContainer(ctx, dockerClient, "single-scan", containerConfig, hostConfig, files)if err != nil {- return trivyContainerResultInternal{}, fmt.Errorf("failed to prepare trivy scan container: %w", err)+ return nil, fmt.Errorf("failed to prepare trivy scan container: %w", err)}+ defer vuln.RemoveContainer(ctx, dockerClient, containerID, "trivy scan container")slog.DebugContext(ctx, "started trivy scan container", "containerId", containerID)s.setPhase(imageID, vulnerability.ScanPhaseScanningImage)@@ -1025,7 +909,7 @@ func (s *Service) runTrivyContainerInternal(})if err != nil {logsCancel()- return trivyContainerResultInternal{}, fmt.Errorf("failed to stream trivy logs: %w", err)+ return nil, fmt.Errorf("failed to stream trivy logs: %w", err)}var closeLogsOnce sync.Once@@ -1042,303 +926,63 @@ func (s *Service) runTrivyContainerInternal(logDone := docker.StartStdCopy(logs, stdoutFile, stderrFile)startTime := time.Now()- waitTimeout := vuln.ContainerWaitTimeout(s.getTrivyScanTimeoutArg())- waitCtx, waitCancel := context.WithTimeout(ctx, waitTimeout)+ waitCtx, waitCancel := context.WithTimeout(ctx, vuln.ContainerWaitTimeout(s.scanTimeoutArg()))defer waitCancel()- statusCode, err := s.waitForTrivyContainer(waitCtx, dockerClient, containerID)+ waitResult := dockerClient.ContainerWait(waitCtx, containerID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})+ statusCode, err := vuln.AwaitContainerWaitResponse(waitCtx, waitResult.Result, waitResult.Error)if err != nil {closeLogs()if logErr := docker.WaitStdCopy(logDone); logErr != nil {slog.DebugContext(ctx, "trivy log stream ended with error while handling wait failure", "error", logErr)}- return trivyContainerResultInternal{}, err- }-- // Wait for the log copy goroutine to complete naturally. On some Docker- // variants (e.g. Synology with Docker API 1.43), the daemon may not have- // fully flushed container stdout before ContainerWait returns. Cancelling- // the stream immediately after ContainerWait can truncate the JSON output- // and cause "unexpected EOF" or empty-output decode errors.- //- // Strategy: wait up to logDrainDeadline for the stream to close on its own,- // then force-close it (for Docker variants that keep the stream open after- // container exit, e.g. older Docker builds that never send EOF).+ switch {+ case errors.Is(err, context.DeadlineExceeded):+ return nil, fmt.Errorf("trivy container wait timed out: %w", err)+ case errors.Is(err, context.Canceled):+ return nil, fmt.Errorf("scan cancelled: %w", err)+ default:+ return nil, fmt.Errorf("trivy container wait failed: %w", err)+ }+ }++ // Some daemons (e.g. Synology, API 1.43) flush stdout after ContainerWait returns, so let the stream drain+ // before force-closing it for daemons that never send EOF; cancelling early truncates the JSON report.const logDrainDeadline = 30 * time.SecondlogDrainTimer := time.NewTimer(logDrainDeadline)defer logDrainTimer.Stop()select {case logErr := <-logDone:- closeLogs() // clean up context / connection+ closeLogs()if logErr != nil && !docker.IsExpectedStreamEndError(logErr) {- return trivyContainerResultInternal{}, fmt.Errorf("failed to read trivy logs: %w", logErr)+ return nil, fmt.Errorf("failed to read trivy logs: %w", logErr)}case <-logDrainTimer.C:// Stream did not close on its own; force-close so the goroutine exits.slog.DebugContext(ctx, "trivy log stream did not close after container exit; force-closing", "containerId", containerID)closeLogs()if logErr := docker.WaitStdCopy(logDone); logErr != nil {- return trivyContainerResultInternal{}, fmt.Errorf("failed to read trivy logs: %w", logErr)+ return nil, fmt.Errorf("failed to read trivy logs: %w", logErr)}}- duration := time.Since(startTime).Milliseconds()- success = true- return trivyContainerResultInternal{- stdoutFile: stdoutFile, stderrFile: stderrFile, duration: duration,+ run := trivyContainerResult{+ stdoutFile: stdoutFile, stderrFile: stderrFile, duration: time.Since(startTime).Milliseconds(),statusCode: statusCode, containerID: containerID,- }, nil-}--func (s *Service) waitForTrivyContainer(- ctx context.Context,- dockerClient *client.Client,- containerID string,-) (int64, error) {- waitResult := dockerClient.ContainerWait(ctx, containerID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})-- statusCode, err := vuln.AwaitContainerWaitResponse(ctx, waitResult.Result, waitResult.Error)- if err != nil {- if errors.Is(err, context.DeadlineExceeded) {- return 0, fmt.Errorf("trivy container wait timed out: %w", err)- }- if errors.Is(err, context.Canceled) {- return 0, fmt.Errorf("scan cancelled: %w", err)- }- return 0, fmt.Errorf("trivy container wait failed: %w", err)}- return statusCode, nil-}--func (s *Service) Scan(ctx context.Context, scannerImage, imageName, imageID string, slotID int) (*vulnerability.ScanResult, error) {- slog.DebugContext(ctx,- "running trivy scan",- "imageId", imageID,- "imageName", imageName,- "scannerImage", scannerImage,- )-- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- serverURL, serverToken := s.ServerConfig()- var cacheVolume string- if serverURL == "" {- cacheVolume, err = s.ensureTrivyCacheVolumeInternal(ctx)- if err != nil {- return nil, err- }- }-- var cacheDir string- if serverURL == "" {- cacheDir = s.getTrivySingleScanCacheDirForSlotInternal(slotID)- }- if cacheDir != "" {- slog.DebugContext(ctx,- "using slot-based trivy cache dir for scan",- "slotId", slotID,- "cacheDir", cacheDir,- "concurrentScanContainers", s.ConcurrentScanContainers(),- )- }-- outputPath := vuln.NewOutputPath()- files := s.trivyContainerFilesInternal(ctx)- cmdArgs := s.buildTrivyCommandArgs(imageName, cacheDir, outputPath, files)-- runtimeOptions, err := s.resolveTrivyRuntimeOptionsInternal(ctx, dockerClient)+ result, err = s.scanResult(ctx, dockerClient, run, outputPath, imageName, imageID, "container")if err != nil {return nil, err}-- env := append([]string(nil), runtimeOptions.ContainerEnv...)- if files[vuln.RegistryConfigFile] != nil {- env = append(env, "DOCKER_CONFIG="+vuln.RegistryConfigDir)- }- if serverToken != "" {- env = append(env, vuln.ServerTokenEnv(serverToken)...)- }-- containerConfig := vuln.BuildContainerConfig(scannerImage, cmdArgs, env)- resources, cpuSet, applyLimits, err := s.getTrivyContainerResourceOptionsInternal(ctx, dockerClient)- if err != nil {- return nil, err- }- runtimeSecurity := s.getTrivyRuntimeSecurityInternal().OrEmpty()- securityOpts, privileged := runtimeSecurity.securityOpts, runtimeSecurity.privileged- hostConfig := vuln.BuildHostConfig(- cacheVolume,- resources,- cpuSet,- applyLimits,- runtimeOptions,- securityOpts,- privileged,- )- s.setPhase(imageID, vulnerability.ScanPhaseCreatingContainer)-- containerResult, err := s.runTrivyContainerInternal(ctx, dockerClient, imageID, containerConfig, hostConfig, files)- if err != nil {- return nil, err- }- defer vuln.CleanupLogTempFiles(ctx, containerResult.stdoutFile, containerResult.stderrFile)- defer vuln.RemoveContainer(ctx, dockerClient, containerResult.containerID, "trivy scan container")-- if containerResult.statusCode != 0 {- errMsg := vuln.ReadTempFileExcerpt(containerResult.stderrFile)- if errMsg == "" {- errMsg = vuln.ReadTempFileExcerpt(containerResult.stdoutFile)- }- if errMsg == "" {- errMsg = fmt.Sprintf("exit status %d", containerResult.statusCode)- }- slog.WarnContext(ctx,- "trivy scan diagnostics",- "image", imageName,- "imageId", imageID,- "reason", "container non-zero exit",- "exitCode", containerResult.statusCode,- "stdoutBytes", vuln.TempFileSize(containerResult.stdoutFile),- "stderrBytes", vuln.TempFileSize(containerResult.stderrFile),- )- return nil, vuln.FormatExitError(containerResult.statusCode, errMsg, imageName)- }-- trivyOutput, outputErr := vuln.ReadOutputFromContainerFile(ctx, dockerClient, containerResult.containerID, outputPath)- if outputErr != nil {- slog.WarnContext(ctx,- "failed to read trivy output file from single-scan container; falling back to stdout parsing",- "containerId", containerResult.containerID,- "outputPath", outputPath,- "error", outputErr,- )- }-- var trivyReport *vulnerability.TrivyReport- if len(trivyOutput) > 0 {- trivyReport, err = vuln.DecodeReportFromBytes(trivyOutput)- } else {- trivyReport, err = vuln.DecodeReportFromFile(containerResult.stdoutFile)- }- if err != nil {- if errors.Is(err, io.EOF) {- errMsg := cmp.Or(vuln.ReadTempFileExcerpt(containerResult.stderrFile), "trivy scan produced no output")- slog.WarnContext(ctx,- "trivy scan diagnostics",- "image", imageName,- "imageId", imageID,- "reason", "container empty output",- "exitCode", containerResult.statusCode,- "stdoutBytes", vuln.TempFileSize(containerResult.stdoutFile),- "stderrBytes", vuln.TempFileSize(containerResult.stderrFile),- )- return nil, fmt.Errorf("trivy scan failed: %s", errMsg)- }-- slog.WarnContext(ctx,- "trivy scan diagnostics",- "image", imageName,- "imageId", imageID,- "reason", "container parse error",- "exitCode", containerResult.statusCode,- "stdoutBytes", vuln.TempFileSize(containerResult.stdoutFile),- "stderrBytes", vuln.TempFileSize(containerResult.stderrFile),- )- return nil, fmt.Errorf("failed to parse trivy output: %w", err)- }-- if trivyReport == nil {- errMsg := cmp.Or(vuln.ReadTempFileExcerpt(containerResult.stderrFile), "trivy scan produced no output")- slog.WarnContext(ctx,- "trivy scan diagnostics",- "image", imageName,- "imageId", imageID,- "reason", "container nil report",- "exitCode", containerResult.statusCode,- "stdoutBytes", vuln.TempFileSize(containerResult.stdoutFile),- "stderrBytes", vuln.TempFileSize(containerResult.stderrFile),- )- return nil, fmt.Errorf("trivy scan failed: %s", errMsg)- }-- // Convert Trivy report to our ScanResult format- result := vulnerability.ConvertTrivyReportToScanResult(trivyReport, imageID, time.Now(), containerResult.duration)result.ScannerVersion = s.Version(ctx)- result.HasReport = s.persistRawTrivyReportInternal(ctx, imageID, trivyOutput, containerResult.stdoutFile)- totalVulnerabilities := 0- if result.Summary != nil {- totalVulnerabilities = result.Summary.Total- }-- slog.DebugContext(ctx,- "trivy scan finished",- "imageId", imageID,- "imageName", imageName,- "durationMs", containerResult.duration,- "status", result.Status,- "vulnerabilities", totalVulnerabilities,- )-return result, nil}-type trivyCPUCapabilitiesInternal struct {+type trivyCPUCapabilities struct {Quota bool `json:"quota"`CPUSet bool `json:"cpuSet"`CPUs int `json:"cpus"`FetchedAt time.Time `json:"fetchedAt"`}--func (s *Service) getTrivyCPUCapabilitiesInternal(ctx context.Context, dockerClient *client.Client) (trivyCPUCapabilitiesInternal, error) {- key := "trivy:cpu-capabilities:v1:" + kit.SHA256Hex(dockerClient.DaemonHost()+"\x00"+dockerClient.ClientVersion())- var cached trivyCPUCapabilitiesInternal- if s.kvService != nil {- raw, found, err := s.kvService.Get(ctx, key)- if err != nil {- slog.WarnContext(ctx, "failed to read cached Trivy CPU capabilities", "error", err)- } else if found {- if unmarshalErr := json.Unmarshal([]byte(raw), &cached); unmarshalErr != nil || cached.FetchedAt.IsZero() {- slog.WarnContext(ctx, "ignoring invalid cached Trivy CPU capabilities", "error", unmarshalErr)- cached = trivyCPUCapabilitiesInternal{}- } else if age := time.Since(cached.FetchedAt); age >= 0 && age < trivyCPUCapabilitiesCacheTTL {- return cached, nil- }- }- }-- infoCtx, cancel := context.WithTimeout(ctx, trivyCPUCapabilitiesLookupTimeout)- defer cancel()- info, err := dockerClient.Info(infoCtx, client.InfoOptions{})- if err != nil {- if ctx.Err() != nil {- return trivyCPUCapabilitiesInternal{}, ctx.Err()- }- if !cached.FetchedAt.IsZero() {- slog.WarnContext(ctx, "failed to refresh Trivy CPU capabilities; using cached capabilities", "error", err)- return cached, nil- }- return trivyCPUCapabilitiesInternal{}, err- }-- capabilities := trivyCPUCapabilitiesInternal{- Quota: info.Info.CPUCfsQuota && !vuln.IsSynologyDockerHost(info.Info.OperatingSystem),- CPUSet: info.Info.CPUSet,- CPUs: info.Info.NCPU,- FetchedAt: time.Now().UTC(),- }- if s.kvService != nil {- payload, marshalErr := json.Marshal(capabilities)- if marshalErr == nil {- marshalErr = s.kvService.Set(ctx, key, string(payload))- }- if marshalErr != nil {- slog.WarnContext(ctx, "failed to cache Trivy CPU capabilities", "error", marshalErr)- }- }- return capabilities, nil-}diff --git a/backend/internal/vulnerability/children/scan/children/trivy/service_test.go b/backend/internal/vulnerability/children/scan/children/trivy/service_test.goindex 2ea132f40..09feb3a9f 100644--- a/backend/internal/vulnerability/children/scan/children/trivy/service_test.go+++ b/backend/internal/vulnerability/children/scan/children/trivy/service_test.go@@ -24,7 +24,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/vuln")-func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *database.DB) {+func featureSettings(t *testing.T) (*settings.SettingsService, *database.DB) {t.Helper()gdb, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)@@ -38,8 +38,8 @@ func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *databasereturn svc, db}-func TestBuildTrivyContainerResourcesInternal_Defaults(t *testing.T) {- resources, applyLimits := buildTrivyContainerResourcesInternal(nil).Get()+func TestBuildTrivyContainerResources_Defaults(t *testing.T) {+ resources, applyLimits := buildTrivyContainerResources(nil).Get()require.True(t, applyLimits)require.Equal(t, int64(1_000_000_000), resources.NanoCPUs)@@ -47,14 +47,14 @@ func TestBuildTrivyContainerResourcesInternal_Defaults(t *testing.T) {require.Equal(t, int64(0), resources.MemorySwap)}-func TestBuildTrivyContainerResourcesInternal_LimitsDisabled(t *testing.T) {+func TestBuildTrivyContainerResources_LimitsDisabled(t *testing.T) {cfg := &settings.Settings{TrivyResourceLimitsEnabled: settings.SettingVariable{Value: "false"},TrivyCpuLimit: settings.SettingVariable{Value: "2"},TrivyMemoryLimitMb: settings.SettingVariable{Value: "2048"},}- resources, applyLimits := buildTrivyContainerResourcesInternal(cfg).Get()+ resources, applyLimits := buildTrivyContainerResources(cfg).Get()require.False(t, applyLimits)require.Equal(t, int64(0), resources.NanoCPUs)@@ -62,14 +62,14 @@ func TestBuildTrivyContainerResourcesInternal_LimitsDisabled(t *testing.T) {require.Equal(t, int64(0), resources.MemorySwap)}-func TestBuildTrivyContainerResourcesInternal_CustomLimits(t *testing.T) {+func TestBuildTrivyContainerResources_CustomLimits(t *testing.T) {cfg := &settings.Settings{TrivyResourceLimitsEnabled: settings.SettingVariable{Value: "true"},TrivyCpuLimit: settings.SettingVariable{Value: "2.5"},TrivyMemoryLimitMb: settings.SettingVariable{Value: "2048"},}- resources, applyLimits := buildTrivyContainerResourcesInternal(cfg).Get()+ resources, applyLimits := buildTrivyContainerResources(cfg).Get()require.True(t, applyLimits)require.Equal(t, int64(2_500_000_000), resources.NanoCPUs)@@ -100,7 +100,7 @@ func TestBuildTrivyContainerResourcesInternal_CustomLimits(t *testing.T) {{name: "no Docker client", cpu: "0.5", noClient: true},} {t.Run(tc.name, func(t *testing.T) {- settingsSvc, _ := featureSettingsInternal(t)+ settingsSvc, _ := featureSettings(t)require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), "trivyResourceLimitsEnabled", true))require.NoError(t, settingsSvc.SetStringSetting(t.Context(), "trivyCpuLimit", tc.cpu))require.NoError(t, settingsSvc.SetIntSetting(t.Context(), "trivyMemoryLimitMb", 2048))@@ -145,7 +145,7 @@ func TestBuildTrivyContainerResourcesInternal_CustomLimits(t *testing.T) {if tc.noClient {dockerClient = nil}- actual, cpuSet, apply, err := svc.getTrivyContainerResourceOptionsInternal(ctx, dockerClient)+ actual, cpuSet, apply, err := svc.getTrivyContainerResourceOptions(ctx, dockerClient)if tc.expired || tc.noClient {require.Error(t, err)if tc.expired {@@ -166,7 +166,7 @@ func TestBuildTrivyContainerResourcesInternal_CustomLimits(t *testing.T) {}for _, quotaSupported := range []bool{true, false} {- t.Run(fmt.Sprintf("persistent capabilities quota=%t", quotaSupported), func(t *testing.T) { assertPersistentTrivyCapabilitiesInternal(t, quotaSupported) })+ t.Run(fmt.Sprintf("persistent capabilities quota=%t", quotaSupported), func(t *testing.T) { assertPersistentTrivyCapabilities(t, quotaSupported) })}}@@ -207,38 +207,29 @@ func TestBuildTrivyCommandArgs_IncludesDefaultRepositories(t *testing.T) {require.Contains(t, cmdArgs, "ghcr.io/"+vuln.ChecksBundleRepository)}-func TestGetTrivySingleScanCacheDirForSlotInternal(t *testing.T) {- svc := &Service{}-- require.Equal(t, "/root/.cache/arcane-scan-slots/slot-0", svc.getTrivySingleScanCacheDirForSlotInternal(0))- require.Equal(t, "/root/.cache/arcane-scan-slots/slot-2", svc.getTrivySingleScanCacheDirForSlotInternal(2))- // Negative slot IDs should normalize to slot-0- require.Equal(t, "/root/.cache/arcane-scan-slots/slot-0", svc.getTrivySingleScanCacheDirForSlotInternal(-1))-}--func TestGetTrivyScanSlotChannelInternal_PrefillsSlots(t *testing.T) {- svc := &Service{}-- slotCh := svc.getTrivyScanSlotChannelInternal(3)- require.NotNil(t, slotCh)- require.Equal(t, 3, cap(slotCh))- require.Len(t, slotCh, 3)+func TestAcquireSlot_PrefillsSlots(t *testing.T) {+ settingsSvc, _ := featureSettings(t)+ require.NoError(t, settingsSvc.SetIntSetting(t.Context(), "trivyConcurrentScanContainers", 3))+ svc := &Service{settingsService: settingsSvc}seen := map[int]bool{}- for range 3 {- slotID := <-slotCh+ for i := range 3 {+ slotID, release, err := svc.AcquireSlot(t.Context(), fmt.Sprintf("image-%d", i), false)+ require.NoError(t, err)+ require.NotNil(t, release)+ t.Cleanup(release)seen[slotID] = true}+ require.Equal(t, map[int]bool{0: true, 1: true, 2: true}, seen)- require.True(t, seen[0])- require.True(t, seen[1])- require.True(t, seen[2])- require.Empty(t, slotCh)+ _, release, err := svc.AcquireSlot(t.Context(), "image-3", false)+ require.NoError(t, err)+ require.Nil(t, release, "all slots are taken")}-func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool) {+func assertPersistentTrivyCapabilities(t *testing.T, quotaSupported bool) {t.Helper()- settingsSvc, db := featureSettingsInternal(t)+ settingsSvc, db := featureSettings(t)require.NoError(t, db.AutoMigrate(&kv.KVEntry{}))store := kv.NewKVService(db)require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), "trivyResourceLimitsEnabled", true))@@ -262,7 +253,7 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported boolrequire.NoError(t, err)t.Cleanup(func() { require.NoError(t, dockerClient.Close()) })svc := &Service{settingsService: settingsSvc, kvService: store}- first, firstCPUSet, apply, err := svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)+ first, firstCPUSet, apply, err := svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)require.NoError(t, err)require.True(t, apply)require.Equal(t, int32(1), requests.Load())@@ -277,7 +268,7 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool// A new service instance reuses persisted capabilities without calling Docker.svc = &Service{settingsService: settingsSvc, kvService: kv.NewKVService(db)}status.Store(http.StatusForbidden)- cached, cpuSet, apply, err := svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)+ cached, cpuSet, apply, err := svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)require.NoError(t, err)require.True(t, apply)require.Equal(t, first, cached)@@ -286,7 +277,7 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool// Cache capabilities, not the configured limit.require.NoError(t, settingsSvc.SetStringSetting(t.Context(), "trivyCpuLimit", "0.5"))- cached, cpuSet, _, err = svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)+ cached, cpuSet, _, err = svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)require.NoError(t, err)if quotaSupported {require.Equal(t, int64(500_000_000), cached.NanoCPUs)@@ -298,13 +289,13 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported boolentries, err := store.ListByPrefix(t.Context(), "trivy:cpu-capabilities:v1:")require.NoError(t, err)require.Len(t, entries, 1)- var expired trivyCPUCapabilitiesInternal+ var expired trivyCPUCapabilitiesrequire.NoError(t, json.Unmarshal([]byte(entries[0].Value), &expired))expired.FetchedAt = time.Now().Add(-2 * trivyCPUCapabilitiesCacheTTL)payload, err := json.Marshal(expired)require.NoError(t, err)require.NoError(t, store.Set(t.Context(), entries[0].Key, string(payload)))- stale, staleCPUSet, apply, err := svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)+ stale, staleCPUSet, apply, err := svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)require.NoError(t, err)require.True(t, apply)require.Equal(t, cached, stale)@@ -312,13 +303,13 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported boolrequire.Equal(t, int32(2), requests.Load())status.Store(http.StatusOK)- _, _, _, err = svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)+ _, _, _, err = svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)require.NoError(t, err)require.Equal(t, int32(3), requests.Load())raw, found, err := store.Get(t.Context(), entries[0].Key)require.NoError(t, err)require.True(t, found)- var refreshed trivyCPUCapabilitiesInternal+ var refreshed trivyCPUCapabilitiesrequire.NoError(t, json.Unmarshal([]byte(raw), &refreshed))require.True(t, refreshed.FetchedAt.After(expired.FetchedAt))@@ -332,7 +323,7 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported boolotherClient, newErr := client.New(client.WithHost(endpoint.host), client.WithAPIVersion(endpoint.version))require.NoError(t, newErr)t.Cleanup(func() { require.NoError(t, otherClient.Close()) })- fallback, fallbackCPUSet, fallbackApply, newErr := svc.getTrivyContainerResourceOptionsInternal(t.Context(), otherClient)+ fallback, fallbackCPUSet, fallbackApply, newErr := svc.getTrivyContainerResourceOptions(t.Context(), otherClient)require.NoError(t, newErr)require.True(t, fallbackApply)require.Equal(t, int64(500_000_000), fallback.NanoCPUs)@@ -341,7 +332,7 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool}require.NoError(t, store.Set(t.Context(), entries[0].Key, "invalid JSON"))- fallback, cpuSet, apply, err := svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)+ fallback, cpuSet, apply, err := svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)require.NoError(t, err)require.True(t, apply)require.Equal(t, int64(500_000_000), fallback.NanoCPUs)@@ -352,13 +343,13 @@ func assertPersistentTrivyCapabilitiesInternal(t *testing.T, quotaSupported bool// Cache storage failures must not prevent a live lookup or a safe fallback.require.NoError(t, db.Migrator().DropTable(&kv.KVEntry{}))status.Store(http.StatusOK)- live, liveCPUSet, apply, err := svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)+ live, liveCPUSet, apply, err := svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)require.NoError(t, err)require.True(t, apply)require.Equal(t, cached, live)require.Equal(t, staleCPUSet, liveCPUSet)status.Store(http.StatusInternalServerError)- fallback, cpuSet, apply, err = svc.getTrivyContainerResourceOptionsInternal(t.Context(), dockerClient)+ fallback, cpuSet, apply, err = svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)require.NoError(t, err)require.True(t, apply)require.Equal(t, int64(500_000_000), fallback.NanoCPUs)@@ -377,7 +368,7 @@ func TestVulnerabilityFeatureDisabledWhileWaitingForSlot(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- settingsSvc, _ := featureSettingsInternal(t)+ settingsSvc, _ := featureSettings(t)require.NoError(t, settingsSvc.SetIntSetting(t.Context(), "trivyConcurrentScanContainers", tt.slots))var inspections atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {@@ -432,7 +423,7 @@ func TestVulnerabilityFeatureDisabledWhileWaitingForSlot(t *testing.T) {}func TestVulnerabilityFeatureDisabledRejectsBatchContainers(t *testing.T) {- settingsSvc, _ := featureSettingsInternal(t)+ settingsSvc, _ := featureSettings(t)svc := &Service{settingsService: settingsSvc}require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))diff --git a/backend/internal/vulnerability/module.go b/backend/internal/vulnerability/module.goindex 72f049328..efe19184b 100644--- a/backend/internal/vulnerability/module.go+++ b/backend/internal/vulnerability/module.go@@ -2,23 +2,67 @@package vulnerabilityimport (+ "context"+ "errors""log/slog""net/http""github.com/danielgtaylor/huma/v2""github.com/getarcaneapp/arcane/types/v2/features"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/ignores""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/query""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/scan""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")type Module struct {service *VulnerabilityService}+// New wires vulnerability routes and registers the arcane.vulnerability.* gauges once.func New(service *VulnerabilityService) *Module {+ if service != nil && service.query != nil && service.settingsService != nil {+ meter := otel.Meter(tracing.InstrumentationName)+ findings, findingsErr := meter.Int64ObservableGauge("arcane.vulnerability.findings",+ metric.WithDescription("Vulnerabilities in the latest completed scans of local images by severity"), metric.WithUnit("{vulnerability}"))+ scanned, scannedErr := meter.Int64ObservableGauge("arcane.vulnerability.images.scanned",+ metric.WithDescription("Local images with a completed vulnerability scan"), metric.WithUnit("{image}"))+ actionable, actionableErr := meter.Int64ObservableGauge("arcane.vulnerability.actionable",+ metric.WithDescription("Critical and high vulnerabilities that are not ignored"), metric.WithUnit("{vulnerability}"))+ err := errors.Join(findingsErr, scannedErr, actionableErr)+ if err == nil {+ _, err = meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {+ if !service.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {+ return nil+ }+ if summary, summaryErr := service.query.GetEnvironmentSummary(ctx); summaryErr != nil {+ slog.DebugContext(ctx, "Skipping vulnerability summary metrics", "error", summaryErr)+ } else if summary.Summary != nil {+ o.ObserveInt64(scanned, int64(summary.ScannedImages))+ for severity, n := range map[string]int{+ "critical": summary.Summary.Critical, "high": summary.Summary.High, "medium": summary.Summary.Medium,+ "low": summary.Summary.Low, "unknown": summary.Summary.Unknown,+ } {+ o.ObserveInt64(findings, int64(n), metric.WithAttributes(attribute.String("vulnerability.severity", severity)))+ }+ }+ if count, countErr := service.ActionableCountExcludingIgnored(ctx); countErr != nil {+ slog.DebugContext(ctx, "Skipping actionable vulnerability metric", "error", countErr)+ } else {+ o.ObserveInt64(actionable, int64(count))+ }+ return nil+ }, findings, scanned, actionable)+ }+ if err != nil {+ otel.Handle(err)+ }+ }return &Module{service: service}}diff --git a/backend/pkg/gitutil/git.go b/backend/pkg/gitutil/git.goindex c1ca6a8ae..5dc2dbba2 100644--- a/backend/pkg/gitutil/git.go+++ b/backend/pkg/gitutil/git.go@@ -15,7 +15,6 @@ import ("path/filepath""regexp""slices"- "sort""strings""time"@@ -32,8 +31,13 @@ import ("go.getarcane.app/acfs"acfstypes "go.getarcane.app/acfs/types"kit "go.getarcane.app/kit/pkg"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/trace"gossh "golang.org/x/crypto/ssh""golang.org/x/crypto/ssh/knownhosts"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")const (@@ -54,16 +58,11 @@ const (defaultKnownHostsPath = "/app/data/.ssh/known_hosts")-// go-git's file transport execs the git binary, which doesn't exist in the-// distroless image. Unregister it so a repository URL can never reach it.+// go-git's file transport execs the git binary, which the distroless image lacks, so unregister it.func init() {client.InstallProtocol("file", nil)- // go-git strips multi_ack/multi_ack_detailed from pack negotiation by- // default, which Azure DevOps rejects by sending an incomplete pack- // ("invalid reset option: object not found" on clone). Advertise them and- // keep only thin-pack disabled, matching Flux/ArgoCD; other hosts are- // unaffected.+ // Azure DevOps sends an incomplete pack without multi_ack, so only thin-pack stays disabled (as in Flux/ArgoCD).transport.UnsupportedCapabilities = []capability.Capability{capability.ThinPack,}@@ -76,10 +75,9 @@ type Client struct {var scpLikeURLPattern = regexp.MustCompile(`^[^@/]+@[^:/]+:`)-// normalizeURLInternal coerces a repository URL into a form go-git resolves to a-// network transport, never the file transport (which execs the git binary).+// normalizeURL coerces a repository URL onto a network transport, never the file transport.// Schemeless host-style URLs (e.g. github.com/org/repo.git) get https://.-func normalizeURLInternal(raw string) (string, error) {+func normalizeURL(raw string) (string, error) {url := strings.TrimSpace(raw)if url == "" {return "", errors.New("repository URL is empty")@@ -121,8 +119,8 @@ type AuthConfig struct {SSHHostKeyVerification string // strict, accept_new, skip}-// getAuthInternal returns the appropriate transport.AuthMethod.-func (c *Client) getAuthInternal(ctx context.Context, url string, localConfig AuthConfig) (transport.AuthMethod, error) {+// getAuth returns the appropriate transport.AuthMethod.+func (c *Client) getAuth(ctx context.Context, url string, localConfig AuthConfig) (transport.AuthMethod, error) {switch localConfig.AuthType {case "http":if localConfig.Token != "" {@@ -144,8 +142,7 @@ func (c *Client) getAuthInternal(ctx context.Context, url string, localConfig Aureturn nil, fmt.Errorf("failed to create ssh auth: %w", err)}- // Configure host key verification based on mode- hostKeyCallback, err := c.getSSHHostKeyCallbackInternal(ctx, localConfig.SSHHostKeyVerification)+ hostKeyCallback, err := c.getSSHHostKeyCallback(ctx, localConfig.SSHHostKeyVerification)if err != nil {return nil, fmt.Errorf("failed to configure SSH host key verification: %w", err)}@@ -156,21 +153,18 @@ func (c *Client) getAuthInternal(ctx context.Context, url string, localConfig Aureturn publicKeys, nil}return nil, errors.New("ssh key required for ssh authentication")- case "none":- return nil, nildefault:return nil, nil}}-// getSSHHostKeyCallbackInternal returns the SSH host key callback for the verification mode.-func (c *Client) getSSHHostKeyCallbackInternal(ctx context.Context, mode string) (gossh.HostKeyCallback, error) {+// getSSHHostKeyCallback returns the SSH host key callback for the verification mode.+func (c *Client) getSSHHostKeyCallback(ctx context.Context, mode string) (gossh.HostKeyCallback, error) {if mode == SSHHostKeyVerificationSkip {- // Skip host key verification - intentionally insecure, user explicitly opted in via UIreturn gossh.InsecureIgnoreHostKey(), nil //nolint:gosec // User explicitly chose to skip verification}- knownHostsPaths, pathErr := getKnownHostsPathsInternal(+ knownHostsPaths, pathErr := getKnownHostsPaths(c.workDir,os.Getenv,os.Stat,@@ -183,16 +177,13 @@ func (c *Client) getSSHHostKeyCallbackInternal(ctx context.Context, mode string)return knownhosts.New(knownHostsPaths...)}- // Other modes default to accepting and remembering new host keys.- // Ensure the directory exists- // os.* supports home and override paths outside acfs roots, plus append/flock.+ // Other modes accept and remember new host keys; os.* reaches paths outside acfs roots and supports append/flock.knownHostsPath := knownHostsPaths[0]dir := filepath.Dir(knownHostsPath)if err := os.MkdirAll(dir, 0o700); err != nil {return nil, fmt.Errorf("failed to create known_hosts directory: %w", err)}- // Create the file if it doesn't existif _, err := os.Stat(knownHostsPath); os.IsNotExist(err) {file, openFileErr := os.OpenFile(knownHostsPath, os.O_CREATE|os.O_WRONLY, 0o600)if openFileErr != nil {@@ -204,8 +195,7 @@ func (c *Client) getSSHHostKeyCallbackInternal(ctx context.Context, mode string)}return func(hostname string, remote net.Addr, key gossh.PublicKey) error {- // Re-read known_hosts on each call to handle concurrent modifications.- // Unreadable trust must fail closed so changed keys are never accepted.+ // Re-read known_hosts each call for concurrent edits; unreadable trust fails closed.existingCallback, err := knownhosts.New(knownHostsPaths...)if err != nil {return fmt.Errorf("failed to read known_hosts: %w", err)@@ -213,16 +203,13 @@ func (c *Client) getSSHHostKeyCallbackInternal(ctx context.Context, mode string)existingCallbackErr := existingCallback(hostname, remote, key)if existingCallbackErr == nil {- return nil // Host key matches+ return nil}- // Check if it's a "key mismatch" error vs "unknown host"if keyErr, ok := errors.AsType[*knownhosts.KeyError](existingCallbackErr); ok && len(keyErr.Want) > 0 {- // Host is known but key doesn't match - this is a security concernreturn fmt.Errorf("host key mismatch for %s (possible MITM attack): %w", hostname, existingCallbackErr)}- // Otherwise, host is unknown - we'll add it- // Log persistence failures after closing the file and releasing its lock.+ // Unknown host: save it, logging persistence failures after the file closes and the lock releases.var saveErr errordefer func() {if saveErr != nil {@@ -260,20 +247,18 @@ func (c *Client) getSSHHostKeyCallbackInternal(ctx context.Context, mode string)}, nil}-// getKnownHostsPathsInternal returns trusted known_hosts files; new keys are saved to the first.-func getKnownHostsPathsInternal(+// getKnownHostsPaths returns trusted known_hosts files; new keys are saved to the first.+func getKnownHostsPaths(workDir string,getenv func(string) string,stat func(string) (os.FileInfo, error),userHomeDir func() (string, error),) ([]string, error) {- // Check environment variable firstif localPath := getenv("SSH_KNOWN_HOSTS"); localPath != "" {return []string{localPath}, nil}- // Prefer Arcane's writable persistent data directory when it is available,- // which is the case for published container images and PUID/PGID setups.+ // Prefer Arcane's writable data directory, present in published images and PUID/PGID setups.if info, err := stat(defaultKnownHostsDataDir); err == nil && info.IsDir() {return []string{defaultKnownHostsPath}, nil}@@ -319,74 +304,78 @@ func getKnownHostsPathsInternal(}}+// RepositoryAttribute names the repository on spans without credentials or query tokens.+func RepositoryAttribute(rawURL string) attribute.KeyValue {+ endpoint, err := transport.NewEndpoint(rawURL)+ if err != nil {+ return attribute.String("arcane.git.repository", "")+ }+ endpoint.User, endpoint.Password = "", ""+ if i := strings.IndexAny(endpoint.Path, "?#"); i >= 0 {+ endpoint.Path = endpoint.Path[:i]+ }+ return attribute.String("arcane.git.repository", endpoint.String())+}+// Clone clones a repository to a temporary directory. A depth of 0 fetches the// full history; read-only callers pass 1 to fetch only the branch tip.-func (c *Client) Clone(ctx context.Context, url, branch string, auth AuthConfig, depth int) (string, error) {+func (c *Client) Clone(ctx context.Context, url, branch string, auth AuthConfig, depth int) (repoPath string, err error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "gitrepo.clone", trace.WithAttributes(+ attribute.String("arcane.git.branch", branch),+ attribute.Int("arcane.git.depth", depth),+ ))+ defer func() { tracing.End(span, err) }()+if _, hasDeadline := ctx.Deadline(); !hasDeadline {var cancel context.CancelFuncctx, cancel = context.WithTimeout(ctx, 5*time.Minute)defer cancel()}-- if err := ctx.Err(); err != nil {- return "", err+ if ctxErr := ctx.Err(); ctxErr != nil {+ return "", ctxErr}- // Create a temporary directory- workDir := c.workDir- if workDir == "" {- workDir = os.TempDir()- }- // Ensure the work directory exists- // os.* rather than acfs: this creates the clone staging root itself (under the- // system temp dir by default), which has to exist before acfs could open it.- if err := os.MkdirAll(workDir, 0o755); err != nil {- return "", fmt.Errorf("failed to create work dir: %w", err)- }- tmpDir, err := os.MkdirTemp(workDir, cloneScratchPrefix+"*")+ url, err = normalizeURL(url)if err != nil {- return "", fmt.Errorf("failed to create temp dir: %w", err)- }-- url, err = normalizeURLInternal(url)- if err != nil {- _ = os.RemoveAll(tmpDir)return "", err}-- authMethod, err := c.getAuthInternal(ctx, url, auth)+ span.SetAttributes(RepositoryAttribute(url))+ authMethod, err := c.getAuth(ctx, url, auth)if err != nil {- _ = os.RemoveAll(tmpDir)return "", err}- cloneOptions := &git.CloneOptions{- URL: url,- Progress: nil,- Depth: depth,+ // os.* rather than acfs: the clone staging root must exist before acfs could open it.+ workDir := cmp.Or(c.workDir, os.TempDir())+ if err = os.MkdirAll(workDir, 0o755); err != nil {+ return "", fmt.Errorf("failed to create work dir: %w", err)}-- if authMethod != nil {- cloneOptions.Auth = authMethod+ tmpDir, err := os.MkdirTemp(workDir, cloneScratchPrefix+"*")+ if err != nil {+ return "", fmt.Errorf("failed to create temp dir: %w", err)}+ cloneOptions := &git.CloneOptions{URL: url, Auth: authMethod, Depth: depth}if branch != "" {cloneOptions.ReferenceName = plumbing.NewBranchReferenceName(branch)cloneOptions.SingleBranch = true}- _, err = git.PlainCloneContext(ctx, tmpDir, false, cloneOptions)+ repo, err := git.PlainCloneContext(ctx, tmpDir, false, cloneOptions)if err != nil && depth > 0 && strings.Contains(err.Error(), "shallow") {// Servers without shallow support, go-git's own included, get a full clone.cloneOptions.Depth = 0if err = os.RemoveAll(tmpDir); err == nil {- _, err = git.PlainCloneContext(ctx, tmpDir, false, cloneOptions)+ repo, err = git.PlainCloneContext(ctx, tmpDir, false, cloneOptions)}}if err != nil {_ = os.RemoveAll(tmpDir)return "", fmt.Errorf("failed to clone repository: %w", err)}+ if head, headErr := repo.Head(); headErr == nil {+ span.SetAttributes(attribute.String("arcane.git.commit", head.Hash().String()))+ }return tmpDir, nil}@@ -415,111 +404,79 @@ type BranchInfo struct {IsDefault bool}-// ListBranches lists all branches in a remote repository+// ListBranches lists a remote repository's branches, default branch first.func (c *Client) ListBranches(ctx context.Context, url string, auth AuthConfig) ([]BranchInfo, error) {- if err := ctx.Err(); err != nil {- return nil, err- }- refs, err := c.listRemoteReferencesInternal(ctx, url, auth)+ refs, err := c.listRemoteReferences(ctx, url, auth)if err != nil {return nil, err}- var branches []BranchInfo+ // HEAD is a symbolic reference to the default branch.var defaultBranch string-- // Find the default branch (HEAD points to it)- for _, ref := range refs {- if ref.Name().String() == "HEAD" {- // HEAD is a symbolic reference that points to the default branch- if ref.Target().IsBranch() {- defaultBranch = ref.Target().Short()- }- break- }+ if i := slices.IndexFunc(refs, func(ref *plumbing.Reference) bool { return ref.Name().String() == "HEAD" }); i >= 0 && refs[i].Target().IsBranch() {+ defaultBranch = refs[i].Target().Short()}- // Collect all branches+ var branches []BranchInfoseen := make(map[string]bool)for _, ref := range refs {- if ref.Name().IsBranch() {- branchName := ref.Name().Short()- if seen[branchName] {- continue- }- seen[branchName] = true-- branches = append(branches, BranchInfo{- Name: branchName,- IsDefault: branchName == defaultBranch,- })+ branchName := ref.Name().Short()+ if !ref.Name().IsBranch() || seen[branchName] {+ continue}+ seen[branchName] = true+ branches = append(branches, BranchInfo{Name: branchName, IsDefault: branchName == defaultBranch})}- // Sort branches with default first- sort.Slice(branches, func(i, j int) bool {- if branches[i].IsDefault {- return true- }- if branches[j].IsDefault {- return false+ slices.SortFunc(branches, func(a, b BranchInfo) int {+ if a.IsDefault != b.IsDefault {+ return kit.Ternary(a.IsDefault, -1, 1)}- return branches[i].Name < branches[j].Name+ return strings.Compare(a.Name, b.Name)})-return branches, nil}// ProbeRemote verifies that a remote repository is reachable without cloning it.func (c *Client) ProbeRemote(ctx context.Context, url string, auth AuthConfig) error {- if err := ctx.Err(); err != nil {- return err- }-- _, err := c.listRemoteReferencesInternal(ctx, url, auth)+ _, err := c.listRemoteReferences(ctx, url, auth)return err}-func (c *Client) listRemoteReferencesInternal(ctx context.Context, url string, auth AuthConfig) ([]*plumbing.Reference, error) {- url, err := normalizeURLInternal(url)+func (c *Client) listRemoteReferences(ctx context.Context, url string, auth AuthConfig) (refs []*plumbing.Reference, err error) {+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "gitrepo.list_remote")+ defer func() { tracing.End(span, err) }()++ if ctxErr := ctx.Err(); ctxErr != nil {+ return nil, ctxErr+ }+ url, err = normalizeURL(url)if err != nil {return nil, err}-- authMethod, err := c.getAuthInternal(ctx, url, auth)+ span.SetAttributes(RepositoryAttribute(url))+ authMethod, err := c.getAuth(ctx, url, auth)if err != nil {return nil, err}- // Create a remote without cloning- rem := git.NewRemote(nil, &config.RemoteConfig{- Name: "origin",- URLs: []string{url},- })-- listOptions := &git.ListOptions{}- if authMethod != nil {- listOptions.Auth = authMethod- }-listCtx, cancel := context.WithTimeout(ctx, 60*time.Second)defer cancel()- refs, err := rem.ListContext(listCtx, listOptions)+ rem := git.NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{url}})+ refs, err = rem.ListContext(listCtx, &git.ListOptions{Auth: authMethod})if err != nil {return nil, fmt.Errorf("failed to list remote references: %w", err)}-+ span.SetAttributes(attribute.Int("arcane.result.count", len(refs)))return refs, nil}// ValidatePath ensures the path is safe and doesn't escape the repofunc ValidatePath(repoPath, requestedPath string) error {- // Clean the pathscleanRepoPath := filepath.Clean(repoPath)cleanRequestedPath := filepath.Clean(filepath.Join(repoPath, requestedPath))- // Check if the requested path is within the repo using relative path validationrel, err := filepath.Rel(cleanRepoPath, cleanRequestedPath)if err != nil {return fmt.Errorf("invalid path: %w", err)@@ -531,9 +488,8 @@ func ValidatePath(repoPath, requestedPath string) error {return nil}-// BrowseTree returns the file tree at the specified path. The clone directory-// is the confinement root: paths that escape it, and symbolic links pointing-// outside it, are rejected rather than followed.+// BrowseTree returns the file tree at targetPath, confined to the clone directory: escaping paths and+// symlinks pointing outside it are rejected rather than followed.func (c *Client) BrowseTree(ctx context.Context, repoPath, targetPath string) ([]gitops.FileTreeNode, error) {if err := ctx.Err(); err != nil {return nil, err@@ -558,7 +514,6 @@ func (c *Client) BrowseTree(ctx context.Context, repoPath, targetPath string) ([if errErr := ctx.Err(); errErr != nil {return nil, errErr}- // Skip .git directoryif child.Name == ".git" {continue}@@ -576,9 +531,7 @@ func (c *Client) BrowseTree(ctx context.Context, repoPath, targetPath string) ([return nodes, nil}-// Cleanup removes a temporary repository directory-// Stays on os.RemoveAll: acfs refuses to remove its own root, and here the whole-// clone directory itself is what gets deleted.+// Cleanup removes a temporary repository directory with os.RemoveAll, since acfs refuses to remove its own root.func (c *Client) Cleanup(repoPath string) error {return os.RemoveAll(repoPath)}@@ -593,10 +546,7 @@ func (c *Client) Discard(ctx context.Context, repoPath string) {// PurgeScratchDirs removes clone scratch dirs ("gitops-*") under the work dir// whose mtime is older than maxAge. maxAge <= 0 removes all (boot sweep).func (c *Client) PurgeScratchDirs(ctx context.Context, maxAge time.Duration) (int, error) {- root := c.workDir- if root == "" {- root = os.TempDir()- }+ root := cmp.Or(c.workDir, os.TempDir())entries, err := os.ReadDir(root)if err != nil {@@ -634,27 +584,13 @@ func (c *Client) PurgeScratchDirs(ctx context.Context, maxAge time.Duration) (inreturn removed, nil}-// CommitInfo holds information about a git commit-type CommitInfo struct {- Hash string- Author string- Message string- Date time.Time-}-// TestConnection tests if the repository can be accessed with the given credentialsfunc (c *Client) TestConnection(ctx context.Context, url, branch string, auth AuthConfig) error {- if err := ctx.Err(); err != nil {- return err- }-tmpDir, err := c.Clone(ctx, url, branch, auth, 1)if err != nil {return err}- defer func() {- _ = c.Cleanup(tmpDir)- }()+ c.Discard(ctx, tmpDir)return nil}@@ -685,10 +621,7 @@ type SyncFileInfo struct {Content []byteSize int64IsBinary bool- // Executable mirrors git's +x bit so callers can preserve it on the- // destination. Required for lifecycle hooks: a script committed as- // 100755 must arrive in the project workspace runnable, otherwise the- // lifecycle runner fails to exec it.+ // Executable mirrors git's +x bit so lifecycle hook scripts arrive runnable.Executable bool}@@ -700,10 +633,8 @@ type DirectoryWalkResult struct {SkippedBinaries int}-// WalkDirectory walks the directory containing the compose file and returns all files.-// It enforces limits on file count, total size, and skips large binary files.-// The composePath is the path to the compose file within the repo - the directory-// containing this file will be walked.+// WalkDirectory returns every file in the compose file's directory, enforcing file count and+// total size limits and skipping large binary files.func (c *Client) WalkDirectory(ctx context.Context, repoPath, composePath string,maxFiles int, maxTotalSize, maxBinarySize int64,) (*DirectoryWalkResult, error) {@@ -711,12 +642,10 @@ func (c *Client) WalkDirectory(ctx context.Context, repoPath, composePath stringreturn nil, err}- // Validate compose pathif err := ValidatePath(repoPath, composePath); err != nil {return nil, fmt.Errorf("invalid compose path: %w", err)}- // Get the directory containing the compose filesyncDir := filepath.Dir(filepath.Join(repoPath, composePath))result := &DirectoryWalkResult{@@ -803,18 +732,13 @@ func IsBinaryContent(content []byte) bool {return false}- // Check first 512 bytes (or less if file is smaller)- checkSize := min(len(content), 512)-- // Use net/http's content type detection+ checkSize := min(len(content), binarySniffBytes)contentType := nethttp.DetectContentType(content[:checkSize])- // Text types are not binaryif strings.HasPrefix(contentType, "text/") {return false}- // Common text-based application typestextAppTypes := []string{"application/json","application/xml",@@ -830,13 +754,11 @@ func IsBinaryContent(content []byte) bool {}}- // For application/octet-stream, do additional null-byte check- // Text files rarely have null bytes, so their presence indicates binary+ // Text files rarely contain null bytes, so they mark octet-stream content as binary.if contentType == "application/octet-stream" {return slices.Contains(content[:checkSize], 0)}- // Everything else is considered binaryreturn strings.HasPrefix(contentType, "application/") ||strings.HasPrefix(contentType, "image/") ||strings.HasPrefix(contentType, "video/") ||diff --git a/backend/pkg/gitutil/git_test.go b/backend/pkg/gitutil/git_test.goindex cb95984e7..545a16902 100644--- a/backend/pkg/gitutil/git_test.go+++ b/backend/pkg/gitutil/git_test.go@@ -109,7 +109,7 @@ func TestGetKnownHostsPath(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- result, err := getKnownHostsPathsInternal(+ result, err := getKnownHostsPaths(tt.workDir,func(key string) string {require.Equal(t, "SSH_KNOWN_HOSTS", key)@@ -165,7 +165,7 @@ func TestGetSSHHostKeyCallback(t *testing.T) {t.Run("skip mode returns InsecureIgnoreHostKey", func(t *testing.T) {t.Setenv("SSH_KNOWN_HOSTS", filepath.Join(t.TempDir(), "missing", "known_hosts"))- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationSkip)+ callback, err := client.getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationSkip)require.NoError(t, err,"unexpected error: %v", err)@@ -185,7 +185,7 @@ func TestGetSSHHostKeyCallback(t *testing.T) {knownHostsPath := filepath.Join(tmpDir, "known_hosts")t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), "")+ callback, err := client.getSSHHostKeyCallback(t.Context(), "")require.NoError(t, err,"unexpected error: %v", err)@@ -199,7 +199,7 @@ func TestGetSSHHostKeyCallback(t *testing.T) {knownHostsPath := filepath.Join(tmpDir, "known_hosts")t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)+ callback, err := client.getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)require.NoError(t, err,"unexpected error: %v", err)@@ -218,7 +218,7 @@ func TestAddHostKey(t *testing.T) {key := generateTestPublicKey(t)t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)- callback, err := NewClient("").getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)+ callback, err := NewClient("").getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)require.NoError(t, err)err = callback("example.com:22", &net.TCPAddr{}, key)@@ -241,7 +241,7 @@ func TestAddHostKey(t *testing.T) {t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)key := generateTestPublicKey(t)- callback, callbackErr := NewClient("").getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)+ callback, callbackErr := NewClient("").getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)require.NoError(t, callbackErr)var wg sync.WaitGrouperrChan := make(chan error, 10)@@ -283,7 +283,7 @@ func TestCreateAcceptNewHostKeyCallback(t *testing.T) {t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)client := NewClient("")- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)+ callback, err := client.getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)require.NoError(t, err,"unexpected error: %v", err)@@ -305,7 +305,7 @@ func TestCreateAcceptNewHostKeyCallback(t *testing.T) {t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)client := NewClient("")- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)+ callback, err := client.getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)require.NoError(t, err,"unexpected error: %v", err)@@ -334,7 +334,7 @@ func TestCreateAcceptNewHostKeyCallback(t *testing.T) {t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)client := NewClient("")- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)+ callback, err := client.getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)require.NoError(t, err,"unexpected error: %v", err)@@ -349,7 +349,7 @@ func TestCreateAcceptNewHostKeyCallback(t *testing.T) {"first callback returned error: %v", err)// A new client must reuse the persisted trust file.- callback, err = NewClient("").getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationStrict)+ callback, err = NewClient("").getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationStrict)require.NoError(t, err)err = callback("192.168.1.1:22", addr, key)@@ -363,7 +363,7 @@ func TestCreateAcceptNewHostKeyCallback(t *testing.T) {t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)client := NewClient("")- callback, err := client.getSSHHostKeyCallbackInternal(t.Context(), SSHHostKeyVerificationAcceptNew)+ callback, err := client.getSSHHostKeyCallback(t.Context(), SSHHostKeyVerificationAcceptNew)require.NoError(t, err,"unexpected error: %v", err)@@ -431,7 +431,7 @@ func TestNewClient(t *testing.T) {})}-func writeFileInternal(t *testing.T, dir, name string, content []byte) {+func writeFile(t *testing.T, dir, name string, content []byte) {t.Helper()targetPath := filepath.Join(dir, name){@@ -453,9 +453,9 @@ func minimalCompose() []byte {func TestWalkDirectory_BasicWalk(t *testing.T) {tmpDir := t.TempDir()- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())- writeFileInternal(t, tmpDir, "file1.txt", []byte("hello world"))- writeFileInternal(t, tmpDir, "file2.txt", []byte("another file"))+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())+ writeFile(t, tmpDir, "file1.txt", []byte("hello world"))+ writeFile(t, tmpDir, "file2.txt", []byte("another file"))client := NewClient("")result, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 0, 0)@@ -472,9 +472,9 @@ func TestWalkDirectory_BasicWalk(t *testing.T) {func TestWalkDirectory_PreservesExecutableBit(t *testing.T) {tmpDir := t.TempDir()- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())- writeFileInternal(t, tmpDir, "scripts/hook.sh", []byte("#!/bin/sh\necho hi\n"))- writeFileInternal(t, tmpDir, "README.md", []byte("readme"))+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())+ writeFile(t, tmpDir, "scripts/hook.sh", []byte("#!/bin/sh\necho hi\n"))+ writeFile(t, tmpDir, "README.md", []byte("readme")){err := os.Chmod(filepath.Join(tmpDir, "scripts", "hook.sh"), 0o755)require.NoError(t, err,@@ -509,11 +509,11 @@ func TestWalkDirectory_PreservesExecutableBit(t *testing.T) {func TestWalkDirectory_MaxFilesLimit(t *testing.T) {tmpDir := t.TempDir()- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())- writeFileInternal(t, tmpDir, "a.txt", []byte("a"))- writeFileInternal(t, tmpDir, "b.txt", []byte("b"))- writeFileInternal(t, tmpDir, "c.txt", []byte("c"))- writeFileInternal(t, tmpDir, "d.txt", []byte("d"))+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())+ writeFile(t, tmpDir, "a.txt", []byte("a"))+ writeFile(t, tmpDir, "b.txt", []byte("b"))+ writeFile(t, tmpDir, "c.txt", []byte("c"))+ writeFile(t, tmpDir, "d.txt", []byte("d"))client := NewClient("")_, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 3, 0, 0)@@ -527,11 +527,11 @@ func TestWalkDirectory_MaxFilesLimit(t *testing.T) {func TestWalkDirectory_MaxFilesUnlimited(t *testing.T) {tmpDir := t.TempDir()- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())- writeFileInternal(t, tmpDir, "a.txt", []byte("a"))- writeFileInternal(t, tmpDir, "b.txt", []byte("b"))- writeFileInternal(t, tmpDir, "c.txt", []byte("c"))- writeFileInternal(t, tmpDir, "d.txt", []byte("d"))+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())+ writeFile(t, tmpDir, "a.txt", []byte("a"))+ writeFile(t, tmpDir, "b.txt", []byte("b"))+ writeFile(t, tmpDir, "c.txt", []byte("c"))+ writeFile(t, tmpDir, "d.txt", []byte("d"))client := NewClient("")result, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 0, 0)@@ -545,9 +545,9 @@ func TestWalkDirectory_MaxFilesUnlimited(t *testing.T) {func TestWalkDirectory_MaxTotalSizeLimit(t *testing.T) {tmpDir := t.TempDir()- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())- writeFileInternal(t, tmpDir, "big1.txt", []byte(strings.Repeat("x", 40)))- writeFileInternal(t, tmpDir, "big2.txt", []byte(strings.Repeat("y", 40)))+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())+ writeFile(t, tmpDir, "big1.txt", []byte(strings.Repeat("x", 40)))+ writeFile(t, tmpDir, "big2.txt", []byte(strings.Repeat("y", 40)))client := NewClient("")_, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 50, 0)@@ -561,9 +561,9 @@ func TestWalkDirectory_MaxTotalSizeLimit(t *testing.T) {func TestWalkDirectory_MaxTotalSizeUnlimited(t *testing.T) {tmpDir := t.TempDir()- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())- writeFileInternal(t, tmpDir, "big1.txt", []byte(strings.Repeat("x", 500)))- writeFileInternal(t, tmpDir, "big2.txt", []byte(strings.Repeat("y", 500)))+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())+ writeFile(t, tmpDir, "big1.txt", []byte(strings.Repeat("x", 500)))+ writeFile(t, tmpDir, "big2.txt", []byte(strings.Repeat("y", 500)))client := NewClient("")result, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 0, 0)@@ -577,10 +577,10 @@ func TestWalkDirectory_MaxTotalSizeUnlimited(t *testing.T) {func TestWalkDirectory_MaxBinarySizeSkips(t *testing.T) {tmpDir := t.TempDir()- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())// Binary content: null bytes cause IsBinaryContent to return truebinaryContent := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13}- writeFileInternal(t, tmpDir, "data.bin", binaryContent)+ writeFile(t, tmpDir, "data.bin", binaryContent)client := NewClient("")result, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 0, 5)@@ -594,9 +594,9 @@ func TestWalkDirectory_MaxBinarySizeSkips(t *testing.T) {func TestWalkDirectory_MaxBinarySizeUnlimited(t *testing.T) {tmpDir := t.TempDir()- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())binaryContent := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13}- writeFileInternal(t, tmpDir, "data.bin", binaryContent)+ writeFile(t, tmpDir, "data.bin", binaryContent)client := NewClient("")result, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 0, 0)@@ -613,8 +613,8 @@ func TestWalkDirectory_MaxBinarySizeUnlimited(t *testing.T) {func TestWalkDirectory_LargeTextFileNotSkippedByBinaryLimit(t *testing.T) {tmpDir := t.TempDir()- writeFileInternal(t, tmpDir, "compose.yaml", minimalCompose())- writeFileInternal(t, tmpDir, "notes.txt", []byte(strings.Repeat("plain text\n", 32)))+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())+ writeFile(t, tmpDir, "notes.txt", []byte(strings.Repeat("plain text\n", 32)))client := NewClient("")result, err := client.WalkDirectory(t.Context(), tmpDir, "compose.yaml", 0, 0, 16)@@ -631,8 +631,8 @@ func TestWalkDirectory_LargeTextFileNotSkippedByBinaryLimit(t *testing.T) {func TestWalkDirectory_ComposeInSubdirectory(t *testing.T) {tmpDir := t.TempDir()- writeFileInternal(t, tmpDir, "subdir/docker-compose.yml", minimalCompose())- writeFileInternal(t, tmpDir, "subdir/dynamic_config.yml", []byte("http:\n routers: {}\n"))+ writeFile(t, tmpDir, "subdir/docker-compose.yml", minimalCompose())+ writeFile(t, tmpDir, "subdir/dynamic_config.yml", []byte("http:\n routers: {}\n"))client := NewClient("")result, err := client.WalkDirectory(t.Context(), tmpDir, "subdir/docker-compose.yml", 0, 0, 0)@@ -661,8 +661,8 @@ func TestWalkDirectory_ComposeInSubdirectory(t *testing.T) {func TestWalkDirectory_NestedSiblingFile(t *testing.T) {tmpDir := t.TempDir()- writeFileInternal(t, tmpDir, "subdir/docker-compose.yml", minimalCompose())- writeFileInternal(t, tmpDir, "subdir/config/dynamic_config.yml", []byte("tls:\n certificates: []\n"))+ writeFile(t, tmpDir, "subdir/docker-compose.yml", minimalCompose())+ writeFile(t, tmpDir, "subdir/config/dynamic_config.yml", []byte("tls:\n certificates: []\n"))client := NewClient("")result, err := client.WalkDirectory(t.Context(), tmpDir, "subdir/docker-compose.yml", 0, 0, 0)@@ -691,8 +691,8 @@ func TestWalkDirectory_NestedSiblingFile(t *testing.T) {func TestWalkDirectory_SpecialCharsInPath(t *testing.T) {tmpDir := t.TempDir()- writeFileInternal(t, tmpDir, "traefik (nl10)/docker-compose.yml", minimalCompose())- writeFileInternal(t, tmpDir, "traefik (nl10)/config/dynamic_config.yml", []byte("http:\n middlewares: {}\n"))+ writeFile(t, tmpDir, "traefik (nl10)/docker-compose.yml", minimalCompose())+ writeFile(t, tmpDir, "traefik (nl10)/config/dynamic_config.yml", []byte("http:\n middlewares: {}\n"))client := NewClient("")result, err := client.WalkDirectory(t.Context(), tmpDir, "traefik (nl10)/docker-compose.yml", 0, 0, 0)@@ -836,7 +836,7 @@ func TestNormalizeURL(t *testing.T) {for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {- got, err := normalizeURLInternal(tc.in)+ got, err := normalizeURL(tc.in)if tc.wantErr {require.Error(t, err,diff --git a/backend/pkg/gitutil/git_write.go b/backend/pkg/gitutil/git_write.goindex c05d5859d..49c9e4872 100644--- a/backend/pkg/gitutil/git_write.go+++ b/backend/pkg/gitutil/git_write.go@@ -210,7 +210,7 @@ func (c *Client) CheckoutForWrite(ctx context.Context, url, branch string, authif strings.TrimSpace(branch) == "" {return nil, errors.New("branch is required")}- normalized, err := normalizeURLInternal(url)+ normalized, err := normalizeURL(url)if err != nil {return nil, err}@@ -361,7 +361,7 @@ func stageCommitFilesInternal(ctx context.Context, checkout *WriteCheckout, work// pushBranchInternal pushes the checkout branch without force and verifies the// remote now points at commit.func (c *Client) pushBranchInternal(ctx context.Context, checkout *WriteCheckout, commit string, auth AuthConfig) error {- authMethod, err := c.getAuthInternal(ctx, checkout.url, auth)+ authMethod, err := c.getAuth(ctx, checkout.url, auth)if err != nil {return err}@@ -400,7 +400,7 @@ func isPushRejectedInternal(err error) bool {// RemoteBranchHead resolves the remote head of branch without cloning.func (c *Client) RemoteBranchHead(ctx context.Context, url, branch string, auth AuthConfig) (string, bool, error) {- refs, err := c.listRemoteReferencesInternal(ctx, url, auth)+ refs, err := c.listRemoteReferences(ctx, url, auth)if err != nil {return "", false, kit.Ternary(errors.Is(err, transport.ErrEmptyRemoteRepository), nil, err)}diff --git a/backend/pkg/libarcane/edge/client.go b/backend/pkg/libarcane/edge/client.goindex 2c50f2795..e601e7727 100644--- a/backend/pkg/libarcane/edge/client.go+++ b/backend/pkg/libarcane/edge/client.go@@ -2,8 +2,8 @@ package edgeimport ("bytes"+ "cmp""context"- "crypto/tls""errors""fmt""io"@@ -18,7 +18,6 @@ import ("github.com/cenkalti/backoff/v5""github.com/coder/websocket"- "github.com/samber/mo"kit "go.getarcane.app/kit/pkg"wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"@@ -46,36 +45,47 @@ const (// DefaultRequestTimeout is the timeout for executing local requestsDefaultRequestTimeout = 5 * time.Minute// DefaultGRPCRegistrationTimeout bounds how long the agent waits for the- // manager to acknowledge tunnel registration on any transport before- // treating it as a failed transport attempt.+ // manager to acknowledge tunnel registration on any transport.DefaultGRPCRegistrationTimeout = 10 * time.Second// DefaultWebSocketPreferenceTTL keeps websocket as the preferred transport// for a short period after a successful auto-mode fallback.DefaultWebSocketPreferenceTTL = 2 * time.Minute- // maxWebSocketPreferenceTTL caps the exponential backoff applied to the- // websocket preference window while gRPC keeps failing, so a persistently- // broken gRPC path is retried at most this often but never disabled.+ // maxWebSocketPreferenceTTL caps the backoff of the websocket preference window+ // while gRPC keeps failing, so a broken gRPC path is retried but never disabled.maxWebSocketPreferenceTTL = 30 * time.MinutedefaultCommandChunkSize = 256 * 1024-- // errTunnelRegistrationTimeout marks a registration attempt where the manager- // accepted the connection but never answered the register message.-- // errEstablishedTunnelSessionEnded marks a session that the manager accepted- // and later dropped, as opposed to a transport that never connected.-)var (+ // errTunnelRegistrationTimeout marks a manager that accepted the connection but never answered registration.errTunnelRegistrationTimeout = errors.New("timed out waiting for tunnel registration response")-+ // errEstablishedTunnelSessionEnded marks an accepted session the manager later dropped.errEstablishedTunnelSessionEnded = errors.New("established edge tunnel session ended"))-func (t *commandRequestTransfer) stopInternal() {- if t == nil {- return- }+// localDialSkipHeaders lists headers not forwarded when the agent dials its own server for a+// proxied WebSocket: handshake headers and browser headers that fail the local origin check.+var localDialSkipHeaders = map[string]bool{+ "Sec-Websocket-Key": true,+ "Sec-Websocket-Version": true,+ "Sec-Websocket-Extensions": true,+ "Upgrade": true,+ "Connection": true,+ "Host": true,+ "Origin": true,+ "Cookie": true,+ "Authorization": true,+ "Referer": true,+ "Sec-Fetch-Dest": true,+ "Sec-Fetch-Mode": true,+ "Sec-Fetch-Site": true,+ "Sec-Fetch-User": true,+ "Sec-Ch-Ua": true,+ "Sec-Ch-Ua-Mobile": true,+ "Sec-Ch-Ua-Platform": true,+}++func (t *commandRequestTransfer) stop() {t.timerMu.Lock()defer t.timerMu.Unlock()if t.timer != nil {@@ -90,19 +100,6 @@ func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {reconnectInterval = 5 * time.Second}- managerURL := ""- if managerBaseURL := strings.TrimRight(httpx.ManagerBaseURL(cfg.ManagerApiUrl), "/"); managerBaseURL != "" {- // Convert HTTP to WebSocket URL- managerURL = HTTPToWebSocketURL(managerBaseURL) + "/api/tunnel/connect"- }- managerGRPCAddr := httpx.ManagerGRPCAddr(cfg.ManagerApiUrl)-- // Get local port for WebSocket dialing- localPort := cfg.Port- if localPort == "" {- localPort = "3552" // Default port- }-return &TunnelClient{cfg: cfg,handler: handler,@@ -111,9 +108,8 @@ func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {registrationTimeout: DefaultGRPCRegistrationTimeout,websocketPreferenceTTL: DefaultWebSocketPreferenceTTL,healthySessionDuration: healthyTunnelSessionDuration,- managerURL: managerURL,- managerGRPCAddr: managerGRPCAddr,- localPort: localPort,+ managerGRPCAddr: httpx.ManagerGRPCAddr(cfg.ManagerApiUrl),+ localPort: cmp.Or(cfg.Port, "3552"),requestTimeout: DefaultRequestTimeout,agentInstanceID: uuid.New().String(),}@@ -125,50 +121,43 @@ func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error)defer close(errCh)}slog.InfoContext(ctx, "Starting edge agent session client", StartupLogAttrs(c.cfg)...)- // A fixed reconnect interval meant a permanently broken or unauthorized agent- // hammered the manager's auth and DB path forever. Back off exponentially- // (with jitter) up to maxReconnectInterval, and reset only once a session has- // stayed up long enough to count as healthy.+ // Back off exponentially (with jitter) so a broken or unauthorized agent does not hammer the+ // manager, resetting only once a session has stayed up long enough to count as healthy.reconnectBackoff := backoff.NewExponentialBackOff()reconnectBackoff.InitialInterval = c.reconnectIntervalreconnectBackoff.MaxInterval = maxReconnectInterval+ serve := kit.Ternary(UsePollEdgeTransport(c.cfg), c.connectAndServePoll, c.connectAndServeManagedTunnel)for {- select {- case <-ctx.Done():+ if ctx.Err() != nil {slog.InfoContext(ctx, "Edge tunnel client shutting down")return- default:- sessionStart := time.Now()- if err := c.connectAndServe(ctx); err != nil {- slog.WarnContext(ctx, "Edge tunnel disconnected", "error", err)- if errCh != nil {- select {- case errCh <- err:- default:- }+ }+ sessionStart := time.Now()+ if err := serve(ctx); err != nil {+ slog.WarnContext(ctx, "Edge tunnel disconnected", "error", err)+ if errCh != nil {+ select {+ case errCh <- err:+ default:}}+ }+ if time.Since(sessionStart) >= c.healthySessionDuration {+ reconnectBackoff.Reset()+ }- if time.Since(sessionStart) >= healthyTunnelSessionDuration {- reconnectBackoff.Reset()- }-- // Wait before reconnecting- delay := reconnectBackoff.NextBackOff()- reconnectTimer := time.NewTimer(delay)- select {- case <-ctx.Done():- reconnectTimer.Stop()- return- case <-reconnectTimer.C:- reconnectTimer.Stop()- slog.InfoContext(ctx, "Attempting to reconnect edge tunnel", "delay", delay)- }+ delay := reconnectBackoff.NextBackOff()+ select {+ case <-ctx.Done():+ return+ case <-time.After(delay):+ slog.InfoContext(ctx, "Attempting to reconnect edge tunnel", "delay", delay)}}}+// StartupLogAttrs returns the slog attributes describing the agent's edge transport configuration.func StartupLogAttrs(cfg *Config) []any {if cfg == nil {return []any{@@ -179,53 +168,47 @@ func StartupLogAttrs(cfg *Config) []any {}}- controlPlane := kit.Ternary(UsePollEdgeTransport(cfg), EdgeTransportPoll, "managed")-+ poll := UsePollEdgeTransport(cfg)+ managerGRPCAddr := strings.TrimSpace(httpx.ManagerGRPCAddr(cfg.ManagerApiUrl))+ managerBaseURL := strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl))managedSessionTransports := make([]string, 0, 2)- if UseGRPCEdgeTransport(cfg) || (UsePollEdgeTransport(cfg) && strings.TrimSpace(httpx.ManagerGRPCAddr(cfg.ManagerApiUrl)) != "") {+ if UseGRPCEdgeTransport(cfg) || (poll && managerGRPCAddr != "") {managedSessionTransports = append(managedSessionTransports, EdgeTransportGRPC)}- if UseWebSocketEdgeTransport(cfg) || (UsePollEdgeTransport(cfg) && strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)) != "") {+ if UseWebSocketEdgeTransport(cfg) || (poll && managerBaseURL != "") {managedSessionTransports = append(managedSessionTransports, EdgeTransportWebSocket)}attrs := []any{- "control_plane", controlPlane,+ "control_plane", kit.Ternary(poll, EdgeTransportPoll, "managed"),"managed_session_transports", managedSessionTransports,"security_mode", NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode),}-if managerAPIURL := strings.TrimSpace(cfg.ManagerApiUrl); managerAPIURL != "" {attrs = append(attrs, "manager_api_url", managerAPIURL)}- if managerGRPCAddr := strings.TrimSpace(httpx.ManagerGRPCAddr(cfg.ManagerApiUrl)); managerGRPCAddr != "" {+ if managerGRPCAddr != "" {attrs = append(attrs, "manager_grpc_addr", managerGRPCAddr)}- if managerBaseURL := strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)); managerBaseURL != "" {+ if managerBaseURL != "" {attrs = append(attrs, "manager_base_url", managerBaseURL)}-return attrs}-// connectAndServe establishes a connection and handles messages.-func (c *TunnelClient) connectAndServe(ctx context.Context) error {- if UsePollEdgeTransport(c.cfg) {- return c.connectAndServePoll(ctx)- }- return c.connectAndServeManagedTunnelInternal(ctx)-}--func (c *TunnelClient) connectAndServeManagedTunnelInternal(ctx context.Context) error {- transports := c.managedTunnelTransportsInternal()+func (c *TunnelClient) connectAndServeManagedTunnel(ctx context.Context) error {+ transports := c.managedTunnelTransports()if transports.grpc {- if preferredUntil, ok := c.preferredWebSocketUntilInternal(time.Now()).Get(); ok {+ c.transportPreferenceMu.RLock()+ preferredUntil, grpcFailureStreak := c.preferWebSocketUntil, c.grpcFailureStreak+ c.transportPreferenceMu.RUnlock()+ if transports.websocket && time.Now().Before(preferredUntil) {slog.InfoContext(ctx, "Temporarily preferring websocket edge tunnel transport after recent websocket success","preferredUntil", preferredUntil,- "grpcFailureStreak", c.grpcFailureStreakInternal(),- "managerWsUrl", c.managerWebSocketURLInternal(),+ "grpcFailureStreak", grpcFailureStreak,+ "managerWsUrl", c.managerWebSocketURL(),)- return c.connectAndServeWebSocketInternal(ctx)+ return c.connectAndServeWebSocket(ctx)}sessionStart := time.Now()@@ -233,21 +216,21 @@ func (c *TunnelClient) connectAndServeManagedTunnelInternal(ctx context.Context)if ctx.Err() != nil {return ctx.Err()}- // A healthy gRPC session that the manager dropped (restart, redeploy)- // is not a broken gRPC path: let the outer loop back off and retry- // gRPC first instead of racing a websocket dial against the restart.- if errors.Is(err, errEstablishedTunnelSessionEnded) && time.Since(sessionStart) >= c.healthySessionDurationInternal() {+ // A healthy session the manager dropped (restart, redeploy) is not a broken gRPC path:+ // let the outer loop back off and retry gRPC instead of racing a websocket dial.+ if errors.Is(err, errEstablishedTunnelSessionEnded) && time.Since(sessionStart) >= c.healthySessionDuration {return err}- c.noteGRPCTunnelFailureInternal()+ c.transportPreferenceMu.Lock()+ c.grpcFailureStreak+++ c.transportPreferenceMu.Unlock()if transports.websocket {- managerWSURL := c.managerWebSocketURLInternal()slog.WarnContext(ctx, "gRPC edge tunnel connection failed, falling back to websocket transport","error", err,"managerGrpcAddr", c.managerGRPCAddr,- "managerWsUrl", managerWSURL,+ "managerWsUrl", c.managerWebSocketURL(),)- if wsErr := c.connectAndServeWebSocketInternal(ctx); wsErr != nil {+ if wsErr := c.connectAndServeWebSocket(ctx); wsErr != nil {// Keep both transport failures in the chain for errors.Is/errors.As traversal.return fmt.Errorf("gRPC edge tunnel failed: %w; websocket fallback failed: %w", err, wsErr)}@@ -258,48 +241,31 @@ func (c *TunnelClient) connectAndServeManagedTunnelInternal(ctx context.Context)return nil}if transports.websocket {- return c.connectAndServeWebSocketInternal(ctx)+ return c.connectAndServeWebSocket(ctx)}return errors.New("no edge tunnel transport is available")}-func (c *TunnelClient) managedTunnelTransportsInternal() managedTunnelTransportsInternal {+func (c *TunnelClient) managedTunnelTransports() managedTunnelTransportsInternal {if c == nil || c.cfg == nil {return managedTunnelTransportsInternal{}}managerGRPCAvailable := strings.TrimSpace(c.managerGRPCAddr) != ""- managerWebSocketAvailable := c.managerWebSocketURLInternal() != ""- transport := NormalizeEdgeTransport(c.cfg.EdgeTransport)-- switch transport {- case EdgeTransportAuto:- return managedTunnelTransportsInternal{- grpc: managerGRPCAvailable,- websocket: managerWebSocketAvailable,- }- case EdgeTransportPoll:- return managedTunnelTransportsInternal{- grpc: managerGRPCAvailable,- websocket: managerWebSocketAvailable,- }+ managerWebSocketAvailable := c.managerWebSocketURL() != ""+ switch NormalizeEdgeTransport(c.cfg.EdgeTransport) {+ case EdgeTransportAuto, EdgeTransportPoll:+ return managedTunnelTransportsInternal{grpc: managerGRPCAvailable, websocket: managerWebSocketAvailable}case EdgeTransportWebSocket:- return managedTunnelTransportsInternal{- websocket: managerWebSocketAvailable,- }+ return managedTunnelTransportsInternal{websocket: managerWebSocketAvailable}case EdgeTransportGRPC:- return managedTunnelTransportsInternal{- grpc: managerGRPCAvailable,- }+ return managedTunnelTransportsInternal{grpc: managerGRPCAvailable}default:return managedTunnelTransportsInternal{}}}-func (c *TunnelClient) managerWebSocketURLInternal() string {- if c == nil {- return ""- }+func (c *TunnelClient) managerWebSocketURL() string {if managerURL := strings.TrimSpace(c.managerURL); managerURL != "" {return managerURL}@@ -313,129 +279,19 @@ func (c *TunnelClient) managerWebSocketURLInternal() string {return HTTPToWebSocketURL(managerBaseURL) + "/api/tunnel/connect"}-func (c *TunnelClient) registrationTimeoutInternal() time.Duration {- if c == nil || c.registrationTimeout <= 0 {- return DefaultGRPCRegistrationTimeout- }- return c.registrationTimeout-}--func (c *TunnelClient) requestTimeoutInternal() time.Duration {- if c == nil || c.requestTimeout <= 0 {- return DefaultRequestTimeout- }- return c.requestTimeout-}--func (c *TunnelClient) healthySessionDurationInternal() time.Duration {- if c == nil || c.healthySessionDuration <= 0 {- return healthyTunnelSessionDuration- }- return c.healthySessionDuration-}--func (c *TunnelClient) websocketPreferenceTTLInternal() time.Duration {- if c == nil || c.websocketPreferenceTTL <= 0 {- return DefaultWebSocketPreferenceTTL- }- return c.websocketPreferenceTTL-}--func (c *TunnelClient) preferredWebSocketUntilInternal(now time.Time) mo.Option[time.Time] {- // A nil client yields a zero transports struct, so this also guards nil.- transports := c.managedTunnelTransportsInternal()- if !transports.grpc || !transports.websocket {- return mo.None[time.Time]()- }-- c.transportPreferenceMu.RLock()- defer c.transportPreferenceMu.RUnlock()-- if c.preferWebSocketUntil.IsZero() || !now.Before(c.preferWebSocketUntil) {- return mo.None[time.Time]()- }-- return mo.Some(c.preferWebSocketUntil)-}--func (c *TunnelClient) markTransportConnectedInternal(transport string) {- if c == nil {- return- }-- c.transportPreferenceMu.Lock()- defer c.transportPreferenceMu.Unlock()-- switch transport {- case EdgeTransportGRPC:- c.preferWebSocketUntil = time.Time{}- c.grpcFailureStreak = 0- case EdgeTransportWebSocket:- if transports := c.managedTunnelTransportsInternal(); transports.grpc && transports.websocket {- // Back the preference window off exponentially while gRPC keeps- // failing so each reconnect does not re-pay the registration- // timeout against a persistently broken gRPC path.- ttl := c.websocketPreferenceTTLInternal()- if streak := min(c.grpcFailureStreak, 4); streak > 0 {- ttl = min(ttl<<streak, maxWebSocketPreferenceTTL)- }- c.preferWebSocketUntil = time.Now().Add(ttl)- }- }-}--func (c *TunnelClient) noteGRPCTunnelFailureInternal() {- if c == nil {- return- }- c.transportPreferenceMu.Lock()- defer c.transportPreferenceMu.Unlock()- c.grpcFailureStreak++-}--func (c *TunnelClient) grpcFailureStreakInternal() int {- if c == nil {- return 0- }- c.transportPreferenceMu.RLock()- defer c.transportPreferenceMu.RUnlock()- return c.grpcFailureStreak-}--func (c *TunnelClient) registerMessageInternal() *TunnelMessage {- capabilities := AdvertisedEdgeCommands()- capabilities = append(capabilities, tunnelCapabilityChunkedRequest, tunnelCapabilityProtoParity, tunnelCapabilityCommandCredit)- registration, _ := c.registration.Load()- return &TunnelMessage{- Type: MessageTypeRegister,- AgentToken: c.cfg.AgentToken,- AgentInstance: c.agentInstanceID,- Capabilities: capabilities,- ResumeSession: registration.sessionID,- }-}--func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context, conn TunnelConnection) (*TunnelMessage, error) {- if c == nil {- return nil, errors.New("edge tunnel connection is not initialized")- }- if conn == nil {- return nil, errors.New("edge tunnel connection is not initialized")- }-+func (c *TunnelClient) awaitRegistration(ctx context.Context, conn TunnelConnection) (*TunnelMessage, error) {type registrationResult struct {msg *TunnelMessageerr error}- timeout := c.registrationTimeoutInternal()recvCh := make(chan registrationResult, 1)go func() {msg, err := conn.Receive()recvCh <- registrationResult{msg: msg, err: err}}()- timer := time.NewTimer(timeout)+ timer := time.NewTimer(c.registrationTimeout)defer timer.Stop()select {@@ -444,7 +300,7 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context, conn Tunnereturn nil, ctx.Err()case <-timer.C:_ = conn.Close()- return nil, fmt.Errorf("after %s: %w", timeout, errTunnelRegistrationTimeout)+ return nil, fmt.Errorf("after %s: %w", c.registrationTimeout, errTunnelRegistrationTimeout)case result := <-recvCh:if result.err != nil {return nil, fmt.Errorf("failed to receive tunnel registration response: %w", result.err)@@ -466,9 +322,9 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context, conn Tunne}}-// serveTunnelSessionInternal runs the shared register/heartbeat/message-// lifecycle on an established tunnel connection, for every transport.-func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn TunnelConnection, managerAddr string) error {+// serveTunnelSession runs the shared register/heartbeat/message lifecycle on an established+// tunnel connection, for every transport.+func (c *TunnelClient) serveTunnelSession(ctx context.Context, conn TunnelConnection, managerAddr string) error {box := &connBox{conn: conn}c.conn.Store(box)setActiveAgentTunnelConn(conn)@@ -477,8 +333,24 @@ func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn TunnstopCloseWatch := context.AfterFunc(connCtx, func() { _ = conn.Close() })defer func() {connCancel()- c.closeAllStreams()- c.clearCommandRequestTransfersInternal(conn)+ // Tear down streams and pending transfers so a reconnect cannot leak goroutines or sockets.+ c.activeStreams.Range(func(key, value any) bool {+ streamID, idOK := key.(string)+ stream, streamOK := value.(*activeWSStream)+ if idOK && streamOK {+ c.closeWebSocketStream(streamID, stream)+ }+ return true+ })+ c.requestTransfers.Range(func(transferID, value any) bool {+ switch transfer, ok := value.(*commandRequestTransfer); {+ case !ok:+ c.requestTransfers.Delete(transferID)+ case transfer.conn == conn && c.requestTransfers.CompareAndDelete(transferID, transfer):+ transfer.stop()+ }+ return true+ })_ = conn.Close()stopCloseWatch()drainCtx, cancelDrain := context.WithTimeout(context.WithoutCancel(ctx), tunnelSessionWorkerDrainTimeout)@@ -490,14 +362,22 @@ func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn TunnclearActiveAgentTunnelConn(conn)}()- if err := conn.Send(c.registerMessageInternal()); err != nil {+ registration, _ := c.registration.Load()+ registerMsg := &TunnelMessage{+ Type: MessageTypeRegister,+ AgentToken: c.cfg.AgentToken,+ AgentInstance: c.agentInstanceID,+ Capabilities: append(AdvertisedEdgeCommands(), tunnelCapabilityChunkedRequest, tunnelCapabilityProtoParity, tunnelCapabilityCommandCredit),+ ResumeSession: registration.sessionID,+ }+ if err := conn.Send(registerMsg); err != nil {// A rejected gRPC stream can report EOF from Send; Recv carries the RPC status.if conn.Transport() != EdgeTransportGRPC || !errors.Is(err, io.EOF) {return fmt.Errorf("failed to send %s tunnel register message: %w", conn.Transport(), err)}}- registerMsg, err := c.awaitRegistrationInternal(ctx, conn)+ registerResp, err := c.awaitRegistration(ctx, conn)if err != nil {return err}@@ -505,17 +385,25 @@ func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn Tunnslog.InfoContext(ctx, "Edge tunnel connected to manager","transport", conn.Transport(),"managerAddr", managerAddr,- "environmentId", registerMsg.EnvironmentID,- "sessionId", registerMsg.SessionID,+ "environmentId", registerResp.EnvironmentID,+ "sessionId", registerResp.SessionID,)- c.markTransportConnectedInternal(conn.Transport())+ c.transportPreferenceMu.Lock()+ switch conn.Transport() {+ case EdgeTransportGRPC:+ c.preferWebSocketUntil, c.grpcFailureStreak = time.Time{}, 0+ case EdgeTransportWebSocket:+ if transports := c.managedTunnelTransports(); transports.grpc && transports.websocket {+ // Back off the preference window while gRPC keeps failing so reconnects do not re-pay its timeout.+ ttl := min(c.websocketPreferenceTTL<<min(c.grpcFailureStreak, 4), maxWebSocketPreferenceTTL)+ c.preferWebSocketUntil = time.Now().Add(ttl)+ }+ }+ c.transportPreferenceMu.Unlock()workers.Go(func() { c.heartbeatLoop(connCtx, conn) })- if messageLoopErr := c.messageLoop(connCtx, conn, &workers); messageLoopErr != nil {- return fmt.Errorf("%w: %w", errEstablishedTunnelSessionEnded, messageLoopErr)- }- return nil+ return fmt.Errorf("%w: %w", errEstablishedTunnelSessionEnded, c.messageLoop(connCtx, conn, &workers))}// heartbeatLoop sends periodic heartbeats@@ -532,12 +420,7 @@ func (c *TunnelClient) heartbeatLoop(ctx context.Context, conn TunnelConnection)return}- msg := &TunnelMessage{- ID: uuid.New().String(),- Type: MessageTypeHeartbeat,- }-- if err := conn.Send(msg); err != nil {+ if err := conn.Send(&TunnelMessage{ID: uuid.New().String(), Type: MessageTypeHeartbeat}); err != nil {slog.WarnContext(ctx, "Failed to send heartbeat", "error", err)// Force reconnect so the manager does not keep stale state without heartbeats.if closeErr := conn.Close(); closeErr != nil {@@ -553,96 +436,92 @@ func (c *TunnelClient) heartbeatLoop(ctx context.Context, conn TunnelConnection)// messageLoop processes incoming messages from the managerfunc (c *TunnelClient) messageLoop(ctx context.Context, conn TunnelConnection, workers *sync.WaitGroup) error {for {- select {- case <-ctx.Done():- return ctx.Err()- default:- msg, err := conn.Receive()- if err != nil {- return fmt.Errorf("failed to receive message: %w", err)- }+ if err := ctx.Err(); err != nil {+ return err+ }+ msg, err := conn.Receive()+ if err != nil {+ return fmt.Errorf("failed to receive message: %w", err)+ }- switch msg.Type {- case MessageTypeRequest:- workers.Go(func() { c.handleRequest(ctx, conn, msg) })- case MessageTypeCommandRequest:- if transferID := strings.TrimSpace(msg.Metadata[bodyTransferMetadataKey]); transferID != "" {- c.beginCommandRequestTransferInternal(ctx, conn, transferID, msg)- } else {- workers.Go(func() { c.handleCommandRequest(ctx, conn, msg) })+ switch msg.Type {+ case MessageTypeRequest:+ workers.Go(func() { c.handleRequest(ctx, conn, msg) })+ case MessageTypeCommandRequest:+ if transferID := strings.TrimSpace(msg.Metadata[bodyTransferMetadataKey]); transferID != "" {+ c.beginCommandRequestTransfer(ctx, conn, transferID, msg)+ } else {+ workers.Go(func() { c.handleCommandRequest(ctx, conn, msg) })+ }+ case MessageTypeFileChunk:+ c.handleCommandRequestChunk(ctx, msg, workers)+ case MessageTypeWebSocketStart, MessageTypeStreamOpen:+ c.handleWebSocketStart(ctx, conn, msg, workers)+ case MessageTypeWebSocketData, MessageTypeStreamData:+ c.handleStreamData(ctx, msg)+ case MessageTypeWebSocketClose, MessageTypeStreamClose:+ if value, ok := c.activeStreams.Load(msg.ID); ok {+ if stream, isStream := value.(*activeWSStream); isStream {+ c.closeWebSocketStream(msg.ID, stream)+ slog.DebugContext(ctx, "Closed WebSocket stream", "streamId", msg.ID)}- case MessageTypeFileChunk:- c.handleCommandRequestChunkInternal(ctx, msg, workers)- case MessageTypeWebSocketStart:- c.handleWebSocketStart(ctx, conn, msg, workers)- case MessageTypeStreamOpen:- c.handleStreamOpen(ctx, conn, msg, workers)- case MessageTypeWebSocketData:- c.handleWebSocketData(ctx, msg)- case MessageTypeStreamData:- c.handleStreamData(ctx, msg)- case MessageTypeWebSocketClose:- c.handleWebSocketClose(ctx, msg)- case MessageTypeStreamClose:- c.handleStreamClose(ctx, msg)- case MessageTypeCancelRequest:- slog.DebugContext(ctx, "Ignoring edge cancel request on agent", "id", msg.ID)- case MessageTypeCommandCredit:- if value, ok := c.commandRecorders.Load(msg.ID); ok {- if recorder, isRecorder := value.(*commandResponseRecorder); isRecorder {- recorder.inFlight.Add(-msg.Credit)- select {- case recorder.credited <- struct{}{}:- default:- }+ }+ case MessageTypeCancelRequest:+ slog.DebugContext(ctx, "Ignoring edge cancel request on agent", "id", msg.ID)+ case MessageTypeCommandCredit:+ if value, ok := c.commandRecorders.Load(msg.ID); ok {+ if recorder, isRecorder := value.(*commandResponseRecorder); isRecorder {+ recorder.inFlight.Add(-msg.Credit)+ select {+ case recorder.credited <- struct{}{}:+ default:}}- case MessageTypeResponse, MessageTypeHeartbeat, MessageTypeStreamEnd, MessageTypeEvent, MessageTypeCommandAck, MessageTypeCommandOutput, MessageTypeCommandComplete:- slog.DebugContext(ctx, "Ignoring message type on agent", "type", msg.Type)- case MessageTypeHeartbeatAck:- slog.DebugContext(ctx, "Received heartbeat ack")- case MessageTypeRegisterResponse:- if !msg.Accepted {- return fmt.Errorf("manager rejected tunnel registration: %s", msg.Error)- }- slog.InfoContext(ctx, "Edge tunnel re-registered",- "transport", conn.Transport(),- "environmentId", msg.EnvironmentID,- )- case MessageTypeRegister:- slog.DebugContext(ctx, "Ignoring register message on agent")- default:- slog.WarnContext(ctx, "Unknown message type", "type", msg.Type)}+ case MessageTypeResponse, MessageTypeHeartbeat, MessageTypeStreamEnd, MessageTypeEvent, MessageTypeCommandAck, MessageTypeCommandOutput, MessageTypeCommandComplete:+ slog.DebugContext(ctx, "Ignoring message type on agent", "type", msg.Type)+ case MessageTypeHeartbeatAck:+ slog.DebugContext(ctx, "Received heartbeat ack")+ case MessageTypeRegisterResponse:+ if !msg.Accepted {+ return fmt.Errorf("manager rejected tunnel registration: %s", msg.Error)+ }+ slog.InfoContext(ctx, "Edge tunnel re-registered",+ "transport", conn.Transport(),+ "environmentId", msg.EnvironmentID,+ )+ case MessageTypeRegister:+ slog.DebugContext(ctx, "Ignoring register message on agent")+ default:+ slog.WarnContext(ctx, "Unknown message type", "type", msg.Type)}}}-func (c *TunnelClient) beginCommandRequestTransferInternal(ctx context.Context, conn TunnelConnection, transferID string, msg *TunnelMessage) {+func (c *TunnelClient) beginCommandRequestTransfer(ctx context.Context, conn TunnelConnection, transferID string, msg *TunnelMessage) {if c == nil || conn == nil || msg == nil {return}- timeout := c.requestTimeoutInternal()transfer := &commandRequestTransfer{request: msg, conn: conn}transfer.timerMu.Lock()previous, loaded := c.requestTransfers.Swap(transferID, transfer)- transfer.timer = time.AfterFunc(timeout, func() {+ transfer.timer = time.AfterFunc(c.requestTimeout, func() {if c.requestTransfers.CompareAndDelete(transferID, transfer) {slog.WarnContext(ctx, "Command body transfer expired", "transferId", transferID, "commandId", msg.ID)- c.sendCommandCompleteInternal(conn, msg.ID, http.StatusRequestTimeout, "command body transfer timed out")+ c.sendCommandComplete(conn, msg.ID, http.StatusRequestTimeout, "command body transfer timed out")}})transfer.timerMu.Unlock()if loaded {if previous, ok := previous.(*commandRequestTransfer); ok {- previous.stopInternal()- c.sendCommandCompleteInternal(previous.conn, previous.request.ID, http.StatusBadRequest, "duplicate command body transfer ID")+ previous.stop()+ c.sendCommandComplete(previous.conn, previous.request.ID, http.StatusBadRequest, "duplicate command body transfer ID")}}}-func (c *TunnelClient) handleCommandRequestChunkInternal(ctx context.Context, msg *TunnelMessage, workers *sync.WaitGroup) {+func (c *TunnelClient) handleCommandRequestChunk(ctx context.Context, msg *TunnelMessage, workers *sync.WaitGroup) {if c == nil || msg == nil {return}@@ -660,8 +539,8 @@ func (c *TunnelClient) handleCommandRequestChunkInternal(ctx context.Context, ms}if msg.Sequence != transfer.nextSequence {if c.requestTransfers.CompareAndDelete(msg.ID, transfer) {- transfer.stopInternal()- c.sendCommandCompleteInternal(transfer.conn, transfer.request.ID, http.StatusBadRequest, "command body chunks arrived out of order")+ transfer.stop()+ c.sendCommandComplete(transfer.conn, transfer.request.ID, http.StatusBadRequest, "command body chunks arrived out of order")}return}@@ -673,36 +552,15 @@ func (c *TunnelClient) handleCommandRequestChunkInternal(ctx context.Context, ms}if c.requestTransfers.CompareAndDelete(msg.ID, transfer) {- transfer.stopInternal()+ transfer.stop()transfer.request.Body = append([]byte(nil), transfer.body.Bytes()...)workers.Go(func() { c.handleCommandRequest(ctx, transfer.conn, transfer.request) })}}-func (c *TunnelClient) clearCommandRequestTransfersInternal(conn TunnelConnection) {- if c == nil {- return- }-- c.requestTransfers.Range(func(transferID, value any) bool {- transfer, ok := value.(*commandRequestTransfer)- if !ok {- c.requestTransfers.Delete(transferID)- return true- }- if transfer.conn != conn {- return true- }- if c.requestTransfers.CompareAndDelete(transferID, transfer) {- transfer.stopInternal()- }- return true- })-}-func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {if !ValidateEdgeCommand(msg.Command, msg.Method, msg.Path, false) {- c.sendCommandCompleteInternal(conn, msg.ID, http.StatusBadRequest, "unsupported edge command")+ c.sendCommandComplete(conn, msg.ID, http.StatusBadRequest, "unsupported edge command")return}if conn == nil {@@ -713,12 +571,12 @@ func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConnreturn}- reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeoutInternal())+ reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeout)defer cancel()req, err := c.buildLocalHTTPRequest(reqCtx, msg)if err != nil {- c.sendCommandCompleteInternal(conn, msg.ID, http.StatusInternalServerError, fmt.Sprintf("failed to create request: %v", err))+ c.sendCommandComplete(conn, msg.ID, http.StatusInternalServerError, fmt.Sprintf("failed to create request: %v", err))return}@@ -744,30 +602,17 @@ func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConnfunc (c *TunnelClient) buildLocalHTTPRequest(ctx context.Context, msg *TunnelMessage) (*http.Request, error) {var body io.Reader- var bodyBytes []byteif len(msg.Body) > 0 {- bodyBytes = append([]byte(nil), msg.Body...)- body = bytes.NewReader(bodyBytes)- }-- path := msg.Path- if msg.Query != "" {- path = path + "?" + msg.Query+ body = bytes.NewReader(append([]byte(nil), msg.Body...))}+ path := msg.Path + kit.Ternary(msg.Query != "", "?"+msg.Query, "")req, err := http.NewRequestWithContext(ctx, msg.Method, path, body)if err != nil {return nil, err}- if bodyBytes != nil {- req.GetBody = func() (io.ReadCloser, error) {- return io.NopCloser(bytes.NewReader(bodyBytes)), nil- }- }-- // Set headers. Note: Go's net/http does not populate req.Host from- // Header.Set("Host", ...) — it must be set explicitly on the field.+ // net/http ignores Header.Set("Host", ...); the field must be set explicitly.for k, v := range msg.Headers {if http.CanonicalHeaderKey(k) == "Host" {req.Host = v@@ -779,9 +624,9 @@ func (c *TunnelClient) buildLocalHTTPRequest(ctx context.Context, msg *TunnelMesreturn req, nil}-// agentAuthCredentialsInternal returns the canonical header/metadata credential+// agentAuthCredentials returns the canonical header/metadata credential// set every agent transport presents, so the three transports cannot drift.-func agentAuthCredentialsInternal(token string) map[string]string {+func agentAuthCredentials(token string) map[string]string {return map[string]string{HeaderAgentToken: token,HeaderAPIKey: token,@@ -789,50 +634,44 @@ func agentAuthCredentialsInternal(token string) map[string]string {}}-// handleRequest processes an incoming request and sends back a response+// handleRequest serves a tunneled request locally, streaming the response on gRPC connections.func (c *TunnelClient) handleRequest(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {- if isGRPCConnection(conn) {- c.handleRequestStreaming(ctx, conn, msg)- return- }-- reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeoutInternal())+ reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeout)defer cancel()reqCtx = context.WithValue(reqCtx, internalTunnelRequestContextKey{}, true)+ streaming := conn.Transport() == EdgeTransportGRPC- slog.DebugContext(reqCtx, "Processing tunneled request", "id", msg.ID, "method", msg.Method, "path", msg.Path, "bodyLength", len(msg.Body))+ slog.DebugContext(reqCtx, "Processing tunneled request", "id", msg.ID, "method", msg.Method, "path", msg.Path, "bodyLength", len(msg.Body), "streaming", streaming)req, err := c.buildLocalHTTPRequest(reqCtx, msg)if err != nil {- c.sendErrorResponse(conn, msg.ID, http.StatusInternalServerError, fmt.Sprintf("failed to create request: %v", err))+ _ = conn.Send(&TunnelMessage{+ ID: msg.ID,+ Type: MessageTypeResponse,+ Status: http.StatusInternalServerError,+ Body: fmt.Appendf(nil, "failed to create request: %v", err),+ })return}- // Use a response recorder to capture the response- rw := &responseRecorder{- headers: make(http.Header),- statusCode: http.StatusOK,- }-- // Execute the request through the local handler- c.handler.ServeHTTP(rw, req)-- // Build response message- respHeaders := make(map[string]string)- for k, v := range rw.headers {- if len(v) > 0 {- respHeaders[k] = v[0]+ if streaming {+ recorder := &streamingResponseRecorder{requestID: msg.ID, conn: conn, headers: make(http.Header), statusCode: http.StatusOK}+ c.handler.ServeHTTP(recorder, req)+ if closeErr := recorder.Close(); closeErr != nil {+ slog.WarnContext(reqCtx, "Failed to finalize streamed response", "id", msg.ID, "error", closeErr)}+ return}+ rw := &responseRecorder{headers: make(http.Header), statusCode: http.StatusOK}+ c.handler.ServeHTTP(rw, req)resp := &TunnelMessage{ID: msg.ID,Type: MessageTypeResponse,Status: rw.statusCode,- Headers: respHeaders,+ Headers: flattenResponseHeaders(rw.headers),Body: rw.body.Bytes(),}-if sendErr := conn.Send(resp); sendErr != nil {slog.ErrorContext(reqCtx, "Failed to send response", "id", msg.ID, "error", sendErr)} else {@@ -840,48 +679,19 @@ func (c *TunnelClient) handleRequest(ctx context.Context, conn TunnelConnection,}}-func isGRPCConnection(conn TunnelConnection) bool {- if conn == nil {- return false- }- return conn.Transport() == EdgeTransportGRPC-}--func (c *TunnelClient) handleRequestStreaming(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {- reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeoutInternal())- defer cancel()- reqCtx = context.WithValue(reqCtx, internalTunnelRequestContextKey{}, true)-- slog.DebugContext(reqCtx, "Processing tunneled request (streaming)", "id", msg.ID, "method", msg.Method, "path", msg.Path, "bodyLength", len(msg.Body))-- req, err := c.buildLocalHTTPRequest(reqCtx, msg)- if err != nil {- c.sendErrorResponse(conn, msg.ID, http.StatusInternalServerError, fmt.Sprintf("failed to create request: %v", err))- return- }-- recorder := newStreamingResponseRecorder(msg.ID, conn)- c.handler.ServeHTTP(recorder, req)-- if closeErr := recorder.Close(); closeErr != nil {- slog.WarnContext(reqCtx, "Failed to finalize streamed response", "id", msg.ID, "error", closeErr)- }-}--// handleWebSocketStart handles a WebSocket stream start request from the manager.+// handleWebSocketStart proxies a manager WebSocket or stream-open request to the local server.func (c *TunnelClient) handleWebSocketStart(ctx context.Context, conn TunnelConnection, msg *TunnelMessage, workers *sync.WaitGroup) {- if msg.Command == "" {- if commandName, ok := ResolveEdgeCommandName(http.MethodGet, msg.Path, true).Get(); ok {- msg.Command = commandName- }- }streamID := msg.ID+ if msg.Type == MessageTypeStreamOpen && !ValidateEdgeCommand(msg.Command, http.MethodGet, msg.Path, true) {+ c.sendStreamCloseMessage(conn, streamID, "unsupported edge stream")+ return+ }slog.DebugContext(ctx, "Starting WebSocket stream", "streamId", streamID, "path", msg.Path)- localURL := c.buildLocalWebSocketURLInternal(msg)- headers := c.buildLocalWebSocketHeadersInternal(msg)-- ws, resp, err := c.dialLocalWebSocket(ctx, localURL, headers)+ localURL := c.buildLocalWebSocketURL(msg)+ dialCtx, cancelDial := context.WithTimeout(ctx, 30*time.Second)+ defer cancelDial()+ ws, resp, err := websocket.Dial(dialCtx, localURL, &websocket.DialOptions{HTTPHeader: c.buildLocalWebSocketHeaders(msg)})// coder/websocket leaves resp.Body nil on a successful handshake.if resp != nil && resp.Body != nil {defer func() { _ = resp.Body.Close() }()@@ -892,111 +702,58 @@ func (c *TunnelClient) handleWebSocketStart(ctx context.Context, conn TunnelConnattrs = append(attrs, "status", resp.StatusCode)if resp.Body != nil {buf := make([]byte, 512)- n, _ := resp.Body.Read(buf)- if n > 0 {+ if n, _ := resp.Body.Read(buf); n > 0 {attrs = append(attrs, "response_body", string(buf[:n]))}}}slog.ErrorContext(ctx, "Failed to dial local WebSocket", attrs...)- c.sendWebSocketClose(conn, streamID)+ c.sendStreamCloseMessage(conn, streamID, "")return}+ // Local frames are forwarded into tunnel messages, so allow that size instead of the 32KB default.+ ws.SetReadLimit(maxGRPCTunnelMessageSize)streamCtx, cancel := context.WithCancel(ctx)- stream := c.registerStream(conn, streamID, ws, cancel)+ stream := &activeWSStream{ws: ws, conn: conn, cancel: cancel, dataCh: make(chan wsPayload, 100)}+ c.activeStreams.Store(streamID, stream)- workers.Go(func() { c.startLocalWebSocketReadLoop(ctx, streamCtx, streamID, ws, stream) })+ workers.Go(func() {+ defer c.closeWebSocketStream(streamID, stream)+ for {+ msgType, data, readErr := ws.Read(streamCtx)+ if readErr != nil {+ if !wshub.IsExpectedClose(readErr) {+ slog.DebugContext(ctx, "Local WebSocket read error", "error", readErr)+ }+ c.sendStreamCloseMessage(conn, streamID, "")+ return+ }+ if sendErr := conn.Send(&TunnelMessage{ID: streamID, Type: MessageTypeWebSocketData, Body: data, WSMessageType: int(msgType)}); sendErr != nil {+ slog.DebugContext(ctx, "Failed to send WebSocket data to manager", "error", sendErr)+ return+ }+ }+ })workers.Go(func() { c.startLocalWebSocketWriteLoop(ctx, streamCtx, ws, stream, cancel) })}-func (c *TunnelClient) handleStreamOpen(ctx context.Context, conn TunnelConnection, msg *TunnelMessage, workers *sync.WaitGroup) {- if !ValidateEdgeCommand(msg.Command, http.MethodGet, msg.Path, true) {- c.sendStreamCloseMessage(conn, msg.ID, "unsupported edge stream")- return- }-- c.handleWebSocketStart(ctx, conn, &TunnelMessage{- ID: msg.ID,- Type: MessageTypeWebSocketStart,- Command: msg.Command,- Path: msg.Path,- Query: msg.Query,- Headers: msg.Headers,- }, workers)-}--func (c *TunnelClient) buildLocalWebSocketURLInternal(msg *TunnelMessage) string {- path := msg.Path- if msg.Query != "" {- path = path + "?" + msg.Query- }- host := c.localWebSocketHostInternal()- return "ws://" + net.JoinHostPort(host, c.localPort) + path-}--func (c *TunnelClient) localWebSocketHostInternal() string {- listenHost := strings.TrimSpace(c.cfg.Listen)- if listenHost == "" {- return "localhost"- }-- // LISTEN may be just a host ("0.0.0.0"), host:port ("0.0.0.0:3552"),- // IPv6 ("::"), or bracketed IPv6 with port ("[::]:3552").- if strings.HasPrefix(listenHost, ":") {- return "localhost"+func (c *TunnelClient) buildLocalWebSocketURL(msg *TunnelMessage) string {+ // LISTEN may be a host, host:port, IPv6, or bracketed IPv6 with port.+ host := strings.TrimSpace(c.cfg.Listen)+ if splitHost, _, err := net.SplitHostPort(host); err == nil {+ host = splitHost}- if host, _, err := net.SplitHostPort(listenHost); err == nil {- listenHost = host+ if host = strings.Trim(host, "[]"); slices.Contains([]string{"", "0.0.0.0", "::"}, host) {+ host = "localhost"}-- trimmed := strings.Trim(listenHost, "[]")-- switch trimmed {- case "", "0.0.0.0", "::":- return "localhost"- default:- return trimmed- }-}--// localDialSkipHeaders lists headers that must not be forwarded when the-// agent dials its own local HTTP server for a proxied WebSocket stream.-// This includes:-// - Standard WebSocket handshake headers (coder/websocket sets its own)-// - Browser-specific headers that were forwarded through the tunnel from-// the manager. These cause handshake failures because the agent's-// WebSocket upgrader validates the Origin against localhost, not the-// browser's remote origin.-var localDialSkipHeaders = map[string]bool{- // WebSocket handshake (coder/websocket adds its own)- "Sec-Websocket-Key": true,- "Sec-Websocket-Version": true,- "Sec-Websocket-Extensions": true,- "Upgrade": true,- "Connection": true,- "Host": true,-- // Browser headers forwarded through the tunnel that are invalid- // for a server-to-server local dial.- "Origin": true,- "Cookie": true,- "Authorization": true,- "Referer": true,- "Sec-Fetch-Dest": true,- "Sec-Fetch-Mode": true,- "Sec-Fetch-Site": true,- "Sec-Fetch-User": true,- "Sec-Ch-Ua": true,- "Sec-Ch-Ua-Mobile": true,- "Sec-Ch-Ua-Platform": true,+ return "ws://" + net.JoinHostPort(host, c.localPort) + msg.Path + kit.Ternary(msg.Query != "", "?"+msg.Query, "")}-func (c *TunnelClient) buildLocalWebSocketHeadersInternal(msg *TunnelMessage) http.Header {+func (c *TunnelClient) buildLocalWebSocketHeaders(msg *TunnelMessage) http.Header {headers := http.Header{}for k, v := range msg.Headers {- canonicalKey := http.CanonicalHeaderKey(k)- if !localDialSkipHeaders[canonicalKey] {+ if canonicalKey := http.CanonicalHeaderKey(k); !localDialSkipHeaders[canonicalKey] {headers.Set(canonicalKey, v)}}@@ -1009,32 +766,6 @@ func (c *TunnelClient) buildLocalWebSocketHeadersInternal(msg *TunnelMessage) htreturn headers}-func (c *TunnelClient) dialLocalWebSocket(ctx context.Context, localURL string, headers http.Header) (*websocket.Conn, *http.Response, error) {- dialCtx, cancel := context.WithTimeout(ctx, 30*time.Second)- defer cancel()-- ws, resp, err := websocket.Dial(dialCtx, localURL, &websocket.DialOptions{HTTPHeader: headers})- if err != nil {- return nil, resp, err- }- // Local stream frames (exec output, log lines) are forwarded into tunnel- // messages capped at maxGRPCTunnelMessageSize; allow the same size here- // instead of coder/websocket's 32KB default.- ws.SetReadLimit(maxGRPCTunnelMessageSize)- return ws, resp, nil-}--func (c *TunnelClient) registerStream(conn TunnelConnection, streamID string, ws *websocket.Conn, cancel context.CancelFunc) *activeWSStream {- stream := &activeWSStream{- ws: ws,- conn: conn,- cancel: cancel,- dataCh: make(chan wsPayload, 100),- }- c.activeStreams.Store(streamID, stream)- return stream-}-func (c *TunnelClient) closeWebSocketStream(streamID string, stream *activeWSStream) {stream.mu.Lock()if stream.closed {@@ -1050,45 +781,6 @@ func (c *TunnelClient) closeWebSocketStream(streamID string, stream *activeWSStrc.activeStreams.Delete(streamID)}-// closeAllStreams tears down every active WebSocket stream. It is called when-// the message loop exits so a reconnect cannot leak stream goroutines, local-// sockets, or activeStreams entries. closeWebSocketStream is idempotent, so it-// is safe to race a concurrent manager-driven stream close.-func (c *TunnelClient) closeAllStreams() {- c.activeStreams.Range(func(key, value any) bool {- streamID, ok := key.(string)- if !ok {- return true- }- if stream, localOk := value.(*activeWSStream); localOk {- c.closeWebSocketStream(streamID, stream)- }- return true- })-}--func (c *TunnelClient) startLocalWebSocketReadLoop(ctx, streamCtx context.Context, streamID string, ws *websocket.Conn, stream *activeWSStream) {- defer func() {- c.closeWebSocketStream(streamID, stream)- }()-- for {- msgType, data, err := ws.Read(streamCtx)- if err != nil {- if !wshub.IsExpectedClose(err) {- slog.DebugContext(ctx, "Local WebSocket read error", "error", err)- }- c.sendWebSocketClose(stream.conn, streamID)- return- }-- if sendWebSocketDataErr := c.sendWebSocketData(stream.conn, streamID, int(msgType), data); sendWebSocketDataErr != nil {- slog.DebugContext(ctx, "Failed to send WebSocket data to manager", "error", sendWebSocketDataErr)- return- }- }-}-func (c *TunnelClient) startLocalWebSocketWriteLoop(ctx, streamCtx context.Context, ws *websocket.Conn, stream *activeWSStream, cancel context.CancelFunc) {for {select {@@ -1098,12 +790,11 @@ func (c *TunnelClient) startLocalWebSocketWriteLoop(ctx, streamCtx context.Conteif !ok {return}- msgType := websocket.MessageType(payload.messageType)- if msgType != websocket.MessageText && msgType != websocket.MessageBinary {+ if !isForwardableWSMessage(payload.messageType) {slog.WarnContext(ctx, "Dropping WebSocket message with unsupported type", "messageType", payload.messageType)continue}- if err := ws.Write(streamCtx, msgType, payload.data); err != nil {+ if err := ws.Write(streamCtx, websocket.MessageType(payload.messageType), payload.data); err != nil {slog.DebugContext(ctx, "Failed to write to local WebSocket", "error", err)cancel()return@@ -1112,38 +803,11 @@ func (c *TunnelClient) startLocalWebSocketWriteLoop(ctx, streamCtx context.Conte}}-func (c *TunnelClient) sendWebSocketData(conn TunnelConnection, streamID string, msgType int, data []byte) error {- if conn == nil {- return ErrNoActiveAgentTunnel- }- wsDataMsg := &TunnelMessage{- ID: streamID,- Type: MessageTypeWebSocketData,- Body: data,- WSMessageType: msgType,- }- return conn.Send(wsDataMsg)-}--func (c *TunnelClient) sendWebSocketClose(conn TunnelConnection, streamID string) {- c.sendStreamCloseMessage(conn, streamID, "")-}-func (c *TunnelClient) sendStreamCloseMessage(conn TunnelConnection, streamID, message string) {if conn == nil {return}- closeMsg := &TunnelMessage{- ID: streamID,- Type: MessageTypeStreamClose,- Error: message,- }- _ = conn.Send(closeMsg)-}--// handleWebSocketData handles incoming WebSocket data from the manager.-func (c *TunnelClient) handleWebSocketData(ctx context.Context, msg *TunnelMessage) {- c.handleStreamData(ctx, msg)+ _ = conn.Send(&TunnelMessage{ID: streamID, Type: MessageTypeStreamClose, Error: message})}func (c *TunnelClient) handleStreamData(ctx context.Context, msg *TunnelMessage) {@@ -1171,39 +835,7 @@ func (c *TunnelClient) handleStreamData(ctx context.Context, msg *TunnelMessage)}}-// handleWebSocketClose handles WebSocket close from the manager.-func (c *TunnelClient) handleWebSocketClose(ctx context.Context, msg *TunnelMessage) {- c.handleStreamClose(ctx, msg)-}--func (c *TunnelClient) handleStreamClose(ctx context.Context, msg *TunnelMessage) {- streamRaw, ok := c.activeStreams.Load(msg.ID)- if !ok {- return- }- stream, ok := streamRaw.(*activeWSStream)- if !ok {- return- }- c.closeWebSocketStream(msg.ID, stream)- slog.DebugContext(ctx, "Closed WebSocket stream", "streamId", msg.ID)-}--// sendErrorResponse sends an error response-func (c *TunnelClient) sendErrorResponse(conn TunnelConnection, requestID string, status int, message string) {- if conn == nil {- return- }- resp := &TunnelMessage{- ID: requestID,- Type: MessageTypeResponse,- Status: status,- Body: []byte(message),- }- _ = conn.Send(resp)-}--func (c *TunnelClient) sendCommandCompleteInternal(conn TunnelConnection, commandID string, status int, message string) {+func (c *TunnelClient) sendCommandComplete(conn TunnelConnection, commandID string, status int, message string) {if conn == nil {return}@@ -1222,7 +854,6 @@ func (r *commandResponseRecorder) Header() http.Header {func (r *commandResponseRecorder) Write(b []byte) (int, error) {r.mu.Lock()defer r.mu.Unlock()- originalLen := len(b)if r.flushErr != nil {return 0, r.flushErr@@ -1239,19 +870,12 @@ func (r *commandResponseRecorder) Write(b []byte) (int, error) {if err := r.flushBufferLocked(); err != nil {return 0, err}-- for len(b) > 0 {- chunk := b- if len(chunk) > defaultCommandChunkSize {- chunk = chunk[:defaultCommandChunkSize]- }+ for chunk := range slices.Chunk(b, defaultCommandChunkSize) {if err := r.sendOutputLocked(chunk); err != nil {return 0, err}- b = b[len(chunk):]}-- return originalLen, nil+ return len(b), nil}func (r *commandResponseRecorder) WriteHeader(statusCode int) {@@ -1286,37 +910,24 @@ func (r *commandResponseRecorder) Close() error {return r.flushErr}- if !r.streaming {- err := r.conn.Send(&TunnelMessage{- ID: r.commandID,- Type: MessageTypeCommandComplete,- Status: r.statusCode,- Headers: flattenResponseHeadersInternal(r.headers),- Body: append([]byte(nil), r.buffer.Bytes()...),- Streaming: false,- Command: r.commandName,- })- if err != nil {- return err- }- r.closed = true- return nil- }-- if err := r.flushBufferLocked(); err != nil {- return err- }- if err := r.conn.Send(&TunnelMessage{+ complete := &TunnelMessage{ID: r.commandID,Type: MessageTypeCommandComplete,Status: r.statusCode,- Headers: flattenResponseHeadersInternal(r.headers),- Streaming: true,+ Headers: flattenResponseHeaders(r.headers),+ Streaming: r.streaming,Command: r.commandName,- }); err != nil {+ }+ if r.streaming {+ if err := r.flushBufferLocked(); err != nil {+ return err+ }+ } else {+ complete.Body = append([]byte(nil), r.buffer.Bytes()...)+ }+ if err := r.conn.Send(complete); err != nil {return err}-r.closed = truereturn nil}@@ -1353,7 +964,7 @@ func (r *commandResponseRecorder) sendOutputLocked(chunk []byte) error {}if r.sequence == 0 {msg.Status = r.statusCode- msg.Headers = flattenResponseHeadersInternal(r.headers)+ msg.Headers = flattenResponseHeaders(r.headers)}if err := r.conn.Send(msg); err != nil {return err@@ -1362,10 +973,7 @@ func (r *commandResponseRecorder) sendOutputLocked(chunk []byte) error {return nil}-func flattenResponseHeadersInternal(headers http.Header) map[string]string {- if len(headers) == 0 {- return nil- }+func flattenResponseHeaders(headers http.Header) map[string]string {out := make(map[string]string, len(headers))for k, vs := range headers {if len(vs) > 0 {@@ -1387,15 +995,6 @@ func (r *responseRecorder) WriteHeader(statusCode int) {r.statusCode = statusCode}-func newStreamingResponseRecorder(requestID string, conn TunnelConnection) *streamingResponseRecorder {- return &streamingResponseRecorder{- requestID: requestID,- conn: conn,- headers: make(http.Header),- statusCode: http.StatusOK,- }-}-func (r *streamingResponseRecorder) Header() http.Header {return r.headers}@@ -1404,22 +1003,10 @@ func (r *streamingResponseRecorder) Write(b []byte) (int, error) {r.mu.Lock()defer r.mu.Unlock()- if !r.wroteHeader {- if err := r.writeHeaderLocked(r.statusCode); err != nil {- return 0, err- }- }-- if len(b) == 0 {- return 0, nil+ if err := r.writeHeaderLocked(); err != nil {+ return 0, err}-- originalLen := len(b)- for len(b) > 0 {- chunk := b- if len(chunk) > defaultCommandChunkSize {- chunk = chunk[:defaultCommandChunkSize]- }+ for chunk := range slices.Chunk(b, defaultCommandChunkSize) {if err := r.conn.Send(&TunnelMessage{ID: r.requestID,Type: MessageTypeStreamData,@@ -1427,31 +1014,21 @@ func (r *streamingResponseRecorder) Write(b []byte) (int, error) {}); err != nil {return 0, err}- b = b[len(chunk):]}- return originalLen, nil+ return len(b), nil}func (r *streamingResponseRecorder) WriteHeader(statusCode int) {r.mu.Lock()defer r.mu.Unlock()-r.statusCode = statusCode- if r.wroteHeader {- return- }- if err := r.writeHeaderLocked(statusCode); err != nil {- return- }+ _ = r.writeHeaderLocked()}func (r *streamingResponseRecorder) Flush() {r.mu.Lock()defer r.mu.Unlock()-- if !r.wroteHeader {- _ = r.writeHeaderLocked(r.statusCode)- }+ _ = r.writeHeaderLocked()}func (r *streamingResponseRecorder) Close() error {@@ -1461,37 +1038,27 @@ func (r *streamingResponseRecorder) Close() error {if r.closed {return nil}-- if !r.wroteHeader {- if err := r.writeHeaderLocked(r.statusCode); err != nil {- return err- }+ if err := r.writeHeaderLocked(); err != nil {+ return err}-- if err := r.conn.Send(&TunnelMessage{- ID: r.requestID,- Type: MessageTypeStreamEnd,- }); err != nil {+ if err := r.conn.Send(&TunnelMessage{ID: r.requestID, Type: MessageTypeStreamEnd}); err != nil {return err}-r.closed = truereturn nil}-func (r *streamingResponseRecorder) writeHeaderLocked(statusCode int) error {- respHeaders := make(map[string]string)- for k, v := range r.headers {- if len(v) > 0 {- respHeaders[k] = v[0]- }+// writeHeaderLocked sends the response status and headers once, marked as a streamed response.+func (r *streamingResponseRecorder) writeHeaderLocked() error {+ if r.wroteHeader {+ return nil}+ respHeaders := flattenResponseHeaders(r.headers)respHeaders[tunnelStreamHeader] = "1"-if err := r.conn.Send(&TunnelMessage{ID: r.requestID,Type: MessageTypeResponse,- Status: statusCode,+ Status: r.statusCode,Headers: respHeaders,}); err != nil {return err@@ -1505,21 +1072,15 @@ func StartTunnelClient(ctx context.Context, cfg *Config, handler http.Handler) (if !cfg.EdgeAgent {return nil, errors.New("edge tunnel disabled")}-- if UseGRPCEdgeTransport(cfg) {- if httpx.ManagerGRPCAddr(cfg.ManagerApiUrl) == "" {- return nil, errors.New("MANAGER_API_URL with a valid host is required for gRPC transport")- }+ if UseGRPCEdgeTransport(cfg) && httpx.ManagerGRPCAddr(cfg.ManagerApiUrl) == "" {+ return nil, errors.New("MANAGER_API_URL with a valid host is required for gRPC transport")}-if UseWebSocketEdgeTransport(cfg) && strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)) == "" {return nil, errors.New("MANAGER_API_URL is required for websocket transport")}-if UsePollEdgeTransport(cfg) && strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)) == "" {return nil, errors.New("MANAGER_API_URL is required for poll transport")}-if cfg.AgentToken == "" {return nil, errors.New("AGENT_TOKEN is required")}@@ -1538,25 +1099,18 @@ func StartTunnelClient(ctx context.Context, cfg *Config, handler http.Handler) (})}-func (c *TunnelClient) connectAndServeWebSocketInternal(ctx context.Context) error {- managerWSURL := c.managerWebSocketURLInternal()+func (c *TunnelClient) connectAndServeWebSocket(ctx context.Context) error {+ managerWSURL := c.managerWebSocketURL()if managerWSURL == "" {return errors.New("manager WebSocket URL is empty")}- transport := &http.Transport{Proxy: http.ProxyFromEnvironment}- if strings.HasPrefix(strings.ToLower(managerWSURL), "wss://") {- tlsConfig, err := buildManagerClientTLSConfig(c.cfg)- if err != nil {- return fmt.Errorf("failed to configure edge websocket TLS: %w", err)- }- if tlsConfig == nil {- tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12}- }- transport.TLSClientConfig = tlsConfig+ httpClient, err := NewManagerHTTPClient(c.cfg, 0)+ if err != nil {+ return fmt.Errorf("failed to configure edge websocket TLS: %w", err)}headers := http.Header{}- for header, value := range agentAuthCredentialsInternal(c.cfg.AgentToken) {+ for header, value := range agentAuthCredentials(c.cfg.AgentToken) {headers.Set(header, value)}@@ -1566,7 +1120,7 @@ func (c *TunnelClient) connectAndServeWebSocketInternal(ctx context.Context) errdialCtx, dialCancel := context.WithTimeout(ctx, 30*time.Second)defer dialCancel()conn, resp, err := websocket.Dial(dialCtx, managerWSURL, &websocket.DialOptions{- HTTPClient: &http.Client{Transport: transport},+ HTTPClient: httpClient,HTTPHeader: headers,})if err != nil {@@ -1578,5 +1132,5 @@ func (c *TunnelClient) connectAndServeWebSocketInternal(ctx context.Context) errreturn fmt.Errorf("failed to connect to manager websocket endpoint: %w", err)}- return c.serveTunnelSessionInternal(ctx, NewTunnelConn(conn), managerWSURL)+ return c.serveTunnelSession(ctx, NewTunnelConn(conn), managerWSURL)}diff --git a/backend/pkg/libarcane/edge/client_test.go b/backend/pkg/libarcane/edge/client_test.goindex 436b6039a..52e6a2978 100644--- a/backend/pkg/libarcane/edge/client_test.go+++ b/backend/pkg/libarcane/edge/client_test.go@@ -193,10 +193,8 @@ func TestTunnelClient_WebSocketProxy(t *testing.T) {time.Sleep(100 * time.Millisecond)}-// TestTunnelClient_WebSocket_ReconnectClosesStreams verifies that a reconnect-// reclaims the goroutines, local sockets, and activeStreams entries opened on-// the previous connection instead of leaking them. Run under -race it also-// exercises concurrent c.conn access across the reassignment on reconnect.+// TestTunnelClient_WebSocket_ReconnectClosesStreams verifies a reconnect reclaims the previous+// connection's goroutines, sockets, and activeStreams entries, and races c.conn under -race.func TestTunnelClient_WebSocket_ReconnectClosesStreams(t *testing.T) {// Local WS echo server the agent proxies to.localServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -370,50 +368,6 @@ func TestTunnelClient_HandleRequest_Errors(t *testing.T) {time.Sleep(100 * time.Millisecond)}-func TestTunnelClient_InternalHelpers(t *testing.T) {- // Mock connection- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- conn, err := websocket.Accept(w, r, nil)- if err != nil {- return- }- defer func() { _ = conn.CloseNow() }()-- for {- _, _, readErr := conn.Read(r.Context())- if readErr != nil {- return- }- }- }))- defer server.Close()-- cfg := &Config{- ManagerApiUrl: server.URL,- AgentToken: "test-token",- }- client := NewTunnelClient(cfg, nil)-- // Manually connect- serverURL := "ws" + strings.TrimPrefix(server.URL, "http")- conn, _, err := websocket.Dial(t.Context(), serverURL, nil)- require.NoError(t, err)- defer func() { _ = conn.CloseNow() }()-- tunnelConn := NewTunnelConn(conn)- client.conn.Store(&connBox{conn: tunnelConn})-- // Test sendWebSocketData- err = client.sendWebSocketData(tunnelConn, "stream-1", int(websocket.MessageText), []byte("data"))- require.NoError(t, err)-- // Test sendWebSocketClose- client.sendWebSocketClose(tunnelConn, "stream-1")-- // Test sendErrorResponse- client.sendErrorResponse(tunnelConn, "req-1", 500, "error")-}-func TestTunnelClient_BuildLocalWebSocketURL(t *testing.T) {tests := []struct {name string@@ -492,22 +446,22 @@ func TestTunnelClient_BuildLocalWebSocketURL(t *testing.T) {Path: testCase.path,Query: testCase.query,}- assert.Equal(t, testCase.expected, client.buildLocalWebSocketURLInternal(msg))+ assert.Equal(t, testCase.expected, client.buildLocalWebSocketURL(msg))})}}-func TestTunnelClient_GRPCConnectMethodInternal(t *testing.T) {+func TestTunnelClient_GRPCConnectMethod(t *testing.T) {client := NewTunnelClient(&Config{}, http.NotFoundHandler())assert.Equal(t, "/api/tunnel/connect", client.grpcConnectMethodInternal())}-func TestTunnelClient_buildLocalWebSocketHeadersInternal(t *testing.T) {+func TestTunnelClient_buildLocalWebSocketHeaders(t *testing.T) {client := NewTunnelClient(&Config{AgentToken: "agent-token",}, http.NotFoundHandler())- headers := client.buildLocalWebSocketHeadersInternal(&TunnelMessage{+ headers := client.buildLocalWebSocketHeaders(&TunnelMessage{Headers: map[string]string{"sec-websocket-key": "abc","sec-websocket-version": "13",@@ -525,12 +479,12 @@ func TestTunnelClient_buildLocalWebSocketHeadersInternal(t *testing.T) {assert.Equal(t, "agent-token", headers.Get("X-Arcane-Agent-Token"))}-func TestTunnelClient_buildLocalWebSocketHeadersInternal_FiltersBrowserHeaders(t *testing.T) {+func TestTunnelClient_buildLocalWebSocketHeaders_FiltersBrowserHeaders(t *testing.T) {client := NewTunnelClient(&Config{AgentToken: "agent-token",}, http.NotFoundHandler())- headers := client.buildLocalWebSocketHeadersInternal(&TunnelMessage{+ headers := client.buildLocalWebSocketHeaders(&TunnelMessage{Headers: map[string]string{// Browser headers that should be stripped"Origin": "https://docker.example.com",@@ -630,13 +584,13 @@ func TestTunnelClient_DialLocalWebSocket_StripsForwardedBrowserHeaders(t *testin},}- headers := client.buildLocalWebSocketHeadersInternal(msg)+ headers := client.buildLocalWebSocketHeaders(msg)assert.Empty(t, headers.Get("Host"))assert.Empty(t, headers.Get("Origin"))assert.Empty(t, headers.Get("Cookie"))assert.Empty(t, headers.Get("Authorization"))- ws, _, err := client.dialLocalWebSocket(t.Context(), client.buildLocalWebSocketURLInternal(msg), headers)+ ws, _, err := websocket.Dial(t.Context(), client.buildLocalWebSocketURL(msg), &websocket.DialOptions{HTTPHeader: headers})require.NoError(t, err)defer func() { _ = ws.CloseNow() }()@@ -646,20 +600,6 @@ func TestTunnelClient_DialLocalWebSocket_StripsForwardedBrowserHeaders(t *testinassert.Equal(t, "ok", string(body))}-func TestTunnelClient_IsGRPCConnectionInternal(t *testing.T) {- t.Run("nil connection", func(t *testing.T) {- assert.False(t, isGRPCConnection(nil))- })-- t.Run("grpc connection", func(t *testing.T) {- assert.True(t, isGRPCConnection(NewGRPCAgentTunnelConn(nil)))- })-- t.Run("non-grpc connection", func(t *testing.T) {- assert.False(t, isGRPCConnection(&fakeTunnelConnForTransportCheck{}))- })-}-func TestTunnelClient_HandleRequest_GRPCConfigWithWebSocketConnUsesNonStreamingResponse(t *testing.T) {localHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if !assert.Equal(t, "/local/api", r.URL.Path) {@@ -701,30 +641,6 @@ func TestTunnelClient_HeartbeatLoop_ClosesConnectionOnSendFailure(t *testing.T)assert.True(t, conn.closeCalled)}-type fakeTunnelConnForTransportCheck struct{}--func (f *fakeTunnelConnForTransportCheck) Send(_ *TunnelMessage) error {- return nil-}--func (f *fakeTunnelConnForTransportCheck) Receive() (*TunnelMessage, error) {- return nil, nil-}--func (f *fakeTunnelConnForTransportCheck) IsExpectedReceiveError(error) bool {- return false-}--func (f *fakeTunnelConnForTransportCheck) Close() error {- return nil-}--func (f *fakeTunnelConnForTransportCheck) IsClosed() bool {- return false-}--func (f *fakeTunnelConnForTransportCheck) Transport() string { return EdgeTransportWebSocket }-type capturingTunnelConnForHandleRequest struct {sent []*TunnelMessage}@@ -799,7 +715,7 @@ type blockingRegistrationConn struct {receiveStartOnce sync.Once}-func newBlockingRegistrationConnInternal() *blockingRegistrationConn {+func newBlockingRegistrationConn() *blockingRegistrationConn {return &blockingRegistrationConn{closedCh: make(chan struct{}),receiveStarted: make(chan struct{}),@@ -867,7 +783,7 @@ func TestTunnelClient_GRPC_EndToEnd(t *testing.T) {tunnelServer.WaitForCleanupDone()}()- managerURL, stopManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)+ managerURL, stopManager := startTestGRPCTunnelServerOnAPIPath(t, ctx, tunnelServer)defer stopManager()localHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -935,7 +851,7 @@ func TestTunnelClient_GRPC_ChunkedRequestBodyEndToEnd(t *testing.T) {tunnelServer.WaitForCleanupDone()}()- managerURL, stopManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)+ managerURL, stopManager := startTestGRPCTunnelServerOnAPIPath(t, ctx, tunnelServer)defer stopManager()wantBody := bytes.Repeat([]byte("arcane-edge-chunk"), 320*1024)@@ -1078,7 +994,7 @@ func TestTunnelClient_connectAndServeGRPC_RegistrationRejected(t *testing.T) {tunnelServer.WaitForCleanupDone()}()- managerURL, stopManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)+ managerURL, stopManager := startTestGRPCTunnelServerOnAPIPath(t, ctx, tunnelServer)defer stopManager()client := NewTunnelClient(&Config{@@ -1100,7 +1016,7 @@ func TestTunnelClient_connectAndServeGRPC_TimesOutWithoutRegisterResponse(t *tesdefer cancel()service := &stallingTunnelService{}- managerURL, stopManager := startTestTunnelServiceOnAPIPathInternal(t, ctx, service)+ managerURL, stopManager := startTestTunnelServiceOnAPIPath(t, ctx, service)defer stopManager()client := NewTunnelClient(&Config{@@ -1120,29 +1036,29 @@ func TestTunnelClient_connectAndServeGRPC_TimesOutWithoutRegisterResponse(t *tesassert.True(t, active == nil || active.conn.IsClosed())}-func TestTunnelClient_awaitRegistrationInternal_ClosesConnOnContextDone(t *testing.T) {+func TestTunnelClient_awaitRegistration_ClosesConnOnContextDone(t *testing.T) {ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)defer cancel()- conn := newBlockingRegistrationConnInternal()+ conn := newBlockingRegistrationConn()client := &TunnelClient{registrationTimeout: time.Second,}client.conn.Store(&connBox{conn: conn})- msg, err := client.awaitRegistrationInternal(ctx, conn)+ msg, err := client.awaitRegistration(ctx, conn)require.Nil(t, msg)require.ErrorIs(t, err, context.DeadlineExceeded)assert.True(t, conn.IsClosed())assert.EqualValues(t, 1, conn.closeCount.Load())}-func TestTunnelClient_awaitRegistrationInternal_ClosesAttemptConnWhenClientConnChanges(t *testing.T) {+func TestTunnelClient_awaitRegistration_ClosesAttemptConnWhenClientConnChanges(t *testing.T) {ctx, cancel := context.WithCancel(t.Context())defer cancel()- firstConn := newBlockingRegistrationConnInternal()- secondConn := newBlockingRegistrationConnInternal()+ firstConn := newBlockingRegistrationConn()+ secondConn := newBlockingRegistrationConn()t.Cleanup(func() {_ = firstConn.Close()_ = secondConn.Close()@@ -1159,7 +1075,7 @@ func TestTunnelClient_awaitRegistrationInternal_ClosesAttemptConnWhenClientConnC}resultCh := make(chan registrationResult, 1)go func() {- msg, err := client.awaitRegistrationInternal(ctx, firstConn)+ msg, err := client.awaitRegistration(ctx, firstConn)resultCh <- registrationResult{msg: msg, err: err}}()@@ -1207,7 +1123,7 @@ func TestTunnelClient_GRPC_WebSocketProxyEndToEnd(t *testing.T) {tunnelServer.WaitForCleanupDone()}()- managerURL, stopManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)+ managerURL, stopManager := startTestGRPCTunnelServerOnAPIPath(t, ctx, tunnelServer)defer stopManager()headerTokenCh := make(chan string, 1)@@ -1342,7 +1258,7 @@ func TestTunnelClient_GRPC_WebSocketProxyEndToEnd(t *testing.T) {}}-func TestTunnelClient_connectAndServe_WebSocketConfigFallsBackToWebSocket(t *testing.T) {+func TestTunnelClient_connectAndServeManagedTunnel_WebSocketConfigFallsBackToWebSocket(t *testing.T) {ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)defer cancel()@@ -1375,7 +1291,7 @@ func TestTunnelClient_connectAndServe_WebSocketConfigFallsBackToWebSocket(t *tes}client := NewTunnelClient(cfg, http.NotFoundHandler())- err := client.connectAndServe(ctx)+ err := client.connectAndServeManagedTunnel(ctx)require.Error(t, err)select {@@ -1392,16 +1308,16 @@ func TestTunnelClient_managedTunnelTransports_AutoEnablesGRPCAndWebSocket(t *tesAgentToken: "valid-token",}, http.NotFoundHandler())- transports := client.managedTunnelTransportsInternal()+ transports := client.managedTunnelTransports()assert.True(t, transports.grpc)assert.True(t, transports.websocket)}-func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCUnavailable(t *testing.T) {+func TestTunnelClient_connectAndServeManagedTunnel_AutoFallsBackToWebSocketWhenGRPCUnavailable(t *testing.T) {ctx, cancel := context.WithCancel(t.Context())defer cancel()- managerURL, wsConnectedCh, stopManager := startTestWebSocketTunnelManagerInternal(t, ctx)+ managerURL, wsConnectedCh, stopManager := startTestWebSocketTunnelManager(t, ctx)defer stopManager()client := NewTunnelClient(&Config{@@ -1409,14 +1325,14 @@ func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCUnavailablManagerApiUrl: managerURL,AgentToken: "valid-token",}, http.NotFoundHandler())- grpcAddr, releaseGRPCAddr := reserveTCPAddressInternal(t)+ grpcAddr, releaseGRPCAddr := reserveTCPAddress(t)defer releaseGRPCAddr()client.managerGRPCAddr = grpcAddrclient.registrationTimeout = 100 * time.MilliseconderrCh := make(chan error, 1)go func() {- errCh <- client.connectAndServe(ctx)+ errCh <- client.connectAndServeManagedTunnel(ctx)}()select {@@ -1434,14 +1350,14 @@ func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCUnavailabl}}-func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCSetupHangs(t *testing.T) {+func TestTunnelClient_connectAndServeManagedTunnel_AutoFallsBackToWebSocketWhenGRPCSetupHangs(t *testing.T) {ctx, cancel := context.WithCancel(t.Context())defer cancel()- grpcAddr, stopGRPC := startHangingTCPServerInternal(t, ctx)+ grpcAddr, stopGRPC := startHangingTCPServer(t, ctx)defer stopGRPC()- managerURL, wsConnectedCh, stopManager := startTestWebSocketTunnelManagerInternal(t, ctx)+ managerURL, wsConnectedCh, stopManager := startTestWebSocketTunnelManager(t, ctx)defer stopManager()client := NewTunnelClient(&Config{@@ -1454,7 +1370,7 @@ func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCSetupHangserrCh := make(chan error, 1)go func() {- errCh <- client.connectAndServe(ctx)+ errCh <- client.connectAndServeManagedTunnel(ctx)}()select {@@ -1472,11 +1388,11 @@ func TestTunnelClient_connectAndServe_AutoFallsBackToWebSocketWhenGRPCSetupHangs}}-func TestTunnelClient_connectAndServe_GRPCDoesNotFallbackToWebSocket(t *testing.T) {+func TestTunnelClient_connectAndServeManagedTunnel_GRPCDoesNotFallbackToWebSocket(t *testing.T) {ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)defer cancel()- managerURL, wsConnectedCh, stopManager := startTestWebSocketTunnelManagerInternal(t, ctx)+ managerURL, wsConnectedCh, stopManager := startTestWebSocketTunnelManager(t, ctx)defer stopManager()client := NewTunnelClient(&Config{@@ -1484,12 +1400,12 @@ func TestTunnelClient_connectAndServe_GRPCDoesNotFallbackToWebSocket(t *testing.ManagerApiUrl: managerURL,AgentToken: "valid-token",}, http.NotFoundHandler())- grpcAddr, releaseGRPCAddr := reserveTCPAddressInternal(t)+ grpcAddr, releaseGRPCAddr := reserveTCPAddress(t)defer releaseGRPCAddr()client.managerGRPCAddr = grpcAddrclient.registrationTimeout = 100 * time.Millisecond- err := client.connectAndServe(ctx)+ err := client.connectAndServeManagedTunnel(ctx)require.Error(t, err)select {@@ -1499,7 +1415,7 @@ func TestTunnelClient_connectAndServe_GRPCDoesNotFallbackToWebSocket(t *testing.}}-func TestTunnelClient_connectAndServe_OpensGRPCWhenAvailable(t *testing.T) {+func TestTunnelClient_connectAndServeManagedTunnel_OpensGRPCWhenAvailable(t *testing.T) {ctx, cancel := context.WithCancel(t.Context())defer cancel()@@ -1518,7 +1434,7 @@ func TestTunnelClient_connectAndServe_OpensGRPCWhenAvailable(t *testing.T) {go tunnelServer.StartCleanupLoop(ctx)defer tunnelServer.WaitForCleanupDone()- managerURL, stopManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)+ managerURL, stopManager := startTestGRPCTunnelServerOnAPIPath(t, ctx, tunnelServer)defer stopManager()client := NewTunnelClient(&Config{@@ -1529,7 +1445,7 @@ func TestTunnelClient_connectAndServe_OpensGRPCWhenAvailable(t *testing.T) {errCh := make(chan error, 1)go func() {- errCh <- client.connectAndServe(ctx)+ errCh <- client.connectAndServeManagedTunnel(ctx)}()require.Eventually(t, func() bool {@@ -1551,7 +1467,7 @@ func TestTunnelClient_connectAndServe_OpensGRPCWhenAvailable(t *testing.T) {}}-func startTestWebSocketTunnelManagerInternal(t *testing.T, ctx context.Context) (string, <-chan struct{}, func()) {+func startTestWebSocketTunnelManager(t *testing.T, ctx context.Context) (string, <-chan struct{}, func()) {t.Helper()wsConnectedCh := make(chan struct{}, 1)@@ -1595,7 +1511,7 @@ func startTestWebSocketTunnelManagerInternal(t *testing.T, ctx context.Context)return managerServer.URL, wsConnectedCh, managerServer.Close}-func reserveTCPAddressInternal(t *testing.T) (string, func()) {+func reserveTCPAddress(t *testing.T) (string, func()) {t.Helper()lis, err := net.Listen("tcp", "127.0.0.1:0")@@ -1605,7 +1521,7 @@ func reserveTCPAddressInternal(t *testing.T) (string, func()) {}}-func startHangingTCPServerInternal(t *testing.T, ctx context.Context) (string, func()) {+func startHangingTCPServer(t *testing.T, ctx context.Context) (string, func()) {t.Helper()lis, err := net.Listen("tcp", "127.0.0.1:0")@@ -1638,12 +1554,12 @@ func startHangingTCPServerInternal(t *testing.T, ctx context.Context) (string, freturn lis.Addr().String(), stop}-func startTestGRPCTunnelServerOnAPIPathInternal(t *testing.T, ctx context.Context, tunnelServer *TunnelServer) (string, func()) {+func startTestGRPCTunnelServerOnAPIPath(t *testing.T, ctx context.Context, tunnelServer *TunnelServer) (string, func()) {t.Helper()- return startTestTunnelServiceOnAPIPathInternal(t, ctx, tunnelServer)+ return startTestTunnelServiceOnAPIPath(t, ctx, tunnelServer)}-func startTestTunnelServiceOnAPIPathInternal(t *testing.T, ctx context.Context, service tunnelpb.TunnelServiceServer) (string, func()) {+func startTestTunnelServiceOnAPIPath(t *testing.T, ctx context.Context, service tunnelpb.TunnelServiceServer) (string, func()) {t.Helper()var serverOptions []grpc.ServerOption@@ -1702,7 +1618,7 @@ func startTestTunnelServiceOnAPIPathInternal(t *testing.T, ctx context.Context,return "http://" + lis.Addr().String(), cleanup}-func TestTunnelClient_connectAndServe_AutoDoesNotFallbackWhenEstablishedGRPCSessionDrops(t *testing.T) {+func TestTunnelClient_connectAndServeManagedTunnel_AutoDoesNotFallbackWhenEstablishedGRPCSessionDrops(t *testing.T) {ctx, cancel := context.WithCancel(t.Context())defer cancel()@@ -1721,10 +1637,10 @@ func TestTunnelClient_connectAndServe_AutoDoesNotFallbackWhenEstablishedGRPCSessgo tunnelServer.StartCleanupLoop(ctx)defer tunnelServer.WaitForCleanupDone()- grpcManagerURL, stopGRPCManager := startTestGRPCTunnelServerOnAPIPathInternal(t, ctx, tunnelServer)+ grpcManagerURL, stopGRPCManager := startTestGRPCTunnelServerOnAPIPath(t, ctx, tunnelServer)defer stopGRPCManager()- wsManagerURL, wsConnectedCh, stopWSManager := startTestWebSocketTunnelManagerInternal(t, ctx)+ wsManagerURL, wsConnectedCh, stopWSManager := startTestWebSocketTunnelManager(t, ctx)defer stopWSManager()client := NewTunnelClient(&Config{@@ -1739,7 +1655,7 @@ func TestTunnelClient_connectAndServe_AutoDoesNotFallbackWhenEstablishedGRPCSesserrCh := make(chan error, 1)go func() {- errCh <- client.connectAndServe(ctx)+ errCh <- client.connectAndServeManagedTunnel(ctx)}()var tunnel *AgentTunnel@@ -1768,7 +1684,7 @@ func TestTunnelClient_connectAndServe_AutoDoesNotFallbackWhenEstablishedGRPCSessrequire.FailNow(t, "dropped gRPC session must not fall back to websocket")default:}- assert.Zero(t, client.grpcFailureStreakInternal())+ assert.Zero(t, client.grpcFailureStreak)cancel()}@@ -1794,7 +1710,7 @@ func TestTunnelClient_connectAndServePoll_OpensGRPCWhenRequired(t *testing.T) {tunnelServer.WaitForCleanupDone()}()- managerURL, stopManager := startTestPollAndGRPCManagerInternal(t, ctx, tunnelServer, TunnelPollResponse{+ managerURL, stopManager := startTestPollAndGRPCManager(t, ctx, tunnelServer, TunnelPollResponse{Status: TunnelStatusRequired,PollIntervalSeconds: 1,})@@ -1808,7 +1724,7 @@ func TestTunnelClient_connectAndServePoll_OpensGRPCWhenRequired(t *testing.T) {errCh := make(chan error, 1)go func() {- errCh <- client.connectAndServe(ctx)+ errCh <- client.connectAndServePoll(ctx)}()require.Eventually(t, func() bool {@@ -1889,7 +1805,7 @@ func TestTunnelClient_connectAndServePoll_OpensWebSocketWhenRequired(t *testing.}, http.NotFoundHandler())client.registrationTimeout = 100 * time.Millisecond- err := client.connectAndServe(ctx)+ err := client.connectAndServePoll(ctx)require.Error(t, err)require.ErrorIs(t, err, context.DeadlineExceeded)@@ -1902,7 +1818,7 @@ func TestTunnelClient_connectAndServePoll_OpensWebSocketWhenRequired(t *testing.assert.GreaterOrEqual(t, pollCount.Load(), int32(1))}-func TestTunnelClient_pollTunnelControlInternal_UsesConfiguredHTTPClient(t *testing.T) {+func TestTunnelClient_pollTunnelControl_UsesConfiguredHTTPClient(t *testing.T) {t.Parallel()managerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -1974,7 +1890,7 @@ func TestTunnelClient_connectAndServePoll_DoesNotOpenWebSocketWhenIdle(t *testinAgentToken: "valid-token",}, http.NotFoundHandler())- err := client.connectAndServe(ctx)+ err := client.connectAndServePoll(ctx)require.Error(t, err)require.ErrorIs(t, err, context.DeadlineExceeded)@@ -2050,7 +1966,7 @@ func TestTunnelClient_connectAndServePoll_RetriesAfterTransientPollError(t *test}, http.NotFoundHandler())client.registrationTimeout = 100 * time.Millisecond- err := client.connectAndServe(ctx)+ err := client.connectAndServePoll(ctx)require.Error(t, err)require.ErrorIs(t, err, context.DeadlineExceeded)@@ -2063,7 +1979,7 @@ func TestTunnelClient_connectAndServePoll_RetriesAfterTransientPollError(t *testassert.GreaterOrEqual(t, pollCount.Load(), int32(2))}-func TestTunnelClient_stopPollManagedSessionInternal_DeadlineExceededReturnsTimeoutMessage(t *testing.T) {+func TestTunnelClient_stopPollManagedSession_DeadlineExceededReturnsTimeoutMessage(t *testing.T) {t.Parallel()session := &pollManagedTunnelSession{@@ -2079,7 +1995,7 @@ func TestTunnelClient_stopPollManagedSessionInternal_DeadlineExceededReturnsTimeassert.EqualError(t, err, "timed out waiting for poll-managed websocket session to stop")}-func TestTunnelClient_syncPollManagedSessionInternal_IdleUsesBoundedStopTimeout(t *testing.T) {+func TestTunnelClient_syncPollManagedSession_IdleUsesBoundedStopTimeout(t *testing.T) {previousTimeout := defaultPollManagedSessionStopTimeoutdefaultPollManagedSessionStopTimeout = 20 * time.Milliseconddefer func() {@@ -2107,7 +2023,7 @@ func TestTunnelClient_syncPollManagedSessionInternal_IdleUsesBoundedStopTimeout(require.Less(t, time.Since(started), defaultPollManagedSessionStopTimeout)}-func startTestPollAndGRPCManagerInternal(t *testing.T, ctx context.Context, service tunnelpb.TunnelServiceServer, pollResp TunnelPollResponse) (string, func()) {+func startTestPollAndGRPCManager(t *testing.T, ctx context.Context, service tunnelpb.TunnelServiceServer, pollResp TunnelPollResponse) (string, func()) {t.Helper()var serverOptions []grpc.ServerOption@@ -2210,10 +2126,10 @@ func TestTunnelClient_InternalRequestSkipsSlogEcho(t *testing.T) {{name: "streaming response",run: func(t *testing.T, client *TunnelClient) {- conn := &fakeTunnelConn{}+ conn := &fakeTunnelConn{transport: EdgeTransportGRPC}client.conn.Store(&connBox{conn: conn})- client.handleRequestStreaming(t.Context(), conn, &TunnelMessage{+ client.handleRequest(t.Context(), conn, &TunnelMessage{ID: "req-stream",Type: MessageTypeRequest,Method: http.MethodGet,@@ -2314,7 +2230,7 @@ func (f *fakeTunnelConn) Transport() string {return f.transport}-func TestTunnelClient_serveTunnelSessionInternal_GRPCSendEOFSurfacesRegistrationError(t *testing.T) {+func TestTunnelClient_serveTunnelSession_GRPCSendEOFSurfacesRegistrationError(t *testing.T) {conn := &fakeTunnelConn{sendErr: io.EOF,receiveErr: status.Error(codes.Unauthenticated, "invalid agent token"),@@ -2322,7 +2238,7 @@ func TestTunnelClient_serveTunnelSessionInternal_GRPCSendEOFSurfacesRegistration}client := NewTunnelClient(&Config{}, http.NotFoundHandler())- err := client.serveTunnelSessionInternal(t.Context(), conn, "manager.test")+ err := client.serveTunnelSession(t.Context(), conn, "manager.test")require.Error(t, err)assert.Equal(t, codes.Unauthenticated, status.Code(err))assert.Contains(t, err.Error(), "invalid agent token")@@ -2330,7 +2246,7 @@ func TestTunnelClient_serveTunnelSessionInternal_GRPCSendEOFSurfacesRegistrationfunc TestStreamingResponseRecorder_Sequence(t *testing.T) {conn := &fakeTunnelConn{}- r := newStreamingResponseRecorder("req-1", conn)+ r := &streamingResponseRecorder{requestID: "req-1", conn: conn, headers: make(http.Header), statusCode: http.StatusOK}r.Header().Set("Content-Type", "text/plain")@@ -2361,7 +2277,7 @@ func TestStreamingResponseRecorder_Sequence(t *testing.T) {func TestStreamingResponseRecorder_WriteHeaderAndClose(t *testing.T) {conn := &fakeTunnelConn{}- r := newStreamingResponseRecorder("req-2", conn)+ r := &streamingResponseRecorder{requestID: "req-2", conn: conn, headers: make(http.Header), statusCode: http.StatusOK}r.WriteHeader(http.StatusCreated)require.NoError(t, r.Close())@@ -2411,7 +2327,7 @@ func TestConnectAndServeWebSocket_CancelUnblocksMessageLoop(t *testing.T) {ctx, cancel := context.WithCancel(t.Context())done := make(chan error, 1)- go func() { done <- client.connectAndServeWebSocketInternal(ctx) }()+ go func() { done <- client.connectAndServeWebSocket(ctx) }()select {case <-registered:diff --git a/backend/pkg/libarcane/edge/client_transport_grpc.go b/backend/pkg/libarcane/edge/client_transport_grpc.goindex 5f8f7ea85..660ce54a0 100644--- a/backend/pkg/libarcane/edge/client_transport_grpc.go+++ b/backend/pkg/libarcane/edge/client_transport_grpc.go@@ -76,7 +76,7 @@ func (c *TunnelClient) connectAndServeGRPC(ctx context.Context) error {// metadata.New lowercases the keys itself.streamCtx, streamCancel := context.WithCancel(metadata.NewOutgoingContext(ctx,- metadata.New(agentAuthCredentialsInternal(c.cfg.AgentToken))))+ metadata.New(agentAuthCredentials(c.cfg.AgentToken))))defer streamCancel()method := c.grpcConnectMethodInternal()@@ -85,7 +85,7 @@ func (c *TunnelClient) connectAndServeGRPC(ctx context.Context) error {return fmt.Errorf("failed to open tunnel stream: %w", err)}- if serveTunnelSessionErr := c.serveTunnelSessionInternal(ctx, NewGRPCAgentTunnelConn(stream, streamCancel), managerAddr); serveTunnelSessionErr != nil {+ if serveTunnelSessionErr := c.serveTunnelSession(ctx, NewGRPCAgentTunnelConn(stream, streamCancel), managerAddr); serveTunnelSessionErr != nil {if errors.Is(serveTunnelSessionErr, errTunnelRegistrationTimeout) {// The channel already reached Ready, so TCP/TLS works but gRPC// framing was never answered end to end.@@ -104,7 +104,7 @@ func (c *TunnelClient) waitForGRPCReadyInternal(ctx context.Context, conn *grpc.return errors.New("manager gRPC connection is not initialized")}- timeout := c.registrationTimeoutInternal()+ timeout := c.registrationTimeoutreadyCtx, cancel := context.WithTimeout(ctx, timeout)defer cancel()diff --git a/backend/pkg/libarcane/edge/client_transport_poll.go b/backend/pkg/libarcane/edge/client_transport_poll.goindex 76d5e71cf..c552d2dd5 100644--- a/backend/pkg/libarcane/edge/client_transport_poll.go+++ b/backend/pkg/libarcane/edge/client_transport_poll.go@@ -137,7 +137,7 @@ func (c *TunnelClient) syncPollManagedSessionInternal(ctx context.Context, sessislog.InfoContext(ctx, "Poll control plane requested edge tunnel","status", status,"managerGrpcAddr", c.managerGRPCAddr,- "managerWsUrl", c.managerWebSocketURLInternal(),+ "managerWsUrl", c.managerWebSocketURL(),)return c.startPollManagedSessionInternal(ctx), nilcase TunnelStatusIdle:@@ -204,7 +204,7 @@ func (c *TunnelClient) pollTunnelControlInternal(ctx context.Context, httpClientreturn nil, fmt.Errorf("failed to create poll request: %w", err)}req.Header.Set("Content-Type", "application/json")- for header, value := range agentAuthCredentialsInternal(c.cfg.AgentToken) {+ for header, value := range agentAuthCredentials(c.cfg.AgentToken) {req.Header.Set(header, value)}@@ -231,7 +231,7 @@ func (c *TunnelClient) startPollManagedSessionInternal(ctx context.Context) *poldone := make(chan error, 1)go func() {- err := c.connectAndServeManagedTunnelInternal(sessionCtx)+ err := c.connectAndServeManagedTunnel(sessionCtx)if sessionCtx.Err() != nil {err = nil}diff --git a/backend/pkg/libarcane/edge/command_client.go b/backend/pkg/libarcane/edge/command_client.goindex 3b1ff9e12..b491b88b6 100644--- a/backend/pkg/libarcane/edge/command_client.go+++ b/backend/pkg/libarcane/edge/command_client.go@@ -1,6 +1,7 @@package edgeimport (+ "cmp""context""errors""fmt"@@ -8,53 +9,90 @@ import ("math""net/http""slices"+ "sync""time""uuid"++ "go.getarcane.app/kit/pkg"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric"+ "go.opentelemetry.io/otel/propagation"+ "go.opentelemetry.io/otel/trace"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")const (tunnelCapabilityChunkedRequest = "chunked-request"- // tunnelCapabilityProtoParity signals the peer can decode the full- // TunnelMessage vocabulary natively over gRPC (stream_data/stream_end- // manager->agent, cancel_request agent->manager) instead of the legacy- // re-encoded forms.+ // tunnelCapabilityProtoParity signals the peer decodes the full TunnelMessage vocabulary natively over gRPC.tunnelCapabilityProtoParity = "proto-parity-v1"- // Credit-based flow control for command output keeps at most commandCreditWindow bytes- // unconsumed by the manager. Agents offer it; only the manager sends the grant, which older- // managers that echo agent capabilities cannot produce.+ // Command output credit keeps at most commandCreditWindow bytes unconsumed by the manager.+ // Agents offer it; only the manager sends the grant, which older echoing managers cannot produce.tunnelCapabilityCommandCredit = "command-credit-v1"tunnelCapabilityCommandCreditGrant = "command-credit-grant-v1"commandCreditWindow = 8 << 20bodyTransferMetadataKey = "body_transfer_id")+// commandDuration is created once and shared by every CommandClient.+var commandDuration = sync.OnceValue(func() metric.Float64Histogram {+ histogram, err := otel.Meter(tracing.InstrumentationName).Float64Histogram("arcane.edge.command.duration",+ metric.WithDescription("Edge tunnel command round-trip duration"),+ metric.WithUnit("s"),+ metric.WithExplicitBucketBoundaries(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300),+ )+ if err != nil {+ otel.Handle(err)+ }+ return histogram+})+func NewCommandClient() *CommandClient {return &CommandClient{}}-func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *CommandRequest) (*CommandResult, error) {- if ctx == nil {- return nil, errors.New("context is required")- }- if err := validateConnectedTunnelInternal(tunnel); err != nil {+func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *CommandRequest) (result *CommandResult, err error) {+ commandName, err := resolveCommand(ctx, tunnel, req, false)+ if err != nil {return nil, err}- if req == nil {- return nil, errors.New("command request is required")- }+ requestID := cmp.Or(req.ID, uuid.New().String())- commandName := req.Command- if commandName == "" {- resolved, ok := ResolveEdgeCommandName(req.Method, req.Path, false).Get()- if !ok {- return nil, fmt.Errorf("unsupported edge command for %s %s", req.Method, req.Path)+ ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "edge.command "+commandName,+ trace.WithSpanKind(trace.SpanKindClient),+ trace.WithAttributes(attribute.String("arcane.edge.command", commandName), attribute.String("http.request.method", req.Method)),+ )+ started := time.Now()+ defer func() {+ tracing.End(span, err)+ var outcome string+ switch {+ case err == nil:+ outcome = "success"+ case errors.Is(err, context.Canceled):+ outcome = "canceled"+ case errors.Is(err, context.DeadlineExceeded):+ outcome = "timeout"+ default:+ outcome = "error"}- commandName = resolved- }+ attrs := metric.WithAttributes(attribute.String("arcane.edge.command", commandName), attribute.String("arcane.edge.outcome", outcome))+ commandDuration().Record(context.WithoutCancel(ctx), time.Since(started).Seconds(), attrs)+ }()- requestID := req.ID- if requestID == "" {- requestID = uuid.New().String()+ // Replace any inbound trace headers so the agent's in-process request+ // continues this span.+ headers := req.Headers+ if span.SpanContext().IsValid() {+ carrier := make(propagation.MapCarrier, len(req.Headers)+2)+ for k, v := range req.Headers {+ if key := http.CanonicalHeaderKey(k); key != "Traceparent" && key != "Tracestate" {+ carrier[k] = v+ }+ }+ otel.GetTextMapPropagator().Inject(ctx, carrier)+ headers = carrier}timeoutMillis := req.TimeoutMillis@@ -69,7 +107,7 @@ func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *CMethod: req.Method,Path: req.Path,Query: req.Query,- Headers: req.Headers,+ Headers: headers,Body: req.Body,TimeoutMillis: timeoutMillis,SessionID: tunnel.SessionID,@@ -82,9 +120,9 @@ func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *C}defer tunnel.Pending.Delete(requestID)- chunkRequestBody := len(req.Body) > defaultCommandChunkSize && slices.Contains(tunnel.Capabilities, tunnelCapabilityChunkedRequest)- if chunkRequestBody {- transferID := uuid.New().String()+ var transferID string+ if len(req.Body) > defaultCommandChunkSize && slices.Contains(tunnel.Capabilities, tunnelCapabilityChunkedRequest) {+ transferID = uuid.New().String()msg.Body = nilmsg.Metadata = map[string]string{bodyTransferMetadataKey: transferID}}@@ -92,18 +130,17 @@ func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *Cif sendErr := tunnel.Conn.Send(msg); sendErr != nil {return nil, fmt.Errorf("tunnel request failed: %w", sendErr)}- if chunkRequestBody {- transferID := msg.Metadata[bodyTransferMetadataKey]+ if transferID != "" {for sequence, offset := int64(0), 0; offset < len(req.Body); sequence++ {end := min(offset+defaultCommandChunkSize, len(req.Body))- if sendErr2 := tunnel.Conn.Send(&TunnelMessage{+ if sendErr := tunnel.Conn.Send(&TunnelMessage{ID: transferID,Type: MessageTypeFileChunk,Body: req.Body[offset:end],Sequence: sequence,EOF: end == len(req.Body),- }); sendErr2 != nil {- return nil, fmt.Errorf("tunnel request body transfer failed: %w", sendErr2)+ }); sendErr != nil {+ return nil, fmt.Errorf("tunnel request body transfer failed: %w", sendErr)}offset = end}@@ -134,27 +171,10 @@ func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *C}func (c *CommandClient) OpenStream(ctx context.Context, tunnel *AgentTunnel, req *CommandRequest) error {- if ctx == nil {- return errors.New("context is required")- }- if err := validateConnectedTunnelInternal(tunnel); err != nil {+ commandName, err := resolveCommand(ctx, tunnel, req, true)+ if err != nil {return err}- if req == nil {- return errors.New("command request is required")- }- if req.ID == "" {- return errors.New("stream ID is required")- }-- commandName := req.Command- if commandName == "" {- resolved, ok := ResolveEdgeCommandName(http.MethodGet, req.Path, true).Get()- if !ok {- return fmt.Errorf("unsupported edge stream target %q", req.Path)- }- commandName = resolved- }msg := &TunnelMessage{ID: req.ID,@@ -177,7 +197,7 @@ func (c *CommandClient) OpenStream(ctx context.Context, tunnel *AgentTunnel, reqvar DefaultCommandClient = NewCommandClient()// grantCommandCredit queues credit for a command without blocking. A per-tunnel sender coalesces-// queued credits and sends them, so a stalled connection never holds up the response reader.+// queued credits, so a stalled connection never holds up the response reader.func (t *AgentTunnel) grantCommandCredit(ctx context.Context, commandID string, consumed int64) {t.creditOnce.Do(func() {t.credits = make(map[string]int64)@@ -212,9 +232,25 @@ func (t *AgentTunnel) grantCommandCredit(ctx context.Context, commandID string,}}-func validateConnectedTunnelInternal(tunnel *AgentTunnel) error {- if tunnel == nil || tunnel.Conn == nil || tunnel.Conn.IsClosed() {- return errors.New("edge tunnel is not connected")+// resolveCommand validates a command request and resolves its edge command name.+func resolveCommand(ctx context.Context, tunnel *AgentTunnel, req *CommandRequest, stream bool) (string, error) {+ switch {+ case ctx == nil:+ return "", errors.New("context is required")+ case !tunnel.connected():+ return "", errors.New("edge tunnel is not connected")+ case req == nil:+ return "", errors.New("command request is required")+ case stream && req.ID == "":+ return "", errors.New("stream ID is required")+ case req.Command != "":+ return req.Command, nil+ }+ if name, ok := ResolveEdgeCommandName(kit.Ternary(stream, http.MethodGet, req.Method), req.Path, stream).Get(); ok {+ return name, nil+ }+ if stream {+ return "", fmt.Errorf("unsupported edge stream target %q", req.Path)}- return nil+ return "", fmt.Errorf("unsupported edge command for %s %s", req.Method, req.Path)}diff --git a/backend/pkg/libarcane/edge/poll_control.go b/backend/pkg/libarcane/edge/poll_control.goindex 79e38de68..e731852b6 100644--- a/backend/pkg/libarcane/edge/poll_control.go+++ b/backend/pkg/libarcane/edge/poll_control.go@@ -19,14 +19,10 @@ import (const (// DefaultTunnelPollInterval is how often poll-mode agents should check in.DefaultTunnelPollInterval = 2 * time.Second- // DefaultPollRuntimeTTL is how long a poll check-in is considered fresh for- // runtime status reporting when no live tunnel is currently open.+ // DefaultPollRuntimeTTL is the minimum time a poll check-in stays fresh for runtime status.DefaultPollRuntimeTTL = 6 * time.Second- // DefaultTunnelDemandTTL is how long the manager should keep an edge tunnel- // marked as required after a user/API request touches the environment. It is- // deliberately longer than the default environment health interval (2m) so- // health checks refreshing the demand keep idle poll-mode tunnels open- // instead of letting them flap between checks.+ // DefaultTunnelDemandTTL keeps a touched environment's tunnel required; it exceeds the 2m health+ // interval so health checks keep idle poll-mode tunnels open instead of flapping.DefaultTunnelDemandTTL = 5 * time.Minute// TunnelStatusIdle indicates that no reverse tunnel is currently needed.@@ -37,15 +33,9 @@ const (TunnelStatusActive = "ACTIVE")-func pollRuntimeTTLInternal(state PollRuntimeState) time.Duration {- ttl := DefaultPollRuntimeTTL- if state.PollIntervalSeconds > 0 {- intervalTTL := time.Duration(state.PollIntervalSeconds) * time.Second * 3- if intervalTTL > ttl {- ttl = intervalTTL- }- }- return ttl+// pollRuntimeTTL is how long a check-in stays fresh: three poll intervals, never under DefaultPollRuntimeTTL.+func pollRuntimeTTL(state PollRuntimeState) time.Duration {+ return max(DefaultPollRuntimeTTL, time.Duration(state.PollIntervalSeconds)*time.Second*3)}// NewTunnelDemandRegistry creates a new tunnel demand registry.@@ -164,7 +154,7 @@ func (r *PollRuntimeRegistry) Get(envID string, now time.Time) mo.Option[PollRunreturn mo.None[PollRuntimeState]()}- ttl := pollRuntimeTTLInternal(state)+ ttl := pollRuntimeTTL(state)if now.Sub(*state.LastPollAt) > ttl {r.mu.Lock()@@ -173,7 +163,7 @@ func (r *PollRuntimeRegistry) Get(envID string, now time.Time) mo.Option[PollRunr.mu.Unlock()return mo.None[PollRuntimeState]()}- ttl = pollRuntimeTTLInternal(state)+ ttl = pollRuntimeTTL(state)if now.Sub(*state.LastPollAt) > ttl {delete(r.states, envID)r.mu.Unlock()@@ -185,58 +175,21 @@ func (r *PollRuntimeRegistry) Get(envID string, now time.Time) mo.Option[PollRunreturn mo.Some(state)}-func decodeTunnelPollRequestInternal(c *echo.Context) (*TunnelPollRequest, error) {- if c == nil || c.Request() == nil || c.Request().Body == nil {- return &TunnelPollRequest{}, nil- }-- req := c.Request()- defer func() { _ = req.Body.Close() }()-- var pollReq TunnelPollRequest- if err := json.UnmarshalRead(req.Body, &pollReq); err != nil {- if errors.Is(err, http.ErrBodyReadAfterClose) {- return &TunnelPollRequest{}, nil- }- if errors.Is(err, io.EOF) {- return &TunnelPollRequest{}, nil- }- return nil, err- }-- return &pollReq, nil-}--func (s *TunnelServer) pollStatusInternal(envID string) TunnelPollResponse {- hasActiveTunnel := false- activeTransport := ""-- if tunnel, ok := s.registry.Get(envID).Get(); ok && tunnel != nil && tunnel.Conn != nil && !tunnel.Conn.IsClosed() {- hasActiveTunnel = true- activeTransport = tunnel.Conn.Transport()- }-- status := GetDemandRegistry().DesiredStatus(envID, hasActiveTunnel, time.Now())- return TunnelPollResponse{- Status: status,- PollIntervalSeconds: int(DefaultTunnelPollInterval / time.Second),- ActiveTransport: activeTransport,- Connected: hasActiveTunnel,- }-}-// HandlePoll is the HTTP control-plane endpoint used by poll-mode agents.func (s *TunnelServer) HandlePoll(c *echo.Context) error {req := c.Request()ctx := req.Context()- if _, err := decodeTunnelPollRequestInternal(c); err != nil {- return c.JSON(http.StatusBadRequest, map[string]any{"error": "invalid poll payload"})+ if req.Body != nil {+ var pollReq TunnelPollRequest+ err := json.UnmarshalRead(req.Body, &pollReq)+ _ = req.Body.Close()+ if err != nil && !errors.Is(err, http.ErrBodyReadAfterClose) && !errors.Is(err, io.EOF) {+ return c.JSON(http.StatusBadRequest, map[string]any{"error": "invalid poll payload"})+ }}- // In proxy-terminated mTLS deployments, the client certificate is consumed- // by the TLS terminator before this request reaches Arcane. The token is- // still needed as the poll protocol's environment lookup claim.+ // Proxy-terminated mTLS consumes the client certificate, so the token remains the environment claim.token, source := agentToken(req.Header.Values)if token == "" {slog.WarnContext(ctx, "Edge poll request without token")@@ -264,10 +217,18 @@ func (s *TunnelServer) HandlePoll(c *echo.Context) error {return c.JSON(http.StatusUnauthorized, map[string]any{"error": identityErr.Error()})}- pollInterval := DefaultTunnelPollInterval- GetPollRuntimeRegistry().Update(envID, pollInterval, time.Now())+ now := time.Now()+ GetPollRuntimeRegistry().Update(envID, DefaultTunnelPollInterval, now)- resp := s.pollStatusInternal(envID)- resp.PollIntervalSeconds = int(pollInterval / time.Second)+ tunnel, _ := s.registry.Get(envID).Get()+ connected := tunnel.connected()+ resp := TunnelPollResponse{+ Status: GetDemandRegistry().DesiredStatus(envID, connected, now),+ PollIntervalSeconds: int(DefaultTunnelPollInterval / time.Second),+ Connected: connected,+ }+ if connected {+ resp.ActiveTransport = tunnel.Conn.Transport()+ }return c.JSON(http.StatusOK, resp)}diff --git a/backend/pkg/libarcane/edge/registry.go b/backend/pkg/libarcane/edge/registry.goindex 7fd9e75b2..0b4345ae6 100644--- a/backend/pkg/libarcane/edge/registry.go+++ b/backend/pkg/libarcane/edge/registry.go@@ -8,6 +8,8 @@ import ("time""github.com/samber/mo"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency")@@ -32,6 +34,11 @@ func (t *AgentTunnel) UpdateHeartbeat() {t.mu.Unlock()}+// connected reports whether the tunnel has a live, open connection.+func (t *AgentTunnel) connected() bool {+ return t != nil && t.Conn != nil && !t.Conn.IsClosed()+}+// GetLastHeartbeat returns the last heartbeat timefunc (t *AgentTunnel) GetLastHeartbeat() time.Time {t.mu.RLock()@@ -115,20 +122,14 @@ func (r *TunnelRegistry) Register(envID string, tunnel *AgentTunnel) {slog.ErrorContext(ctx, "Failed to register edge agent tunnel", "environmentId", envID, "error", "tunnel is required")return}- previous,- err := r.tunnels.ApplyTyped(ctx,- "register edge tunnel",- func(tunnels map[string]*AgentTunnel) (*AgentTunnel,- bool,- error,- ) {- if r.stopped.Load() {- return nil, false, errors.New("edge tunnel registry stopped")- }- previous := tunnels[envID]- tunnels[envID] = tunnel- return previous, true, nil- })+ previous, err := r.tunnels.ApplyTyped(ctx, "register edge tunnel", func(tunnels map[string]*AgentTunnel) (*AgentTunnel, bool, error) {+ if r.stopped.Load() {+ return nil, false, errors.New("edge tunnel registry stopped")+ }+ previous := tunnels[envID]+ tunnels[envID] = tunnel+ return previous, true, nil+ })if err != nil {slog.ErrorContext(ctx, "Failed to register edge agent tunnel", "environmentId", envID, "error", err)return@@ -140,7 +141,7 @@ func (r *TunnelRegistry) Register(envID string, tunnel *AgentTunnel) {slog.InfoContext(ctx, "Edge agent tunnel registered")}-type registerSessionResultInternal struct {+type registerSessionResult struct {accepted booldrainPrevious boolreason string@@ -158,8 +159,8 @@ func (r *TunnelRegistry) RegisterSession(ctx context.Context, tunnel *AgentTunnereturn false, false, "environment ID is required", nil}- result, err := r.tunnels.ApplyTyped(ctx, "register edge tunnel session", func(tunnels map[string]*AgentTunnel) (registerSessionResultInternal, bool, error) {- var result registerSessionResultInternal+ result, err := r.tunnels.ApplyTyped(ctx, "register edge tunnel session", func(tunnels map[string]*AgentTunnel) (registerSessionResult, bool, error) {+ var result registerSessionResultif r.stopped.Load() {return result, false, errors.New("edge tunnel registry stopped")}@@ -173,7 +174,7 @@ func (r *TunnelRegistry) RegisterSession(ctx context.Context, tunnel *AgentTunnetunnelMetadata := tunnel.MetadataSnapshot()existingStale := staleAfter > 0 && time.Since(existingMetadata.LastHeartbeat) > staleAftersameAgentInstance := existingMetadata.AgentInstance != "" && tunnelMetadata.AgentInstance != "" && existingMetadata.AgentInstance == tunnelMetadata.AgentInstance- if existing.Conn != nil && !existing.Conn.IsClosed() && !existingStale && !sameAgentInstance {+ if existing.connected() && !existingStale && !sameAgentInstance {result.reason = "another edge agent session is already active"return result, false, nil}@@ -218,15 +219,15 @@ func (r *TunnelRegistry) UnregisterCurrent(ctx context.Context, envID string, cureturn false, false}- type resultInternal struct {+ type unregisterResult struct {removed boolactiveReplacement bool}- result, err := r.tunnels.ApplyTyped(ctx, "unregister current edge tunnel", func(tunnels map[string]*AgentTunnel) (resultInternal, bool, error) {- var result resultInternal+ result, err := r.tunnels.ApplyTyped(ctx, "unregister current edge tunnel", func(tunnels map[string]*AgentTunnel) (unregisterResult, bool, error) {+ var result unregisterResultexisting := tunnels[envID]if existing != current {- result.activeReplacement = existing != nil && existing.Conn != nil && !existing.Conn.IsClosed()+ result.activeReplacement = existing.connected()return result, false, nil}delete(tunnels, envID)@@ -284,6 +285,40 @@ func (r *TunnelRegistry) Stop(ctx context.Context) error {return err}+// ObserveAgents reports connected edge agents by transport through the arcane.edge.agents gauge.+// Poll-mode agents count as "poll" while their check-ins are fresh and no tunnel is open.+func (r *TunnelRegistry) ObserveAgents(meter metric.Meter) (metric.Registration, error) {+ gauge, err := meter.Int64ObservableGauge("arcane.edge.agents",+ metric.WithDescription("Connected edge agents by transport"),+ metric.WithUnit("{agent}"),+ )+ if err != nil {+ return nil, err+ }+ return meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {+ counts := map[string]int64{EdgeTransportWebSocket: 0, EdgeTransportGRPC: 0, EdgeTransportPoll: 0}+ tunneled := make(map[string]bool)+ for _, tunnel := range r.tunnels.Values() {+ if tunnel.connected() {+ counts[tunnel.Conn.Transport()]+++ tunneled[tunnel.EnvironmentID] = true+ }+ }+ polls, now := GetPollRuntimeRegistry(), time.Now()+ polls.mu.RLock()+ for envID, state := range polls.states {+ if !tunneled[envID] && state.LastPollAt != nil && now.Sub(*state.LastPollAt) <= pollRuntimeTTL(state) {+ counts[EdgeTransportPoll]+++ }+ }+ polls.mu.RUnlock()+ for transport, count := range counts {+ o.ObserveInt64(gauge, count, metric.WithAttributes(attribute.String("arcane.edge.transport", transport)))+ }+ return nil+ }, gauge)+}+var (defaultRegistryMu sync.RWMutexdefaultRegistry = NewTunnelRegistry()diff --git a/backend/pkg/libarcane/edge/server.go b/backend/pkg/libarcane/edge/server.goindex f8f9469a4..07d5bcbcc 100644--- a/backend/pkg/libarcane/edge/server.go+++ b/backend/pkg/libarcane/edge/server.go@@ -14,6 +14,7 @@ import ("path/filepath""slices""strings"+ "sync""time""uuid"@@ -21,6 +22,9 @@ import ("github.com/labstack/echo/v5""go.getarcane.app/acfs/atomic"kit "go.getarcane.app/kit/pkg"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/trace""google.golang.org/grpc""google.golang.org/grpc/codes""google.golang.org/grpc/credentials"@@ -32,6 +36,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/remenv""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1")@@ -47,6 +52,9 @@ const (maxPendingBacklogMessages = 4096)+// observeAgentsOnce registers the connected-agents gauge for the first (manager) tunnel server.+var observeAgentsOnce sync.Once+// EnvironmentResolver resolves an agent token to an environment ID.type EnvironmentResolver func(ctx context.Context, token string) (environmentID string, err error)@@ -69,6 +77,11 @@ func NewTunnelServerWithRegistry(registry *TunnelRegistry, resolver EnvironmentRif registry == nil {registry = NewTunnelRegistry()}+ observeAgentsOnce.Do(func() {+ if _, err := registry.ObserveAgents(otel.Meter(tracing.InstrumentationName)); err != nil {+ otel.Handle(err)+ }+ })return &TunnelServer{registry: registry,@@ -362,16 +375,31 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, envID string,tunnel.Capabilities = registerMsg.Capabilitiestunnel.Transport = conn.Transport()+ // The span covers the connection lifetime; ctx stays unparented so per-message work doesn't nest under it.+ _, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "edge.tunnel "+tunnel.Transport,+ trace.WithSpanKind(trace.SpanKindServer),+ trace.WithAttributes(+ attribute.String("arcane.edge.transport", tunnel.Transport),+ attribute.String("arcane.environment.id", envID),+ attribute.String("arcane.edge.session_id", tunnel.SessionID),+ attribute.String("arcane.edge.security_mode", securityMode),+ attribute.String("arcane.edge.agent_instance", tunnel.AgentInstance),+ ))+ var spanErr error+ defer func() { tracing.End(span, spanErr) }()+accepted, drainPrevious, rejectReason, err := s.registry.RegisterSession(callbackCtx, tunnel, TunnelStaleTimeout)if err != nil {slog.ErrorContext(ctx, "Failed to register edge agent session", "environmentId", envID, "agentInstanceId", tunnel.AgentInstance, "error", err)_ = tunnel.CloseWithReason("edge agent session registration unavailable")+ spanErr = errreturn}if !accepted {slog.WarnContext(ctx, "Rejected duplicate edge agent session", "environmentId", envID, "agentInstanceId", tunnel.AgentInstance, "reason", rejectReason)_ = conn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, EnvironmentID: envID, Error: rejectReason})_ = tunnel.CloseWithReason(rejectReason)+ spanErr = errors.New(rejectReason)return}@@ -401,6 +429,7 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, envID string,}); sendErr != nil {slog.WarnContext(ctx, "Failed to send register response", "environmentId", envID, "error", sendErr)_ = tunnel.CloseWithReason("")+ spanErr = sendErrreturn}s.updateConnectionStatus(callbackCtx, tunnel, true)@@ -419,6 +448,7 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, envID string,if receiveErr != nil {if !conn.IsExpectedReceiveError(receiveErr) {slog.WarnContext(ctx, "Error receiving from edge tunnel", "environmentId", envID, "error", receiveErr)+ spanErr = receiveErr}return}@@ -458,10 +488,8 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, envID string,}}-// enqueue hands msg to the consumer without blocking the tunnel. Behind a full channel, messages-// wait in order for up to pendingDeliveryTimeout each. A consumer that stalls or falls past the-// backlog limits is failed once, and a stream also tells the agent to stop. Queued messages-// outlive a tunnel close so the collector can still drain a fully received response.+// enqueue hands msg to the consumer without blocking the tunnel; queued messages wait in order and outlive+// a tunnel close. A consumer that stalls past pendingDeliveryTimeout or the backlog limits is failed once.func (p *PendingRequest) enqueue(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage, closeStream bool) {retained := func(m *TunnelMessage) int {size := len(m.Body) + len(m.Error)@@ -582,7 +610,7 @@ func (s *TunnelServer) updateConnectionStatus(ctx context.Context, tunnel *Agents.statusMu.Lock()defer s.statusMu.Unlock()if !connected {- if active, ok := s.registry.Get(tunnel.EnvironmentID).Get(); ok && active != tunnel && active.Conn != nil && !active.Conn.IsClosed() {+ if active, _ := s.registry.Get(tunnel.EnvironmentID).Get(); active != tunnel && active.connected() {return}}diff --git a/backend/pkg/libarcane/edge/tls.go b/backend/pkg/libarcane/edge/tls.goindex a6f386e43..a194ba057 100644--- a/backend/pkg/libarcane/edge/tls.go+++ b/backend/pkg/libarcane/edge/tls.go@@ -33,6 +33,7 @@ import ("go.getarcane.app/acfs/atomic"kit "go.getarcane.app/kit/pkg"libcrypto "go.getarcane.app/sys/crypto"+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"@@ -101,12 +102,7 @@ func NewManagerHTTPClient(cfg *Config, timeout time.Duration) (*http.Client, errif tlsConfig != nil {transport.TLSClientConfig = tlsConfig}-- client := &http.Client{Transport: transport}- if timeout > 0 {- client.Timeout = timeout- }- return client, nil+ return &http.Client{Transport: otelhttp.NewTransport(transport), Timeout: max(timeout, 0)}, nil}// PrepareManagerMTLSAssetsWithContext generates the manager CA when edge mTLS is@@ -345,7 +341,7 @@ func enrollAgentMTLSAssets(ctx context.Context, cfg *Config, assetsDir, certPathif reqErr != nil {return fmt.Errorf("failed to create edge mTLS enrollment request: %w", reqErr)}- for header, value := range agentAuthCredentialsInternal(cfg.AgentToken) {+ for header, value := range agentAuthCredentials(cfg.AgentToken) {req.Header.Set(header, value)}diff --git a/backend/pkg/libarcane/edge/transport.go b/backend/pkg/libarcane/edge/transport.goindex 14cf9ba9c..100523f97 100644--- a/backend/pkg/libarcane/edge/transport.go+++ b/backend/pkg/libarcane/edge/transport.go@@ -83,11 +83,10 @@ func UsePollEdgeTransport(cfg *Config) bool {// GetActiveTunnelTransport returns the currently active tunnel transport for an environment.func GetActiveTunnelTransport(envID string) mo.Option[string] {- tunnel, ok := GetRegistry().Get(envID).Get()- if !ok || tunnel == nil || tunnel.Conn == nil || tunnel.Conn.IsClosed() {+ tunnel, ok := GetActiveTunnel(envID).Get()+ if !ok {return mo.None[string]()}-if transport := tunnel.Conn.Transport(); transport != "" {return mo.Some(transport)}@@ -96,13 +95,11 @@ func GetActiveTunnelTransport(envID string) mo.Option[string] {// GetTunnelRuntimeState returns live metadata for an active tunnel.func GetTunnelRuntimeState(envID string) mo.Option[*TunnelRuntimeState] {- tunnel, ok := GetRegistry().Get(envID).Get()- if !ok || tunnel == nil || tunnel.Conn == nil || tunnel.Conn.IsClosed() {+ tunnel, ok := GetActiveTunnel(envID).Get()+ if !ok {return mo.None[*TunnelRuntimeState]()}-- // One snapshot under a single read lock, so the reported fields all describe- // the same moment and none of them race the tunnel's writers.+ // One snapshot under a single read lock keeps the fields consistent and race-free.meta := tunnel.MetadataSnapshot()return mo.Some(&TunnelRuntimeState{diff --git a/backend/pkg/libarcane/ws/metrics.go b/backend/pkg/libarcane/ws/metrics.goindex fbc5826ef..504ef7f2e 100644--- a/backend/pkg/libarcane/ws/metrics.go+++ b/backend/pkg/libarcane/ws/metrics.go@@ -1,31 +1,39 @@package wsimport (+ "context"+ "maps"+ "slices""strconv""sync""sync/atomic""time"systemtypes "github.com/getarcaneapp/arcane/types/v2/system"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/metric")// WebSocketMetrics tracks active WebSocket connections and their counts.type WebSocketMetrics struct {- projectLogsActive atomic.Int64- containerLogsActive atomic.Int64- containerStats atomic.Int64- containerExec atomic.Int64- systemStats atomic.Int64- serviceLogsActive atomic.Int64- diagnosticsActive atomic.Int64- seq atomic.Uint64- mu sync.RWMutex- connections map[string]systemtypes.WebSocketConnectionInfo+ counters map[string]*atomic.Int64+ seq atomic.Uint64+ mu sync.RWMutex+ connections map[string]systemtypes.WebSocketConnectionInfo}// NewWebSocketMetrics creates a new WebSocketMetrics instance.func NewWebSocketMetrics() *WebSocketMetrics {return &WebSocketMetrics{+ counters: map[string]*atomic.Int64{+ systemtypes.WSKindProjectLogs: {},+ systemtypes.WSKindContainerLogs: {},+ systemtypes.WSKindContainerStats: {},+ systemtypes.WSKindContainerExec: {},+ systemtypes.WSKindSystemStats: {},+ systemtypes.WSKindServiceLogs: {},+ systemtypes.WSKindDiagnostics: {},+ },connections: make(map[string]systemtypes.WebSocketConnectionInfo),}}@@ -33,29 +41,43 @@ func NewWebSocketMetrics() *WebSocketMetrics {// Snapshot returns a point-in-time copy of the active connection counts.func (m *WebSocketMetrics) Snapshot() systemtypes.WebSocketMetricsSnapshot {return systemtypes.WebSocketMetricsSnapshot{- ProjectLogsActive: m.projectLogsActive.Load(),- ContainerLogsActive: m.containerLogsActive.Load(),- ContainerStats: m.containerStats.Load(),- ContainerExec: m.containerExec.Load(),- SystemStats: m.systemStats.Load(),- ServiceLogsActive: m.serviceLogsActive.Load(),- DiagnosticsActive: m.diagnosticsActive.Load(),+ ProjectLogsActive: m.counters[systemtypes.WSKindProjectLogs].Load(),+ ContainerLogsActive: m.counters[systemtypes.WSKindContainerLogs].Load(),+ ContainerStats: m.counters[systemtypes.WSKindContainerStats].Load(),+ ContainerExec: m.counters[systemtypes.WSKindContainerExec].Load(),+ SystemStats: m.counters[systemtypes.WSKindSystemStats].Load(),+ ServiceLogsActive: m.counters[systemtypes.WSKindServiceLogs].Load(),+ DiagnosticsActive: m.counters[systemtypes.WSKindDiagnostics].Load(),}}+// ObserveConnections reports active connection counts by kind through the arcane.websocket.connections gauge.+func (m *WebSocketMetrics) ObserveConnections(meter metric.Meter) error {+ gauge, err := meter.Int64ObservableGauge("arcane.websocket.connections",+ metric.WithDescription("Active WebSocket connections by kind"),+ metric.WithUnit("{connection}"),+ )+ if err != nil {+ return err+ }+ _, err = meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {+ for kind, counter := range m.counters {+ o.ObserveInt64(gauge, counter.Load(), metric.WithAttributes(attribute.String("kind", kind)))+ }+ return nil+ }, gauge)+ return err+}+// Connections returns a snapshot of all tracked WebSocket connections.func (m *WebSocketMetrics) Connections() []systemtypes.WebSocketConnectionInfo {m.mu.RLock()defer m.mu.RUnlock()- result := make([]systemtypes.WebSocketConnectionInfo, 0, len(m.connections))- for _, info := range m.connections {- result = append(result, info)- }- return result+ return slices.AppendSeq(make([]systemtypes.WebSocketConnectionInfo, 0, len(m.connections)), maps.Values(m.connections))}-// RegisterConnection adds a connection to the tracker and increments the-// appropriate kind counter. Returns the assigned connection ID.+// RegisterConnection adds a connection to the tracker and increments its kind counter.+// Returns the assigned connection ID.func (m *WebSocketMetrics) RegisterConnection(info systemtypes.WebSocketConnectionInfo) string {if info.ID == "" {info.ID = "ws-" + strconv.FormatUint(m.seq.Add(1), 10)@@ -66,43 +88,19 @@ func (m *WebSocketMetrics) RegisterConnection(info systemtypes.WebSocketConnectim.mu.Lock()m.connections[info.ID] = infom.mu.Unlock()- m.applyDelta(info.Kind, 1)+ if counter, ok := m.counters[info.Kind]; ok {+ counter.Add(1)+ }return info.ID}-// UnregisterConnection removes a connection from the tracker and decrements-// the appropriate kind counter.+// UnregisterConnection removes a connection from the tracker and decrements its kind counter.func (m *WebSocketMetrics) UnregisterConnection(id string) {- if id == "" {- return- }- var info systemtypes.WebSocketConnectionInfom.mu.Lock()- if existing, ok := m.connections[id]; ok {- info = existing- delete(m.connections, id)- }+ info := m.connections[id]+ delete(m.connections, id)m.mu.Unlock()- if info.Kind != "" {- m.applyDelta(info.Kind, -1)- }-}--func (m *WebSocketMetrics) applyDelta(kind string, delta int64) {- switch kind {- case systemtypes.WSKindProjectLogs:- m.projectLogsActive.Add(delta)- case systemtypes.WSKindContainerLogs:- m.containerLogsActive.Add(delta)- case systemtypes.WSKindContainerStats:- m.containerStats.Add(delta)- case systemtypes.WSKindContainerExec:- m.containerExec.Add(delta)- case systemtypes.WSKindSystemStats:- m.systemStats.Add(delta)- case systemtypes.WSKindServiceLogs:- m.serviceLogsActive.Add(delta)- case systemtypes.WSKindDiagnostics:- m.diagnosticsActive.Add(delta)+ if counter, ok := m.counters[info.Kind]; ok {+ counter.Add(-1)}}diff --git a/backend/pkg/libarcane/ws/proxy.go b/backend/pkg/libarcane/ws/proxy.goindex bc14da766..29b9c77e1 100644--- a/backend/pkg/libarcane/ws/proxy.go+++ b/backend/pkg/libarcane/ws/proxy.go@@ -10,12 +10,17 @@ import ("github.com/coder/websocket"httpxtypes "github.com/getarcaneapp/arcane/types/v2/httpx"+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx")-// dialClient is shared by every upstream dial instead of building a transport per connection.-var dialClient = httpx.NewHTTPClient(httpxtypes.ClientOptions{TLSHandshakeTimeout: 10 * time.Second})+// dialClient is shared by every upstream dial; its span covers the whole proxied session.+var dialClient = func() *http.Client {+ client := httpx.NewHTTPClient(httpxtypes.ClientOptions{TLSHandshakeTimeout: 10 * time.Second})+ client.Transport = otelhttp.NewTransport(client.Transport)+ return client+}()// ProxyHTTP upgrades the incoming client connection and bridges it to remoteWS.//@@ -50,12 +55,11 @@ func ProxyHTTP(w http.ResponseWriter, r *http.Request, remoteWS string, header h_ = resp.Body.Close()}if err != nil {- slog.ErrorContext(dialCtx, "failed to dial remote websocket", "remoteWs", remoteWS, "err", err, "respStatus", func() int {- if resp != nil {- return resp.StatusCode- }- return 0- }())+ respStatus := 0+ if resp != nil {+ respStatus = resp.StatusCode+ }+ slog.ErrorContext(dialCtx, "failed to dial remote websocket", "remoteWs", remoteWS, "err", err, "respStatus", respStatus)_ = clientConn.Close(websocket.StatusBadGateway, "")return err}@@ -70,53 +74,31 @@ func ProxyHTTP(w http.ResponseWriter, r *http.Request, remoteWS string, header hdefer pumpCancel()errc := make(chan struct{}, 2)-- // client -> remote- go func() {- defer func() { errc <- struct{}{} }()- for {- mt, msg, readErr := clientConn.Read(pumpCtx)- if readErr != nil {- closeRelay(r.Context(), remoteConn, readErr)- return- }- if writeErr := remoteConn.Write(pumpCtx, mt, msg); writeErr != nil {- return- }- }- }()-- // remote -> client- go func() {+ pump := func(src, dst *websocket.Conn) {defer func() { errc <- struct{}{} }()for {- mt, msg, readErr := remoteConn.Read(pumpCtx)+ mt, msg, readErr := src.Read(pumpCtx)if readErr != nil {- closeRelay(r.Context(), clientConn, readErr)+ // Forward the peer's close status so the terminating reason survives the proxy hop.+ if ce, ok := errors.AsType[websocket.CloseError](readErr); ok {+ code := ce.Code+ if code == websocket.StatusNoStatusRcvd {+ code = websocket.StatusNormalClosure+ }+ if closeErr := dst.Close(code, ce.Reason); closeErr != nil {+ slog.DebugContext(r.Context(), "failed to relay websocket close", "status", int(code), "err", closeErr)+ }+ }return}- if writeErr := clientConn.Write(pumpCtx, mt, msg); writeErr != nil {+ if writeErr := dst.Write(pumpCtx, mt, msg); writeErr != nil {return}}- }()+ }+ go pump(clientConn, remoteConn)+ go pump(remoteConn, clientConn)<-errcreturn nil}--// closeRelay forwards a peer's close status to the other side of the-// bridge so the terminating reason survives the proxy hop.-func closeRelay(ctx context.Context, conn *websocket.Conn, err error) {- var ce websocket.CloseError- if !errors.As(err, &ce) {- return- }- code := ce.Code- if code == websocket.StatusNoStatusRcvd {- code = websocket.StatusNormalClosure- }- if closeErr := conn.Close(code, ce.Reason); closeErr != nil {- slog.DebugContext(ctx, "failed to relay websocket close", "status", int(code), "err", closeErr)- }-}diff --git a/backend/pkg/remenv/client.go b/backend/pkg/remenv/client.goindex c996e9d1c..c3c0fe059 100644--- a/backend/pkg/remenv/client.go+++ b/backend/pkg/remenv/client.go@@ -11,6 +11,8 @@ import ("net/http""strings""time"++ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp")const (@@ -22,9 +24,8 @@ const (var ErrEnvironmentUnavailable = errors.New("edge agent is not connected")-// ExtractBearerToken returns the token portion of an "Authorization: Bearer <token>"-// header value. It is case-insensitive on the scheme and trims surrounding whitespace.-// Returns an empty string if the header is empty or does not use the Bearer scheme.+// ExtractBearerToken returns the trimmed token from an "Authorization: Bearer <token>" value,+// or "" when the scheme (case-insensitive) is not Bearer.func ExtractBearerToken(headerValue string) string {headerValue = strings.TrimSpace(headerValue)if headerValue == "" {@@ -36,9 +37,7 @@ func ExtractBearerToken(headerValue string) string {return strings.TrimSpace(headerValue[len(bearerScheme):])}-// RedactedTokenFingerprint returns a short, redacted identifier for a token-// suitable for debug logs. It is intentionally not reversible and never-// reveals the full secret.+// RedactedTokenFingerprint returns a short, non-reversible token identifier for debug logs.func RedactedTokenFingerprint(token string) string {token = strings.TrimSpace(token)if len(token) <= 10 {@@ -93,42 +92,38 @@ type Client struct {}func NewClient(httpClient *http.Client, tunnel TunnelTransport) *Client {- if httpClient == nil {- httpClient = &http.Client{Timeout: 30 * time.Second}+ client := &http.Client{Timeout: 30 * time.Second}+ if httpClient != nil {+ copied := *httpClient+ client = &copied}+ client.Transport = otelhttp.NewTransport(client.Transport)return &Client{- httpClient: httpClient,+ httpClient: client,tunnel: tunnel,}}func (c *Client) Do(ctx context.Context, req Request) (*Response, error) {if req.IsEdge {- return c.doViaTunnelInternal(ctx, req)- }-- return c.doDirectHTTPInternal(ctx, req)-}--func (c *Client) doViaTunnelInternal(ctx context.Context, req Request) (*Response, error) {- if c.tunnel == nil {- return nil, &TransportError{Err: errors.New("edge transport unavailable")}- }-- if err := c.tunnel.EnsureAvailable(ctx, req.EnvironmentID); err != nil {- return nil, &TransportError{Err: err}- }-- resp, err := c.tunnel.Do(ctx, req.EnvironmentID, req.Method, req.Path, cloneHeaders(req.Headers), req.Body)- if err != nil {- return nil, &TransportError{Err: err}+ if c.tunnel == nil {+ return nil, &TransportError{Err: errors.New("edge transport unavailable")}+ }+ if err := c.tunnel.EnsureAvailable(ctx, req.EnvironmentID); err != nil {+ return nil, &TransportError{Err: err}+ }+ headers := maps.Clone(req.Headers)+ if headers == nil {+ headers = map[string]string{}+ }+ resp, err := c.tunnel.Do(ctx, req.EnvironmentID, req.Method, req.Path, headers, req.Body)+ if err != nil {+ return nil, &TransportError{Err: err}+ }+ return resp, nil}- return resp, nil-}--func (c *Client) doDirectHTTPInternal(ctx context.Context, req Request) (*Response, error) {var bodyReader io.Readerif len(req.Body) > 0 {bodyReader = bytes.NewReader(req.Body)@@ -154,26 +149,23 @@ func (c *Client) doDirectHTTPInternal(ctx context.Context, req Request) (*Responreturn nil, &TransportError{Err: fmt.Errorf("failed to read response body: %w", err)}}- return &Response{- StatusCode: resp.StatusCode,- Body: respBody,- Headers: flattenHeaders(resp.Header),- }, nil+ headers := make(map[string]string, len(resp.Header))+ for key, values := range resp.Header {+ if len(values) > 0 {+ headers[key] = values[0]+ }+ }+ return &Response{StatusCode: resp.StatusCode, Body: respBody, Headers: headers}, nil}func (r *Response) RequireSuccess() error {- if r != nil && r.StatusCode >= http.StatusOK && r.StatusCode < http.StatusMultipleChoices {- return nil- }-if r == nil {return &StatusError{}}-- return &StatusError{- StatusCode: r.StatusCode,- Body: r.Body,+ if r.StatusCode >= http.StatusOK && r.StatusCode < http.StatusMultipleChoices {+ return nil}+ return &StatusError{StatusCode: r.StatusCode, Body: r.Body}}type TransportError struct {@@ -189,16 +181,15 @@ func (e *TransportError) Unavailable() bool {if errors.Is(e.Err, context.DeadlineExceeded) || errors.Is(e.Err, ErrEnvironmentUnavailable) {return true}- var networkErr net.Error- if errors.As(e.Err, &networkErr) && networkErr.Timeout() {+ if networkErr, ok := errors.AsType[net.Error](e.Err); ok && networkErr.Timeout() {return true}- var operationErr *net.OpError- if !errors.As(e.Err, &operationErr) || operationErr.Op != "dial" {+ operationErr, ok := errors.AsType[*net.OpError](e.Err)+ if !ok || operationErr.Op != "dial" {return false}- var addressErr *net.AddrError- return !errors.As(operationErr, &addressErr)+ _, isAddressErr := errors.AsType[*net.AddrError](operationErr)+ return !isAddressErr}func (e *TransportError) Error() string {@@ -248,14 +239,8 @@ func (e *DecodeError) Unwrap() error {return e.Err}-// ApplyAgentTokenHeaders sets the agent token in both X- header forms on the-// given http.Header. It deliberately does NOT set the Authorization header:-// these helpers are used by the manager when proxying user requests through-// to remote environments, and the Authorization header in that case carries-// the calling user's bearer or session token. Direct agent->manager call-// sites (poll, websocket dial, grpc dial, mTLS enroll) set-// `Authorization: Bearer <token>` themselves so the token survives reverse-// proxies that strip non-standard X- headers.+// ApplyAgentTokenHeaders sets the agent token in both X- header forms. It never sets+// Authorization, which carries the proxied user's own token.func ApplyAgentTokenHeaders(headers http.Header, accessToken *string) {if headers == nil || accessToken == nil || strings.TrimSpace(*accessToken) == "" {return@@ -265,8 +250,7 @@ func ApplyAgentTokenHeaders(headers http.Header, accessToken *string) {headers.Set(HeaderAPIKey, *accessToken)}-// ApplyAgentTokenHeaderMap mirrors ApplyAgentTokenHeaders for map-based-// header carriers. Same rationale: do not write Authorization here.+// ApplyAgentTokenHeaderMap mirrors ApplyAgentTokenHeaders for map-based header carriers.func ApplyAgentTokenHeaderMap(headers map[string]string, accessToken *string) {if headers == nil || accessToken == nil || strings.TrimSpace(*accessToken) == "" {return@@ -275,27 +259,3 @@ func ApplyAgentTokenHeaderMap(headers map[string]string, accessToken *string) {headers[HeaderAgentToken] = *accessTokenheaders[HeaderAPIKey] = *accessToken}--func flattenHeaders(headers http.Header) map[string]string {- if len(headers) == 0 {- return map[string]string{}- }-- out := make(map[string]string, len(headers))- for key, values := range headers {- if len(values) > 0 {- out[key] = values[0]- }- }- return out-}--func cloneHeaders(headers map[string]string) map[string]string {- if len(headers) == 0 {- return map[string]string{}- }-- out := make(map[string]string, len(headers))- maps.Copy(out, headers)- return out-}diff --git a/backend/pkg/scheduler/analytics_job.go b/backend/pkg/scheduler/analytics_job.goindex 46916f152..70deba030 100644--- a/backend/pkg/scheduler/analytics_job.go+++ b/backend/pkg/scheduler/analytics_job.go@@ -16,6 +16,7 @@ import ("github.com/cenkalti/backoff/v5"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"kit "go.getarcane.app/kit/pkg"+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/kv"@@ -54,6 +55,10 @@ func NewAnalyticsJob(if httpClient == nil {httpClient = &http.Client{Timeout: 30 * time.Second}}+ // Trace a copy so the injected shared client keeps its raw transport.+ traced := *httpClient+ traced.Transport = otelhttp.NewTransport(httpClient.Transport)+ httpClient = &tracedheartbeatURL := kit.Ternary(!cfg.Environment.IsProdEnvironment(), devHeartbeatEndpoint, defaultHeartbeatEndpoint)return &AnalyticsJob{settingsService: settingsService,diff --git a/backend/pkg/utils/httpx/safe_remote.go b/backend/pkg/utils/httpx/safe_remote.goindex ca282c40a..465136c70 100644--- a/backend/pkg/utils/httpx/safe_remote.go+++ b/backend/pkg/utils/httpx/safe_remote.go@@ -8,148 +8,116 @@ import ("net/http""net/netip""net/url"+ "slices""strings""time"+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"+"github.com/getarcaneapp/arcane/backend/v2/internal/common")type LookupIPFunc func(ctx context.Context, host string) ([]net.IP, error)-// blockedRemotePrefixes contains special-use ranges that should never be treated-// as valid public registry destinations. We intentionally keep this list explicit-// because Go exposes helpers for some classes (for example private, loopback, and-// link-local) but does not provide one "publicly routable on the internet" check-// that covers the full SSRF threat model. These prefixes complement the net.IP-// helper checks in isBlockedIPInternal and make the policy reviewable in one place.-var blockedRemotePrefixes = mustParsePrefixesInternal(- "0.0.0.0/8",- "100.64.0.0/10",- "127.0.0.0/8",- "169.254.0.0/16",- "192.0.0.0/24",- "192.0.2.0/24",- "198.18.0.0/15",- "198.51.100.0/24",- "203.0.113.0/24",- "224.0.0.0/4",- "240.0.0.0/4",- "::/128",- "::1/128",- "2001:db8::/32",- "fc00::/7",- "fe80::/10",- "ff00::/8",-)+// blockedRemotePrefixes lists special-use ranges the netip.Addr helpers in resolveAllowedIPs do not cover,+// keeping the SSRF policy reviewable in one place.+var blockedRemotePrefixes = []netip.Prefix{+ netip.MustParsePrefix("0.0.0.0/8"),+ netip.MustParsePrefix("100.64.0.0/10"),+ netip.MustParsePrefix("127.0.0.0/8"),+ netip.MustParsePrefix("169.254.0.0/16"),+ netip.MustParsePrefix("192.0.0.0/24"),+ netip.MustParsePrefix("192.0.2.0/24"),+ netip.MustParsePrefix("198.18.0.0/15"),+ netip.MustParsePrefix("198.51.100.0/24"),+ netip.MustParsePrefix("203.0.113.0/24"),+ netip.MustParsePrefix("224.0.0.0/4"),+ netip.MustParsePrefix("240.0.0.0/4"),+ netip.MustParsePrefix("::/128"),+ netip.MustParsePrefix("::1/128"),+ netip.MustParsePrefix("2001:db8::/32"),+ netip.MustParsePrefix("fc00::/7"),+ netip.MustParsePrefix("fe80::/10"),+ netip.MustParsePrefix("ff00::/8"),+}func DefaultLookupIP(ctx context.Context, host string) ([]net.IP, error) {return net.DefaultResolver.LookupIP(ctx, "ip", host)}func ValidateSafeRemoteURL(ctx context.Context, rawURL string, lookupIP LookupIPFunc) (*url.URL, error) {- if lookupIP == nil {- lookupIP = DefaultLookupIP- }-- parsed, err := url.Parse(rawURL)+ parsed, err := ValidateOutboundHTTPURL(rawURL)if err != nil {- return nil, common.Classify(common.ErrUnsafeRemoteURL, fmt.Errorf("Remote URL is not allowed: %w", err)) //nolint:staticcheck // Preserve the existing error message.- }-- scheme := strings.ToLower(parsed.Scheme)- if scheme != "http" && scheme != "https" {- //nolint:staticcheck // Preserve the existing error message.- return nil,- common.Classify(common.ErrUnsafeRemoteURL,- fmt.Errorf("Remote URL is not allowed: %w",- fmt.Errorf("unsupported URL scheme %q",- scheme)))+ return nil, unsafeRemoteURLError(err)}- if parsed.User != nil {- //nolint:staticcheck // Preserve the existing error message.- return nil,- common.Classify(common.ErrUnsafeRemoteURL,- fmt.Errorf("Remote URL is not allowed: %w",- errors.New("URL credentials are not allowed")))+ host := strings.ToLower(parsed.Hostname())+ if host == "localhost" || strings.HasSuffix(host, ".localhost") {+ return nil, unsafeRemoteURLError(errors.New("missing or blocked hostname"))}- host := parsed.Hostname()- if host == "" || isBlockedHostnameInternal(host) {- //nolint:staticcheck // Preserve the existing error message.- return nil,- common.Classify(common.ErrUnsafeRemoteURL,- fmt.Errorf("Remote URL is not allowed: %w",- errors.New("missing or blocked hostname")))- }-- ips, err := resolveAllowedIPsInternal(ctx, host, lookupIP)- if err != nil || len(ips) == 0 {- if err == nil {- err = errors.New("host did not resolve to an allowed IP")- }- return nil, common.Classify(common.ErrUnsafeRemoteURL, fmt.Errorf("Remote URL is not allowed: %w", err)) //nolint:staticcheck // Preserve the existing error message.+ if _, err = resolveAllowedIPs(ctx, parsed.Hostname(), lookupIP); err != nil {+ return nil, err}-return parsed, nil}func NewSafeOutboundHTTPClient(base *http.Client, lookupIP LookupIPFunc) (*http.Client, error) {- if lookupIP == nil {- lookupIP = DefaultLookupIP- }if base == nil {base = http.DefaultClient}- transport, err := cloneHTTPTransportInternal(base.Transport)- if err != nil {- return nil, err+ var transport *http.Transport+ switch t := base.Transport.(type) {+ case nil:+ defaultTransport, ok := http.DefaultTransport.(*http.Transport)+ if !ok {+ return nil, errors.New("http.DefaultTransport is not *http.Transport")+ }+ transport = defaultTransport.Clone()+ case *http.Transport:+ transport = t.Clone()+ default:+ return nil, fmt.Errorf("unsupported HTTP transport type %T", base.Transport)}baseDial := transport.DialContextif baseDial == nil {- dialer := &net.Dialer{- Timeout: 30 * time.Second,- KeepAlive: 30 * time.Second,- }+ dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}baseDial = dialer.DialContext}transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {- host, port, splitHostPortErr := net.SplitHostPort(address)- if splitHostPortErr != nil {- return nil, common.Classify(common.ErrUnsafeRemoteURL, fmt.Errorf("Remote URL is not allowed: %w", splitHostPortErr)) //nolint:staticcheck // Preserve the existing error message.+ host, port, err := net.SplitHostPort(address)+ if err != nil {+ return nil, unsafeRemoteURLError(err)}- ips, splitHostPortErr := resolveAllowedIPsInternal(ctx, host, lookupIP)- if splitHostPortErr != nil {- return nil, splitHostPortErr+ ips, err := resolveAllowedIPs(ctx, host, lookupIP)+ if err != nil {+ return nil, err}- var lastErr error+ var dialErr errorfor _, ip := range ips {- conn, dialErr := baseDial(ctx, network, net.JoinHostPort(ip.String(), port))- if dialErr == nil {+ conn, connErr := baseDial(ctx, network, net.JoinHostPort(ip.String(), port))+ if connErr == nil {return conn, nil}- lastErr = dialErr- }-- if lastErr != nil {- return nil, lastErr+ dialErr = connErr}- return nil, errors.New("failed to resolve remote host")+ return nil, dialErr}client := *base- client.Transport = transport+ // Instrumented after the SSRF dialer is installed; the wrapper never dials itself.+ client.Transport = otelhttp.NewTransport(transport)client.CheckRedirect = func(req *http.Request, via []*http.Request) error {if len(via) >= 10 {return errors.New("stopped after 10 redirects")}- if _, validateSafeRemoteURLErr := ValidateSafeRemoteURL(req.Context(), req.URL.String(), lookupIP); validateSafeRemoteURLErr != nil {- return validateSafeRemoteURLErr+ if _, err := ValidateSafeRemoteURL(req.Context(), req.URL.String(), lookupIP); err != nil {+ return err}if base.CheckRedirect != nil {return base.CheckRedirect(req, via)@@ -160,37 +128,18 @@ func NewSafeOutboundHTTPClient(base *http.Client, lookupIP LookupIPFunc) (*http.return &client, nil}-func cloneHTTPTransportInternal(base http.RoundTripper) (*http.Transport, error) {- switch t := base.(type) {- case nil:- defaultTransport, ok := http.DefaultTransport.(*http.Transport)- if !ok {- return nil, errors.New("http.DefaultTransport is not *http.Transport")+// resolveAllowedIPs resolves host (or parses an IP literal) and rejects it if any address is blocked.+func resolveAllowedIPs(ctx context.Context, host string, lookupIP LookupIPFunc) ([]net.IP, error) {+ literal, _, _ := strings.Cut(host, "%")+ ips := []net.IP{net.ParseIP(literal)}+ if ips[0] == nil {+ if lookupIP == nil {+ lookupIP = DefaultLookupIP}- return defaultTransport.Clone(), nil- case *http.Transport:- return t.Clone(), nil- default:- return nil, fmt.Errorf("unsupported HTTP transport type %T", base)- }-}--func resolveAllowedIPsInternal(ctx context.Context, host string, lookupIP LookupIPFunc) ([]net.IP, error) {- if parsedIP := parseIPLiteralInternal(host); parsedIP != nil {- if isBlockedIPInternal(parsedIP) {- //nolint:staticcheck // Preserve the existing error message.- return nil,- common.Classify(common.ErrUnsafeRemoteURL,- fmt.Errorf("Remote URL is not allowed: %w",- fmt.Errorf("blocked IP address %s",- parsedIP)))+ var err error+ if ips, err = lookupIP(ctx, host); err != nil {+ return nil, unsafeRemoteURLError(err)}- return []net.IP{parsedIP}, nil- }-- ips, err := lookupIP(ctx, host)- if err != nil {- return nil, err}allowed := make([]net.IP, 0, len(ips))@@ -198,66 +147,22 @@ func resolveAllowedIPsInternal(ctx context.Context, host string, lookupIP Lookupif ip == nil {continue}- if isBlockedIPInternal(ip) {- //nolint:staticcheck // Preserve the existing error message.- return nil,- common.Classify(common.ErrUnsafeRemoteURL,- fmt.Errorf("Remote URL is not allowed: %w",- fmt.Errorf("blocked IP address %s",- ip)))+ addr, ok := netip.AddrFromSlice(ip)+ addr = addr.Unmap()+ blocked := !ok || addr.IsLoopback() || addr.IsPrivate() || addr.IsMulticast() || addr.IsUnspecified() ||+ addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() || slices.ContainsFunc(blockedRemotePrefixes, func(p netip.Prefix) bool { return p.Contains(addr) })+ if blocked {+ return nil, unsafeRemoteURLError(fmt.Errorf("blocked IP address %s", ip))}allowed = append(allowed, ip)}if len(allowed) == 0 {- //nolint:staticcheck // Preserve the existing error message.- return nil,- common.Classify(common.ErrUnsafeRemoteURL,- fmt.Errorf("Remote URL is not allowed: %w",- errors.New("host did not resolve to an allowed IP")))+ return nil, unsafeRemoteURLError(errors.New("host did not resolve to an allowed IP"))}-return allowed, nil}-func parseIPLiteralInternal(host string) net.IP {- host = strings.Trim(strings.TrimSpace(host), "[]")- if zoneIdx := strings.Index(host, "%"); zoneIdx != -1 {- host = host[:zoneIdx]- }- return net.ParseIP(host)-}--func isBlockedHostnameInternal(host string) bool {- host = strings.ToLower(strings.TrimSpace(host))- return host == "localhost" || strings.HasSuffix(host, ".localhost")-}--func isBlockedIPInternal(ip net.IP) bool {- if ip.IsLoopback() || ip.IsPrivate() || ip.IsMulticast() || ip.IsUnspecified() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {- return true- }-- // net.IP helper methods do not cover every special-use range we want to keep- // off-limits for remote registry fetching, so we also check the explicit prefix list above.- addr, ok := netip.AddrFromSlice(ip)- if !ok {- return true- }-- for _, prefix := range blockedRemotePrefixes {- if prefix.Contains(addr.Unmap()) {- return true- }- }-- return false-}--func mustParsePrefixesInternal(raw ...string) []netip.Prefix {- prefixes := make([]netip.Prefix, 0, len(raw))- for _, cidr := range raw {- prefixes = append(prefixes, netip.MustParsePrefix(cidr))- }- return prefixes+func unsafeRemoteURLError(err error) error {+ return common.Classify(common.ErrUnsafeRemoteURL, fmt.Errorf("Remote URL is not allowed: %w", err)) //nolint:staticcheck // Preserve the existing error message.}diff --git a/backend/pkg/utils/httpx/safe_remote_test.go b/backend/pkg/utils/httpx/safe_remote_test.goindex d6446cedd..764c9d66d 100644--- a/backend/pkg/utils/httpx/safe_remote_test.go+++ b/backend/pkg/utils/httpx/safe_remote_test.go@@ -65,7 +65,12 @@ func TestNewSafeOutboundHTTPClient_BlocksUnsafeRedirectTarget(t *testing.T) {}))defer server.Close()- baseClient := newRedirectTestClient(server.Listener.Addr().String())+ transport := http.DefaultTransport.(*http.Transport).Clone()+ transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {+ dialer := &net.Dialer{}+ return dialer.DialContext(ctx, network, server.Listener.Addr().String())+ }+ baseClient := &http.Client{Transport: transport, Timeout: 5 * time.Second}lookupIP := func(ctx context.Context, host string) ([]net.IP, error) {return []net.IP{net.ParseIP("93.184.216.34")}, nil}@@ -77,16 +82,3 @@ func TestNewSafeOutboundHTTPClient_BlocksUnsafeRedirectTarget(t *testing.T) {require.Error(t, err)require.ErrorIs(t, err, common.ErrUnsafeRemoteURL)}--func newRedirectTestClient(listenerAddr string) *http.Client {- transport := http.DefaultTransport.(*http.Transport).Clone()- transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {- dialer := &net.Dialer{}- return dialer.DialContext(ctx, network, listenerAddr)- }-- return &http.Client{- Transport: transport,- Timeout: 5 * time.Second,- }-}diff --git a/backend/pkg/utils/notifications/deliver.go b/backend/pkg/utils/notifications/deliver.goindex 520ff029f..f32fb8046 100644--- a/backend/pkg/utils/notifications/deliver.go+++ b/backend/pkg/utils/notifications/deliver.go@@ -6,6 +6,7 @@ import ("errors""fmt""net/mail"+ "regexp""github.com/getarcaneapp/arcane/backend/v2/internal/database")@@ -64,9 +65,26 @@ func Deliver(ctx context.Context, provider NotificationProvider, config databaseif !ok {return false, nil}- return true, deliver(ctx, config, c)+ if deliverErr := deliver(ctx, config, c); deliverErr != nil {+ return true, deliveryError{message: webhookURL.ReplaceAllString(deliverErr.Error(), "${1}${2}"), err: deliverErr}+ }+ return true, nil+}++// webhookURL matches http(s) URLs in provider errors. Webhook tokens often sit in the path or userinfo,+// so only the scheme and host are kept.+var webhookURL = regexp.MustCompile(`(https?://)(?:[^@/?#\s"']*@)?([^/?#\s"']+)[^\s"']*`)++// deliveryError reports a provider failure without its webhook URL while keeping the cause for errors.Is.+type deliveryError struct {+ message string+ err error}+func (e deliveryError) Error() string { return e.message }++func (e deliveryError) Unwrap() error { return e.err }+func deliverDiscord(ctx context.Context, config database.JSON, c Content) error {discordConfig, err := DecodeConfig[DiscordConfig](config, "Discord")if err != nil {diff --git a/backend/pkg/utils/notifications/generic_sender.go b/backend/pkg/utils/notifications/generic_sender.goindex 59175fb0f..01ba63c64 100644--- a/backend/pkg/utils/notifications/generic_sender.go+++ b/backend/pkg/utils/notifications/generic_sender.go@@ -16,25 +16,18 @@ import ("time""github.com/nicholas-fedor/shoutrrr"- shoutrrrTypes "github.com/nicholas-fedor/shoutrrr/pkg/types"- kit "go.getarcane.app/kit/pkg"+ "github.com/nicholas-fedor/shoutrrr/pkg/types"+ "go.getarcane.app/kit/pkg")-// genericPayloadTemplateID is the Shoutrrr template ID under which a user's-// payload template is registered on the generic service instance.+// genericPayloadTemplateID names the Shoutrrr template a user's payload template is registered under.const genericPayloadTemplateID = "arcane"-// genericHTTPClient bounds every generic webhook request. Shoutrrr's generic-// service falls back to a client with no timeout when none is injected, and-// the payload-template path sends through the located service directly,-// bypassing the router's own per-send timeout.+// genericHTTPClient bounds generic webhook sends; Shoutrrr's default client has no timeout.var genericHTTPClient = &http.Client{Timeout: 15 * time.Second}-// resolveWebhookURLInternal parses and normalises the configured webhook URL,-// adding a default scheme when the user omitted one. It is the single source-// of truth for scheme normalisation and host validation used by both-// BuildGenericURL and sendGenericDirectInternal.-func resolveWebhookURLInternal(config GenericConfig) (*url.URL, error) {+// resolveWebhookURL parses the configured webhook URL, adding a default scheme when omitted.+func resolveWebhookURL(config GenericConfig) (*url.URL, error) {if config.WebhookURL == "" {return nil, errors.New("webhook URL is empty")}@@ -67,48 +60,24 @@ func resolveWebhookURLInternal(config GenericConfig) (*url.URL, error) {return parsed, nil}-// BuildGenericURL converts GenericConfig to Shoutrrr URL format for generic webhooks+// BuildGenericURL converts GenericConfig to Shoutrrr URL format for generic webhooks.func BuildGenericURL(config GenericConfig) (string, error) {- webhookURL, err := resolveWebhookURLInternal(config)+ webhookURL, err := resolveWebhookURL(config)if err != nil {return "", err}- // Start from the user's existing query parameters. Shoutrrr's generic- // service preserves any query keys it does not recognise, so provider- // tokens embedded in the webhook URL (e.g. PushPlus's `?token=...`) flow- // straight through to the outbound HTTP request untouched.- //- // For Shoutrrr config keys (template, contenttype, method, titlekey,- // messagekey, disabletls) we only fill in defaults / configured values- // when the user has not already set the same key inline in the URL.- // That way an explicit `?template=custom` or `?disabletls=yes` from the- // user is always respected and never silently overwritten by the- // provider settings or the URL-scheme-derived TLS flag.+ // Unknown query keys (e.g. PushPlus `?token=`) pass through; Shoutrrr config keys set inline win over settings.query := webhookURL.Query()-setDefault := func(key, value string) {- if value == "" {- return- }- if query.Get(key) != "" {- return+ if value != "" && query.Get(key) == "" {+ query.Set(key, value)}- query.Set(key, value)}- // Default to the JSON template — Shoutrrr's JSON template marshals the- // notification params as a flat JSON object at the root level, which is- // the format most providers (PushPlus, custom APIs, Home Assistant, etc.)- // expect. When the user configured a payload template, point Shoutrrr at- // the named template registered at send time instead, and default the- // content type to JSON since that is what payload templates target.+ // Payload templates use the named template registered at send time and default to JSON content.hasPayloadTemplate := strings.TrimSpace(config.PayloadTemplate) != ""- if hasPayloadTemplate {- setDefault("template", genericPayloadTemplateID)- } else {- setDefault("template", "json")- }+ setDefault("template", kit.Ternary(hasPayloadTemplate, genericPayloadTemplateID, "json"))setDefault("contenttype", config.ContentType)if hasPayloadTemplate {setDefault("contenttype", "application/json")@@ -117,8 +86,6 @@ func BuildGenericURL(config GenericConfig) (string, error) {setDefault("titlekey", config.TitleKey)setDefault("messagekey", config.MessageKey)- // Determine TLS setting from the webhook URL scheme (http/https) when the- // user has not already passed `disabletls` explicitly.switch strings.ToLower(webhookURL.Scheme) {case "http":setDefault("disabletls", "yes")@@ -126,12 +93,9 @@ func BuildGenericURL(config GenericConfig) (string, error) {setDefault("disabletls", "no")}- // Add custom headers as query parameters with @ prefix- if len(config.CustomHeaders) > 0 {- for key, value := range config.CustomHeaders {- // Shoutrrr uses @ prefix for headers- query.Set("@"+key, value)- }+ // Shoutrrr reads headers from @-prefixed query keys.+ for key, value := range config.CustomHeaders {+ query.Set("@"+key, value)}shoutrrrURL := &url.URL{@@ -144,10 +108,8 @@ func BuildGenericURL(config GenericConfig) (string, error) {return shoutrrrURL.String(), nil}-// EventVars builds the per-event variables exposed to a generic webhook-// payload template. The names must stay clear of Shoutrrr generic config keys-// (title, template, contenttype, method, messagekey, titlekey, disabletls) —-// a colliding param would mutate the per-send service config.+// EventVars builds the per-event payload template variables.+// Names must not collide with Shoutrrr generic config keys, which would mutate the per-send config.func EventVars(environmentName, environmentID string, event NotificationEventType) map[string]string {return map[string]string{"environment": environmentName,@@ -157,11 +119,7 @@ func EventVars(environmentName, environmentID string, event NotificationEventTyp}}-// jsonEscapeString escapes value for embedding inside a JSON string literal:-// json.Marshal yields a quoted string, and the surrounding quotes are trimmed-// so the template author keeps control of the quoting. Applied only on the-// payload-template path — the default template=json path is escaped by-// Shoutrrr's own json.Marshal.+// jsonEscapeString escapes value for embedding inside a JSON string literal, without the surrounding quotes.func jsonEscapeString(value string) string {encoded, err := json.Marshal(value)if err != nil {@@ -170,56 +128,29 @@ func jsonEscapeString(value string) string {return strings.TrimSuffix(strings.TrimPrefix(string(encoded), `"`), `"`)}-// genericTemplateDataInternal builds the variable map a payload template is-// rendered with on the direct-HTTP path: the JSON-escaped title and message-// under the configured titlekey/messagekey — mirroring how Shoutrrr's-// createSendParams keys the send params — plus the escaped per-event vars.-func genericTemplateDataInternal(config GenericConfig, title, message string, vars map[string]string) map[string]string {- titleKey := cmp.Or(config.TitleKey, "title")- messageKey := cmp.Or(config.MessageKey, "message")-- data := make(map[string]string, len(vars)+2)- for key, value := range vars {- data[key] = jsonEscapeString(value)- }- data[titleKey] = jsonEscapeString(title)- data[messageKey] = jsonEscapeString(message)- return data-}--// RenderGenericPayloadTemplate renders the configured payload template with-// the same variables the Shoutrrr send path exposes. Used by the direct-HTTP-// path (SuccessBodyContains) and by save-time validation.+// RenderGenericPayloadTemplate renders the payload template with the JSON-escaped title, message, and event vars.func RenderGenericPayloadTemplate(config GenericConfig, title, message string, vars map[string]string) (string, error) {tmpl, err := template.New(genericPayloadTemplateID).Parse(config.PayloadTemplate)if err != nil {return "", fmt.Errorf("invalid webhook payload template: %w", err)}+ // Keyed by titlekey/messagekey to mirror Shoutrrr's send params.+ data := make(map[string]string, len(vars)+2)+ for key, value := range vars {+ data[key] = jsonEscapeString(value)+ }+ data[cmp.Or(config.TitleKey, "title")] = jsonEscapeString(title)+ data[cmp.Or(config.MessageKey, "message")] = jsonEscapeString(message)+var rendered bytes.Buffer- if executeErr := tmpl.Execute(&rendered, genericTemplateDataInternal(config, title, message, vars)); executeErr != nil {+ if executeErr := tmpl.Execute(&rendered, data); executeErr != nil {return "", fmt.Errorf("failed to render webhook payload template: %w", executeErr)}return rendered.String(), nil}-// isJSONContentType reports whether the configured content type denotes JSON.-// An empty content type counts as JSON because that is the default applied to-// outgoing generic webhook requests.-func isJSONContentType(contentType string) bool {- if contentType == "" {- return true- }- // Ignore any parameters such as "; charset=utf-8".- mediaType := strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0]))- return mediaType == "application/json" || strings.HasSuffix(mediaType, "+json")-}--// ValidateGenericPayloadTemplate checks a configured payload template at save-// time: it must parse, execute against sample variables, and — when the-// content type is JSON (the default) — render valid JSON. Shoutrrr renders-// the template internally at send time, so validating here surfaces mistakes-// against the user's own configuration instead of as an opaque send failure.+// ValidateGenericPayloadTemplate checks that a payload template renders, and renders valid JSON for JSON content types.func ValidateGenericPayloadTemplate(config GenericConfig) error {if strings.TrimSpace(config.PayloadTemplate) == "" {return nil@@ -231,27 +162,25 @@ func ValidateGenericPayloadTemplate(config GenericConfig) error {return err}- if isJSONContentType(config.ContentType) && !jsontext.Value(rendered).IsValid() {+ // An empty content type defaults to JSON.+ mediaType, _, _ := strings.Cut(config.ContentType, ";")+ mediaType = strings.ToLower(strings.TrimSpace(mediaType))+ isJSON := config.ContentType == "" || mediaType == "application/json" || strings.HasSuffix(mediaType, "+json")+ if isJSON && !jsontext.Value(rendered).IsValid() {return errors.New("webhook payload template did not render valid JSON")}return nil}-// SendGenericWithTitle sends a message with title via Shoutrrr Generic webhook.-// When config.SuccessBodyContains is set the response body is also inspected —-// this is necessary for providers (e.g. PushPlus) that always return HTTP 200-// but embed a success/failure indicator inside the JSON body.+// SendGenericWithTitle sends a message with title via the Shoutrrr generic webhook.+// SuccessBodyContains also checks the response body, for providers that always return HTTP 200.func SendGenericWithTitle(ctx context.Context, config GenericConfig, title, message string, vars map[string]string) error {if config.WebhookURL == "" {return errors.New("webhook URL is empty")}- // When the caller needs response-body validation we make the HTTP request- // ourselves so that we can inspect the body. Otherwise we delegate to- // shoutrrr, which preserves the existing behaviour for everyone who does- // not set SuccessBodyContains.if config.SuccessBodyContains != "" {- return sendGenericDirectInternal(ctx, config, title, message, vars)+ return sendGenericDirect(ctx, config, title, message, vars)}shoutrrrURL, err := BuildGenericURL(config)@@ -259,16 +188,37 @@ func SendGenericWithTitle(ctx context.Context, config GenericConfig, title, messreturn fmt.Errorf("failed to build shoutrrr Generic URL: %w", err)}- senderOptions := shoutrrrTypes.SenderOptions{HTTPClient: genericHTTPClient}+ senderOptions := types.SenderOptions{HTTPClient: genericHTTPClient}- // A user-supplied inline `?template=<id>` in the webhook URL wins over the- // default "arcane" ID, so the payload template must be registered under the- // effective ID for Shoutrrr to resolve it. An inline `template=json`- // selects Shoutrrr's built-in JSON template instead, which does its own- // escaping — send through the plain path so params are not double-escaped.+ // Register the payload template under the URL's effective template ID; an inline json template escapes on its own.if strings.TrimSpace(config.PayloadTemplate) != "" {- if templateID := effectiveGenericTemplateIDInternal(shoutrrrURL); templateID != "json" && templateID != "JSON" {- return sendGenericTemplatedInternal(config, shoutrrrURL, templateID, senderOptions, title, message, vars)+ templateID := genericPayloadTemplateID+ if parsed, parseErr := url.Parse(shoutrrrURL); parseErr == nil {+ templateID = cmp.Or(parsed.Query().Get("template"), genericPayloadTemplateID)+ }+ if templateID != "json" && templateID != "JSON" {+ // Templates are per service instance, which router.Send does not expose, so send through the located service.+ sender, createErr := shoutrrr.CreateSenderWithOptions(senderOptions)+ if createErr != nil {+ return fmt.Errorf("failed to create shoutrrr Generic sender: %w", createErr)+ }+ service, locateErr := sender.Locate(shoutrrrURL)+ if locateErr != nil {+ return fmt.Errorf("failed to initialize shoutrrr Generic service: %w", locateErr)+ }+ if setTemplateStringErr := service.SetTemplateString(templateID, config.PayloadTemplate); setTemplateStringErr != nil {+ return fmt.Errorf("invalid webhook payload template: %w", setTemplateStringErr)+ }++ // text/template does no escaping, so every value is JSON-escaped to keep the payload valid.+ params := types.Params{"title": jsonEscapeString(title)}+ for key, value := range vars {+ params[key] = jsonEscapeString(value)+ }+ if sendErr := service.Send(jsonEscapeString(message), ¶ms); sendErr != nil {+ return fmt.Errorf("failed to send Generic webhook message via shoutrrr: %w", sendErr)+ }+ return nil}}@@ -277,13 +227,9 @@ func SendGenericWithTitle(ctx context.Context, config GenericConfig, title, messreturn fmt.Errorf("failed to create shoutrrr Generic sender: %w", err)}- // Build params with title. Always use "title" as the param key — Shoutrrr's- // generic service maps it to the configured titlekey in the JSON payload.- params := shoutrrrTypes.Params{}- params["title"] = cmp.Or(title, params["title"])-- errs := sender.Send(message, ¶ms)- for _, err := range errs {+ // Shoutrrr maps the "title" param to the configured titlekey.+ params := types.Params{"title": title}+ for _, err := range sender.Send(message, ¶ms) {if err != nil {return fmt.Errorf("failed to send Generic webhook message with title via shoutrrr: %w", err)}@@ -291,64 +237,10 @@ func SendGenericWithTitle(ctx context.Context, config GenericConfig, title, messreturn nil}-// effectiveGenericTemplateIDInternal returns the template ID the built-// Shoutrrr URL selects for payload rendering. BuildGenericURL always sets a-// template query key, so an empty result only happens on an unparseable URL;-// fall back to the default ID in that case.-func effectiveGenericTemplateIDInternal(shoutrrrURL string) string {- parsed, err := url.Parse(shoutrrrURL)- if err != nil {- return genericPayloadTemplateID- }- if id := parsed.Query().Get("template"); id != "" {- return id- }- return genericPayloadTemplateID-}--// sendGenericTemplatedInternal sends through Shoutrrr's generic service with-// the user's payload template registered on the service instance under-// templateID — the ID the URL's template key selects. Templates are resolved-// per service instance, which router.Send does not expose, so the service is-// obtained via Locate — the full initService path (scheme extraction, config-// parse, HTTP client injection) — and sent on directly.-func sendGenericTemplatedInternal(config GenericConfig, shoutrrrURL, templateID string, opts shoutrrrTypes.SenderOptions, title, message string, vars map[string]string) error {- sender, err := shoutrrr.CreateSenderWithOptions(opts)- if err != nil {- return fmt.Errorf("failed to create shoutrrr Generic sender: %w", err)- }-- service, err := sender.Locate(shoutrrrURL)- if err != nil {- return fmt.Errorf("failed to initialize shoutrrr Generic service: %w", err)- }-- if setTemplateStringErr := service.SetTemplateString(templateID, config.PayloadTemplate); setTemplateStringErr != nil {- return fmt.Errorf("invalid webhook payload template: %w", setTemplateStringErr)- }-- // Shoutrrr renders the registered template over the send params, remapping- // the "title" param to the configured titlekey and adding the message under- // the configured messagekey. Every value is JSON-escaped here, so- // {"text": "{{.message}}"} stays valid JSON for hostile message content —- // text/template itself does no escaping.- params := shoutrrrTypes.Params{"title": jsonEscapeString(title)}- for key, value := range vars {- params[key] = jsonEscapeString(value)- }-- if sendErr := service.Send(jsonEscapeString(message), ¶ms); sendErr != nil {- return fmt.Errorf("failed to send Generic webhook message via shoutrrr: %w", sendErr)- }- return nil-}--// sendGenericDirectInternal makes the webhook HTTP call directly, giving access-// to the response body so that provider-level success/failure can be detected-// even when the HTTP status is always 200. It renders the payload template when-// one is configured so the two features compose.-func sendGenericDirectInternal(ctx context.Context, config GenericConfig, title, message string, vars map[string]string) error {- webhookURL, err := resolveWebhookURLInternal(config)+// sendGenericDirect calls the webhook directly so the response body can be checked for SuccessBodyContains.+// Kept separate from SendGenericWithTitle to stay under the gocognit limit.+func sendGenericDirect(ctx context.Context, config GenericConfig, title, message string, vars map[string]string) error {+ webhookURL, err := resolveWebhookURL(config)if err != nil {return err}@@ -361,11 +253,8 @@ func sendGenericDirectInternal(ctx context.Context, config GenericConfig, title,}body = []byte(rendered)} else {- // Build JSON payload using the configured message/title keys.- msgKey := cmp.Or(config.MessageKey, "message")titleKey := cmp.Or(config.TitleKey, "title")-- payload := map[string]string{msgKey: message}+ payload := map[string]string{cmp.Or(config.MessageKey, "message"): message}payload[titleKey] = cmp.Or(title, payload[titleKey])body, err = json.Marshal(payload)@@ -375,15 +264,11 @@ func sendGenericDirectInternal(ctx context.Context, config GenericConfig, title,}method := cmp.Or(strings.ToUpper(config.Method), http.MethodPost)-req, err := http.NewRequestWithContext(ctx, method, webhookURL.String(), bytes.NewReader(body))if err != nil {return fmt.Errorf("failed to create webhook request: %w", err)}-- contentType := cmp.Or(config.ContentType, "application/json")- req.Header.Set("Content-Type", contentType)-+ req.Header.Set("Content-Type", cmp.Or(config.ContentType, "application/json"))for k, v := range config.CustomHeaders {req.Header.Set(k, v)}diff --git a/backend/pkg/utils/tracing/tracing.go b/backend/pkg/utils/tracing/tracing.gonew file mode 100644index 000000000..78df00458--- /dev/null+++ b/backend/pkg/utils/tracing/tracing.go@@ -0,0 +1,23 @@+// Package tracing holds shared helpers for Arcane's own OpenTelemetry spans.+package tracing++import (+ "context"+ "errors"++ "go.opentelemetry.io/otel/codes"+ "go.opentelemetry.io/otel/trace"+)++// InstrumentationName identifies spans, metrics, and logs emitted by Arcane code.+const InstrumentationName = "github.com/getarcaneapp/arcane/backend"++// End records err on span, marks the span failed, and ends it. Context+// cancellation is treated as a normal outcome.+func End(span trace.Span, err error) {+ if err != nil && !errors.Is(err, context.Canceled) {+ span.RecordError(err)+ span.SetStatus(codes.Error, err.Error())+ }+ span.End()+}diff --git a/docker/examples/compose.otel.yaml b/docker/examples/compose.otel.yamlnew file mode 100644index 000000000..10f2e94ae--- /dev/null+++ b/docker/examples/compose.otel.yaml@@ -0,0 +1,33 @@+services:+ arcane:+ image: ghcr.io/getarcaneapp/manager:latest+ container_name: arcane+ ports:+ - '3552:3552'+ volumes:+ - /var/run/docker.sock:/var/run/docker.sock+ - arcane-data:/app/data+ environment:+ - ENCRYPTION_KEY=replace_me_with_a_random_32_char_value+ - TZ=UTC+ - OTEL_TRACES_EXPORTER=otlp+ - OTEL_METRICS_EXPORTER=otlp+ - OTEL_LOGS_EXPORTER=otlp+ - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318+ # Keep 25% of new traces; traces started by a sampled parent are always kept.+ # - OTEL_TRACES_SAMPLER=parentbased_traceidratio+ # - OTEL_TRACES_SAMPLER_ARG=0.25+ depends_on:+ - otel-collector+ restart: unless-stopped++ otel-collector:+ image: otel/opentelemetry-collector-contrib:latest+ container_name: otel-collector+ command: ['--config=/etc/otelcol/config.yaml']+ volumes:+ - ./otel/collector.yaml:/etc/otelcol/config.yaml:ro+ restart: unless-stopped++volumes:+ arcane-data:diff --git a/docker/examples/otel/collector.yaml b/docker/examples/otel/collector.yamlnew file mode 100644index 000000000..c11d34bc0--- /dev/null+++ b/docker/examples/otel/collector.yaml@@ -0,0 +1,38 @@+receivers:+ otlp:+ protocols:+ http:+ endpoint: 0.0.0.0:4318+ grpc:+ endpoint: 0.0.0.0:4317++processors:+ memory_limiter:+ check_interval: 1s+ limit_mib: 256+ spike_limit_mib: 64+ batch: {}++exporters:+ debug:+ verbosity: basic+ # Send to your backend instead of (or alongside) debug output.+ # otlphttp:+ # endpoint: https://otlp.example.com+ # headers:+ # authorization: Bearer ${env:OTLP_TOKEN}++service:+ pipelines:+ traces:+ receivers: [otlp]+ processors: [memory_limiter, batch]+ exporters: [debug]+ metrics:+ receivers: [otlp]+ processors: [memory_limiter, batch]+ exporters: [debug]+ logs:+ receivers: [otlp]+ processors: [memory_limiter, batch]+ exporters: [debug]diff --git a/frontend/package.json b/frontend/package.jsonindex 3aaf80eb4..235b57f64 100644--- a/frontend/package.json+++ b/frontend/package.json@@ -33,6 +33,20 @@"@fontsource-variable/montserrat": "^5.3.0","@internationalized/date": "^3.12.4","@lezer/highlight": "^1.2.5",+ "@opentelemetry/api": "^1.9.1",+ "@opentelemetry/api-logs": "^0.223.0",+ "@opentelemetry/exporter-logs-otlp-http": "^0.223.0",+ "@opentelemetry/exporter-metrics-otlp-http": "^0.223.0",+ "@opentelemetry/exporter-trace-otlp-http": "^0.223.0",+ "@opentelemetry/instrumentation": "^0.223.0",+ "@opentelemetry/instrumentation-document-load": "^0.68.0",+ "@opentelemetry/instrumentation-fetch": "^0.223.0",+ "@opentelemetry/opentelemetry-browser-detector": "^0.223.0",+ "@opentelemetry/resources": "^2.12.0",+ "@opentelemetry/sdk-logs": "^0.223.0",+ "@opentelemetry/sdk-metrics": "^2.12.0",+ "@opentelemetry/sdk-trace-web": "^2.12.0",+ "@opentelemetry/semantic-conventions": "^1.43.0","@simplewebauthn/browser": "^14.0.0","@sveltejs/kit": "3.0.0","@tanstack/svelte-query": "^6.3.1",@@ -61,6 +75,7 @@"tailwind-variants": "^3.3.1","temporal-polyfill": "^1.0.5","vaul-svelte": "1.0.0-next.7",+ "web-vitals": "^6.2.3","yaml": "^2.9.1","zod": "^4.6.5"},diff --git a/frontend/src/hooks.client.ts b/frontend/src/hooks.client.tsindex 2696ac99b..73779f2c5 100644--- a/frontend/src/hooks.client.ts+++ b/frontend/src/hooks.client.ts@@ -1,12 +1,14 @@import type { HandleClientError } from '@sveltejs/kit/hooks';import { extractApiErrorMessage } from '#lib/utils/api.js';+import { recordError } from '#lib/utils/telemetry.js';export const handleError: HandleClientError = async ({ kind, error }) => {if (kind !== 'unknown') {return;}+ recordError(error, 'sveltekit.handleError');if (error && typeof error === 'object' && 'response' in error) {const responseStatus = (error as { response?: { status?: number } }).response?.status;const apiErrorMessage = extractApiErrorMessage(error);diff --git a/frontend/src/lib/services/api-service.ts b/frontend/src/lib/services/api-service.tsindex 07920c0aa..4d0d6cea6 100644--- a/frontend/src/lib/services/api-service.ts+++ b/frontend/src/lib/services/api-service.ts@@ -1,4 +1,4 @@-import ky, { HTTPError as KyHTTPError, NetworkError, TimeoutError, type Options as KyOptions, type SearchParamsOption } from 'ky';+import ky, { HTTPError as KyHTTPError, TimeoutError, type Options as KyOptions, type SearchParamsOption } from 'ky';import { toast } from 'svelte-sonner';import { m } from '#lib/paraglide/messages.js';@@ -39,18 +39,13 @@ export class SessionCancelledError extends Error {// Browsers may surface a logout abort as a plain AbortError instead of the abort reason.export function isSessionCancelledError(error: unknown): boolean {if (error instanceof SessionCancelledError) return true;- return error instanceof Error && error.name === 'AbortError' && sessionControllerInternal.signal.aborted;+ return error instanceof Error && error.name === 'AbortError' && sessionController.signal.aborted;}export class APIError extends Error {config: InternalRequestConfig & { method?: string; url?: string };request?: { url: string };- response?: {- data: any;- headers: Headers;- raw: Response;- status: number;- };+ response?: APIResponse;status?: number;constructor(@@ -63,99 +58,68 @@ export class APIError extends Error {cause?: unknown;}) {- super(message);+ super(message, { cause: options.cause });this.name = options.name ?? 'APIError';- if (options.cause !== undefined) {- (this as Error & { cause?: unknown }).cause = options.cause;- }this.config = options.config;this.request = options.requestUrl ? { url: options.requestUrl } : undefined;- this.response = options.response- ? {- data: options.response.data,- headers: options.response.headers,- raw: options.response.raw,- status: options.response.status- }- : undefined;+ this.response = options.response;this.status = options.response?.status;}}-const problemMessageFactoriesInternal: Record<string, () => string> = {+const problemMessageFactories: Record<string, () => string> = {'urn:arcane:problem:password-policy:basic': m.security_password_policy_basic_tooltip,'urn:arcane:problem:password-policy:standard': m.security_password_policy_standard_tooltip,'urn:arcane:problem:password-policy:strong': m.security_password_policy_strong_tooltip};-function asRecordInternal(value: unknown): Record<string, unknown> | undefined {+function asRecord(value: unknown): Record<string, unknown> | undefined {return value !== null && typeof value === 'object' ? (value as Record<string, unknown>) : undefined;}-function nonEmptyStringInternal(value: unknown): string | undefined {+function nonEmptyString(value: unknown): string | undefined {return typeof value === 'string' && value.trim() ? value : undefined;}export function extractServerMessage(data: unknown, includeErrors = false): string | undefined {- const outer = asRecordInternal(data);- const innerValue = outer?.['data'] ?? data;- const stringValue = nonEmptyStringInternal(innerValue);+ const innerValue = asRecord(data)?.['data'] ?? data;+ const stringValue = nonEmptyString(innerValue);if (stringValue) return stringValue;- const inner = asRecordInternal(innerValue);+ const inner = asRecord(innerValue);if (!inner) return undefined;if (inner['code'] === 'feature_disabled' && inner['feature'] === 'vulnerabilityManagement') {return m.features_vulnerability_disabled();}- const problemType = nonEmptyStringInternal(inner['type']);- const localizedMessage = problemType ? problemMessageFactoriesInternal[problemType] : undefined;+ const problemType = nonEmptyString(inner['type']);+ const localizedMessage = problemType ? problemMessageFactories[problemType] : undefined;if (localizedMessage) return localizedMessage();for (const key of ['error', 'message', 'detail', 'error_description']) {- const message = nonEmptyStringInternal(inner[key]);+ const message = nonEmptyString(inner[key]);if (message) return message;}- if (includeErrors && Array.isArray(inner['errors']) && inner['errors'].length > 0) {- const first = inner['errors'][0];- const firstString = nonEmptyStringInternal(first);- if (firstString) return firstString;-- const firstError = asRecordInternal(first);- return nonEmptyStringInternal(firstError?.['message']) ?? nonEmptyStringInternal(firstError?.['error']);- }-- return undefined;+ if (!includeErrors || !Array.isArray(inner['errors'])) return undefined;+ const first: unknown = inner['errors'][0];+ const firstError = asRecord(first);+ return nonEmptyString(first) ?? nonEmptyString(firstError?.['message']) ?? nonEmptyString(firstError?.['error']);}-// Batch id applied to mutating requests STARTED synchronously inside-// runWithActivityBatchId, so the backend groups the resulting activities under-// one Activity Center row. The scope never spans an await: fn must kick off-// its requests synchronously and return their promises, so an unrelated action-// started from any other task can never inherit the batch id.-let activeActivityBatchIdInternal: string | null = null;+// Batch id for mutating requests started synchronously inside runWithActivityBatchId; it never spans an await,+// so fn must start its requests synchronously and return their promises.+let activeActivityBatchId: string | null = null;export function runWithActivityBatchId<T>(batchId: string, fn: () => T): T {- activeActivityBatchIdInternal = batchId;+ activeActivityBatchId = batchId;try {return fn();} finally {- activeActivityBatchIdInternal = null;+ activeActivityBatchId = null;}}-function isBodyInit(value: unknown): value is BodyInit {- return (- value instanceof FormData ||- value instanceof URLSearchParams ||- value instanceof Blob ||- value instanceof ArrayBuffer ||- ArrayBuffer.isView(value) ||- typeof value === 'string'- );-}-async function parseResponseBody(response: Response, responseType: APIRequestConfig['responseType'] = 'json'): Promise<any> {if (responseType === 'blob') {return response.blob();@@ -182,187 +146,92 @@ async function parseResponseBody(response: Response, responseType: APIRequestCon}}-async function parseErrorResponseBody(error: KyHTTPError): Promise<any> {- if (error.data !== undefined) {- return error.data;- }-- if (error.response.bodyUsed) {- return undefined;- }-- return parseResponseBody(error.response.clone());-}--function normalizeUrl(baseURL: string, url: string): string {- if (/^[a-z]+:\/\//i.test(url)) {- return url;- }-- const trimmedBase = baseURL.replace(/\/+$/, '');- const trimmedUrl = url.replace(/^\/+/, '');-- if (/^[a-z]+:\/\//i.test(trimmedBase)) {- return new URL(trimmedUrl, `${trimmedBase}/`).toString();- }-- return `${trimmedBase}/${trimmedUrl}`;-}--function getRequestPath(url: string, baseURL: string): string {- let reqUrl = url;- try {- if (/^https?:\/\//i.test(reqUrl)) {- reqUrl = new URL(reqUrl).pathname;- } else if (baseURL && /^https?:\/\//i.test(baseURL)) {- reqUrl = new URL(reqUrl.replace(/^\/+/, ''), `${baseURL.replace(/\/+$/, '')}/`).pathname;- }- } catch {- // ignore URL parse errors and fall back to the raw request URL- }-- if (reqUrl.startsWith('/api')) {- reqUrl = reqUrl.slice(4) || '/';- }-- return reqUrl;-}-let tokenRefreshHandler: (() => Promise<string | null>) | null = null;-// Set true while a manager self-update / fleet "Update All" is running. During that-// window the backend briefly restarts and returns version-mismatch 401s; we must not-// bounce the user to /login — the session is recoverable once the backend is back.-let upgradeInProgressInternal = false;+// True while a manager self-update or fleet "Update All" runs; its version-mismatch 401s+// are recoverable and must not bounce the user to /login.+let upgradeInProgress = false;// A confirmed manager restart still needs document recovery after polling ends.-let upgradeReloadPendingInternal = false;-let upgradeReloadStartedInternal = false;-let unauthorizedRedirectStartedInternal = false;-const skipAuthPathsInternal = [+let upgradeReloadPending = false;+let upgradeReloadStarted = false;+let unauthorizedRedirectStarted = false;+const skipAuthPaths = ['/auth/login','/auth/logout','/auth/refresh','/auth/oidc',- '/auth/oidc/login',- '/auth/oidc/callback','/auth/passkey','/auth/mfa','/auth/auto-login',- '/auth/auto-login-config','/settings/public'];// Each session owns one controller; logout aborts it and keeps it aborted until// the next sign-in, so protected requests fail fast in between.-let sessionControllerInternal = new AbortController();--// Requests needed to sign in again, which must work after logout.-function isSessionFreePathInternal(path: string): boolean {- if (path.startsWith('/auth/refresh')) return false;- return (- skipAuthPathsInternal.some((prefix) => path.startsWith(prefix)) ||- ['/oidc/url', '/oidc/callback', '/oidc/status'].includes(path) ||- path.startsWith('/app-version') ||- path.endsWith('/settings/public')- );-}+let sessionController = new AbortController();/** Combines a caller signal with the current session, so logout cancels the request. */export function withSessionSignal(signal?: AbortSignal | null): AbortSignal {- const session = sessionControllerInternal.signal;+ const session = sessionController.signal;return signal ? AbortSignal.any([signal, session]) : session;}export function currentSessionSignal(): AbortSignal {- return sessionControllerInternal.signal;+ return sessionController.signal;}-type UnauthorizedActionInternal = 'none' | 'redirect' | 'reload' | 'retry';--function isAuthPagePathInternal(pathname: string): boolean {+export function isAuthPagePath(pathname: string): boolean {return ['/login', '/logout', '/oidc', '/auth/oidc', '/mobile/passkey'].some((prefix) => pathname.startsWith(prefix));}+type UnauthorizedAction = 'none' | 'redirect' | 'reload' | 'retry';+export async function handleUnauthorizedResponseInternal(requestPath: string,retry = false,serverMsg?: string | null-): Promise<UnauthorizedActionInternal> {- const session = sessionControllerInternal.signal;+): Promise<UnauthorizedAction> {+ const session = sessionController.signal;if (typeof window === 'undefined' || retry || session.aborted) {return 'none';}const isVersionMismatch = serverMsg?.toLowerCase().includes('application has been updated') ?? false;- const isAuthApi = skipAuthPathsInternal.some((path) => requestPath.startsWith(path));+ const isAuthApi = skipAuthPaths.some((path) => requestPath.startsWith(path));const pathname = window.location.pathname || '/';- const isOnAuthPage = isAuthPagePathInternal(pathname);- if (isAuthApi || isOnAuthPage) {+ if (isAuthApi || isAuthPagePath(pathname)) {return 'none';}- if (unauthorizedRedirectStartedInternal) return 'redirect';+ if (unauthorizedRedirectStarted) return 'redirect';- // During a server self-update the backend briefly returns version-mismatch 401s.- // Refresh first so the new backend rotates both tokens, then replace the stale- // frontend document exactly once. Only transport failures are recoverable restart- // noise; a rejected or missing refresh token is a terminal authentication failure.- const recoverable = isVersionMismatch || upgradeInProgressInternal;+ // During a self-update, refresh so the new backend rotates tokens, then reload the stale document once.+ // Only transport failures are recoverable; a rejected refresh is terminal.+ const recoverable = isVersionMismatch || upgradeInProgress;if (!tokenRefreshHandler) return 'none';const operationResult = await tryCatch((async () => {await tokenRefreshHandler();- if (!isVersionMismatch || (!upgradeInProgressInternal && !upgradeReloadPendingInternal)) return 'retry';- if (!upgradeReloadStartedInternal) {- upgradeReloadStartedInternal = true;+ if (!isVersionMismatch || (!upgradeInProgress && !upgradeReloadPending)) return 'retry';+ if (!upgradeReloadStarted) {+ upgradeReloadStarted = true;window.location.reload();}return 'reload';})());if (session.aborted) return 'none';- if (operationResult.error !== null) {- const error = operationResult.error;- const isTransientRefreshFailure =- error instanceof APIError && (error.name === 'NetworkError' || error.name === 'TimeoutError');- if (recoverable && isTransientRefreshFailure) return 'none';- if (!unauthorizedRedirectStartedInternal) {- unauthorizedRedirectStartedInternal = true;- window.location.replace(`/login?redirect=${encodeURIComponent(pathname)}`);- }- return 'redirect';- } else {- return operationResult.data;- }-}+ if (operationResult.error === null) return operationResult.data;-function buildRequestOptionsInternal(method: string, data: unknown, config: InternalRequestConfig): KyOptions {- const headers = new Headers(config.headers);- const isMutation = !['GET', 'HEAD'].includes(method.toUpperCase());- if (activeActivityBatchIdInternal && isMutation && !headers.has('X-Arcane-Batch-Id')) {- headers.set('X-Arcane-Batch-Id', activeActivityBatchIdInternal);+ const error = operationResult.error;+ const isTransientRefreshFailure = error instanceof APIError && (error.name === 'NetworkError' || error.name === 'TimeoutError');+ if (recoverable && isTransientRefreshFailure) return 'none';+ if (!unauthorizedRedirectStarted) {+ unauthorizedRedirectStarted = true;+ window.location.replace(`/login?redirect=${encodeURIComponent(pathname)}`);}- const options: KyOptions = {- cache: config.cache,- method,- headers,- retry: config.retry ?? 0,- searchParams: config.params,- signal: config.signal,- timeout: config.timeout ?? false- };-- const bodyData = data !== undefined ? data : config.data;- if (bodyData !== undefined && bodyData !== null) {- if (isBodyInit(bodyData)) {- options.body = bodyData;- } else {- options.json = bodyData;- }- }-- return options;+ return 'redirect';}class APIClient {@@ -389,23 +258,58 @@ class APIClient {config: InternalRequestConfig = {}): Promise<APIResponse<T>> {const baseURL = config.baseURL ?? this.defaults.baseURL;- const requestUrl = normalizeUrl(baseURL, url);- const requestConfig = {- ...config,- baseURL,- method,- url- };- const session = isSessionFreePathInternal(getRequestPath(url, baseURL)) ? undefined : sessionControllerInternal.signal;+ const requestUrl = /^[a-z]+:\/\//i.test(url) ? url : `${baseURL.replace(/\/+$/, '')}/${url.replace(/^\/+/, '')}`;+ const requestConfig = { ...config, baseURL, method, url };+ let requestPath = requestUrl;+ try {+ requestPath = new URL(requestUrl, 'http://localhost').pathname;+ } catch {+ // Fall back to the raw request URL.+ }+ if (requestPath.startsWith('/api')) requestPath = requestPath.slice(4) || '/';++ // Requests needed to sign in again must keep working after logout.+ const isSessionFree =+ !requestPath.startsWith('/auth/refresh') &&+ (skipAuthPaths.some((prefix) => requestPath.startsWith(prefix)) ||+ ['/oidc/url', '/oidc/callback', '/oidc/status'].includes(requestPath) ||+ requestPath.startsWith('/app-version') ||+ requestPath.endsWith('/settings/public'));+ const session = isSessionFree ? undefined : sessionController.signal;const signal = session ? withSessionSignal(config.signal) : config.signal;+ const upperMethod = method.toUpperCase();const operationResult = await tryCatch((async () => {- const options = buildRequestOptionsInternal(method, data, { ...config, signal });+ const headers = new Headers(config.headers);+ if (activeActivityBatchId && !['GET', 'HEAD'].includes(upperMethod) && !headers.has('X-Arcane-Batch-Id')) {+ headers.set('X-Arcane-Batch-Id', activeActivityBatchId);+ }+ const options: KyOptions = {+ cache: config.cache,+ method,+ headers,+ retry: config.retry ?? 0,+ searchParams: config.params,+ signal,+ timeout: config.timeout ?? false+ };+ const body = data ?? config.data;+ if (+ body instanceof FormData ||+ body instanceof URLSearchParams ||+ body instanceof Blob ||+ body instanceof ArrayBuffer ||+ ArrayBuffer.isView(body) ||+ typeof body === 'string'+ ) {+ options.body = body as BodyInit;+ } else if (body !== undefined && body !== null) {+ options.json = body;+ }const response = await this.client(requestUrl, options);- const parsed =- method.toUpperCase() === 'HEAD' ? undefined : await parseResponseBody(response.clone(), config.responseType);+ const parsed = upperMethod === 'HEAD' ? undefined : await parseResponseBody(response.clone(), config.responseType);if (session?.aborted) throw new SessionCancelledError();return {data: parsed as T,@@ -415,84 +319,32 @@ class APIClient {};})());- if (operationResult.error !== null) {- const error = operationResult.error;- if (session?.aborted) {- throw new SessionCancelledError();- }- if (error instanceof KyHTTPError) {- return this.handleHttpError<T>(error, method, url, data, config, requestConfig, requestUrl, session);- }-- if (error instanceof TimeoutError) {- throw new APIError('Request timed out', {- cause: error,- config: requestConfig,- name: 'TimeoutError',- requestUrl- });- }-- if (error instanceof NetworkError) {- throw new APIError(error.message, {- cause: error,- config: requestConfig,- name: 'NetworkError',- requestUrl- });- }-- if (error instanceof Error) {- throw new APIError(error.message, {- cause: error,- config: requestConfig,- name: error.name || 'APIError',- requestUrl- });- }+ if (operationResult.error === null) return operationResult.data;- throw new APIError('Unknown error', {+ const error = operationResult.error;+ if (session?.aborted) throw new SessionCancelledError();+ if (!(error instanceof KyHTTPError)) {+ let message = 'Unknown error';+ if (error instanceof TimeoutError) message = 'Request timed out';+ else if (error instanceof Error) message = error.message;+ throw new APIError(message, {cause: error,config: requestConfig,+ name: (error instanceof Error && error.name) || undefined,requestUrl});- } else {- return operationResult.data;}- }- private async handleHttpError<T>(- error: KyHTTPError,- method: string,- url: string,- data: unknown,- config: InternalRequestConfig,- requestConfig: InternalRequestConfig & { baseURL: string; method: string; url: string },- requestUrl: string,- session: AbortSignal | undefined- ): Promise<APIResponse<T>> {const errorResponse = error.response;- const parsed = await parseErrorResponseBody(error);+ const parsed =+ error.data !== undefined || errorResponse.bodyUsed ? error.data : await parseResponseBody(errorResponse.clone());if (session?.aborted) throw new SessionCancelledError();- const response: APIResponse = {- data: parsed,- headers: errorResponse.headers,- raw: errorResponse,- status: errorResponse.status- };-- if (errorResponse.status === 401 && typeof window !== 'undefined' && !config._retry) {- const action = await handleUnauthorizedResponseInternal(- getRequestPath(url, requestConfig.baseURL),- !!config._retry,- extractServerMessage(parsed)- );++ if (errorResponse.status === 401) {+ const action = await handleUnauthorizedResponseInternal(requestPath, config._retry, extractServerMessage(parsed));if (session?.aborted) throw new SessionCancelledError();if (action === 'retry') {- return this.performRequest<T>(method, url, data, {- ...config,- _retry: true- });+ return this.performRequest<T>(method, url, data, { ...config, _retry: true });}if (action === 'redirect' || action === 'reload') {return new Promise(() => {});@@ -509,7 +361,7 @@ class APIClient {config: requestConfig,name: 'HTTPError',requestUrl,- response+ response: { data: parsed, headers: errorResponse.headers, raw: errorResponse, status: errorResponse.status }});}@@ -551,35 +403,31 @@ abstract class BaseAPIService {// Toggled by the update flows (Update All / local update center) so an in-progress// self-update restart is treated as a recoverable reconnect, not a logout.static setUpgradeInProgress(value: boolean) {- if (!upgradeReloadPendingInternal && (!value || !upgradeInProgressInternal)) {- upgradeReloadStartedInternal = false;+ if (!upgradeReloadPending && (!value || !upgradeInProgress)) {+ upgradeReloadStarted = false;}- upgradeInProgressInternal = value;+ upgradeInProgress = value;}/** Cancels the current session's work and blocks protected requests until beginSession. */static endSession() {- sessionControllerInternal.abort(new SessionCancelledError());+ sessionController.abort(new SessionCancelledError());}static beginSession() {- if (sessionControllerInternal.signal.aborted) {- sessionControllerInternal = new AbortController();+ if (sessionController.signal.aborted) {+ sessionController = new AbortController();}}static confirmUpgradeRestart() {- upgradeReloadPendingInternal = true;+ upgradeReloadPending = true;}protected async handleResponse<T>(promise: Promise<APIResponse>): Promise<T> {- const response = await promise;- const payload = response.data;- const extracted =- payload && typeof payload === 'object' && 'data' in payload && (payload as any).data !== undefined- ? (payload as any).data- : payload;- return extracted as T;+ const payload = (await promise).data;+ const inner = asRecord(payload)?.['data'];+ return (inner !== undefined ? inner : payload) as T;}}diff --git a/frontend/src/lib/types/settings.ts b/frontend/src/lib/types/settings.tsindex 04c5d0d03..8b003b285 100644--- a/frontend/src/lib/types/settings.ts+++ b/frontend/src/lib/types/settings.ts@@ -80,6 +80,9 @@ export type Settings = {autoInjectEnv: boolean;backupVolumeName?: string;edgeMTLSManagerCAAvailable?: boolean;+ frontendTracingEnabled?: boolean;+ frontendMetricsEnabled?: boolean;+ frontendLogsEnabled?: boolean;authLocalEnabled: boolean;authSessionTimeout: number;diff --git a/frontend/src/lib/utils/settings.svelte.ts b/frontend/src/lib/utils/settings.svelte.tsindex 8f8f25499..cf93c966d 100644--- a/frontend/src/lib/utils/settings.svelte.ts+++ b/frontend/src/lib/utils/settings.svelte.ts@@ -12,7 +12,6 @@ const LOCAL_SETTING_KEYS = new Set(['oidcAutoRedirectToProvider','oidcMergeAccounts','oidcSkipTlsVerify',- 'oidcAutoRedirectToProvider','oidcClientId','oidcClientSecret','oidcIssuerUrl',@@ -21,6 +20,9 @@ const LOCAL_SETTING_KEYS = new Set(['oidcProviderName','oidcProviderLogoUrl','edgeMTLSManagerCAAvailable',+ 'frontendTracingEnabled',+ 'frontendMetricsEnabled',+ 'frontendLogsEnabled','experimentalFeaturesEnabled','developmentBrandingEnabled','apnsEnabled'@@ -46,33 +48,39 @@ export function preventDefault<T extends Event>(fn: (event: T) => unknown) {}export function createForm<T extends z.ZodType<Record<string, unknown>>>(schema: T, initialValues: z.infer<T>) {- const inputs = $state<FormInputs<z.infer<T>>>(initializeInputs(initialValues));+ const schemaKeys = Object.keys(schema instanceof z.ZodObject ? schema.shape : {});+ const inputs = $state(+ Object.fromEntries(+ schemaKeys+ .filter((key) => Object.hasOwn(initialValues, key))+ .map((key) => [key, { value: initialValues[key as keyof z.infer<T>], error: null }])+ ) as FormInputs<z.infer<T>>+ );let errors = $state.raw<z.ZodError<z.infer<T>>>();- function initializeInputs(values: z.infer<T>): FormInputs<z.infer<T>> {- const fields = {} as FormInputs<z.infer<T>>;- const schemaShape = schema instanceof z.ZodObject ? schema.shape : {};- for (const key of Object.keys(schemaShape) as (keyof z.infer<T>)[]) {- if (Object.prototype.hasOwnProperty.call(values, key)) {- fields[key] = { value: values[key], error: null };- }- }- return fields;+ function parse() {+ const values = Object.fromEntries(+ Object.entries(inputs).map(([key, input]) => {+ const value: unknown = input.value;+ if (typeof value === 'string') return [key, value.trim()];+ if (Array.isArray(value)) return [key, value.map((item: unknown) => (typeof item === 'string' ? item.trim() : item))];+ return [key, value];+ })+ );+ return { values, result: schema.safeParse(values) };}function validate() {- const values = Object.fromEntries(Object.entries(inputs).map(([key, input]) => [key, input.value]));- const result = schema.safeParse(values);+ const { result } = parse();errors = result.error;for (const key of Object.keys(inputs) as (keyof z.infer<T>)[]) {inputs[key].error = result.error?.issues.find((issue) => issue.path[0] === key)?.message ?? null;}- return result.success ? data() : null;+ return result.success ? (result.data as z.infer<T>) : null;}function data() {- const values = Object.fromEntries(Object.entries(inputs).map(([key, input]) => [key, trimValue(input.value)]));- const result = schema.safeParse(values);+ const { values, result } = parse();return (result.success ? result.data : values) as z.infer<T>;}@@ -86,12 +94,6 @@ export function createForm<T extends z.ZodType<Record<string, unknown>>>(schema:inputs[key].value = value;}- function trimValue(value: unknown): unknown {- if (typeof value === 'string') return value.trim();- if (Array.isArray(value)) return value.map((item: unknown) => (typeof item === 'string' ? item.trim() : item));- return value;- }-return {schema,inputs,diff --git a/frontend/src/lib/utils/telemetry.ts b/frontend/src/lib/utils/telemetry.tsnew file mode 100644index 000000000..fab0a28a6--- /dev/null+++ b/frontend/src/lib/utils/telemetry.ts@@ -0,0 +1,237 @@+import type { NavigationType } from '$app/navigation';+import type { Context, Counter, Histogram, Span } from '@opentelemetry/api';+import type { LogRecord } from '@opentelemetry/api-logs';+import type { Resource } from '@opentelemetry/resources';+import type { Metric } from 'web-vitals';++type Signal = 'traces' | 'metrics' | 'logs';++const SERVICE_NAME = 'arcane-frontend';+// Keeps each export well under the relay's 512 KiB request limit.+const MAX_EXPORT_BATCH_SIZE = 32;+// Aborted fetches and logout cancellations are expected, not failures.+const CANCELLED_ERRORS = ['AbortError', 'SessionCancelledError'];++// Query strings and fragments can carry OIDC codes and tokens, so they never leave the browser.+function stripUrlQuery(value: string): string {+ return value.replace(/(https?:\/\/[^\s?#"']*)[?#][^\s"']*/g, '$1').replace(/^(\/[^\s?#]*)[?#].*$/, '$1');+}++let otel: typeof import('@opentelemetry/api') | undefined;+let resource: Promise<Resource> | undefined;+const initializations: Partial<Record<Signal, Promise<void>>> = {};+const providers: { forceFlush(): Promise<void> }[] = [];+let route = '/';+let navigationStart = 0;+let navigated = false;+let pageSpan: Span | undefined;+let pageContext: Context | undefined;+let navigationMetrics: { count: Counter; duration: Histogram } | undefined;+let emitErrorLog: ((record: LogRecord) => void) | undefined;++const initializers: Record<Signal, (resource: Resource) => Promise<void>> = {+ traces: async (resource) => {+ const [api, sdk, exporter, instrumentation, documentLoad, fetch] = await Promise.all([+ import('@opentelemetry/api'),+ import('@opentelemetry/sdk-trace-web'),+ import('@opentelemetry/exporter-trace-otlp-http'),+ import('@opentelemetry/instrumentation'),+ import('@opentelemetry/instrumentation-document-load'),+ import('@opentelemetry/instrumentation-fetch')+ ]);+ // Spans started outside any active span (fetches, logs) belong to the current pageview.+ class PageContextManager extends sdk.StackContextManager {+ override active(): Context {+ const active = super.active();+ return pageContext && !api.trace.getSpan(active) ? pageContext : active;+ }+ }+ const provider = new sdk.WebTracerProvider({+ resource,+ spanProcessors: [+ {+ onStart: () => {},+ onEnd: (span) => {+ for (const attributes of [span.attributes, ...span.events.map((event) => event.attributes ?? {})]) {+ for (const [key, value] of Object.entries(attributes)) {+ if (key === 'url.query') delete attributes[key];+ else if (typeof value === 'string') attributes[key] = stripUrlQuery(value);+ }+ }+ },+ forceFlush: () => Promise.resolve(),+ shutdown: () => Promise.resolve()+ },+ new sdk.BatchSpanProcessor(new exporter.OTLPTraceExporter({ url: '/api/telemetry/traces' }), {+ maxExportBatchSize: MAX_EXPORT_BATCH_SIZE+ })+ ]+ });+ provider.register({ contextManager: new PageContextManager() });+ providers.push(provider);+ instrumentation.registerInstrumentations({+ tracerProvider: provider,+ instrumentations: [+ new documentLoad.DocumentLoadInstrumentation(),+ new fetch.FetchInstrumentation({+ // The SSE stream never ends, and tracing it would keep the connection open after the app cancels it.+ ignoreUrls: [/\/api\/telemetry\//, /\/api\/stream(\?|$)/],+ applyCustomAttributesOnSpan: (span, _request, result) => {+ if (result instanceof Response || (result instanceof Error && CANCELLED_ERRORS.includes(result.name))) return;+ span.setAttribute('error.type', result instanceof Error ? result.name : 'Error');+ span.setStatus({ code: api.SpanStatusCode.ERROR, message: stripUrlQuery(result.message) });+ }+ })+ ]+ });+ otel = api;+ // Tracing can finish loading after the first navigation; start the pageview it missed.+ if (navigated) startPageView(route, navigationStart);+ },+ metrics: async (resource) => {+ const [sdk, exporter, vitals] = await Promise.all([+ import('@opentelemetry/sdk-metrics'),+ import('@opentelemetry/exporter-metrics-otlp-http'),+ import('web-vitals')+ ]);+ const provider = new sdk.MeterProvider({+ resource,+ readers: [+ new sdk.PeriodicExportingMetricReader({+ exporter: new exporter.OTLPMetricExporter({ url: '/api/telemetry/metrics' }),+ exportIntervalMillis: 60_000+ })+ ]+ });+ providers.push(provider);+ const meter = provider.getMeter(SERVICE_NAME);+ navigationMetrics = {+ count: meter.createCounter('browser.navigation.count', { description: 'Client-side navigations' }),+ duration: meter.createHistogram('browser.navigation.duration', {+ unit: 'ms',+ description: 'Time until a navigation completes'+ })+ };+ const webVitals: Record<Metric['name'], Histogram> = {+ CLS: meter.createHistogram('browser.web_vital.cls', {+ advice: { explicitBucketBoundaries: [0, 0.01, 0.05, 0.1, 0.15, 0.25, 0.5, 1] }+ }),+ FCP: meter.createHistogram('browser.web_vital.fcp', { unit: 'ms' }),+ INP: meter.createHistogram('browser.web_vital.inp', { unit: 'ms' }),+ LCP: meter.createHistogram('browser.web_vital.lcp', { unit: 'ms' }),+ TTFB: meter.createHistogram('browser.web_vital.ttfb', { unit: 'ms' })+ };+ const report = (metric: Metric) =>+ webVitals[metric.name].record(metric.value, { 'http.route': route, 'web_vital.rating': metric.rating });+ vitals.onCLS(report);+ vitals.onFCP(report);+ vitals.onINP(report);+ vitals.onLCP(report);+ vitals.onTTFB(report);+ },+ logs: async (resource) => {+ const [api, sdk, exporter] = await Promise.all([+ import('@opentelemetry/api-logs'),+ import('@opentelemetry/sdk-logs'),+ import('@opentelemetry/exporter-logs-otlp-http')+ ]);+ const provider = new sdk.LoggerProvider({+ resource,+ processors: [+ new sdk.BatchLogRecordProcessor({+ exporter: new exporter.OTLPLogExporter({ url: '/api/telemetry/logs' }),+ maxExportBatchSize: MAX_EXPORT_BATCH_SIZE+ })+ ]+ });+ providers.push(provider);+ const logger = provider.getLogger(SERVICE_NAME);+ emitErrorLog = (record) => logger.emit({ ...record, severityNumber: api.SeverityNumber.ERROR, severityText: 'ERROR' });+ window.addEventListener('error', (event) => recordError(event.error ?? event.message, 'window.error'));+ window.addEventListener('unhandledrejection', (event) => recordError(event.reason, 'window.unhandledrejection'));+ }+};++// Lazily loads the SDK for each enabled signal once; nothing loads while every signal is off.+export function setTelemetry(enabled: Record<Signal, boolean>, version: string): Promise<void> {+ if (typeof window === 'undefined') return Promise.resolve();+ const signals = (Object.keys(initializers) as Signal[]).filter((signal) => enabled[signal]);+ if (signals.length === 0) return Promise.resolve();++ const loaded = (resource ??= Promise.all([+ import('@opentelemetry/resources'),+ import('@opentelemetry/semantic-conventions'),+ import('@opentelemetry/opentelemetry-browser-detector'),+ import('#lib/utils/streaming.js')+ ]).then(([resources, semconv, detector, streaming]) => {+ const flush = (event: Event) => {+ if (event.type === 'pagehide') pageSpan?.end();+ else if (document.visibilityState !== 'hidden') return;+ for (const provider of providers) void provider.forceFlush().catch(() => {});+ };+ window.addEventListener('visibilitychange', flush);+ window.addEventListener('pagehide', flush);+ return resources+ .resourceFromAttributes({+ [semconv.ATTR_SERVICE_NAME]: SERVICE_NAME,+ [semconv.ATTR_SERVICE_VERSION]: version,+ // Each page session owns its cumulative metric series, so tabs never overwrite each other.+ [semconv.ATTR_SERVICE_INSTANCE_ID]: streaming.streamCacheBuster()+ })+ .merge(resources.detectResources({ detectors: [detector.browserDetector] }));+ }));+ // A failed chunk download is forgotten so the next navigation retries instead of staying off.+ loaded.catch(() => {+ if (resource === loaded) resource = undefined;+ });+ return Promise.all(+ signals.map(+ (signal) =>+ (initializations[signal] ??= loaded.then(initializers[signal]).catch(() => {+ delete initializations[signal];+ }))+ )+ ).then(() => {});+}++// Starts the pageview span that parents this navigation's requests.+export function startPageView(routeId: string | null | undefined, startTime = performance.now()): void {+ route = (routeId ?? '').replace(/\/\([^)]*\)/g, '') || '/';+ navigationStart = startTime;+ navigated = true;+ if (!otel) return;+ pageSpan?.end();+ pageSpan = otel.trace+ .getTracer(SERVICE_NAME)+ .startSpan(`pageview ${route}`, { root: true, startTime, attributes: { 'http.route': route } });+ pageContext = otel.trace.setSpan(otel.ROOT_CONTEXT, pageSpan);+}++// The initial 'enter' navigation starts its pageview here and is measured from the document's time origin.+export function finishNavigation(routeId: string | null | undefined, type: NavigationType): void {+ if (type === 'enter') startPageView(routeId, 0);+ const attributes = { 'http.route': route, 'navigation.type': type };+ navigationMetrics?.count.add(1, attributes);+ navigationMetrics?.duration.record(performance.now() - navigationStart, attributes);+}++// Emits a sanitized error log with the current route and trace context.+export function recordError(error: unknown, source: string): void {+ if (!emitErrorLog || (error instanceof Error && CANCELLED_ERRORS.includes(error.name))) return;+ try {+ const message = stripUrlQuery(error instanceof Error ? error.message : String(error)).slice(0, 1_000);+ emitErrorLog({+ eventName: 'exception',+ body: message,+ attributes: {+ 'exception.type': error instanceof Error ? error.name : typeof error,+ 'exception.message': message,+ 'exception.stacktrace': error instanceof Error && error.stack ? stripUrlQuery(error.stack).slice(0, 4_000) : undefined,+ 'http.route': route,+ 'error.source': source+ }+ });+ } catch {+ // Telemetry must never affect the app.+ }+}diff --git a/frontend/src/routes/+layout.svelte b/frontend/src/routes/+layout.svelteindex c937507e0..b206cb592 100644--- a/frontend/src/routes/+layout.svelte+++ b/frontend/src/routes/+layout.svelte@@ -1,6 +1,6 @@<script lang="ts">- import { browser, dev } from '$app/env';- import { refreshAll } from '$app/navigation';+ import { dev } from '$app/env';+ import { afterNavigate, beforeNavigate, refreshAll } from '$app/navigation';import { navigating, page } from '$app/state';import { QueryClientProvider } from '@tanstack/svelte-query';import { SvelteQueryDevtools } from '@tanstack/svelte-query-devtools';@@ -17,9 +17,10 @@import { IsMobile } from '#lib/hooks/is-mobile.svelte.js';import { IsTablet } from '#lib/hooks/is-tablet.svelte.js';import { m } from '#lib/paraglide/messages.js';+ import { isAuthPagePath } from '#lib/services/api-service.js';import settingsStore from '#lib/stores/config-store.svelte.js';import { environmentStore } from '#lib/stores/environment.store.svelte.js';- import { cn } from '#lib/utils.js';+ import { finishNavigation, startPageView } from '#lib/utils/telemetry.js';import type { LayoutProps } from './$types';@@ -28,40 +29,31 @@let { data, children }: LayoutProps = $props();onMount(() => {- if (!dev && browser && 'serviceWorker' in navigator) {+ if (!dev && 'serviceWorker' in navigator) {navigator.serviceWorker.register('/service-worker.js', { type: 'module' });}});- const settings = $derived(data.settings);+ beforeNavigate((navigation) => {+ if (!navigation.willUnload) startPageView(navigation.to?.route.id);+ });++ afterNavigate((navigation) => finishNavigation(navigation.to?.route.id, navigation.type));const isMobile = new IsMobile();const isTablet = new IsTablet();- // Mirror the effective layout onto <html> so CSS (sidebar visibility, etc.) can follow the- // per-device layout preference instead of raw viewport breakpoints. `$effect.pre` runs- // before the first DOM commit, so the attribute is present at first paint.+ // Mirror the per-device layout onto <html> for CSS; `$effect.pre` sets it before first paint.$effect.pre(() => {document.documentElement.setAttribute('data-layout', isMobile.current ? 'mobile' : 'desktop');});- const isNavigating = $derived(navigating.type !== null);- const isAuthPage = $derived(- String(page.url.pathname).startsWith('/login') ||- String(page.url.pathname).startsWith('/logout') ||- String(page.url.pathname).startsWith('/oidc') ||- String(page.url.pathname).startsWith('/mobile/passkey')- );+ const isAuthPage = $derived(isAuthPagePath(page.url.pathname));- const autoLoginEnabled = $derived(settingsStore.autoLoginEnabled.current);const showPasswordChangeDialog = $derived(- !!(data.user && data.user.requiresPasswordChange && !isAuthPage && !autoLoginEnabled)+ !!data.user?.requiresPasswordChange && !isAuthPage && !settingsStore.autoLoginEnabled.current);- function handlePasswordChangeSuccess() {- refreshAll();- }-const resourceName = $derived.by((): string | undefined => {switch (page.route.id) {case '/(app)/projects/[projectId]':@@ -94,13 +86,13 @@</svelte:head><QueryClientProvider client={data.queryClient}>- <div class={cn('flex min-h-dvh flex-col', 'bg-transparent')}>- {#if !settings && data.user && page.route.id !== '/(app)/environments/[id]'}+ <div class="flex min-h-dvh flex-col bg-transparent">+ {#if !data.settings && data.user && page.route.id !== '/(app)/environments/[id]'}<Error message={m.error_occurred()} showButton={true} />{:else}<Tooltip.Provider>{@render children()}- <FirstLoginPasswordDialog open={showPasswordChangeDialog} onSuccess={handlePasswordChangeSuccess} />+ <FirstLoginPasswordDialog open={showPasswordChangeDialog} onSuccess={() => refreshAll()} />{#if dev}<SvelteQueryDevtools />{/if}@@ -123,5 +115,5 @@}}/><ConfirmDialog />- <LoadingIndicator active={isNavigating} thickness="h-1.5" />+ <LoadingIndicator active={navigating.type !== null} thickness="h-1.5" /></QueryClientProvider>diff --git a/frontend/src/routes/+layout.ts b/frontend/src/routes/+layout.tsindex fcc5ec44c..4bd92ace5 100644--- a/frontend/src/routes/+layout.ts+++ b/frontend/src/routes/+layout.ts@@ -17,10 +17,10 @@ import { featureStore } from '#lib/stores/features.store.svelte.js';import userStore from '#lib/stores/user-store.svelte.js';import { versionStore } from '#lib/stores/version.store.svelte.js';import type { PermissionsManifest, User } from '#lib/types/auth.js';-import type { SearchPaginationSortRequest } from '#lib/types/shared.js';import { isAuthRejectionError } from '#lib/utils/api.js';import { getAuthRedirectPath, userHasPermission } from '#lib/utils/auth.js';import { getEffectiveLandingPage } from '#lib/utils/navigation.js';+import { setTelemetry } from '#lib/utils/telemetry.js';import { tryCatch } from '#lib/utils/try-catch.js';import type { LayoutLoad } from './$types';@@ -44,14 +44,8 @@ const queryClient = new QueryClient({let authenticatedUserId: string | null | undefined;-// The layout load re-runs on every navigation and re-checks the session. A-// transient failure of that check (network blip, 5xx, 429) must not sign the-// user out of the SPA: that clears every cache, bounces through /login and-// lands on the landing page. Only an explicit rejection means the session is-// gone; anything else keeps the user we already know about.-function resolveUserAfterLoadFailureInternal(error: unknown): User | null {- if (isAuthRejectionError(error) || isSessionCancelledError(error)) return null;- return userStore.current;+function dataOrNull<T>(promise: Promise<T>): Promise<T | null> {+ return tryCatch(promise).then((result) => (result.error ? null : result.data));}export const load: LayoutLoad = async ({ url }) => {@@ -76,9 +70,11 @@ export const load: LayoutLoad = async ({ url }) => {}): Promise.resolve(null);let [user, autoLoginConfig] = await Promise.all([- tryCatch(userService.getCurrentUser()).then((result) =>- result.error ? resolveUserAfterLoadFailureInternal(result.error) : result.data- ),+ // Only an explicit rejection signs the user out; transient check failures keep the known user.+ tryCatch(userService.getCurrentUser()).then((result): User | null => {+ if (!result.error) return result.data;+ return isAuthRejectionError(result.error) || isSessionCancelledError(result.error) ? null : userStore.current;+ }),autoLoginConfigRequest]);@@ -115,17 +111,8 @@ export const load: LayoutLoad = async ({ url }) => {let permissionsManifestLoadFailed = false;if (user) {// Initialize environment store (required for settings service)- const environmentRequestOptions: SearchPaginationSortRequest = {- pagination: {- page: 1,- limit: -1- }- };-- const environmentsRequest = tryCatch(environmentManagementService.getEnvironments(environmentRequestOptions));- const permissionsManifestRequest = tryCatch(roleService.getPermissionsManifest()).then((result) =>- result.error ? null : result.data- );+ const environmentsRequest = tryCatch(environmentManagementService.getEnvironments({ pagination: { page: 1, limit: -1 } }));+ const permissionsManifestRequest = dataOrNull(roleService.getPermissionsManifest());const environments = await environmentsRequest;if (!environments.error) {await environmentStore.initialize(environments.data.data);@@ -135,23 +122,15 @@ export const load: LayoutLoad = async ({ url }) => {await environmentStore.initialize([]);}- const settingsRequest = userHasPermission(user, 'settings:read')- ? tryCatch(settingsService.getSettings()).then(async (result) => {- if (!result.error) return result.data;- const publicSettings = await tryCatch(settingsService.getPublicSettings(environmentStore.selected?.id ?? '0'));- return publicSettings.error ? null : publicSettings.data;- })- : tryCatch(settingsService.getPublicSettings(environmentStore.selected?.id ?? '0')).then((result) =>- result.error ? null : result.data- );+ const settingsRequest = (+ userHasPermission(user, 'settings:read') ? dataOrNull(settingsService.getSettings()) : Promise.resolve(null)+ ).then((loaded) => loaded ?? dataOrNull(settingsService.getPublicSettings(environmentStore.selected?.id ?? '0')));featureStore.connect(queryClient);const currentEnvironmentId = await environmentStore.getCurrentEnvironmentId();const featuresRequest = featureStore.refresh(currentEnvironmentId);// Optional discovery: skip it without swarm:read and never toast a denial.const swarmStatusRequest = userHasPermission(user, 'swarm:read', currentEnvironmentId)- ? tryCatch(swarmService.getSwarmStatus(currentEnvironmentId, { suppressAccessDeniedToast: true })).then((result) =>- result.error ? null : result.data- )+ ? dataOrNull(swarmService.getSwarmStatus(currentEnvironmentId, { suppressAccessDeniedToast: true })): Promise.resolve(null);const [loadedSettings, loadedSwarmStatus, loadedPermissionsManifest] = await Promise.all([settingsRequest,@@ -159,9 +138,7 @@ export const load: LayoutLoad = async ({ url }) => {permissionsManifestRequest]);// Keep recovery/settings pages reachable while a selected agent is offline.- settings =- loadedSettings ??- (await tryCatch(settingsService.getPublicSettings()).then((result) => (result.error ? null : result.data)));+ settings = loadedSettings ?? (await dataOrNull(settingsService.getPublicSettings()));await featuresRequest;swarmEnabled = loadedSwarmStatus?.enabled;permissionsManifest = loadedPermissionsManifest;@@ -171,7 +148,7 @@ export const load: LayoutLoad = async ({ url }) => {await environmentStore.initialize([]);// Try to fetch public settings for login page configuration- settings = await tryCatch(settingsService.getPublicSettings()).then((result) => (result.error ? null : result.data));+ settings = await dataOrNull(settingsService.getPublicSettings());}if (settings) {@@ -180,6 +157,18 @@ export const load: LayoutLoad = async ({ url }) => {const versionInformation = await versionInformationRequest;versionStore.seed(versionInformation);+ // Waiting lets the first page view be traced, but a stalled SDK download must never hold up the app.+ await Promise.race([+ setTelemetry(+ {+ traces: settings?.frontendTracingEnabled ?? false,+ metrics: settings?.frontendMetricsEnabled ?? false,+ logs: settings?.frontendLogsEnabled ?? false+ },+ versionInformation.currentVersion+ ),+ new Promise((resolve) => setTimeout(resolve, 2_000))+ ]);const redirectPath = getAuthRedirectPath(url.pathname,diff --git a/go.work.sum b/go.work.sumindex a2c6e1b08..240b6b557 100644--- a/go.work.sum+++ b/go.work.sum@@ -2327,8 +2327,6 @@ github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzagithub.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ=github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI=-github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos=-github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM=github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=@@ -2874,10 +2872,6 @@ go.mongodb.org/mongo-driver/v2 v2.8.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybgo.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=-go.opentelemetry.io/contrib/bridges/otelslog v0.20.1 h1:5sHc4ToTFjfSZCtGAAM6jPunICAmJX73htv372T4ipc=-go.opentelemetry.io/contrib/bridges/otelslog v0.20.1/go.mod h1:oa6kgvyz/3GYW04dohd0++xJIH4xdQY8PAbpeCMaM8M=-go.opentelemetry.io/contrib/bridges/prometheus v0.71.0 h1:9qgxsFLskbDMXl8WMqThoF6w8yGJgCumn9qRc67OmnI=-go.opentelemetry.io/contrib/bridges/prometheus v0.71.0/go.mod h1:2rCjF4F2siiTeLCzJsaGZ3CK0XIoimCSKXEBPdv+Je0=go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs=go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts=go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE=@@ -2888,8 +2882,6 @@ go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfgo.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs=go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw=go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s=-go.opentelemetry.io/contrib/exporters/autoexport v0.71.0 h1:VCsJbp0YLyPtx2tu5Vgv2a2/qLoaMCj8hT2uZ34+Mx0=-go.opentelemetry.io/contrib/exporters/autoexport v0.71.0/go.mod h1:qxZqn7e10f6ajmMCkg/47rMS7qQYfaOl2nj/4aytHUQ=go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM=go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo=go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc=@@ -2903,6 +2895,12 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=go.opentelemetry.io/contrib/propagators/envcar v0.71.0 h1:r7TFdi1B3O5ai64tONEuBTyxeQaLKVYf75vLiV7ZsDc=go.opentelemetry.io/contrib/propagators/envcar v0.71.0/go.mod h1:uAnx0mCFm63a47qI4f/hcoI72gVYrGdxEBL+6c500Ns=+go.opentelemetry.io/contrib/propagators/jaeger v1.19.0 h1:mGrx7XEAE+7ybCLM0T6iRl/jUTuHg6qKUJAtsAlknec=+go.opentelemetry.io/contrib/propagators/jaeger v1.19.0/go.mod h1:cHWVPhYWMZOanEf1qexqMIRhr4TKVjZWBKwZTL/tdR4=+go.opentelemetry.io/contrib/propagators/opencensus v0.44.0 h1:foCvUO/MnWlwcS2GFCR/27k25UwolVwp3uYMGBguEvo=+go.opentelemetry.io/contrib/propagators/opencensus v0.44.0/go.mod h1:IUCrK+YXh4EO4dbh/l9NbWUHValpE3odollsVTjfpc4=+go.opentelemetry.io/contrib/propagators/ot v1.19.0 h1:vODRLMlKN4ApM8ri0UDk8nnEeISuwxpf67sE7PmOHhE=+go.opentelemetry.io/contrib/propagators/ot v1.19.0/go.mod h1:S2Uc7th2ZmLiHu0lrCmDCgTQ/y5Nbbis+TNjR1jjm4Q=go.opentelemetry.io/contrib/zpages v0.62.0 h1:9fUYTLmrK0x/lweM2uM+BOx069jLx8PxVqWhegGJ9Bo=go.opentelemetry.io/contrib/zpages v0.62.0/go.mod h1:C8kXoiC1Ytvereztus2R+kqdSa6W/MZ8FfS8Zwj+LiM=go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=@@ -2916,12 +2914,12 @@ go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXgo.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=+go.opentelemetry.io/otel/bridge/opencensus v0.41.0 h1:VBpeaTbrvLFHvRtsyCJXjsTaicBNrAFdmctiN1k6WNI=+go.opentelemetry.io/otel/bridge/opencensus v0.41.0/go.mod h1:yCQB5IKRhgjlbTLc91+ixcZc2/8BncGGJ+CS3dZJwtY=go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4=go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI=-go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 h1:Bu39F5tzJct+f2IZbB8989fwyTps3c8e7EsUQsz+vs8=-go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0/go.mod h1:dJUwod88EsFgYCqrDHaSPzhiY9pBUpt0d85/qSfua7k=-go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 h1:lYk7RmxdLK865qLwibroNGldHa1U7SWKYYvNjlK7PIo=-go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0/go.mod h1:6GvlND0H0xdUJanOtIAn0xfwLkauh1tmsYEEVSMDdqY=+go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU=+go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU=go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4=go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A=@@ -2941,14 +2939,8 @@ go.opentelemetry.io/otel/exporters/prometheus v0.66.0 h1:vkrK8PAznv2NKt2r+kdu252go.opentelemetry.io/otel/exporters/prometheus v0.66.0/go.mod h1:V/UB6D3vMF/UBOL5igAsAYnk1nG/bzYYTzvsB16cy7o=go.opentelemetry.io/otel/exporters/prometheus v0.67.0 h1:7IefDa35e6V3NoiqIeLDMDxMFyZDk5qcoC0Ax4cC16E=go.opentelemetry.io/otel/exporters/prometheus v0.67.0/go.mod h1:nsPI1awTg5Vmg1YrommL2mVarVGlqc4yXOoKAkPRD0c=-go.opentelemetry.io/otel/exporters/prometheus v0.68.0 h1:QOf2IftqQwITVRJpnn0M7M9ZCbgWfxz4P7i9C9yc2N4=-go.opentelemetry.io/otel/exporters/prometheus v0.68.0/go.mod h1:bgSvqu2TWGXiz7yr5UTMfObH8oqxJWHTnubQ3ef9BO4=-go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0 h1:kvMAiLEudKmk+CSG+iYbU8vTUGNNDaf/V09OO5lrTwI=-go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0/go.mod h1:L9Dlksri+MdT1cb2gIiA1cJJYW3Y92ipvDjNxYEyaDI=go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0 h1:lSZHgNHfbmQTPfuTmWVkEu8J8qXaQwuV30pjCcAUvP8=go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw=-go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 h1:PR9eAf7o0dQs3hshZNZpE9aW2dXWX/KdDf6pJilVD3U=-go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0/go.mod h1:2Z4KyNdH1uuzivdinyfGsxzNNT/Rl45pwtVwfYVI0xk=go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 h1:UGZ1QwZWY67Z6BmckTU+9Rxn04m2bD3gD6Mk0OIOCPk=go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0/go.mod h1:fcwWuDuaObkkChiDlhEpSq9+X1C0omv+s5mBtToAQ64=go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE=@@ -2957,10 +2949,6 @@ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0 h1:bl2S7Ubua0Nms+Dgo.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0/go.mod h1:L0hRV50XdVIODHUfWEqGRCXQvj2rV82STVo12FMFBU0=go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68=go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo=-go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 h1:KdRxPiAoMptR3vfWzvjjvutTsSiwbC2uG0496rzZNfo=-go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0/go.mod h1:K/qSA+3G7Eovxi4K09wzrAgkWRnosS0DAOZeEpve7sM=-go.opentelemetry.io/otel/log v0.22.0 h1:5DBNnfvaJ6CVdkJ+Jle8Tzs50aSSv49TXGj9XRsEYw0=-go.opentelemetry.io/otel/log v0.22.0/go.mod h1:gzOt/R67vF2GniAqWu8Qv0SXy89f71muHcrkz76PCdc=go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=@@ -2978,8 +2966,6 @@ go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmgo.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=-go.opentelemetry.io/otel/sdk/log v0.22.0 h1:PRL+s6P63XT4E/bheEflopPUpVxuvANqZwtt89yhoGk=-go.opentelemetry.io/otel/sdk/log v0.22.0/go.mod h1:JNp0sBELrjCTcu5W3GzABVypeU6vDJjBS+X0JISuz+g=go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=@@ -3492,6 +3478,10 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=+gorm.io/driver/clickhouse v0.7.0 h1:BCrqvgONayvZRgtuA6hdya+eAW5P2QVagV3OlEp1vtA=+gorm.io/driver/clickhouse v0.7.0/go.mod h1:TmNo0wcVTsD4BBObiRnCahUgHJHjBIwuRejHwYt3JRs=+gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=+gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 041663a08..a8e860e1a 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -207,7 +207,7 @@ importers:version: '@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])'vite-plus:specifier: 'catalog:'- version: 1.1.0(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])+ version: 1.1.0(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])email-templates:dependencies:@@ -244,7 +244,7 @@ importers:version: '@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])'vite-plus:specifier: 'catalog:'- version: 1.1.0(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])+ version: 1.1.0(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])frontend:dependencies:@@ -293,12 +293,54 @@ importers:'@lezer/highlight':specifier: ^1.2.5version: 1.2.5+ '@opentelemetry/api':+ specifier: ^1.9.1+ version: 1.9.1+ '@opentelemetry/api-logs':+ specifier: ^0.223.0+ version: 0.223.0+ '@opentelemetry/exporter-logs-otlp-http':+ specifier: ^0.223.0+ version: 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/exporter-metrics-otlp-http':+ specifier: ^0.223.0+ version: 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/exporter-trace-otlp-http':+ specifier: ^0.223.0+ version: 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/instrumentation':+ specifier: ^0.223.0+ version: 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/instrumentation-document-load':+ specifier: ^0.68.0+ version: 0.68.0(@opentelemetry/[email protected])+ '@opentelemetry/instrumentation-fetch':+ specifier: ^0.223.0+ version: 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/opentelemetry-browser-detector':+ specifier: ^0.223.0+ version: 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/resources':+ specifier: ^2.12.0+ version: 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-logs':+ specifier: ^0.223.0+ version: 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-metrics':+ specifier: ^2.12.0+ version: 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace-web':+ specifier: ^2.12.0+ version: 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions':+ specifier: ^1.43.0+ version: 1.43.0'@simplewebauthn/browser':specifier: ^14.0.0version: 14.0.0'@sveltejs/kit':specifier: ^3.0.0- version: 3.0.1(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))+ version: 3.0.1(@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))'@tanstack/svelte-query':specifier: ^6.3.1version: 6.3.1([email protected](@typescript-eslint/[email protected]))@@ -331,7 +373,7 @@ importers:version: 0.7.2bits-ui:specifier: ^2.19.5- version: 2.19.5(@internationalized/[email protected])(@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))+ version: 2.19.5(@internationalized/[email protected])(@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))clsx:specifier: ^2.1.1version: 2.1.1@@ -340,7 +382,7 @@ importers:version: 8.6.0([email protected](@typescript-eslint/[email protected]))formsnap:specifier: ^2.0.1- version: 2.0.1([email protected](@typescript-eslint/[email protected]))([email protected](@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@types/[email protected])(@typescript/[email protected])([email protected](@typescript-eslint/[email protected])))+ version: 2.0.1([email protected](@typescript-eslint/[email protected]))([email protected](@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@types/[email protected])(@typescript/[email protected])([email protected](@typescript-eslint/[email protected])))ky:specifier: ^2.1.0version: 2.1.0@@ -352,7 +394,7 @@ importers:version: 1.1.0([email protected](@typescript-eslint/[email protected]))runed:specifier: ^0.37.1- version: 0.37.1(@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))([email protected])+ version: 0.37.1(@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))([email protected])svelte:specifier: ^5.57.2version: 5.57.2(@typescript-eslint/[email protected])@@ -377,6 +419,9 @@ importers:vaul-svelte:specifier: 1.0.0-next.7version: 1.0.0-next.7([email protected](@typescript-eslint/[email protected]))+ web-vitals:+ specifier: ^6.2.3+ version: 6.2.3yaml:specifier: ^2.9.1version: 2.9.1@@ -395,7 +440,7 @@ importers:version: 0.2.0(@typescript-eslint/[email protected](@typescript/[email protected])([email protected]([email protected])))([email protected]([email protected]))'@sveltejs/adapter-static':specifier: ^4.0.0- version: 4.0.0(@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))+ version: 4.0.0(@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))'@sveltejs/vite-plugin-svelte':specifier: ^7.3.1version: 7.3.1(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))@@ -437,10 +482,10 @@ importers:version: 1.8.1([email protected](@typescript-eslint/[email protected]))svelte-toolbelt:specifier: ^0.10.6- version: 0.10.6(@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))+ version: 0.10.6(@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))sveltekit-superforms:specifier: ^3.0.0- version: 3.0.0(@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@types/[email protected])(@typescript/[email protected])([email protected](@typescript-eslint/[email protected]))+ version: 3.0.0(@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@types/[email protected])(@typescript/[email protected])([email protected](@typescript-eslint/[email protected]))tailwindcss:specifier: ^4.3.3version: 4.3.3@@ -461,7 +506,7 @@ importers:version: '@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected])'vite-plus:specifier: 'catalog:'- version: 1.1.0(@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])+ version: 1.1.0(@opentelemetry/[email protected])(@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])tests:devDependencies:@@ -482,7 +527,7 @@ importers:version: '@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])'vite-plus:specifier: 'catalog:'- version: 1.1.0(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])+ version: 1.1.0(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])packages:@@ -1139,6 +1184,120 @@ packages:'@emnapi/core': ^1.7.1 || ^2.0.0-alpha.4'@emnapi/runtime': ^1.7.1 || ^2.0.0-alpha.4+ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-Oepm6etVPFCn0VWB1t833iOTgz1eYBDUPxcrOR/XvwzD1+Ag9VzSPSrrKAFNJ/haatj+8CXbG47pZXvdNQKDJw==}+ engines: {node: '>=8.0.0'}++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==}+ engines: {node: '>=8.0.0'}++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-1HsSAuvT4/my0QrXWsyXFdWjKaPeCHsismKZgEEIb8NK13LP6DrMRlsUbvD/0F9mDWK3xDom9AMv3IscxgOnOQ==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.0.0 <1.10.0'++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-YQh9+t/JmV9OeRkoLhPb4yvcrWCX6lAOmXFos/JKoD+KITZYZoxf0acnwAk6YDI9O7fX68O2ka38hx1G9exl8w==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': ^1.3.0++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-hNC8yai/+xCNyTuiOy6JFepXXA/sb/2jCzQRY7c2cUWz6owaidihlHUpO20B9/dj8JKPNo8rcn7ikTy7Cj+JaQ==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': ^1.3.0++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-N8NWZaoQuF9e307S6xoVoI1lvxOOFkmBnynD42r02CPCfxoC95cJExAKBFrTD4C5iuhqthbzOZJN/W29ez4o/A==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': ^1.3.0++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-pkUPmaW0Mlim8Eetfflhc1oSdvs6SGS2oqGw+eOqNUGWqqpOkCzT/s6STvvMk4wzo2L8OIOMFTntDRatVRg71g==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': ^1.3.0++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-D2bRIBvw8IM1AWE/5BZB6kEemmCP4H89Gd2Lg4Yf44rlwwFxJ9Y25GJEwb4slW3r5lPq8YDj0nIU5fGiFCovtQ==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': ^1.3.0++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-tttBo88TbwK1zpsH6VeJoxLQopxOjW3cnrP50fsQJRZftkSKDDwjolXXBvow2jy0LKXlNRkDQh0NT6yI/q+BEQ==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': ^1.3.0++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-96KHQMIpTO6hhFjPKKxnQS+yQCemo2P0BbpVon56X+GZYmk8mPBwn8P0D9lB9RncITb5GQVnVA9jNFu9tgNRlQ==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': ^1.3.0++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-HlJMrJXjOqdQeoYNDHEvFTpaY2p9uWDK1GPl5AZ4wI5easzfoppTduOv47M8eiF+4ew42vfI/Xvt7THYgESnRw==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': ^1.3.0++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-oxmmPL5/MbSsO8K588V9n/XLIe7FZTK30Fo+/U/lAnt0ZIK/hoPM03n2WpSTZVbUBGTUE7aLyxnAzh7nE2PD4Q==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': ^1.3.0++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-+rSRoOdln6NhXTVcBdHSlZxJDNzwxadeweK2ltZO3tVtfp6aklABc06xclDk6ovRR+EiGgIcSbh/6NgjFhnb3A==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.3.0 <1.10.0'++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-ApiAfUiUh6fmJt1GbsBuoeDn8D8IL42UvGyPUr8o1Z89ZeIiOCE1rZ78KkIUjEX3OV8Oorl9ez5Qjo43ZynDyw==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.4.0 <1.10.0'++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-mVMaEHwCUJuY3S4Esn+nT0yQA3qCEGh+SOPJsXygU0Gl453WcsDNjtEa8EfdewMwoLyr1tlofB4fXytVpiXuow==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.9.0 <1.10.0'++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-JOERgZCxRM06FEJ5kEWxlSRChah6MmXDx3KwCruqEv/iEyLkTokCSemf2y1w8oIHtfrr8EVTpzebq/31vn2Cpg==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.3.0 <1.10.0'++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-y0hCzbnBHScmioF0drJOyaPk1sQf2wwbIuy3rWlM0Fcx8mLfPDvFsQ7tJ+zMm8yAqI06rSlFKkL7ZpcrFOqAdQ==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.0.0 <1.10.0'++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-ImVuRQ6faOsTi1Wna88s2avgrrf/iDgXPigIeynkUhjstMuoQUDOJXgkez2j1mlU4btjXcXBGa+q7qKFYR3GsA==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.3.0 <1.10.0'++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==}+ engines: {node: '>=14'}++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-zUx54ocAoj2Pw2vVnUVp5dJgvp5SindWwekmoB2AUjZJGEeBnCcZ1FiqohWLvdqq0xIHm+fQUpPf5hffU3U4kA==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.3.0 <1.10.0'+'@oxc-minify/[email protected]':resolution: {integrity: sha512-gk1jzky1vLeOcK7w9Ib2RT/UooZ7O21s8YSeXCsy4tzsmguKb3Cbsv/Ra2fKhMp4mKeKCzvC1Xa41OfSBWfWdQ==}engines: {node: ^20.19.0 || >=22.12.0}@@ -2685,6 +2844,9 @@ packages:resolution: {integrity: sha512-+6vJA3L98yv+IdfKGZHBNiGW5KHn22e/JwID0Strsz8h4S/csAu/OuICwxrg44k5MRiZHWIo8XXuJgQTriRP4w==}+ resolution: {integrity: sha512-lmNyBzi6iYiqyrExTctfmjd8tInx+dHu9PBomTfH5RXS53c27JMFJgzNv5TyAnm3PgGf+MLy0+Ws41/HlVxdKQ==}+resolution: {integrity: sha512-AwNusCCam51q703dW82x95tOqQp6oC9HNUl724KxJJOfnKscI8dOloXFgyez7LbTTKWuRBA37FScqVbJEoq8Yw==}@@ -2929,6 +3091,9 @@ packages:resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==}+ resolution: {integrity: sha512-yWGe1O/j+almnV4hOYrmZo8u58JdnmXEMneRQAYkWtqSiT86yoMIEKF4rpqEDj52tbYl3xceBD3DQIBwcsC9Pw==}+resolution: {integrity: sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==}engines: {node: '>=18'}@@ -3125,6 +3290,10 @@ packages:resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==}engines: {node: '>= 4'}+ resolution: {integrity: sha512-WseIA/4o56+GYO3RmlOPvejYi771OFeJEeK/ejROsr9bdjW+E5NXTOD1Di8TL40Qgg3JJX9aa97Hf8iiSVIKZw==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==}@@ -3467,6 +3636,9 @@ packages:peerDependencies:svelte: ^5.27.0+ resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==}+resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==}engines: {node: '>=4'}@@ -3714,6 +3886,10 @@ packages:resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==}engines: {node: '>=0.10.0'}+ resolution: {integrity: sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==}+ engines: {node: '>=9.3.0 || >=8.10.0 <9.0.0'}+resolution: {integrity: sha512-9q8oUiBYeXIDLWNK5DfCWlkL0EW3oGbk845VdKlPeia28l751VpfesaB/+7pI6rnbx1I6rqoZ2fZxptOJLxILA==}peerDependencies:@@ -4250,6 +4426,9 @@ packages:resolution: {integrity: sha512-4Nsy8K5Tr6SPDH9jhKJOHf7ChDrc1zufZTVSF7x72hwuEXBqxqk9G6cK+K2NRUtB3iELRJqjXb4JPDMBjMTl2Q==}engines: {node: ^22.22.2 || ^24.15.0 || >=26.0.0}+ resolution: {integrity: sha512-Dmy+vdO1UPMqXiAtZSNayhl4f794Bxfd94W4HIj2HlYtczEQfEOon969memlcpcHrTuOKLa5baFjX7WbgmOq+A==}+resolution: {integrity: sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==}engines: {node: '>=20'}@@ -5011,6 +5190,140 @@ snapshots:'@tybys/wasm-util': 0.10.4optional: true+ '@opentelemetry/[email protected]':+ dependencies:+ '@opentelemetry/api': 1.9.1++ '@opentelemetry/[email protected]': {}++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/semantic-conventions': 1.43.0++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/otlp-exporter-base': 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-logs': 0.223.0(@opentelemetry/[email protected])++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/otlp-exporter-base': 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-metrics': 2.12.0(@opentelemetry/[email protected])++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/otlp-exporter-base': 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace': 2.12.0(@opentelemetry/[email protected])++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/instrumentation': 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace-web': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions': 1.43.0+ transitivePeerDependencies:+ - supports-color++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/instrumentation': 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions': 1.43.0+ '@opentelemetry/web-common': 0.223.0(@opentelemetry/[email protected])+ transitivePeerDependencies:+ - supports-color++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/api-logs': 0.223.0+ import-in-the-middle: 3.5.2+ require-in-the-middle: 8.0.1+ transitivePeerDependencies:+ - supports-color++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions': 1.43.0++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/[email protected])++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/api-logs': 0.223.0+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-logs': 0.223.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-metrics': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace': 2.12.0(@opentelemetry/[email protected])++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions': 1.43.0++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/api-logs': 0.223.0+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions': 1.43.0++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions': 1.43.0++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace-base': 2.12.0(@opentelemetry/[email protected])++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/resources': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions': 1.43.0++ '@opentelemetry/[email protected]': {}++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.12.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions': 1.43.0+'@oxc-minify/[email protected]':optional: true@@ -5394,11 +5707,11 @@ snapshots:dependencies:acorn: 8.19.0- '@sveltejs/[email protected](@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))':+ '@sveltejs/[email protected](@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))':dependencies:- '@sveltejs/kit': 3.0.1(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))+ '@sveltejs/kit': 3.0.1(@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))- '@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))':+ '@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))':dependencies:'@rolldown/pluginutils': 1.0.1'@standard-schema/spec': 1.1.0@@ -5412,6 +5725,7 @@ snapshots:svelte: 5.57.2(@typescript-eslint/[email protected])vite: '@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected])'optionalDependencies:+ '@opentelemetry/api': 1.9.1typescript: '@typescript/[email protected]''@sveltejs/[email protected]': {}@@ -5797,7 +6111,7 @@ snapshots:'@testing-library/dom': 10.4.2'@testing-library/user-event': 14.6.7(@testing-library/[email protected])'@vitest/browser': 5.0.3(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])- vitest: 5.0.3(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])+ vitest: 5.0.3(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])transitivePeerDependencies:- bufferutil- msw@@ -5809,7 +6123,7 @@ snapshots:'@testing-library/dom': 10.4.2'@testing-library/user-event': 14.6.7(@testing-library/[email protected])'@vitest/browser': 5.0.3(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])- vitest: 5.0.3(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])+ vitest: 5.0.3(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])transitivePeerDependencies:- bufferutil- msw@@ -5826,7 +6140,7 @@ snapshots:pngjs: 7.0.0sirv: 3.0.2tinyrainbow: 3.2.0- vitest: 5.0.3(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])+ vitest: 5.0.3(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])ws: 8.22.0transitivePeerDependencies:- bufferutil@@ -5844,7 +6158,7 @@ snapshots:pngjs: 7.0.0sirv: 3.0.2tinyrainbow: 3.2.0- vitest: 5.0.3(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])+ vitest: 5.0.3(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])ws: 8.22.0transitivePeerDependencies:- bufferutil@@ -5891,7 +6205,7 @@ snapshots:pathe: 2.0.3sirv: 3.0.2tinyrainbow: 3.2.0- vitest: 5.0.3(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])+ vitest: 5.0.3(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])'@vitest/[email protected]':dependencies:@@ -6145,15 +6459,15 @@ snapshots:- [email protected](@internationalized/[email protected])(@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected])):+ [email protected](@internationalized/[email protected])(@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected])):dependencies:'@floating-ui/core': 1.8.0'@floating-ui/dom': 1.8.0'@internationalized/date': 3.12.4esm-env: 1.2.2- runed: 0.35.1(@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))+ runed: 0.35.1(@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))svelte: 5.57.2(@typescript-eslint/[email protected])- svelte-toolbelt: 0.10.6(@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))+ svelte-toolbelt: 0.10.6(@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))tabbable: 6.5.0transitivePeerDependencies:- '@sveltejs/kit'@@ -6185,6 +6499,8 @@ snapshots:+ [email protected]: {}+dependencies:'@types/validator': 13.15.10@@ -6417,6 +6733,8 @@ snapshots:+ [email protected]: {}+optionalDependencies:'@esbuild/aix-ppc64': 0.28.2@@ -6600,11 +6918,11 @@ snapshots:- [email protected]([email protected](@typescript-eslint/[email protected]))([email protected](@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@types/[email protected])(@typescript/[email protected])([email protected](@typescript-eslint/[email protected]))):+ [email protected]([email protected](@typescript-eslint/[email protected]))([email protected](@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@types/[email protected])(@typescript/[email protected])([email protected](@typescript-eslint/[email protected]))):dependencies:svelte: 5.57.2(@typescript-eslint/[email protected])svelte-toolbelt: 0.5.0([email protected](@typescript-eslint/[email protected]))- sveltekit-superforms: 3.0.0(@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@types/[email protected])(@typescript/[email protected])([email protected](@typescript-eslint/[email protected]))+ sveltekit-superforms: 3.0.0(@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@types/[email protected])(@typescript/[email protected])([email protected](@typescript-eslint/[email protected]))optional: true@@ -6658,6 +6976,12 @@ snapshots:+ dependencies:+ cjs-module-lexer: 2.3.0+ es-module-lexer: 3.0.3+ module-details-from-path: 1.0.4+@@ -6938,6 +7262,8 @@ snapshots:svelte: 5.57.2(@typescript-eslint/[email protected])svelte-toolbelt: 0.7.1([email protected](@typescript-eslint/[email protected]))+ [email protected]: {}+@@ -7024,7 +7350,7 @@ snapshots:'@oxc-parser/binding-win32-x64-msvc': 0.148.0optional: true- [email protected]([email protected](@typescript-eslint/[email protected]))([email protected](@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])):+ [email protected]([email protected](@typescript-eslint/[email protected]))([email protected](@opentelemetry/[email protected])(@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])):dependencies:tinypool: 2.2.0optionalDependencies:@@ -7048,9 +7374,9 @@ snapshots:'@oxfmt/binding-win32-ia32-msvc': 0.72.0'@oxfmt/binding-win32-x64-msvc': 0.72.0svelte: 5.57.2(@typescript-eslint/[email protected])- vite-plus: 1.1.0(@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])+ vite-plus: 1.1.0(@opentelemetry/[email protected])(@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])- [email protected]([email protected](@typescript-eslint/[email protected]))([email protected](@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])):+ [email protected]([email protected](@typescript-eslint/[email protected]))([email protected](@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])):dependencies:tinypool: 2.2.0optionalDependencies:@@ -7074,7 +7400,7 @@ snapshots:'@oxfmt/binding-win32-ia32-msvc': 0.72.0'@oxfmt/binding-win32-x64-msvc': 0.72.0svelte: 5.57.2(@typescript-eslint/[email protected])- vite-plus: 1.1.0(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])+ vite-plus: 1.1.0(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])optionalDependencies:@@ -7085,7 +7411,7 @@ snapshots:'@oxlint-tsgolint/win32-arm64': 7.0.2003'@oxlint-tsgolint/win32-x64': 7.0.2003- [email protected]([email protected])([email protected](@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])):+ [email protected]([email protected])([email protected](@opentelemetry/[email protected])(@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])):optionalDependencies:'@oxlint/binding-android-arm-eabi': 1.87.0'@oxlint/binding-android-arm64': 1.87.0@@ -7107,9 +7433,9 @@ snapshots:'@oxlint/binding-win32-ia32-msvc': 1.87.0'@oxlint/binding-win32-x64-msvc': 1.87.0oxlint-tsgolint: 7.0.2003- vite-plus: 1.1.0(@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])+ vite-plus: 1.1.0(@opentelemetry/[email protected])(@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])- [email protected]([email protected])([email protected](@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])):+ [email protected]([email protected])([email protected](@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])):optionalDependencies:'@oxlint/binding-android-arm-eabi': 1.87.0'@oxlint/binding-android-arm64': 1.87.0@@ -7131,7 +7457,7 @@ snapshots:'@oxlint/binding-win32-ia32-msvc': 1.87.0'@oxlint/binding-win32-x64-msvc': 1.87.0oxlint-tsgolint: 7.0.2003- vite-plus: 1.1.0(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])+ vite-plus: 1.1.0(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected])dependencies:@@ -7291,6 +7617,13 @@ snapshots:+ dependencies:+ debug: 4.4.3+ module-details-from-path: 1.0.4+ transitivePeerDependencies:+ - supports-color+dependencies:esm-env: 1.2.2@@ -7311,23 +7644,23 @@ snapshots:esm-env: 1.2.2svelte: 5.57.2(@typescript-eslint/[email protected])- [email protected](@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected])):+ [email protected](@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected])):dependencies:dequal: 2.0.3esm-env: 1.2.2lz-string: 1.5.0svelte: 5.57.2(@typescript-eslint/[email protected])optionalDependencies:- '@sveltejs/kit': 3.0.1(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))+ '@sveltejs/kit': 3.0.1(@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))- [email protected](@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))([email protected]):+ [email protected](@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))([email protected]):dependencies:dequal: 2.0.3esm-env: 1.2.2lz-string: 1.5.0svelte: 5.57.2(@typescript-eslint/[email protected])optionalDependencies:- '@sveltejs/kit': 3.0.1(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))+ '@sveltejs/kit': 3.0.1(@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))zod: 4.6.5@@ -7491,10 +7824,10 @@ snapshots:runed: 0.28.0([email protected](@typescript-eslint/[email protected]))svelte: 5.57.2(@typescript-eslint/[email protected])- [email protected](@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected])):+ [email protected](@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected])):dependencies:clsx: 2.1.1- runed: 0.35.1(@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))+ runed: 0.35.1(@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))([email protected](@typescript-eslint/[email protected]))style-to-object: 1.0.14svelte: 5.57.2(@typescript-eslint/[email protected])transitivePeerDependencies:@@ -7540,9 +7873,9 @@ snapshots:transitivePeerDependencies:- '@typescript-eslint/types'- [email protected](@sveltejs/[email protected](@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@types/[email protected])(@typescript/[email protected])([email protected](@typescript-eslint/[email protected])):+ [email protected](@sveltejs/[email protected](@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@types/[email protected])(@typescript/[email protected])([email protected](@typescript-eslint/[email protected])):dependencies:- '@sveltejs/kit': 3.0.1(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))+ '@sveltejs/kit': 3.0.1(@opentelemetry/[email protected])(@sveltejs/[email protected](@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected])))(@typescript/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected](@typescript-eslint/[email protected]))devalue: 6.0.2esm-env: 1.2.2memoize-weak: 1.0.2@@ -7762,7 +8095,7 @@ snapshots:svelte: 5.57.2(@typescript-eslint/[email protected])svelte-toolbelt: 0.7.1([email protected](@typescript-eslint/[email protected]))- [email protected](@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected]):+ [email protected](@opentelemetry/[email protected])(@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected]):dependencies:'@oxc-project/types': 0.153.0'@oxlint/plugins': 1.79.0@@ -7773,11 +8106,11 @@ snapshots:'@vitest/snapshot': 5.0.3'@vitest/spy': 5.0.3'@vitest/utils': 5.0.3- oxfmt: 0.72.0([email protected](@typescript-eslint/[email protected]))([email protected](@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected]))- oxlint: 1.87.0([email protected])([email protected](@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected]))+ oxfmt: 0.72.0([email protected](@typescript-eslint/[email protected]))([email protected](@opentelemetry/[email protected])(@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected]))+ oxlint: 1.87.0([email protected])([email protected](@opentelemetry/[email protected])(@types/[email protected])(@typescript/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected]))oxlint-tsgolint: 7.0.2003vite: '@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected])'- vitest: 5.0.3(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])+ vitest: 5.0.3(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])optionalDependencies:'@voidzero-dev/vite-plus-darwin-arm64': 1.1.0'@voidzero-dev/vite-plus-darwin-x64': 1.1.0@@ -7822,7 +8155,7 @@ snapshots:- vue-tsc- yaml- [email protected](@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected]):+ [email protected](@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected]):dependencies:'@oxc-project/types': 0.153.0'@oxlint/plugins': 1.79.0@@ -7833,11 +8166,11 @@ snapshots:'@vitest/snapshot': 5.0.3'@vitest/spy': 5.0.3'@vitest/utils': 5.0.3- oxfmt: 0.72.0([email protected](@typescript-eslint/[email protected]))([email protected](@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected]))- oxlint: 1.87.0([email protected])([email protected](@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected]))+ oxfmt: 0.72.0([email protected](@typescript-eslint/[email protected]))([email protected](@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected]))+ oxlint: 1.87.0([email protected])([email protected](@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected])([email protected])([email protected]))oxlint-tsgolint: 7.0.2003vite: '@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])'- vitest: 5.0.3(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])+ vitest: 5.0.3(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])optionalDependencies:'@voidzero-dev/vite-plus-darwin-arm64': 1.1.0'@voidzero-dev/vite-plus-darwin-x64': 1.1.0@@ -7886,7 +8219,7 @@ snapshots:optionalDependencies:vite: '@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected])'- [email protected](@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected]):+ [email protected](@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))([email protected]):dependencies:'@types/chai': 5.2.3'@vitest/mocker': 5.0.3(@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected]))@@ -7903,6 +8236,7 @@ snapshots:vite: '@voidzero-dev/[email protected](@types/[email protected])(@typescript/[email protected])([email protected])([email protected])([email protected])([email protected])'why-is-node-running: 3.2.1optionalDependencies:+ '@opentelemetry/api': 1.9.1'@types/node': 26.6.4'@vitest/browser-preview': 5.0.3(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])'@vitest/ui': 5.0.3([email protected])@@ -7910,7 +8244,7 @@ snapshots:transitivePeerDependencies:- msw- [email protected](@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected]):+ [email protected](@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected]):dependencies:'@types/chai': 5.2.3'@vitest/mocker': 5.0.3(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))@@ -7927,6 +8261,7 @@ snapshots:vite: '@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])'why-is-node-running: 3.2.1optionalDependencies:+ '@opentelemetry/api': 1.9.1'@types/node': 26.6.4'@vitest/browser-preview': 5.0.3(@voidzero-dev/[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected])'@vitest/ui': 5.0.3([email protected])@@ -7940,6 +8275,8 @@ snapshots:dependencies:xml-name-validator: 5.0.0+ [email protected]: {}+