Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cyclops): keep serving available through PostgreSQL cutover (#6804)
r33drichardsa90e68305577
diff --git a/libs/fleet/backend/auth/auth.go b/libs/fleet/backend/auth/auth.goindex 1205439fcc..1e0ad0502d 100644--- a/libs/fleet/backend/auth/auth.go+++ b/libs/fleet/backend/auth/auth.go@@ -38,9 +38,12 @@ package authimport ("context"+ "errors""fmt"+ "io""log""log/slog"+ "net""net/http""slices""strings"@@ -50,6 +53,7 @@ import (keyfunc "github.com/MicahParks/keyfunc/v2"jwt "github.com/golang-jwt/jwt/v5"+ "github.com/jackc/pgx/v5/pgconn")var (@@ -70,6 +74,57 @@ type GitHubTrustResolver interface {ResolveGitHubTrustPolicies(ctx context.Context, repository string) ([]GitHubTrustPolicy, error)}+var ErrDatabaseUnavailable = errors.New("database unavailable")++type databaseContextKey struct{}++const DatabaseRequestTimeout = 15 * time.Second++func DatabaseContext(parent context.Context) (context.Context, context.CancelFunc) {+ if databaseContext, ok := parent.Value(databaseContextKey{}).(context.Context); ok {+ return databaseContext, func() {}+ }+ if deadline, ok := parent.Deadline(); ok && time.Until(deadline) <= DatabaseRequestTimeout {+ return context.WithValue(parent, databaseContextKey{}, parent), func() {}+ }+ databaseContext, cancel := context.WithTimeout(parent, DatabaseRequestTimeout)+ return context.WithValue(databaseContext, databaseContextKey{}, databaseContext), cancel+}++func IsDatabaseUnavailable(err error) bool {+ return errors.Is(err, ErrDatabaseUnavailable) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)+}++func DatabaseUnavailable(err error) error {+ if errors.Is(err, ErrDatabaseUnavailable) {+ return err+ }+ return errors.Join(ErrDatabaseUnavailable, err)+}++func ClassifyDatabaseError(err error) error {+ if err == nil || errors.Is(err, ErrDatabaseUnavailable) {+ return err+ }+ if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {+ return DatabaseUnavailable(err)+ }++ var postgresError *pgconn.PgError+ if errors.As(err, &postgresError) {+ return err+ }+ var connectError *pgconn.ConnectError+ if errors.As(err, &connectError) {+ return DatabaseUnavailable(err)+ }+ var networkError net.Error+ if errors.As(err, &networkError) || errors.Is(err, net.ErrClosed) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {+ return DatabaseUnavailable(err)+ }+ return err+}+var githubTrustResolver GitHubTrustResolverfunc SetGitHubTrustResolver(resolver GitHubTrustResolver) {diff --git a/libs/fleet/backend/auth/github_oidc_test.go b/libs/fleet/backend/auth/github_oidc_test.goindex fe7b59fb7e..e77c9a3675 100644--- a/libs/fleet/backend/auth/github_oidc_test.go+++ b/libs/fleet/backend/auth/github_oidc_test.go@@ -6,15 +6,19 @@ import ("crypto/rsa""encoding/base64""encoding/json"+ "errors"+ "net""net/http""net/http/httptest""strings"+ "syscall""testing""time""cyclops-cs-backend/config"jwt "github.com/golang-jwt/jwt/v5"+ "github.com/jackc/pgx/v5/pgconn")const (@@ -242,3 +246,199 @@ func bigEndianBytes(v int) []byte {}return out}++func TestTokenAuthMiddlewareDoesNotExtendGitHubResolverDeadline(t *testing.T) {+ signingKey, jwksURL := newGitHubJWKS(t)+ t.Cleanup(func() {+ SetGitHubTrustResolver(nil)+ })+ if err := Init(&config.AuthConfiguration{+ Issuer: "https://issuer.example.test/realms/cyclops-cs",+ JWKSUri: jwksURL,+ SigningAlgs: []string{"RS256"},+ SPAClientID: "cyclops-cs-spa",+ KeyClientPfx: "key-",+ UserKeyClientPfx: "ukey-",+ GitHubOIDCIssuer: testGitHubIssuer,+ GitHubOIDCJWKSUri: jwksURL,+ GitHubOIDCAudience: testGitHubAudience,+ GitHubOIDCEnabled: true,+ GitHubOIDCAlgs: []string{"RS256"},+ }); err != nil {+ t.Fatalf("Init err = %v", err)+ }+ resolverDeadline := make(chan time.Time, 1)+ SetGitHubTrustResolver(githubTrustResolverFunc(func(ctx context.Context, _ string) ([]GitHubTrustPolicy, error) {+ deadline, ok := ctx.Deadline()+ if !ok {+ t.Fatal("resolver context has no deadline")+ }+ resolverDeadline <- deadline+ return []GitHubTrustPolicy{{ID: "p1", OwnerSub: "user-123", Repository: "trycua/cloud", AllowedNamespaces: []string{"ns-a"}, Enabled: true}}, nil+ }))+ raw := signGitHubToken(t, signingKey, jwt.MapClaims{+ "iss": testGitHubIssuer, "aud": testGitHubAudience, "sub": "repo:trycua/cloud:ref:refs/heads/main",+ "repository": "trycua/cloud", "exp": time.Now().Add(time.Hour).Unix(), "iat": time.Now().Add(-time.Minute).Unix(),+ })+ parent, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second))+ defer cancel()+ wantDeadline, ok := parent.Deadline()+ if !ok {+ t.Fatal("request context has no deadline")+ }+ request := httptest.NewRequest(http.MethodGet, "/", nil).WithContext(parent)+ request.Header.Set("Authorization", "Bearer "+raw)+ response := httptest.NewRecorder()++ TokenAuthMiddleware(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})).ServeHTTP(response, request)++ if response.Code != http.StatusOK {+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())+ }+ if gotDeadline := <-resolverDeadline; !gotDeadline.Equal(wantDeadline) {+ t.Fatalf("resolver deadline = %s, want request deadline %s", gotDeadline, wantDeadline)+ }+}++func TestTokenAuthMiddlewareReturnsServiceUnavailableForGitHubTrustDatabaseOutage(t *testing.T) {+ signingKey, jwksURL := newGitHubJWKS(t)+ t.Cleanup(func() {+ SetGitHubTrustResolver(nil)+ })+ if err := Init(&config.AuthConfiguration{+ Issuer: "https://issuer.example.test/realms/cyclops-cs",+ JWKSUri: jwksURL,+ SigningAlgs: []string{"RS256"},+ SPAClientID: "cyclops-cs-spa",+ KeyClientPfx: "key-",+ UserKeyClientPfx: "ukey-",+ GitHubOIDCIssuer: testGitHubIssuer,+ GitHubOIDCJWKSUri: jwksURL,+ GitHubOIDCAudience: testGitHubAudience,+ GitHubOIDCEnabled: true,+ GitHubOIDCAlgs: []string{"RS256"},+ }); err != nil {+ t.Fatalf("Init err = %v", err)+ }+ SetGitHubTrustResolver(githubTrustResolverFunc(func(context.Context, string) ([]GitHubTrustPolicy, error) {+ return nil, ErrDatabaseUnavailable+ }))+ raw := signGitHubToken(t, signingKey, jwt.MapClaims{+ "iss": testGitHubIssuer, "aud": testGitHubAudience, "sub": "repo:trycua/cloud:ref:refs/heads/main",+ "repository": "trycua/cloud", "exp": time.Now().Add(time.Hour).Unix(), "iat": time.Now().Add(-time.Minute).Unix(),+ })+ request := httptest.NewRequest(http.MethodGet, "/", nil)+ request.Header.Set("Authorization", "Bearer "+raw)+ response := httptest.NewRecorder()++ TokenAuthMiddleware(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {+ t.Fatal("next handler must not run")+ })).ServeHTTP(response, request)++ if response.Code != http.StatusServiceUnavailable {+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())+ }+}++func TestClassifyDatabaseErrorRecognizesTransportButNotPostgresErrors(t *testing.T) {+ connectionRefused := &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}+ classified := ClassifyDatabaseError(errors.New("resolve policies: " + connectionRefused.Error()))+ if errors.Is(classified, ErrDatabaseUnavailable) {+ t.Fatal("string-only network error must not be classified as database unavailable")+ }++ classified = ClassifyDatabaseError(errors.Join(errors.New("resolve policies"), connectionRefused))+ if !errors.Is(classified, ErrDatabaseUnavailable) {+ t.Fatalf("classified error = %v, want database unavailable sentinel", classified)+ }+ if !errors.Is(classified, connectionRefused) {+ t.Fatalf("classified error = %v, want original transport error", classified)+ }++ postgresError := &pgconn.PgError{Code: "42501", Message: "permission denied"}+ classified = ClassifyDatabaseError(errors.Join(errors.New("resolve policies"), postgresError))+ if errors.Is(classified, ErrDatabaseUnavailable) {+ t.Fatalf("postgres authorization error = %v, must not be classified as unavailable", classified)+ }+}++func TestTokenAuthMiddlewareReturnsServiceUnavailableForImmediateGitHubTrustConnectionFailure(t *testing.T) {+ signingKey, jwksURL := newGitHubJWKS(t)+ t.Cleanup(func() {+ SetGitHubTrustResolver(nil)+ })+ if err := Init(&config.AuthConfiguration{+ Issuer: "https://issuer.example.test/realms/cyclops-cs",+ JWKSUri: jwksURL,+ SigningAlgs: []string{"RS256"},+ SPAClientID: "cyclops-cs-spa",+ KeyClientPfx: "key-",+ UserKeyClientPfx: "ukey-",+ GitHubOIDCIssuer: testGitHubIssuer,+ GitHubOIDCJWKSUri: jwksURL,+ GitHubOIDCAudience: testGitHubAudience,+ GitHubOIDCEnabled: true,+ GitHubOIDCAlgs: []string{"RS256"},+ }); err != nil {+ t.Fatalf("Init err = %v", err)+ }+ connectionRefused := &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}+ SetGitHubTrustResolver(githubTrustResolverFunc(func(context.Context, string) ([]GitHubTrustPolicy, error) {+ return nil, ClassifyDatabaseError(connectionRefused)+ }))+ raw := signGitHubToken(t, signingKey, jwt.MapClaims{+ "iss": testGitHubIssuer, "aud": testGitHubAudience, "sub": "repo:trycua/cloud:ref:refs/heads/main",+ "repository": "trycua/cloud", "exp": time.Now().Add(time.Hour).Unix(), "iat": time.Now().Add(-time.Minute).Unix(),+ })+ request := httptest.NewRequest(http.MethodGet, "/", nil)+ request.Header.Set("Authorization", "Bearer "+raw)+ response := httptest.NewRecorder()++ TokenAuthMiddleware(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {+ t.Fatal("next handler must not run")+ })).ServeHTTP(response, request)++ if response.Code != http.StatusServiceUnavailable {+ t.Fatalf("status = %d, want 503; body = %s", response.Code, response.Body.String())+ }+}++func TestTokenAuthMiddlewareKeepsGitHubTrustPostgresErrorsUnauthorized(t *testing.T) {+ signingKey, jwksURL := newGitHubJWKS(t)+ t.Cleanup(func() {+ SetGitHubTrustResolver(nil)+ })+ if err := Init(&config.AuthConfiguration{+ Issuer: "https://issuer.example.test/realms/cyclops-cs",+ JWKSUri: jwksURL,+ SigningAlgs: []string{"RS256"},+ SPAClientID: "cyclops-cs-spa",+ KeyClientPfx: "key-",+ UserKeyClientPfx: "ukey-",+ GitHubOIDCIssuer: testGitHubIssuer,+ GitHubOIDCJWKSUri: jwksURL,+ GitHubOIDCAudience: testGitHubAudience,+ GitHubOIDCEnabled: true,+ GitHubOIDCAlgs: []string{"RS256"},+ }); err != nil {+ t.Fatalf("Init err = %v", err)+ }+ SetGitHubTrustResolver(githubTrustResolverFunc(func(context.Context, string) ([]GitHubTrustPolicy, error) {+ return nil, &pgconn.PgError{Code: "42501", Message: "permission denied"}+ }))+ raw := signGitHubToken(t, signingKey, jwt.MapClaims{+ "iss": testGitHubIssuer, "aud": testGitHubAudience, "sub": "repo:trycua/cloud:ref:refs/heads/main",+ "repository": "trycua/cloud", "exp": time.Now().Add(time.Hour).Unix(), "iat": time.Now().Add(-time.Minute).Unix(),+ })+ request := httptest.NewRequest(http.MethodGet, "/", nil)+ request.Header.Set("Authorization", "Bearer "+raw)+ response := httptest.NewRecorder()++ TokenAuthMiddleware(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {+ t.Fatal("next handler must not run")+ })).ServeHTTP(response, request)++ if response.Code != http.StatusUnauthorized {+ t.Fatalf("status = %d, want 401; body = %s", response.Code, response.Body.String())+ }+}diff --git a/libs/fleet/backend/auth/middlewares.go b/libs/fleet/backend/auth/middlewares.goindex 7285439cb3..ef31231e33 100644--- a/libs/fleet/backend/auth/middlewares.go+++ b/libs/fleet/backend/auth/middlewares.go@@ -362,6 +362,10 @@ func TokenAuthMiddleware(next http.Handler) http.Handler {user, err := validateWithContext(r.Context(), raw)if err != nil {result = err.Error()+ if IsDatabaseUnavailable(err) {+ writeJSONErr(w, http.StatusServiceUnavailable, "authentication unavailable")+ return+ }writeJSONErr(w, http.StatusUnauthorized, "auth token is invalid")return}diff --git a/libs/fleet/backend/cmd/manifest-contract/main.go b/libs/fleet/backend/cmd/manifest-contract/main.gonew file mode 100644index 0000000000..353b63d644--- /dev/null+++ b/libs/fleet/backend/cmd/manifest-contract/main.go@@ -0,0 +1,279 @@+package main++import (+ "bufio"+ "fmt"+ "io"+ "os"+ "regexp"+ "strings"++ "sigs.k8s.io/yaml"+)++var (+ servingNames = []string{"cyclops-cs", "cyclops-cs-backend"}+ projectorImageTag = regexp.MustCompile(`^main-[0-9]+$`)+)++type manifestObject struct {+ APIVersion string `json:"apiVersion"`+ Kind string `json:"kind"`+ Metadata struct {+ Name string `json:"name"`+ Namespace string `json:"namespace"`+ } `json:"metadata"`+ Spec struct {+ Replicas int `json:"replicas"`+ ProgressDeadlineSeconds int `json:"progressDeadlineSeconds"`+ Strategy struct {+ Type string `json:"type"`+ RollingUpdate struct {+ MaxUnavailable int `json:"maxUnavailable"`+ MaxSurge int `json:"maxSurge"`+ } `json:"rollingUpdate"`+ } `json:"strategy"`+ Selector labelSelector `json:"selector"`+ Template struct {+ Metadata struct {+ Labels map[string]string `json:"labels"`+ } `json:"metadata"`+ Spec struct {+ TopologySpreadConstraints []topologySpreadConstraint `json:"topologySpreadConstraints"`+ Containers []container `json:"containers"`+ } `json:"spec"`+ } `json:"template"`+ MinAvailable int `json:"minAvailable"`+ } `json:"spec"`+}++type labelSelector struct {+ MatchLabels map[string]string `json:"matchLabels"`+}++type topologySpreadConstraint struct {+ MaxSkew int `json:"maxSkew"`+ TopologyKey string `json:"topologyKey"`+ WhenUnsatisfiable string `json:"whenUnsatisfiable"`+ LabelSelector labelSelector `json:"labelSelector"`+}++type container struct {+ Name string `json:"name"`+ Image string `json:"image"`+ ReadinessProbe probe `json:"readinessProbe"`+ LivenessProbe probe `json:"livenessProbe"`+}++type probe struct {+ HTTPGet struct {+ Path string `json:"path"`+ } `json:"httpGet"`+}++func main() {+ if err := run(os.Args[1:], os.Stdin); err != nil {+ fmt.Fprintln(os.Stderr, err)+ os.Exit(1)+ }+}++func run(args []string, stdin io.Reader) error {+ if len(args) == 0 {+ return verify(stdin)+ }++ mode := "serving"+ path := args[0]+ if args[0] == "projector" {+ mode = "projector"+ if len(args) == 1 {+ return verifyProjector(stdin)+ }+ if len(args) != 2 {+ return fmt.Errorf("usage: manifest-contract [manifest.yaml] | manifest-contract projector [manifest.yaml]")+ }+ path = args[1]+ } else if len(args) != 1 {+ return fmt.Errorf("usage: manifest-contract [manifest.yaml] | manifest-contract projector [manifest.yaml]")+ }++ file, err := os.Open(path)+ if err != nil {+ return err+ }+ defer file.Close()++ if mode == "projector" {+ return verifyProjector(file)+ }+ return verify(file)+}++func verify(reader io.Reader) error {+ objects, err := parseDocuments(reader)+ if err != nil {+ return err+ }++ for _, name := range servingNames {+ deployment, err := findObject(objects, "Deployment", name)+ if err != nil {+ return err+ }+ if err := verifyDeployment(deployment, name); err != nil {+ return err+ }++ pdb, err := findObject(objects, "PodDisruptionBudget", name)+ if err != nil {+ return err+ }+ if err := verifyPDB(pdb, name); err != nil {+ return err+ }+ }++ for _, object := range objects {+ if object.Metadata.Name == "k8s-state-projector" || object.Metadata.Name == "cyclops-cs-k8s-state-projector" {+ return fmt.Errorf("serving manifest must not contain state projector")+ }+ }++ backend, err := findObject(objects, "Deployment", "cyclops-cs-backend")+ if err != nil {+ return err+ }+ return verifyBackendProbes(backend)+}++func parseDocuments(reader io.Reader) ([]manifestObject, error) {+ var documents []manifestObject+ var current strings.Builder+ scanner := bufio.NewScanner(reader)+ scanner.Buffer(make([]byte, 64*1024), 10*1024*1024)+ for scanner.Scan() {+ line := scanner.Text()+ if strings.TrimSpace(line) == "---" {+ if err := appendDocument(&documents, current.String()); err != nil {+ return nil, err+ }+ current.Reset()+ continue+ }+ current.WriteString(line)+ current.WriteByte('\n')+ }+ if err := scanner.Err(); err != nil {+ return nil, err+ }+ if err := appendDocument(&documents, current.String()); err != nil {+ return nil, err+ }+ return documents, nil+}++func appendDocument(documents *[]manifestObject, source string) error {+ if strings.TrimSpace(source) == "" {+ return nil+ }++ var object manifestObject+ if err := yaml.Unmarshal([]byte(source), &object); err != nil {+ return fmt.Errorf("parse manifest document: %w", err)+ }+ *documents = append(*documents, object)+ return nil+}++func findObject(objects []manifestObject, kind, name string) (manifestObject, error) {+ var matches []manifestObject+ for _, object := range objects {+ if object.Kind == kind && object.Metadata.Name == name {+ matches = append(matches, object)+ }+ }+ if len(matches) != 1 {+ return manifestObject{}, fmt.Errorf("expected exactly one %s %q, found %d", kind, name, len(matches))+ }+ return matches[0], nil+}++func verifyDeployment(deployment manifestObject, name string) error {+ if deployment.APIVersion != "apps/v1" || deployment.Metadata.Namespace != "cyclops-cs" {+ return fmt.Errorf("Deployment %q has wrong apiVersion or namespace", name)+ }+ if deployment.Spec.Replicas != 2 || deployment.Spec.ProgressDeadlineSeconds != 600 {+ return fmt.Errorf("Deployment %q must set replicas 2 and progressDeadlineSeconds 600", name)+ }+ if deployment.Spec.Strategy.Type != "RollingUpdate" || deployment.Spec.Strategy.RollingUpdate.MaxUnavailable != 0 || deployment.Spec.Strategy.RollingUpdate.MaxSurge != 1 {+ return fmt.Errorf("Deployment %q must use RollingUpdate maxUnavailable 0 and maxSurge 1", name)+ }+ if deployment.Spec.Selector.MatchLabels["app"] != name {+ return fmt.Errorf("Deployment %q selector app label must equal %q", name, name)+ }+ if deployment.Spec.Template.Metadata.Labels["app"] != name {+ return fmt.Errorf("Deployment %q pod template app label must equal %q", name, name)+ }+ for _, constraint := range deployment.Spec.Template.Spec.TopologySpreadConstraints {+ if constraint.MaxSkew == 1 && constraint.TopologyKey == "kubernetes.io/hostname" && constraint.WhenUnsatisfiable == "ScheduleAnyway" && constraint.LabelSelector.MatchLabels["app"] == name {+ return nil+ }+ }+ return fmt.Errorf("Deployment %q must spread pods by app label %q", name, name)+}++func verifyPDB(pdb manifestObject, name string) error {+ if pdb.APIVersion != "policy/v1" || pdb.Metadata.Namespace != "cyclops-cs" {+ return fmt.Errorf("PodDisruptionBudget %q has wrong apiVersion or namespace", name)+ }+ if pdb.Spec.MinAvailable != 1 || pdb.Spec.Selector.MatchLabels["app"] != name {+ return fmt.Errorf("PodDisruptionBudget %q must select app %q with minAvailable 1", name, name)+ }+ return nil+}++func verifyBackendProbes(backend manifestObject) error {+ for _, container := range backend.Spec.Template.Spec.Containers {+ if container.Name == "cyclops-cs-backend" && container.ReadinessProbe.HTTPGet.Path == "/healthz" && container.LivenessProbe.HTTPGet.Path == "/healthz" {+ return nil+ }+ }+ return fmt.Errorf("Deployment %q backend container must use /healthz readiness and liveness probes", backend.Metadata.Name)+}++func verifyProjector(reader io.Reader) error {+ objects, err := parseDocuments(reader)+ if err != nil {+ return err+ }++ projector, err := findObject(objects, "Deployment", "k8s-state-projector")+ if err != nil {+ return err+ }+ if projector.APIVersion != "apps/v1" || projector.Metadata.Namespace != "cyclops-cs" {+ return fmt.Errorf("projector Deployment has wrong apiVersion or namespace")+ }++ for _, container := range projector.Spec.Template.Spec.Containers {+ if container.Name != "projector" {+ continue+ }+ return verifyProjectorImage(container.Image)+ }+ return fmt.Errorf("projector Deployment must contain projector container")+}++func verifyProjectorImage(image string) error {+ const repository = "296062593712.dkr.ecr.us-west-2.amazonaws.com/cyclops-cs-backend"++ imageRepository, tag, found := strings.Cut(image, ":")+ if !found || imageRepository != repository {+ return fmt.Errorf("projector container image repository must equal %q", repository)+ }+ if !projectorImageTag.MatchString(tag) {+ return fmt.Errorf("projector container image tag must match ^main-[0-9]+$")+ }+ return nil+}diff --git a/libs/fleet/backend/cmd/manifest-contract/main_test.go b/libs/fleet/backend/cmd/manifest-contract/main_test.gonew file mode 100644index 0000000000..256afd1f13--- /dev/null+++ b/libs/fleet/backend/cmd/manifest-contract/main_test.go@@ -0,0 +1,218 @@+package main++import (+ "strings"+ "testing"+)++func TestVerifyAcceptsServingObjects(t *testing.T) {+ t.Parallel()++ if err := verify(strings.NewReader(validManifest())); err != nil {+ t.Fatalf("verify() error = %v, want nil", err)+ }+}++func TestVerifyRejectsPodTemplateLabelMismatch(t *testing.T) {+ t.Parallel()++ manifest := strings.Replace(+ validManifest(),+ " app: cyclops-cs\n spec:\n topologySpreadConstraints:",+ " app: wrong-app\n spec:\n topologySpreadConstraints:",+ 1,+ )++ err := verify(strings.NewReader(manifest))+ if err == nil {+ t.Fatal("verify() error = nil, want pod template label mismatch")+ }+ if !strings.Contains(err.Error(), "pod template app label") {+ t.Fatalf("verify() error = %q, want pod template app label failure", err)+ }+}++func TestVerifyRejectsStateProjector(t *testing.T) {+ t.Parallel()++ err := verify(strings.NewReader(validManifest() + `+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: k8s-state-projector+ namespace: cyclops-cs+`))+ if err == nil {+ t.Fatal("verify() error = nil, want state projector rejection")+ }+ if !strings.Contains(err.Error(), "state projector") {+ t.Fatalf("verify() error = %q, want state projector failure", err)+ }+}++func TestVerifyProjectorAcceptsExpectedImage(t *testing.T) {+ t.Parallel()++ manifest := strings.Replace(validProjectorManifest(), "PROJECTOR_IMAGE", "296062593712.dkr.ecr.us-west-2.amazonaws.com/cyclops-cs-backend:main-1786507556", 1)+ if err := verifyProjector(strings.NewReader(manifest)); err != nil {+ t.Fatalf("verifyProjector() error = %v, want nil", err)+ }+}++func TestVerifyProjectorRejectsWrongImage(t *testing.T) {+ t.Parallel()++ tests := []struct {+ name string+ image string+ want string+ }{+ {+ name: "repository",+ image: "296062593712.dkr.ecr.us-west-2.amazonaws.com/wrong-repository:main-1786507556",+ want: "repository",+ },+ {+ name: "tag",+ image: "296062593712.dkr.ecr.us-west-2.amazonaws.com/cyclops-cs-backend:latest",+ want: "tag",+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ manifest := strings.Replace(validProjectorManifest(), "PROJECTOR_IMAGE", test.image, 1)++ err := verifyProjector(strings.NewReader(manifest))+ if err == nil {+ t.Fatal("verifyProjector() error = nil, want image validation failure")+ }+ if !strings.Contains(err.Error(), test.want) {+ t.Fatalf("verifyProjector() error = %q, want %q failure", err, test.want)+ }+ })+ }+}++func validProjectorManifest() string {+ return `apiVersion: v1+kind: ServiceAccount+metadata:+ name: k8s-state-projector+ namespace: cyclops-cs+---+apiVersion: rbac.authorization.k8s.io/v1+kind: ClusterRole+metadata:+ name: cyclops-cs-k8s-state-projector+---+apiVersion: rbac.authorization.k8s.io/v1+kind: ClusterRoleBinding+metadata:+ name: cyclops-cs-k8s-state-projector+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: k8s-state-projector+ namespace: cyclops-cs+spec:+ template:+ spec:+ containers:+ - name: projector+ image: PROJECTOR_IMAGE+`+}++func validManifest() string {+ return `apiVersion: apps/v1+kind: Deployment+metadata:+ name: cyclops-cs+ namespace: cyclops-cs+spec:+ replicas: 2+ progressDeadlineSeconds: 600+ strategy:+ type: RollingUpdate+ rollingUpdate:+ maxUnavailable: 0+ maxSurge: 1+ selector:+ matchLabels:+ app: cyclops-cs+ template:+ metadata:+ labels:+ app: cyclops-cs+ spec:+ topologySpreadConstraints:+ - maxSkew: 1+ topologyKey: kubernetes.io/hostname+ whenUnsatisfiable: ScheduleAnyway+ labelSelector:+ matchLabels:+ app: cyclops-cs+---+apiVersion: policy/v1+kind: PodDisruptionBudget+metadata:+ name: cyclops-cs+ namespace: cyclops-cs+spec:+ minAvailable: 1+ selector:+ matchLabels:+ app: cyclops-cs+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: cyclops-cs-backend+ namespace: cyclops-cs+spec:+ replicas: 2+ progressDeadlineSeconds: 600+ strategy:+ type: RollingUpdate+ rollingUpdate:+ maxUnavailable: 0+ maxSurge: 1+ selector:+ matchLabels:+ app: cyclops-cs-backend+ template:+ metadata:+ labels:+ app: cyclops-cs-backend+ spec:+ topologySpreadConstraints:+ - maxSkew: 1+ topologyKey: kubernetes.io/hostname+ whenUnsatisfiable: ScheduleAnyway+ labelSelector:+ matchLabels:+ app: cyclops-cs-backend+ containers:+ - name: cyclops-cs-backend+ readinessProbe:+ httpGet:+ path: /healthz+ livenessProbe:+ httpGet:+ path: /healthz+---+apiVersion: policy/v1+kind: PodDisruptionBudget+metadata:+ name: cyclops-cs-backend+ namespace: cyclops-cs+spec:+ minAvailable: 1+ selector:+ matchLabels:+ app: cyclops-cs-backend+`+}diff --git a/libs/fleet/backend/database/migrator.go b/libs/fleet/backend/database/migrator.goindex dab6c57722..823e1aa008 100644--- a/libs/fleet/backend/database/migrator.go+++ b/libs/fleet/backend/database/migrator.go@@ -772,6 +772,10 @@ func reconcileStaticRoleContracts(ctx context.Context, transaction pgx.Tx) ([]roreturn nil, nil, fmt.Errorf("read migration role: %w", err)}membershipContracts := staticMembershipContracts(migrationOwner)+ membershipEvents, err := reconcileStaticMembershipContracts(ctx, transaction, migrationOwner, membershipContracts)+ if err != nil {+ return nil, nil, err+ }if err := validateStaticRoleMemberships(ctx, transaction, migrationOwner, membershipContracts); err != nil {return nil, nil, err}@@ -789,45 +793,79 @@ func reconcileStaticRoleContracts(ctx context.Context, transaction pgx.Tx) ([]roroleEvents = append(roleEvents, roleReconciliationEvent{Role: contract.role, DurationMS: time.Since(started).Milliseconds()})}- membershipEvents := make([]membershipReconciliationEvent, 0, len(membershipContracts))- for _, contract := range membershipContracts {- started := time.Now()+ if err := reconcileStaticRoleSettings(ctx, transaction); err != nil {+ return nil, nil, err+ }+ return roleEvents, membershipEvents, nil+}++func reconcileStaticMembershipContracts(ctx context.Context, transaction pgx.Tx, migrationOwner string, membershipContracts []staticMembershipContract) ([]membershipReconciliationEvent, error) {+ grantsByContract := make([][]staticMembershipGrant, len(membershipContracts))+ for index, contract := range membershipContracts {grants, err := readStaticMembershipGrants(ctx, transaction, contract)if err != nil {- return nil, nil, err+ return nil, err}+ grantsByContract[index] = grants+ }+ repairs, err := staticMembershipReconciliationPlan(migrationOwner, membershipContracts, grantsByContract)+ if err != nil {+ return nil, err+ }- var migrationOwnerGrant *staticMembershipGrant- for index := range grants {- grant := &grants[index]- if grant.grantor == migrationOwner {- migrationOwnerGrant = grant- continue- }- if allowsImplicitCreatorAdminMembership(contract.role, contract.member, migrationOwner, *grant) {- continue- }- return nil, nil, foreignStaticMembershipGrantError(contract, grant.grantor, migrationOwner)+ membershipEvents := make([]membershipReconciliationEvent, 0, len(membershipContracts))+ for index, contract := range membershipContracts {+ started := time.Now()+ if !isStaticMembershipRepair(contract, repairs) {+ membershipEvents = append(membershipEvents, membershipReconciliationEvent{Role: contract.role, DurationMS: time.Since(started).Milliseconds()})+ continue}- if migrationOwnerGrant == nil || migrationOwnerGrant.admin != contract.admin || migrationOwnerGrant.inherit != contract.inherit || migrationOwnerGrant.set != contract.set {- ddl := newRuntimeDDL(transaction)- if migrationOwnerGrant != nil {- if err := ddl.revokeStaticMembership(ctx, contract, migrationOwner); err != nil {- return nil, nil, fmt.Errorf("remove drifted static role membership %s -> %s: %w", contract.role, contract.member, err)- }- }- if err := ddl.grantStaticMembership(ctx, contract, migrationOwner); err != nil {- return nil, nil, fmt.Errorf("reconcile static role membership %s -> %s: %w", contract.role, contract.member, err)+ authoritativeGrant, err := authoritativeStaticMembershipGrant(contract, migrationOwner, grantsByContract[index])+ if err != nil && !errors.Is(err, errStaticMembershipGrantMissing) && !errors.Is(err, errStaticMembershipGrantOwnerOptions) {+ return nil, err+ }++ ddl := newRuntimeDDL(transaction)+ if authoritativeGrant != nil {+ if err := ddl.revokeStaticMembership(ctx, contract, migrationOwner); err != nil {+ return nil, fmt.Errorf("remove drifted static role membership %s -> %s: %w", contract.role, contract.member, err)}}+ if err := ddl.grantStaticMembership(ctx, contract, migrationOwner); err != nil {+ return nil, fmt.Errorf("reconcile static role membership %s -> %s: %w", contract.role, contract.member, err)+ }membershipEvents = append(membershipEvents, membershipReconciliationEvent{Role: contract.role, DurationMS: time.Since(started).Milliseconds()})}+ return membershipEvents, nil+}- if err := reconcileStaticRoleSettings(ctx, transaction); err != nil {- return nil, nil, err+func staticMembershipReconciliationPlan(migrationOwner string, membershipContracts []staticMembershipContract, grantsByContract [][]staticMembershipGrant) ([]staticMembershipContract, error) {+ if len(membershipContracts) != len(grantsByContract) {+ return nil, fmt.Errorf("static membership reconciliation plan has %d contracts and %d grant sets", len(membershipContracts), len(grantsByContract))}- return roleEvents, membershipEvents, nil++ repairs := make([]staticMembershipContract, 0, len(membershipContracts))+ for index, contract := range membershipContracts {+ _, err := authoritativeStaticMembershipGrant(contract, migrationOwner, grantsByContract[index])+ if err == nil {+ continue+ }+ if !errors.Is(err, errStaticMembershipGrantMissing) && !errors.Is(err, errStaticMembershipGrantOwnerOptions) {+ return nil, err+ }+ repairs = append(repairs, contract)+ }+ return repairs, nil+}++func isStaticMembershipRepair(contract staticMembershipContract, repairs []staticMembershipContract) bool {+ for _, repair := range repairs {+ if repair == contract {+ return true+ }+ }+ return false}func foreignStaticMembershipGrantError(contract staticMembershipContract, grantor, migrationOwner string) error {@@ -840,10 +878,9 @@ func validateStaticRoleMemberships(ctx context.Context, transaction pgx.Tx, migrif err != nil {return err}- for _, grant := range grants {- if grant.grantor != migrationOwner && contract.member != migrationOwner && !allowsImplicitCreatorAdminMembership(contract.role, contract.member, migrationOwner, grant) {- return foreignStaticMembershipGrantError(contract, grant.grantor, migrationOwner)- }+ _, err = authoritativeStaticMembershipGrant(contract, migrationOwner, grants)+ if err != nil && !errors.Is(err, errStaticMembershipGrantMissing) && !errors.Is(err, errStaticMembershipGrantOwnerOptions) {+ return err}}@@ -873,8 +910,16 @@ func validateStaticRoleMembers(ctx context.Context, transaction pgx.Tx, migratioif err != nil {return err}- allowMigrationOwnerGrant := isDeclaredStaticMembership(role, migrationOwner, contracts)- if !staticCreatorAdminMembershipsAreExact(migrationOwner, creatorGrants, allowMigrationOwnerGrant) {+ declaredContract, allowMigrationOwnerGrant := declaredStaticMembershipContract(role, migrationOwner, contracts)+ var declaredGrant *staticMembershipGrant+ if allowMigrationOwnerGrant {+ var err error+ declaredGrant, err = authoritativeStaticMembershipGrant(declaredContract, migrationOwner, creatorGrants)+ if err != nil {+ return err+ }+ }+ if !staticCreatorAdminMembershipsAreExact(migrationOwner, creatorGrants, allowMigrationOwnerGrant, declaredGrant) {return staticCreatorAdminMembershipError(role, migrationOwner)}@@ -965,7 +1010,7 @@ func validateQueryTenantRoleMembers(ctx context.Context, transaction pgx.Tx, migif err != nil {return nil, err}- if !staticCreatorAdminMembershipsAreExact(migrationOwner, creatorGrants, false) {+ if !staticCreatorAdminMembershipsAreExact(migrationOwner, creatorGrants, false, nil) {return nil, staticCreatorAdminMembershipError("k8s_query_tenant", migrationOwner)}for _, tenant := range dynamicTenants {@@ -1038,10 +1083,14 @@ func validateRegisteredTenantRoleMemberships(ctx context.Context, transaction pgreturn nil}-func staticCreatorAdminMembershipsAreExact(member string, grants []staticMembershipGrant, allowOwnerGrant bool) bool {+func staticCreatorAdminMembershipsAreExact(member string, grants []staticMembershipGrant, allowOwnerGrant bool, declaredGrant *staticMembershipGrant) bool {foreignCount := 0ownerCount := 0for _, grant := range grants {+ if declaredGrant != nil && grant == *declaredGrant {+ declaredGrant = nil+ continue+ }if grant.grantor == member {ownerCount++continue@@ -1094,13 +1143,18 @@ func allowsRegisteredTenantCreatorAdminMembership(tenant dynamicTenantRole, membgrant.admin && !grant.inherit && !grant.set}-func isDeclaredStaticMembership(role, member string, contracts []staticMembershipContract) bool {+func declaredStaticMembershipContract(role, member string, contracts []staticMembershipContract) (staticMembershipContract, bool) {for _, contract := range contracts {if contract.role == role && contract.member == member {- return true+ return contract, true}}- return false+ return staticMembershipContract{}, false+}++func isDeclaredStaticMembership(role, member string, contracts []staticMembershipContract) bool {+ _, ok := declaredStaticMembershipContract(role, member, contracts)+ return ok}func isLowerHex(value string) bool {@@ -1255,6 +1309,40 @@ func allowsImplicitCreatorAdminMembership(role, member, migrationOwner string, g!grant.set}+var (+ errStaticMembershipGrantMissing = errors.New("static membership grant missing")+ errStaticMembershipGrantOwnerOptions = errors.New("static membership grant owner options drifted")+)++func authoritativeStaticMembershipGrant(contract staticMembershipContract, migrationOwner string, grants []staticMembershipGrant) (*staticMembershipGrant, error) {+ nonImplicitGrants := make([]staticMembershipGrant, 0, len(grants))+ for _, grant := range grants {+ if !allowsImplicitCreatorAdminMembership(contract.role, contract.member, migrationOwner, grant) {+ nonImplicitGrants = append(nonImplicitGrants, grant)+ }+ }+ if len(nonImplicitGrants) == 0 {+ return nil, fmt.Errorf("static role membership %s -> %s: %w", contract.role, contract.member, errStaticMembershipGrantMissing)+ }+ for _, grant := range nonImplicitGrants {+ if grant.grantor != migrationOwner && !grant.grantorSuperuser {+ return nil, foreignStaticMembershipGrantError(contract, grant.grantor, migrationOwner)+ }+ }+ if len(nonImplicitGrants) != 1 {+ return nil, fmt.Errorf("static role membership %s -> %s has %d non-implicit grants; revoke duplicate grants without CASCADE before rerunning the migration", contract.role, contract.member, len(nonImplicitGrants))+ }++ grant := nonImplicitGrants[0]+ if grant.admin != contract.admin || grant.inherit != contract.inherit || grant.set != contract.set {+ if grant.grantor == migrationOwner {+ return &grant, fmt.Errorf("static role membership %s -> %s: %w", contract.role, contract.member, errStaticMembershipGrantOwnerOptions)+ }+ return nil, fmt.Errorf("static role membership %s -> %s has options that do not match its contract", contract.role, contract.member)+ }+ return &grant, nil+}+func readStaticMembershipGrants(ctx context.Context, transaction pgx.Tx, contract staticMembershipContract) ([]staticMembershipGrant, error) {rows, err := transaction.Query(ctx, `select grantor_role.rolname, grantor_role.rolsuper, membership.admin_option, membership.inherit_option, membership.set_optiondiff --git a/libs/fleet/backend/database/migrator_pg_test.go b/libs/fleet/backend/database/migrator_pg_test.goindex d0e5d4289f..9be1ac17d7 100644--- a/libs/fleet/backend/database/migrator_pg_test.go+++ b/libs/fleet/backend/database/migrator_pg_test.go@@ -570,6 +570,38 @@ func TestRunFailsClosedForStaticRoleCreateDBDrift(t *testing.T) {}}+func TestRunPreservesMaintenanceSuperuserStaticMembershipGrant(t *testing.T) {+ maintenanceURL := requireMigratorIntegration(t)+ ctx := context.Background()+ migrationURL, credentials := isolatedMigrationDatabase(t, ctx, maintenanceURL)+ if err := Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials}); err != nil {+ t.Fatal(err)+ }++ migrationOwner := currentRole(t, ctx, migrationURL)+ maintenance := connect(t, ctx, maintenanceURLForDatabase(t, maintenanceURL, migrationURL))+ defer maintenance.Close(ctx)+ migrationOwnerIdentifier := pgx.Identifier{migrationOwner}.Sanitize()+ if _, err := maintenance.Exec(ctx, "revoke k8s_query_tenant from k8s_role_admin granted by "+migrationOwnerIdentifier+" restrict"); err != nil {+ t.Fatal("remove migration-owner static membership")+ }+ if _, err := maintenance.Exec(ctx, `grant k8s_query_tenant to k8s_role_admin with admin true, inherit false, set false`); err != nil {+ t.Fatal("create maintenance-superuser static membership")+ }++ before := staticMembershipRows(t, ctx, maintenance, "k8s_query_tenant", "k8s_role_admin")+ if len(before) != 1 || before[0].grantor == migrationOwner || !before[0].grantorSuperuser || !before[0].admin || before[0].inherit || before[0].set {+ t.Fatalf("maintenance-superuser static membership = %+v, want one exact grant from a superuser other than %s", before, migrationOwner)+ }++ if err := Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials}); err != nil {+ t.Fatal(err)+ }+ if after := staticMembershipRows(t, ctx, maintenance, "k8s_query_tenant", "k8s_role_admin"); !reflect.DeepEqual(after, before) {+ t.Fatalf("maintenance-superuser static membership changed during rerun: before=%+v after=%+v", before, after)+ }+}+func TestRunFailsClosedForForeignGrantorMembership(t *testing.T) {maintenanceURL := os.Getenv("CYCLOPS_TEST_DATABASE_URL")if maintenanceURL == "" {@@ -632,6 +664,134 @@ func TestRunFailsClosedForForeignGrantorMembership(t *testing.T) {}}+func TestRunPreflightLeavesEarlierRepairableStaticMembershipUnchanged(t *testing.T) {+ maintenanceURL := requireMigratorIntegration(t)++ ctx := context.Background()++ for _, testCase := range []struct {+ name string+ introduceFirst func(t *testing.T, maintenance *pgx.Conn, migrationOwner string)+ introduceLater func(t *testing.T, maintenance *pgx.Conn, migrationURL, migrationOwner string)+ wantError string+ }{+ {+ name: "missing before foreign membership",+ introduceFirst: func(t *testing.T, maintenance *pgx.Conn, migrationOwner string) {+ t.Helper()+ identifier := pgx.Identifier{migrationOwner}.Sanitize()+ if _, err := maintenance.Exec(ctx, "revoke k8s_state_owner from "+identifier+" granted by "+identifier+" restrict"); err != nil {+ t.Fatal("remove repairable state-owner membership")+ }+ },+ introduceLater: func(t *testing.T, maintenance *pgx.Conn, migrationURL, migrationOwner string) {+ t.Helper()+ foreignGrantor := "foreign_grantor_" + testToken(t)[:12]+ foreignIdentifier := pgx.Identifier{foreignGrantor}.Sanitize()+ t.Cleanup(func() {+ cleanupCtx := context.Background()+ connection := connect(t, cleanupCtx, maintenanceURLForDatabase(t, maintenanceURL, migrationURL))+ defer connection.Close(cleanupCtx)+ if _, err := connection.Exec(cleanupCtx, "drop owned by "+foreignIdentifier); err != nil {+ t.Errorf("drop foreign grantor-owned objects: %v", err)+ }+ dropRole(t, cleanupCtx, maintenanceURL, foreignGrantor)+ })+ if _, err := maintenance.Exec(ctx, "create role "+foreignIdentifier); err != nil {+ t.Fatal("create foreign grantor")+ }+ if _, err := maintenance.Exec(ctx, "grant k8s_query_tenant to "+foreignIdentifier+" with admin true, inherit false, set false"); err != nil {+ t.Fatal("grant query tenant admin to foreign grantor")+ }+ identifier := pgx.Identifier{migrationOwner}.Sanitize()+ if _, err := maintenance.Exec(ctx, "revoke k8s_query_tenant from k8s_role_admin granted by "+identifier+" restrict"); err != nil {+ t.Fatal("remove migration-owner query tenant membership")+ }+ if _, err := maintenance.Exec(ctx, "set role "+foreignIdentifier); err != nil {+ t.Fatal("set foreign grantor role")+ }+ if _, err := maintenance.Exec(ctx, `grant k8s_query_tenant to k8s_role_admin with admin true, inherit false, set false`); err != nil {+ t.Fatal("create foreign static membership")+ }+ if _, err := maintenance.Exec(ctx, `reset role`); err != nil {+ t.Fatal("reset foreign grantor role")+ }+ },+ wantError: "has grantor foreign_grantor_",+ },+ {+ name: "SET drift before duplicate membership",+ introduceFirst: func(t *testing.T, maintenance *pgx.Conn, migrationOwner string) {+ t.Helper()+ identifier := pgx.Identifier{migrationOwner}.Sanitize()+ if _, err := maintenance.Exec(ctx, "revoke k8s_state_owner from "+identifier+" granted by "+identifier+" restrict"); err != nil {+ t.Fatal("remove state-owner membership before SET drift")+ }+ if _, err := maintenance.Exec(ctx, "grant k8s_state_owner to "+identifier+" with admin false, inherit false, set false granted by "+identifier); err != nil {+ t.Fatal("introduce repairable state-owner SET drift")+ }+ },+ introduceLater: func(t *testing.T, maintenance *pgx.Conn, migrationURL, migrationOwner string) {+ t.Helper()+ if _, err := maintenance.Exec(ctx, `grant k8s_query_tenant to k8s_role_admin with admin true, inherit false, set false`); err != nil {+ t.Fatal("create postgres-grantor duplicate static membership")+ }+ },+ wantError: "has 2 non-implicit grants",+ },+ } {+ t.Run(testCase.name, func(t *testing.T) {+ migrationURL, credentials := isolatedMigrationDatabase(t, ctx, maintenanceURL)+ if err := Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials}); err != nil {+ t.Fatal("initial Run()", err)+ }+ migrationOwner := currentRole(t, ctx, migrationURL)+ maintenance := connect(t, ctx, maintenanceURL)+ defer maintenance.Close(ctx)+ testCase.introduceFirst(t, maintenance, migrationOwner)+ testCase.introduceLater(t, maintenance, migrationURL, migrationOwner)++ connection := connect(t, ctx, migrationURL)+ defer connection.Close(ctx)+ before := staticMembershipRows(t, ctx, connection, "k8s_state_owner", migrationOwner)+ err := Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials})+ if err == nil || !strings.Contains(err.Error(), testCase.wantError) {+ t.Fatalf("Run() error = %v, want fail-closed error containing %q", err, testCase.wantError)+ }+ if after := staticMembershipRows(t, ctx, connection, "k8s_state_owner", migrationOwner); !reflect.DeepEqual(after, before) {+ t.Fatalf("repairable state-owner membership changed after fail-closed preflight: before=%+v after=%+v", before, after)+ }+ })+ }+}++func staticMembershipRows(t *testing.T, ctx context.Context, connection *pgx.Conn, role, member string) []staticMembershipGrant {+ t.Helper()+ rows, err := connection.Query(ctx, `+ select grantor_role.rolname, grantor_role.rolsuper, membership.admin_option, membership.inherit_option, membership.set_option+ from pg_auth_members as membership+ join pg_roles as grantor_role on grantor_role.oid = membership.grantor+ where membership.roleid = $1::regrole and membership.member = $2::regrole+ order by grantor_role.rolname`, role, member)+ if err != nil {+ t.Fatalf("read static membership %s -> %s: %v", role, member, err)+ }+ defer rows.Close()++ var grants []staticMembershipGrant+ for rows.Next() {+ var grant staticMembershipGrant+ if err := rows.Scan(&grant.grantor, &grant.grantorSuperuser, &grant.admin, &grant.inherit, &grant.set); err != nil {+ t.Fatalf("scan static membership %s -> %s: %v", role, member, err)+ }+ grants = append(grants, grant)+ }+ if err := rows.Err(); err != nil {+ t.Fatalf("iterate static membership %s -> %s: %v", role, member, err)+ }+ return grants+}+func TestRunFailsClosedForNonSuperuserStaticCreatorAdminGrantor(t *testing.T) {maintenanceURL := requireMigratorIntegration(t)ctx := context.Background()@@ -667,7 +827,7 @@ func TestRunFailsClosedForNonSuperuserStaticCreatorAdminGrantor(t *testing.T) {t.Fatal("introduce static role repair sentinel")}- assertRunFailsBeforeStaticRoleRepair(t, ctx, migrationURL, credentials, "true PostgreSQL superuser")+ assertRunFailsBeforeStaticRoleRepair(t, ctx, migrationURL, credentials, "has grantor")}func TestRunFailsClosedForRegisteredTenantMembershipDrift(t *testing.T) {@@ -1248,7 +1408,7 @@ func assertStaticCreatorAdminMemberships(t *testing.T, ctx context.Context, migr}rows.Close()allowOwnerGrant := role == "k8s_state_owner" || role == "k8s_reporting_owner"- if !staticCreatorAdminMembershipsAreExact(migrationOwner, grants, allowOwnerGrant) {+ if !staticCreatorAdminMembershipsAreExact(migrationOwner, grants, allowOwnerGrant, nil) {t.Fatalf("static creator memberships for %s = %+v", role, grants)}}diff --git a/libs/fleet/backend/database/migrator_test.go b/libs/fleet/backend/database/migrator_test.goindex e6d3b9543d..08813676b7 100644--- a/libs/fleet/backend/database/migrator_test.go+++ b/libs/fleet/backend/database/migrator_test.go@@ -4,6 +4,7 @@ import ("bytes""crypto/sha256""encoding/hex"+ "errors""log/slog""os""reflect"@@ -175,12 +176,19 @@ func TestAllowsRegisteredTenantCreatorAdminMembership(t *testing.T) {func TestStaticCreatorAdminMembershipsRequireOneForeignSuperuserGrant(t *testing.T) {foreign := staticMembershipGrant{grantor: "bootstrap_owner", grantorSuperuser: true, admin: true, inherit: false, set: false}ownerDrift := staticMembershipGrant{grantor: "migration_owner", admin: true, inherit: true, set: true}- if !staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{foreign}, false) {+ if !staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{foreign}, false, nil) {t.Fatal("expected one foreign superuser creator row to be allowed")}- if !staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{foreign, ownerDrift}, true) {+ if !staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{foreign, ownerDrift}, true, nil) {t.Fatal("expected one repairable migration-owner row to be allowed")}+ delegatedContractGrant := staticMembershipGrant{grantor: "rdsadmin", grantorSuperuser: true, admin: false, inherit: false, set: true}+ if !staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{foreign, delegatedContractGrant}, true, &delegatedContractGrant) {+ t.Fatal("expected an authoritative delegated contract grant to be ignored")+ }+ if staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{foreign, delegatedContractGrant, {grantor: "other_bootstrap_owner", grantorSuperuser: true, admin: true, inherit: false, set: false}}, true, &delegatedContractGrant) {+ t.Fatal("unexpectedly allowed an unrelated foreign creator grant")+ }for _, testCase := range []struct {name string@@ -195,13 +203,85 @@ func TestStaticCreatorAdminMembershipsRequireOneForeignSuperuserGrant(t *testing{name: "duplicate owner row", grants: []staticMembershipGrant{foreign, ownerDrift, {grantor: "migration_owner", admin: false, inherit: false, set: true}}, allowOwnerGrant: true},} {t.Run(testCase.name, func(t *testing.T) {- if staticCreatorAdminMembershipsAreExact("migration_owner", testCase.grants, testCase.allowOwnerGrant) {+ if staticCreatorAdminMembershipsAreExact("migration_owner", testCase.grants, testCase.allowOwnerGrant, nil) {t.Fatal("unexpectedly allowed static creator-admin memberships")}})}}+func TestStaticMembershipReconciliationPlanRepairsOnlyAfterFailClosedPreflight(t *testing.T) {+ stateOwner := staticMembershipContract{role: "k8s_state_owner", member: "migration_owner", admin: false, inherit: false, set: true}+ reportingOwner := staticMembershipContract{role: "k8s_reporting_owner", member: "migration_owner", admin: false, inherit: false, set: true}+ healthyGrant := staticMembershipGrant{grantor: "migration_owner", admin: false, inherit: false, set: true}+ driftedStateOwnerGrant := staticMembershipGrant{grantor: "migration_owner", admin: false, inherit: false, set: false}++ for _, testCase := range []struct {+ name string+ grants [][]staticMembershipGrant+ wantRepairs []staticMembershipContract+ wantError string+ }{+ {+ name: "missing state-owner membership",+ grants: [][]staticMembershipGrant{nil, {healthyGrant}},+ wantRepairs: []staticMembershipContract{stateOwner},+ },+ {+ name: "set-drifted state-owner membership",+ grants: [][]staticMembershipGrant{{driftedStateOwnerGrant}, {healthyGrant}},+ wantRepairs: []staticMembershipContract{stateOwner},+ },+ {+ name: "foreign grant blocks earlier state-owner repair",+ grants: [][]staticMembershipGrant{+ nil,+ {{grantor: "foreign_owner", admin: false, inherit: false, set: true}},+ },+ wantError: "grantor foreign_owner",+ },+ {+ name: "duplicate grant blocks earlier state-owner repair",+ grants: [][]staticMembershipGrant{+ nil,+ {healthyGrant, {grantor: "rdsadmin", grantorSuperuser: true, admin: false, inherit: false, set: true}},+ },+ wantError: "2 non-implicit grants",+ },+ } {+ t.Run(testCase.name, func(t *testing.T) {+ repairs, err := staticMembershipReconciliationPlan("migration_owner", []staticMembershipContract{stateOwner, reportingOwner}, testCase.grants)+ if testCase.wantError != "" {+ if err == nil || !strings.Contains(err.Error(), testCase.wantError) {+ t.Fatalf("staticMembershipReconciliationPlan() error = %v, want containing %q", err, testCase.wantError)+ }+ if repairs != nil {+ t.Fatalf("staticMembershipReconciliationPlan() repairs = %#v, want nil before any mutation", repairs)+ }+ return+ }+ if err != nil {+ t.Fatal(err)+ }+ if !reflect.DeepEqual(repairs, testCase.wantRepairs) {+ t.Fatalf("staticMembershipReconciliationPlan() repairs = %#v, want %#v", repairs, testCase.wantRepairs)+ }+ })+ }+}++func TestStaticCreatorAdminMembershipsAllowDelegatedContractGrantAlongsideImplicitCreatorGrant(t *testing.T) {+ implicitCreatorGrant := staticMembershipGrant{grantor: "bootstrap_owner", grantorSuperuser: true, admin: true, inherit: false, set: false}+ delegatedContractGrant := staticMembershipGrant{grantor: "rdsadmin", grantorSuperuser: true, admin: false, inherit: false, set: true}++ if !staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{implicitCreatorGrant, delegatedContractGrant}, true, &delegatedContractGrant) {+ t.Fatal("expected exact delegated contract and implicit creator grants to coexist")+ }+ foreignCreatorGrant := staticMembershipGrant{grantor: "foreign_creator", grantorSuperuser: true, admin: true, inherit: false, set: false}+ if staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{implicitCreatorGrant, delegatedContractGrant, foreignCreatorGrant}, true, &delegatedContractGrant) {+ t.Fatal("unexpectedly allowed unrelated foreign creator grant")+ }+}func TestCreatorAdminMembershipsRequireOneForeignSuperuserGrant(t *testing.T) {valid := staticMembershipGrant{grantor: "bootstrap_owner", grantorSuperuser: true, admin: true, inherit: false, set: false}if !creatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{valid}) {@@ -289,6 +369,130 @@ func TestAllowsImplicitCreatorAdminMembership(t *testing.T) {}}+func TestAuthoritativeStaticMembershipGrant(t *testing.T) {+ declaredContract := staticMembershipContract{+ role: "k8s_query_tenant", member: "k8s_role_admin",+ admin: true, inherit: false, set: false,+ }+ creatorContract := staticMembershipContract{+ role: "k8s_state_owner", member: "migration_owner",+ admin: false, inherit: false, set: true,+ }+ implicitCreatorGrant := staticMembershipGrant{+ grantor: "bootstrap_owner", grantorSuperuser: true,+ admin: true, inherit: false, set: false,+ }++ for _, testCase := range []struct {+ name string+ contract staticMembershipContract+ grants []staticMembershipGrant+ wantGrantor string+ wantError bool+ wantErrorIs error+ wantReturnedGrant bool+ }{+ {+ name: "migration owner grant",+ contract: declaredContract,+ grants: []staticMembershipGrant{{grantor: "migration_owner", admin: true, inherit: false, set: false}},+ wantGrantor: "migration_owner",+ },+ {+ name: "delegated superuser exact grant",+ contract: declaredContract,+ grants: []staticMembershipGrant{{grantor: "rdsadmin", grantorSuperuser: true, admin: true, inherit: false, set: false}},+ wantGrantor: "rdsadmin",+ },+ {+ name: "implicit creator grant is ignored beside owner grant",+ contract: creatorContract,+ grants: []staticMembershipGrant{implicitCreatorGrant, {grantor: "migration_owner", admin: false, inherit: false, set: true}},+ wantGrantor: "migration_owner",+ },+ {+ name: "missing grant",+ contract: declaredContract,+ wantError: true,+ wantErrorIs: errStaticMembershipGrantMissing,+ },+ {+ name: "implicit creator grant does not satisfy declared contract",+ contract: creatorContract,+ grants: []staticMembershipGrant{implicitCreatorGrant},+ wantError: true,+ wantErrorIs: errStaticMembershipGrantMissing,+ },+ {+ name: "non-superuser grantor",+ contract: declaredContract,+ grants: []staticMembershipGrant{{grantor: "foreign_owner", admin: true, inherit: false, set: false}},+ wantError: true,+ },+ {+ name: "owner admin option drift",+ contract: declaredContract,+ grants: []staticMembershipGrant{{grantor: "migration_owner", admin: false, inherit: false, set: false}},+ wantError: true,+ wantErrorIs: errStaticMembershipGrantOwnerOptions,+ wantReturnedGrant: true,+ },+ {+ name: "owner inherit option drift",+ contract: declaredContract,+ grants: []staticMembershipGrant{{grantor: "migration_owner", admin: true, inherit: true, set: false}},+ wantError: true,+ wantErrorIs: errStaticMembershipGrantOwnerOptions,+ wantReturnedGrant: true,+ },+ {+ name: "owner set option drift",+ contract: declaredContract,+ grants: []staticMembershipGrant{{grantor: "migration_owner", admin: true, inherit: false, set: true}},+ wantError: true,+ wantErrorIs: errStaticMembershipGrantOwnerOptions,+ wantReturnedGrant: true,+ },+ {+ name: "delegated superuser option drift",+ contract: declaredContract,+ grants: []staticMembershipGrant{{grantor: "rdsadmin", grantorSuperuser: true, admin: true, inherit: true, set: false}},+ wantError: true,+ },+ {+ name: "owner and superuser duplicates",+ contract: declaredContract,+ grants: []staticMembershipGrant{+ {grantor: "migration_owner", admin: true, inherit: false, set: false},+ {grantor: "rdsadmin", grantorSuperuser: true, admin: true, inherit: false, set: false},+ },+ wantError: true,+ },+ } {+ t.Run(testCase.name, func(t *testing.T) {+ grant, err := authoritativeStaticMembershipGrant(testCase.contract, "migration_owner", testCase.grants)+ if testCase.wantError {+ if err == nil {+ t.Fatal("expected an error")+ }+ if testCase.wantErrorIs != nil && !errors.Is(err, testCase.wantErrorIs) {+ t.Fatalf("error = %v, want errors.Is(_, %v)", err, testCase.wantErrorIs)+ }+ if (grant != nil) != testCase.wantReturnedGrant {+ t.Fatalf("returned grant = %#v, want present %t", grant, testCase.wantReturnedGrant)+ }+ return+ }+ if err != nil {+ t.Fatal(err)+ }+ if grant.grantor != testCase.wantGrantor {+ t.Fatalf("grantor = %q, want %q", grant.grantor, testCase.wantGrantor)+ }+ })+ }+}+func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {files, err := embeddedMigrations()if err != nil {diff --git a/libs/fleet/backend/database/runtime_ddl.go b/libs/fleet/backend/database/runtime_ddl.goindex 6b830f1467..72016e6bcf 100644--- a/libs/fleet/backend/database/runtime_ddl.go+++ b/libs/fleet/backend/database/runtime_ddl.go@@ -299,17 +299,22 @@ func (ddl runtimeDDL) revokeStaticMembership(ctx context.Context, contract stati}func (ddl runtimeDDL) grantStaticMembership(ctx context.Context, contract staticMembershipContract, grantor string) error {- _, err := ddl.transaction.Exec(ctx,- "grant "+pgx.Identifier{contract.role}.Sanitize()+- " to "+pgx.Identifier{contract.member}.Sanitize()+- " with admin "+postgresBoolean(contract.admin)+- ", inherit "+postgresBoolean(contract.inherit)+- ", set "+postgresBoolean(contract.set)+- " granted by "+pgx.Identifier{grantor}.Sanitize(),- )+ _, err := ddl.transaction.Exec(ctx, staticMembershipGrantStatement(contract, grantor))return err}+func staticMembershipGrantStatement(contract staticMembershipContract, grantor string) string {+ statement := "grant " + pgx.Identifier{contract.role}.Sanitize() ++ " to " + pgx.Identifier{contract.member}.Sanitize() ++ " with admin " + postgresBoolean(contract.admin) ++ ", inherit " + postgresBoolean(contract.inherit) ++ ", set " + postgresBoolean(contract.set)+ if !contract.admin {+ statement += " granted by " + pgx.Identifier{grantor}.Sanitize()+ }+ return statement+}+func (ddl runtimeDDL) reconcileStaticRoleSettings(ctx context.Context, role staticRoleContract, actual []staticRoleSetting, desired staticRoleSettingsContract) error {if staticRoleSettingsMatch(actual, desired) {return nildiff --git a/libs/fleet/backend/database/runtime_ddl_test.go b/libs/fleet/backend/database/runtime_ddl_test.goindex b9b0052b92..f6691e00a1 100644--- a/libs/fleet/backend/database/runtime_ddl_test.go+++ b/libs/fleet/backend/database/runtime_ddl_test.go@@ -16,6 +16,39 @@ import (const expectedInsertAppliedMigrationStatement = `insert into cyclops_migrations.applied_migrations (version, filename, sha256) values ($1, $2, $3)`+func TestRuntimeDDLStaticMembershipGrantorSelection(t *testing.T) {+ tests := []struct {+ name string+ contract staticMembershipContract+ want string+ }{+ {+ name: "ADMIN TRUE lets PostgreSQL select the grantor",+ contract: staticMembershipContract{+ role: "k8s_query_tenant", member: "k8s_role_admin",+ admin: true, inherit: false, set: false,+ },+ want: `grant "k8s_query_tenant" to "k8s_role_admin" with admin TRUE, inherit FALSE, set FALSE`,+ },+ {+ name: "ADMIN FALSE retains the migration owner grantor",+ contract: staticMembershipContract{+ role: "k8s_query_tenant", member: "tenant_query_role",+ admin: false, inherit: true, set: false,+ },+ want: `grant "k8s_query_tenant" to "tenant_query_role" with admin FALSE, inherit TRUE, set FALSE granted by "migration_owner"`,+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ if got := staticMembershipGrantStatement(test.contract, "migration_owner"); got != test.want {+ t.Fatalf("staticMembershipGrantStatement() = %q, want %q", got, test.want)+ }+ })+ }+}+func TestMigrationAuthorityExecArguments(t *testing.T) {packageDir := databasePackageDir(t)entries, err := os.ReadDir(packageDir)diff --git a/libs/fleet/backend/go.mod b/libs/fleet/backend/go.modindex 34701ae3a4..bb12006001 100644--- a/libs/fleet/backend/go.mod+++ b/libs/fleet/backend/go.mod@@ -19,6 +19,7 @@ require (github.com/stripe/stripe-go/v85 v85.2.0github.com/swaggo/http-swagger/v2 v2.0.2github.com/swaggo/swag v1.16.6+ sigs.k8s.io/yaml v1.4.0go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0golang.org/x/sync v0.17.0@@ -112,7 +113,6 @@ require (golang.org/x/tools v0.37.0 // indirectgoogle.golang.org/protobuf v1.34.2 // indirectgopkg.in/yaml.v2 v2.4.0 // indirect- sigs.k8s.io/yaml v1.4.0 // indirect)replace github.com/trycua/cloud/pkg/featureflags => ../../pkg/featureflagsdiff --git a/libs/fleet/backend/handlers/database_context.go b/libs/fleet/backend/handlers/database_context.gonew file mode 100644index 0000000000..ce96f9bbc9--- /dev/null+++ b/libs/fleet/backend/handlers/database_context.go@@ -0,0 +1,13 @@+package handlers++import (+ "context"++ "cyclops-cs-backend/auth"+)++const databaseRequestTimeout = auth.DatabaseRequestTimeout++func databaseContext(parent context.Context) (context.Context, context.CancelFunc) {+ return auth.DatabaseContext(parent)+}diff --git a/libs/fleet/backend/handlers/github_trust_policies.go b/libs/fleet/backend/handlers/github_trust_policies.goindex a38eaebe8f..d393cc9871 100644--- a/libs/fleet/backend/handlers/github_trust_policies.go+++ b/libs/fleet/backend/handlers/github_trust_policies.go@@ -47,9 +47,11 @@ func (h Handlers) ListGitHubTrustPolicies(w http.ResponseWriter, r *http.RequestwriteErr(w, http.StatusServiceUnavailable, "github trust policies are not configured")return}- policies, err := h.GitHubTrustPolicies.List(r.Context(), user.ID)+ ctx, cancel := databaseContext(r.Context())+ defer cancel()+ policies, err := h.GitHubTrustPolicies.List(ctx, user.ID)if err != nil {- writeErr(w, http.StatusInternalServerError, "failed to list github trust policies")+ writeGitHubTrustStoreErr(w, err, "failed to list github trust policies")return}resp := GitHubTrustPolicyListResponse{@@ -84,8 +86,10 @@ func (h Handlers) CreateGitHubTrustPolicy(w http.ResponseWriter, r *http.Requestreturn}policy.OwnerSub = user.ID- if err := h.GitHubTrustPolicies.Create(r.Context(), policy); err != nil {- writeErr(w, http.StatusInternalServerError, "failed to create github trust policy")+ ctx, cancel := databaseContext(r.Context())+ defer cancel()+ if err := h.GitHubTrustPolicies.Create(ctx, policy); err != nil {+ writeGitHubTrustStoreErr(w, err, "failed to create github trust policy")return}writeJSON(w, http.StatusCreated, policyResponse(policy))@@ -102,9 +106,11 @@ func (h Handlers) UpdateGitHubTrustPolicy(w http.ResponseWriter, r *http.Requestreturn}id := r.PathValue("id")- current, err := h.GitHubTrustPolicies.Get(r.Context(), user.ID, id)+ ctx, cancel := databaseContext(r.Context())+ defer cancel()+ current, err := h.GitHubTrustPolicies.Get(ctx, user.ID, id)if err != nil {- writeErr(w, http.StatusInternalServerError, "failed to load github trust policy")+ writeGitHubTrustStoreErr(w, err, "failed to load github trust policy")return}if current == nil {@@ -142,8 +148,12 @@ func (h Handlers) UpdateGitHubTrustPolicy(w http.ResponseWriter, r *http.Requestpolicy.ID = current.IDpolicy.OwnerSub = current.OwnerSubpolicy.CreatedAt = current.CreatedAt- if err := h.GitHubTrustPolicies.Update(r.Context(), policy); err != nil {- writeErr(w, http.StatusInternalServerError, "failed to update github trust policy")+ if err := h.GitHubTrustPolicies.Update(ctx, policy); err != nil {+ if errors.Is(err, githubtrust.ErrNotFound) {+ writeErr(w, http.StatusNotFound, "github trust policy not found")+ return+ }+ writeGitHubTrustStoreErr(w, err, "failed to update github trust policy")return}writeJSON(w, http.StatusOK, policyResponse(policy))@@ -159,9 +169,11 @@ func (h Handlers) DeleteGitHubTrustPolicy(w http.ResponseWriter, r *http.RequestwriteErr(w, http.StatusServiceUnavailable, "github trust policies are not configured")return}- found, err := h.GitHubTrustPolicies.Delete(r.Context(), user.ID, r.PathValue("id"))+ ctx, cancel := databaseContext(r.Context())+ defer cancel()+ found, err := h.GitHubTrustPolicies.Delete(ctx, user.ID, r.PathValue("id"))if err != nil {- writeErr(w, http.StatusInternalServerError, "failed to delete github trust policy")+ writeGitHubTrustStoreErr(w, err, "failed to delete github trust policy")return}if !found {@@ -183,8 +195,14 @@ type githubTrustResolver struct {}func (r githubTrustResolver) ResolveGitHubTrustPolicies(ctx context.Context, repository string) ([]auth.GitHubTrustPolicy, error) {- policies, err := r.store.ResolveByRepository(ctx, repository)+ databaseCtx, cancel := databaseContext(ctx)+ defer cancel()+ policies, err := r.store.ResolveByRepository(databaseCtx, repository)if err != nil {+ err = auth.ClassifyDatabaseError(err)+ if auth.IsDatabaseUnavailable(err) {+ return nil, auth.DatabaseUnavailable(err)+ }return nil, err}out := make([]auth.GitHubTrustPolicy, 0, len(policies))@@ -213,6 +231,15 @@ func policyResponse(policy *githubtrust.Policy) GitHubTrustPolicyResponse {}}+func writeGitHubTrustStoreErr(w http.ResponseWriter, err error, fallback string) {+ err = auth.ClassifyDatabaseError(err)+ if auth.IsDatabaseUnavailable(err) {+ writeErr(w, http.StatusServiceUnavailable, "github trust policies unavailable")+ return+ }+ writeErr(w, http.StatusInternalServerError, fallback)+}+func writeTrustPolicyValidationErr(w http.ResponseWriter, err error) {switch {case errors.Is(err, githubtrust.ErrInvalidRepository),diff --git a/libs/fleet/backend/handlers/github_trust_policies_test.go b/libs/fleet/backend/handlers/github_trust_policies_test.goindex e529d449c5..a363cad456 100644--- a/libs/fleet/backend/handlers/github_trust_policies_test.go+++ b/libs/fleet/backend/handlers/github_trust_policies_test.go@@ -5,17 +5,26 @@ import ("context""encoding/json""errors"+ "net""net/http""net/http/httptest"+ "syscall""testing""time""cyclops-cs-backend/auth""cyclops-cs-backend/config""cyclops-cs-backend/githubtrust"++ "github.com/jackc/pgx/v5/pgconn")type fakeGitHubTrustStore struct {+ listContext context.Context+ createContext context.Context+ getContext context.Context+ updateContext context.Context+ deleteContext context.ContextlistByOwner []*githubtrust.PolicycreateInput *githubtrust.PolicyupdateInput *githubtrust.Policy@@ -31,10 +40,11 @@ type fakeGitHubTrustStore struct {resolveResult []*githubtrust.PolicyresolveErr errorresolvedRepoName string+ resolveContext context.Context}func (f *fakeGitHubTrustStore) List(ctx context.Context, ownerSub string) ([]*githubtrust.Policy, error) {- _ = ctx+ f.listContext = ctxif f.listErr != nil {return nil, f.listErr}@@ -44,7 +54,7 @@ func (f *fakeGitHubTrustStore) List(ctx context.Context, ownerSub string) ([]*gi}func (f *fakeGitHubTrustStore) Create(ctx context.Context, policy *githubtrust.Policy) error {- _ = ctx+ f.createContext = ctxif f.createErr != nil {return f.createErr}@@ -54,7 +64,7 @@ func (f *fakeGitHubTrustStore) Create(ctx context.Context, policy *githubtrust.P}func (f *fakeGitHubTrustStore) Get(ctx context.Context, ownerSub, id string) (*githubtrust.Policy, error) {- _ = ctx+ f.getContext = ctx_ = ownerSub_ = idif f.getErr != nil {@@ -68,7 +78,7 @@ func (f *fakeGitHubTrustStore) Get(ctx context.Context, ownerSub, id string) (*g}func (f *fakeGitHubTrustStore) Update(ctx context.Context, policy *githubtrust.Policy) error {- _ = ctx+ f.updateContext = ctxif f.updateErr != nil {return f.updateErr}@@ -78,7 +88,7 @@ func (f *fakeGitHubTrustStore) Update(ctx context.Context, policy *githubtrust.P}func (f *fakeGitHubTrustStore) Delete(ctx context.Context, ownerSub, id string) (bool, error) {- _ = ctx+ f.deleteContext = ctxf.deleteOwner = ownerSubf.deleteID = idif f.deleteErr != nil {@@ -88,7 +98,7 @@ func (f *fakeGitHubTrustStore) Delete(ctx context.Context, ownerSub, id string)}func (f *fakeGitHubTrustStore) ResolveByRepository(ctx context.Context, repository string) ([]*githubtrust.Policy, error) {- _ = ctx+ f.resolveContext = ctxf.resolvedRepoName = repositoryif f.resolveErr != nil {return nil, f.resolveErr@@ -229,3 +239,204 @@ func authConfigForHandlers() config.AuthConfiguration {GitHubOIDCLegacyAudiences: []string{"cyclops-cs"},}}++func TestGitHubTrustPoliciesUseBoundedDatabaseContext(t *testing.T) {+ policy := &githubtrust.Policy{+ ID: "policy-1",+ OwnerSub: "user-123",+ Name: "ci",+ Repository: "trycua/cloud",+ AllowedNamespaces: []string{"ns-a"},+ Enabled: true,+ CreatedAt: time.Date(2026, 6, 26, 10, 0, 0, 0, time.UTC),+ }+ tests := []struct {+ name string+ request *http.Request+ call func(Handlers, http.ResponseWriter, *http.Request)+ context func(*fakeGitHubTrustStore) context.Context+ }{+ {"list", httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil), Handlers.ListGitHubTrustPolicies, func(store *fakeGitHubTrustStore) context.Context { return store.listContext }},+ {"create", httptest.NewRequest(http.MethodPost, "/api/github-trust-policies", bytes.NewBufferString(`{"name":"ci","repository":"trycua/cloud","allowed_namespaces":["ns-a"],"enabled":true}`)), Handlers.CreateGitHubTrustPolicy, func(store *fakeGitHubTrustStore) context.Context { return store.createContext }},+ {"update", httptest.NewRequest(http.MethodPatch, "/api/github-trust-policies/policy-1", bytes.NewBufferString(`{"enabled":false}`)), Handlers.UpdateGitHubTrustPolicy, func(store *fakeGitHubTrustStore) context.Context { return store.updateContext }},+ {"delete", httptest.NewRequest(http.MethodDelete, "/api/github-trust-policies/policy-1", nil), Handlers.DeleteGitHubTrustPolicy, func(store *fakeGitHubTrustStore) context.Context { return store.deleteContext }},+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ store := &fakeGitHubTrustStore{getResult: policy, deleteFound: true}+ request := withUser(test.request, &auth.User{ID: "user-123", AZP: "cyclops-cs-spa"})+ if test.name == "update" || test.name == "delete" {+ request.SetPathValue("id", "policy-1")+ }+ response := httptest.NewRecorder()+ start := time.Now()++ test.call(Handlers{GitHubTrustPolicies: store}, response, request)++ assertBoundedDatabaseContext(t, test.context(store), start)+ if test.name == "update" {+ assertBoundedDatabaseContext(t, store.getContext, start)+ }+ })+ }+}++func assertBoundedDatabaseContext(t *testing.T, ctx context.Context, start time.Time) {+ t.Helper()+ if ctx == nil {+ t.Fatal("store context was not captured")+ }+ deadline, ok := ctx.Deadline()+ if !ok {+ t.Fatal("store context has no deadline")+ }+ if deadline.Before(start) {+ t.Fatalf("deadline = %s, before test start %s", deadline, start)+ }+ if deadline.After(start.Add(databaseRequestTimeout + time.Second)) {+ t.Fatalf("deadline = %s, want no later than %s", deadline, start.Add(databaseRequestTimeout+time.Second))+ }+ if !errors.Is(ctx.Err(), context.Canceled) {+ t.Fatalf("context error after handler returned = %v, want context canceled", ctx.Err())+ }+}++func TestGitHubTrustResolverUsesBoundedDatabaseContext(t *testing.T) {+ store := &fakeGitHubTrustStore{}+ resolver := NewGitHubTrustResolver(store)+ start := time.Now()++ if _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud"); err != nil {+ t.Fatalf("resolve github trust policies: %v", err)+ }++ assertBoundedDatabaseContext(t, store.resolveContext, start)+}++func TestGitHubTrustResolverHonorsExistingDatabaseDeadline(t *testing.T) {+ store := &fakeGitHubTrustStore{}+ resolver := NewGitHubTrustResolver(store)+ parent, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second))+ defer cancel()+ wantDeadline, ok := parent.Deadline()+ if !ok {+ t.Fatal("parent context has no deadline")+ }++ if _, err := resolver.ResolveGitHubTrustPolicies(parent, "trycua/cloud"); err != nil {+ t.Fatalf("resolve github trust policies: %v", err)+ }++ gotDeadline, ok := store.resolveContext.Deadline()+ if !ok {+ t.Fatal("resolver context has no deadline")+ }+ if !gotDeadline.Equal(wantDeadline) {+ t.Fatalf("resolver deadline = %s, want parent deadline %s", gotDeadline, wantDeadline)+ }+}++func TestGitHubTrustResolverNormalizesDatabaseFailures(t *testing.T) {+ store := &fakeGitHubTrustStore{resolveErr: context.DeadlineExceeded}+ resolver := NewGitHubTrustResolver(store)++ _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud")+ if !errors.Is(err, auth.ErrDatabaseUnavailable) {+ t.Fatalf("resolve error = %v, want database unavailable sentinel", err)+ }+ if !errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("resolve error = %v, want original deadline error", err)+ }+}++func TestGitHubTrustResolverClassifiesImmediateNetworkFailures(t *testing.T) {+ connectionRefused := &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}+ store := &fakeGitHubTrustStore{resolveErr: connectionRefused}+ resolver := NewGitHubTrustResolver(store)++ _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud")+ if !errors.Is(err, auth.ErrDatabaseUnavailable) {+ t.Fatalf("resolve error = %v, want database unavailable sentinel", err)+ }+ if !errors.Is(err, connectionRefused) {+ t.Fatalf("resolve error = %v, want original connection error", err)+ }+}++func TestGitHubTrustResolverDoesNotClassifyPostgresErrorsAsUnavailable(t *testing.T) {+ postgresError := &pgconn.PgError{Code: "42501", Message: "permission denied"}+ store := &fakeGitHubTrustStore{resolveErr: postgresError}+ resolver := NewGitHubTrustResolver(store)++ _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud")+ if errors.Is(err, auth.ErrDatabaseUnavailable) {+ t.Fatalf("resolve error = %v, must not be database unavailable", err)+ }+}++func TestGitHubTrustPolicies_DatabaseFailuresReturnServiceUnavailable(t *testing.T) {+ tests := []struct {+ name string+ err error+ }{+ {name: "unavailable", err: auth.ErrDatabaseUnavailable},+ {name: "deadline", err: context.DeadlineExceeded},+ {name: "canceled", err: context.Canceled},+ {name: "connection refused", err: &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}},+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ store := &fakeGitHubTrustStore{listErr: test.err}+ h := Handlers{GitHubTrustPolicies: store}+ request := withUser(httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil), &auth.User{ID: "user-123"})+ response := httptest.NewRecorder()++ h.ListGitHubTrustPolicies(response, request)++ if response.Code != http.StatusServiceUnavailable {+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())+ }+ })+ }+}++func TestGitHubTrustPolicies_PostgresErrorsRemainInternalServerErrors(t *testing.T) {+ store := &fakeGitHubTrustStore{listErr: &pgconn.PgError{Code: "42501", Message: "permission denied"}}+ h := Handlers{GitHubTrustPolicies: store}+ request := withUser(httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil), &auth.User{ID: "user-123"})+ response := httptest.NewRecorder()++ h.ListGitHubTrustPolicies(response, request)++ if response.Code != http.StatusInternalServerError {+ t.Fatalf("status = %d, want 500; body = %s", response.Code, response.Body.String())+ }+}++func TestUpdateGitHubTrustPolicySharesDatabaseBudget(t *testing.T) {+ policy := &githubtrust.Policy{+ ID: "policy-1", OwnerSub: "user-123", Name: "ci", Repository: "trycua/cloud",+ AllowedNamespaces: []string{"ns-a"}, Enabled: true,+ }+ store := &fakeGitHubTrustStore{getResult: policy}+ h := Handlers{GitHubTrustPolicies: store}+ request := withUser(httptest.NewRequest(http.MethodPatch, "/api/github-trust-policies/policy-1", bytes.NewBufferString(`{"enabled":false}`)), &auth.User{ID: "user-123"})+ request.SetPathValue("id", "policy-1")+ response := httptest.NewRecorder()++ h.UpdateGitHubTrustPolicy(response, request)++ if response.Code != http.StatusOK {+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())+ }+ getDeadline, getOK := store.getContext.Deadline()+ updateDeadline, updateOK := store.updateContext.Deadline()+ if !getOK || !updateOK {+ t.Fatalf("deadlines get=%t update=%t, want both", getOK, updateOK)+ }+ if !getDeadline.Equal(updateDeadline) {+ t.Fatalf("deadlines get=%s update=%s, want one shared budget", getDeadline, updateDeadline)+ }+}diff --git a/libs/fleet/backend/handlers/handlers.go b/libs/fleet/backend/handlers/handlers.goindex eecb06abc3..08f5f75542 100644--- a/libs/fleet/backend/handlers/handlers.go+++ b/libs/fleet/backend/handlers/handlers.go@@ -32,8 +32,6 @@ type Handlers struct {StateQueryExecutor StateQueryExecutor- Readiness *Readiness-ChatEnabled boolConversations chat.ConversationStoreModel chat.ModelClient@@ -51,11 +49,11 @@ type Handlers struct {func New(admin *keycloak.Admin, cfg *config.Configuration) Handlers {return Handlers{- Admin: admin,- GatewayCfg: cfg.Gateway,- AuthCfg: cfg.Auth,- KC: cfg.Keycloak,- Stripe: cfg.Stripe,+ Admin: admin,+ GatewayCfg: cfg.Gateway,+ AuthCfg: cfg.Auth,+ KC: cfg.Keycloak,+ Stripe: cfg.Stripe,ChatEnabled: cfg.Chat.Enabled,chatLocks: newConversationLockRegistry(),}diff --git a/libs/fleet/backend/handlers/health.go b/libs/fleet/backend/handlers/health.goindex d6c5d6481f..07b4f4d157 100644--- a/libs/fleet/backend/handlers/health.go+++ b/libs/fleet/backend/handlers/health.go@@ -2,7 +2,6 @@ package handlersimport ("net/http"- "sync/atomic")// HealthResponse is the body of GET /healthz and GET /readyz.@@ -10,20 +9,6 @@ type HealthResponse struct {OK bool `json:"ok"`}-// Readiness tracks whether startup dependencies required for serving traffic-// have completed successfully. It starts fail-closed.-type Readiness struct {- databaseReady atomic.Bool-}--func NewReadiness() *Readiness {- return &Readiness{}-}--func (r *Readiness) MarkReady() {- r.databaseReady.Store(true)-}-// GetHealth godoc//// @Summary Liveness probe@@ -36,9 +21,5 @@ func (h Handlers) GetHealth(w http.ResponseWriter, _ *http.Request) {}func (h Handlers) GetReadiness(w http.ResponseWriter, _ *http.Request) {- if h.Readiness == nil || !h.Readiness.databaseReady.Load() {- writeJSON(w, http.StatusServiceUnavailable, HealthResponse{OK: false})- return- }writeJSON(w, http.StatusOK, HealthResponse{OK: true})}diff --git a/libs/fleet/backend/handlers/health_test.go b/libs/fleet/backend/handlers/health_test.goindex f7f2adfe11..0cf991fb10 100644--- a/libs/fleet/backend/handlers/health_test.go+++ b/libs/fleet/backend/handlers/health_test.go@@ -16,24 +16,10 @@ func TestHealthIsLiveWhenDatabaseIsUnavailable(t *testing.T) {}}-func TestReadinessFailsClosedUntilDatabaseSchemaIsReady(t *testing.T) {- readiness := NewReadiness()- h := Handlers{Readiness: readiness}-- for _, name := range []string{"initial state", "schema check failure"} {- t.Run(name, func(t *testing.T) {- response := httptest.NewRecorder()- h.GetReadiness(response, httptest.NewRequest(http.MethodGet, "/readyz", nil))-- if response.Code < http.StatusBadRequest {- t.Fatalf("status = %d, want non-2xx", response.Code)- }- })- }-- readiness.MarkReady()+func TestReadinessIsReadyWhenDatabaseIsUnavailable(t *testing.T) {response := httptest.NewRecorder()- h.GetReadiness(response, httptest.NewRequest(http.MethodGet, "/readyz", nil))++ Handlers{}.GetReadiness(response, httptest.NewRequest(http.MethodGet, "/readyz", nil))if response.Code != http.StatusOK {t.Fatalf("status = %d, want %d", response.Code, http.StatusOK)diff --git a/libs/fleet/backend/handlers/state_query.go b/libs/fleet/backend/handlers/state_query.goindex efe3280ca8..1c8869c0c5 100644--- a/libs/fleet/backend/handlers/state_query.go+++ b/libs/fleet/backend/handlers/state_query.go@@ -8,6 +8,7 @@ import ("mime""net/http"+ "cyclops-cs-backend/auth""cyclops-cs-backend/identity""cyclops-cs-backend/statequery""github.com/jackc/pgx/v5/pgconn"@@ -63,6 +64,9 @@ func (writer *stateQueryStreamWriter) WriteRow(values []any) error {func (writer *stateQueryStreamWriter) Finish(err error) error {if err != nil {+ if auth.IsDatabaseUnavailable(err) {+ return writer.write(stateQueryErrorEvent{Type: "error", Error: "state query unavailable"})+ }return writer.write(stateQueryErrorEvent{Type: "error", Error: err.Error()})}return writer.write(stateQueryDoneEvent{Type: "done"})@@ -106,9 +110,14 @@ func (h Handlers) QueryState(w http.ResponseWriter, r *http.Request) {user := currentUser(r)stream := newStateQueryStreamWriter(w)- err = stream.Finish(h.StateQueryExecutor.Execute(- r.Context(), identity.PersonalGroup(r.Context(), user.ID), string(body), stream,- ))+ ctx, cancel := databaseContext(r.Context())+ defer cancel()+ err = h.StateQueryExecutor.Execute(ctx, identity.PersonalGroup(r.Context(), user.ID), string(body), stream)+ if err != nil && !stream.started {+ writeErr(w, http.StatusServiceUnavailable, "state query unavailable")+ return+ }+ err = stream.Finish(err)if err != nil {slog.Debug("state query stream failed", "err", err)}diff --git a/libs/fleet/backend/handlers/state_query_test.go b/libs/fleet/backend/handlers/state_query_test.goindex fbcd18f13d..7ba652cec0 100644--- a/libs/fleet/backend/handlers/state_query_test.go+++ b/libs/fleet/backend/handlers/state_query_test.go@@ -7,6 +7,7 @@ import ("net/http/httptest""strings""testing"+ "time""cyclops-cs-backend/auth""cyclops-cs-backend/statequery"@@ -21,6 +22,22 @@ type fakeStateQueryExecutor struct {err error}+type deadlineStateQueryExecutor struct {+ context context.Context+ write bool+ err error+}++func (fake *deadlineStateQueryExecutor) Execute(ctx context.Context, _ string, _ string, writer statequery.ResultWriter) error {+ fake.context = ctx+ if fake.write {+ if err := writer.WriteFieldDescriptions([]pgconn.FieldDescription{{Name: "name", DataTypeOID: 25}}); err != nil {+ return err+ }+ }+ return fake.err+}+func (fake *fakeStateQueryExecutor) Execute(_ context.Context, tenant, query string, writer statequery.ResultWriter) error {fake.tenant, fake.query = tenant, queryif err := writer.WriteFieldDescriptions(fake.fields); err != nil {@@ -110,3 +127,117 @@ func TestQueryStateWritesExecutorErrorAsTerminalEvent(t *testing.T) {t.Fatalf("body = %q, want suffix %q", response.Body.String(), wantSuffix)}}++func TestQueryStateUsesBoundedDatabaseContext(t *testing.T) {+ tests := []struct {+ name string+ write bool+ err error+ wantStatus int+ wantBody string+ wantSuffix string+ wantUnavailable bool+ }{+ {"deadline before stream start", false, context.DeadlineExceeded, http.StatusServiceUnavailable, "", "", true},+ {"executor error before stream start", false, errors.New("stream failed"), http.StatusServiceUnavailable, "", "", true},+ {"deadline after stream start", true, context.DeadlineExceeded, http.StatusOK, "", `{"type":"error","error":"state query unavailable"}` + "\n", true},+ {"executor error after stream start", true, errors.New("stream failed"), http.StatusOK, "", `{"type":"error","error":"stream failed"}` + "\n", false},+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ executor := &deadlineStateQueryExecutor{write: test.write, err: test.err}+ h := Handlers{StateQueryExecutor: executor}+ request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})+ request.Header.Set("Content-Type", "application/sql")+ response := httptest.NewRecorder()+ start := time.Now()++ h.QueryState(response, request)++ assertBoundedDatabaseContext(t, executor.context, start)+ if response.Code != test.wantStatus {+ t.Fatalf("status = %d, want %d; body = %s", response.Code, test.wantStatus, response.Body.String())+ }+ if test.wantBody != "" && response.Body.String() != test.wantBody {+ t.Fatalf("body = %q, want %q", response.Body.String(), test.wantBody)+ }+ if test.wantSuffix != "" && !strings.HasSuffix(response.Body.String(), test.wantSuffix) {+ t.Fatalf("body = %q, want suffix %q", response.Body.String(), test.wantSuffix)+ }+ if test.wantUnavailable && strings.Contains(response.Body.String(), test.err.Error()) {+ t.Fatalf("body exposes internal error: %q", response.Body.String())+ }+ })+ }+}++func TestQueryStateDatabaseFailureBeforeStreamReturnsServiceUnavailable(t *testing.T) {+ executor := &deadlineStateQueryExecutor{err: auth.ErrDatabaseUnavailable}+ h := Handlers{StateQueryExecutor: executor}+ request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})+ request.Header.Set("Content-Type", "application/sql")+ response := httptest.NewRecorder()++ h.QueryState(response, request)++ if response.Code != http.StatusServiceUnavailable {+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())+ }+ if strings.Contains(response.Body.String(), auth.ErrDatabaseUnavailable.Error()) {+ t.Fatalf("body exposes internal error: %s", response.Body.String())+ }+}++func TestQueryStateDatabaseFailureAfterStreamWritesSanitizedTerminalEvent(t *testing.T) {+ executor := &deadlineStateQueryExecutor{write: true, err: auth.ErrDatabaseUnavailable}+ h := Handlers{StateQueryExecutor: executor}+ request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})+ request.Header.Set("Content-Type", "application/sql")+ response := httptest.NewRecorder()++ h.QueryState(response, request)++ if response.Code != http.StatusOK {+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())+ }+ if !strings.HasSuffix(response.Body.String(), `{"type":"error","error":"state query unavailable"}`+"\n") {+ t.Fatalf("body = %q", response.Body.String())+ }+}++type cancellationStateQueryExecutor struct {+ started chan<- struct{}+ done chan<- struct{}+}++func (executor cancellationStateQueryExecutor) Execute(ctx context.Context, _, _ string, _ statequery.ResultWriter) error {+ close(executor.started)+ <-ctx.Done()+ close(executor.done)+ return ctx.Err()+}++func TestQueryStatePropagatesRequestCancellation(t *testing.T) {+ started := make(chan struct{})+ done := make(chan struct{})+ handlerDone := make(chan struct{})+ h := Handlers{StateQueryExecutor: cancellationStateQueryExecutor{started: started, done: done}}+ parent, cancel := context.WithCancel(context.Background())+ defer cancel()+ request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")).WithContext(parent), &auth.User{ID: "alice"})+ request.Header.Set("Content-Type", "application/sql")+ response := httptest.NewRecorder()++ go func() {+ h.QueryState(response, request)+ close(handlerDone)+ }()+ <-started+ cancel()+ <-done+ <-handlerDone+ if response.Code != http.StatusServiceUnavailable {+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())+ }+}diff --git a/libs/fleet/backend/main.go b/libs/fleet/backend/main.goindex aa37ea7cee..c07e2e1ab2 100644--- a/libs/fleet/backend/main.go+++ b/libs/fleet/backend/main.go@@ -25,6 +25,7 @@ import ("log/slog""net/http""os"+ "time""github.com/spf13/cobra"@@ -320,7 +321,6 @@ func run() error {}h := handlers.New(admin, cfg)- h.Readiness = handlers.NewReadiness()if cfg.Stripe.SecretKey != "" {h.Billing = billing.NewService(billing.NewStripeGateway(cfg.Stripe.SecretKey))slog.Info("stripe billing: hosted flows enabled")@@ -331,45 +331,7 @@ func run() error {h.WorkloadAdmin = workloadAdminh.WorkloadAudience = cfg.Keycloak.WorkloadAudienceh.WorkloadTokenURL = cfg.Keycloak.WorkloadTokenURL- databaseReady := false- if cfg.Database.URL != "" {- if err := database.RequireVersion(ctx, cfg.Database.URL, 1); err != nil {- slog.Error("postgres database schema unavailable", "err", err)- } else {- databaseReady = true- h.Readiness.MarkReady()- slog.Info("postgres database schema ready", "version", 1)- }- }-- stateSchemaReady := databaseReady && cfg.Database.StateQueryDSN != "" && cfg.Database.StateQueryTenantPassword != ""- if stateSchemaReady {- executor, err := statequery.NewExecutor(- cfg.Database.StateQueryDSN,- cfg.Database.StateQueryTenantPassword,- )- if err != nil {- slog.Error("kubernetes state query: executor init failed; /api/state/query will return 503", "err", err)- } else {- h.StateQueryExecutor = executor- slog.Info("kubernetes state query: query endpoint enabled")- }- }- // Postgres-backed GitHub OIDC trust policies (CUA-675). A connection- // failure is non-fatal — the routes stay registered and reply 503 until- // the database is reachable, mirroring how the other optional surfaces- // degrade gracefully.- if databaseReady {- if store, err := githubtrust.New(ctx, cfg.Database.URL); err != nil {- slog.Error("github trust policies: init failed; /api/github-trust-policies will return 503", "err", err)- } else if store != nil {- h.GitHubTrustPolicies = store- auth.SetGitHubTrustResolver(handlers.NewGitHubTrustResolver(store))- slog.Info("github trust policies: enabled")- }- } else {- slog.Info("github trust policies: disabled (database schema unavailable)")- }+ initializeDatabaseFeatures(ctx, cfg.Database, &h, defaultDatabaseFeatureDependencies())if cfg.Chat.Enabled {h.Conversations = chat.NewMemoryConversationStore()@@ -385,3 +347,62 @@ func run() error {}return nil}++const databaseStartupTimeout = 5 * time.Second++type databaseFeatureDependencies struct {+ requireVersion func(context.Context, string, int64) error+ newStateQueryExecutor func(string, string) (handlers.StateQueryExecutor, error)+ newGitHubTrustStore func(context.Context, string) (githubtrust.Store, error)+}++func defaultDatabaseFeatureDependencies() databaseFeatureDependencies {+ return databaseFeatureDependencies{+ requireVersion: database.RequireVersion,+ newStateQueryExecutor: func(dsn, tenantPassword string) (handlers.StateQueryExecutor, error) {+ return statequery.NewExecutor(dsn, tenantPassword)+ },+ newGitHubTrustStore: githubtrust.New,+ }+}++func initializeDatabaseFeatures(parent context.Context, cfg config.DatabaseConfiguration, h *handlers.Handlers, dependencies databaseFeatureDependencies) {+ h.StateQueryExecutor = nil+ h.GitHubTrustPolicies = nil+ auth.SetGitHubTrustResolver(nil)++ if cfg.StateQueryDSN != "" && cfg.StateQueryTenantPassword != "" {+ executor, err := dependencies.newStateQueryExecutor(cfg.StateQueryDSN, cfg.StateQueryTenantPassword)+ if err != nil {+ slog.Error("kubernetes state query: executor init failed; /api/state/query will return 503", "err", err)+ } else {+ h.StateQueryExecutor = executor+ slog.Info("kubernetes state query: query endpoint enabled")+ }+ }++ if cfg.URL == "" {+ slog.Info("postgres-backed features disabled (DATABASE_URL unset)")+ return+ }++ ctx, cancel := context.WithTimeout(parent, databaseStartupTimeout)+ defer cancel()++ if err := dependencies.requireVersion(ctx, cfg.URL, 1); err != nil {+ slog.Error("postgres database schema unavailable; database-backed routes will return 503", "err", err)+ return+ }+ slog.Info("postgres database schema ready", "version", 1)++ store, err := dependencies.newGitHubTrustStore(ctx, cfg.URL)+ if err != nil {+ slog.Error("github trust policies: init failed; /api/github-trust-policies will return 503", "err", err)+ return+ }+ if store != nil {+ h.GitHubTrustPolicies = store+ auth.SetGitHubTrustResolver(handlers.NewGitHubTrustResolver(store))+ slog.Info("github trust policies: enabled")+ }+}diff --git a/libs/fleet/backend/main_test.go b/libs/fleet/backend/main_test.goindex 3cfd7f3a46..3570088880 100644--- a/libs/fleet/backend/main_test.go+++ b/libs/fleet/backend/main_test.go@@ -23,7 +23,9 @@ import ("cyclops-cs-backend/auth""cyclops-cs-backend/billing""cyclops-cs-backend/config"+ "cyclops-cs-backend/githubtrust""cyclops-cs-backend/handlers"+ "cyclops-cs-backend/statequery"jwt "github.com/golang-jwt/jwt/v5""github.com/trycua/cloud/pkg/featureflags"@@ -51,7 +53,7 @@ func TestHealthAndReadinessRoutes(t *testing.T) {statusCode int}{{path: "/healthz", statusCode: http.StatusOK},- {path: "/readyz", statusCode: http.StatusServiceUnavailable},+ {path: "/readyz", statusCode: http.StatusOK},} {t.Run(test.path, func(t *testing.T) {response := httptest.NewRecorder()@@ -458,3 +460,164 @@ func TestNginxRoutesChatToBackend(t *testing.T) {t.Fatal("nginx.conf backend matcher does not include chat")}}++func TestInitializeDatabaseFeatures(t *testing.T) {+ tests := []struct {+ name string+ config config.DatabaseConfiguration+ requireVersionError error+ newExecutorError error+ newStoreError error+ wantExecutor bool+ wantStore bool+ wantRequireVersion bool+ wantNewExecutor bool+ wantNewStore bool+ }{+ {+ name: "state query initializes without application database",+ config: config.DatabaseConfiguration{+ StateQueryDSN: "postgres://state-query/state-query",+ StateQueryTenantPassword: "tenant-password",+ },+ wantExecutor: true,+ wantNewExecutor: true,+ },+ {+ name: "state query initializes when application schema validation fails",+ config: config.DatabaseConfiguration{+ URL: "postgres://application/application",+ StateQueryDSN: "postgres://state-query/state-query",+ StateQueryTenantPassword: "tenant-password",+ },+ requireVersionError: fmt.Errorf("schema unavailable"),+ wantExecutor: true,+ wantRequireVersion: true,+ wantNewExecutor: true,+ },+ {+ name: "application database enables GitHub trust store",+ config: config.DatabaseConfiguration{URL: "postgres://application/application"},+ wantStore: true,+ wantRequireVersion: true,+ wantNewStore: true,+ },+ {+ name: "constructor failures are non-fatal and clear stale fields",+ config: config.DatabaseConfiguration{+ URL: "postgres://application/application",+ StateQueryDSN: "postgres://state-query/state-query",+ StateQueryTenantPassword: "tenant-password",+ },+ newExecutorError: fmt.Errorf("invalid state query configuration"),+ newStoreError: fmt.Errorf("store unavailable"),+ wantRequireVersion: true,+ wantNewExecutor: true,+ wantNewStore: true,+ },+ {+ name: "missing database config clears stale fields",+ config: config.DatabaseConfiguration{},+ wantRequireVersion: false,+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ var requireVersionContext context.Context+ var storeContext context.Context+ newExecutorCalls := 0+ newStoreCalls := 0+ dependencies := databaseFeatureDependencies{+ requireVersion: func(ctx context.Context, _ string, _ int64) error {+ requireVersionContext = ctx+ return test.requireVersionError+ },+ newStateQueryExecutor: func(_ string, _ string) (handlers.StateQueryExecutor, error) {+ newExecutorCalls+++ if test.newExecutorError != nil {+ return nil, test.newExecutorError+ }+ return stateQueryExecutorStub{}, nil+ },+ newGitHubTrustStore: func(ctx context.Context, _ string) (githubtrust.Store, error) {+ newStoreCalls+++ storeContext = ctx+ if test.newStoreError != nil {+ return nil, test.newStoreError+ }+ return githubTrustStoreStub{}, nil+ },+ }+ h := handlers.Handlers{+ StateQueryExecutor: stateQueryExecutorStub{},+ GitHubTrustPolicies: githubTrustStoreStub{},+ }+ auth.SetGitHubTrustResolver(handlers.NewGitHubTrustResolver(githubTrustStoreStub{}))+ t.Cleanup(func() { auth.SetGitHubTrustResolver(nil) })++ initializeDatabaseFeatures(context.Background(), test.config, &h, dependencies)++ if got := newExecutorCalls == 1; got != test.wantNewExecutor {+ t.Fatalf("state-query constructor called = %t, want %t", got, test.wantNewExecutor)+ }+ if got := newStoreCalls == 1; got != test.wantNewStore {+ t.Fatalf("GitHub trust store constructor called = %t, want %t", got, test.wantNewStore)+ }+ if got := h.StateQueryExecutor != nil; got != test.wantExecutor {+ t.Fatalf("StateQueryExecutor present = %t, want %t", got, test.wantExecutor)+ }+ if got := h.GitHubTrustPolicies != nil; got != test.wantStore {+ t.Fatalf("GitHubTrustPolicies present = %t, want %t", got, test.wantStore)+ }+ assertStartupContext(t, requireVersionContext, test.wantRequireVersion)+ assertStartupContext(t, storeContext, test.wantNewStore)+ })+ }+}++func assertStartupContext(t *testing.T, ctx context.Context, wantCalled bool) {+ t.Helper()+ if !wantCalled {+ if ctx != nil {+ t.Fatal("initializer received a context unexpectedly")+ }+ return+ }+ if ctx == nil {+ t.Fatal("initializer did not receive a context")+ }+ deadline, ok := ctx.Deadline()+ if !ok {+ t.Fatal("initializer context has no deadline")+ }+ if remaining := time.Until(deadline); remaining > databaseStartupTimeout+100*time.Millisecond {+ t.Fatalf("context deadline is too far away: %s", remaining)+ }+ if err := ctx.Err(); err != context.Canceled {+ t.Fatalf("initializer context error = %v, want %v", err, context.Canceled)+ }+}++type stateQueryExecutorStub struct{}++func (stateQueryExecutorStub) Execute(context.Context, string, string, statequery.ResultWriter) error {+ return nil+}++type githubTrustStoreStub struct{}++func (githubTrustStoreStub) List(context.Context, string) ([]*githubtrust.Policy, error) {+ return nil, nil+}+func (githubTrustStoreStub) Create(context.Context, *githubtrust.Policy) error { return nil }+func (githubTrustStoreStub) Get(context.Context, string, string) (*githubtrust.Policy, error) {+ return nil, nil+}+func (githubTrustStoreStub) Update(context.Context, *githubtrust.Policy) error { return nil }+func (githubTrustStoreStub) Delete(context.Context, string, string) (bool, error) {+ return false, nil+}+func (githubTrustStoreStub) ResolveByRepository(context.Context, string) ([]*githubtrust.Policy, error) {+ return nil, nil+}diff --git a/libs/fleet/backend/startup_migration_contract_test.go b/libs/fleet/backend/startup_migration_contract_test.goindex 0d885cf544..504582a0a6 100644--- a/libs/fleet/backend/startup_migration_contract_test.go+++ b/libs/fleet/backend/startup_migration_contract_test.go@@ -738,23 +738,6 @@ func TestActiveBackendSourcesDoNotOwnMigrations(t *testing.T) {}}-func TestMainRequiresMigrationVersionWithoutRunningMigrations(t *testing.T) {- source := readSource(t, "main.go")- if !strings.Contains(source, "database.RequireVersion(ctx, cfg.Database.URL, 1)") {- t.Fatal("backend must require migration version 1")- }-}--func TestBackendReadinessStartsFailClosedAndRequiresSchema(t *testing.T) {- source := readSource(t, "main.go")- if !strings.Contains(source, "h.Readiness = handlers.NewReadiness()") {- t.Fatal("backend readiness must start fail-closed before the database check")- }- if !strings.Contains(source, "h.Readiness.MarkReady()") {- t.Fatal("backend readiness must become ready after the database schema check")- }-}-func TestBackendDeploymentKeepsCoreAPIReadyWithoutDatabaseSchema(t *testing.T) {source := readSource(t, filepath.Join("..", "..", "clusters", "kopf-k3s", "cyclops-cs", "backend-deployment.yaml"))for _, probe := range []string{"readinessProbe", "livenessProbe"} {