Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cyclops): keep serving available through PostgreSQL cutover (#6804)

r33drichardsa90e68305577
diff --git a/libs/fleet/backend/auth/auth.go b/libs/fleet/backend/auth/auth.go
index 1205439fcc..1e0ad0502d 100644
--- a/libs/fleet/backend/auth/auth.go
+++ b/libs/fleet/backend/auth/auth.go
@@ -38,9 +38,12 @@ package auth
import (
"context"
+ "errors"
"fmt"
+ "io"
"log"
"log/slog"
+ "net"
"net/http"
"slices"
"strings"
@@ -50,6 +53,7 @@ import (
keyfunc "github.com/MicahParks/keyfunc/v2"
jwt "github.com/golang-jwt/jwt/v5"
+ "github.com/jackc/pgx/v5/pgconn"
)
var (
@@ -70,6 +74,57 @@ type GitHubTrustResolver interface {
ResolveGitHubTrustPolicies(ctx context.Context, repository string) ([]GitHubTrustPolicy, error)
}
+var ErrDatabaseUnavailable = errors.New("database unavailable")
+
+type databaseContextKey struct{}
+
+const DatabaseRequestTimeout = 15 * time.Second
+
+func DatabaseContext(parent context.Context) (context.Context, context.CancelFunc) {
+ if databaseContext, ok := parent.Value(databaseContextKey{}).(context.Context); ok {
+ return databaseContext, func() {}
+ }
+ if deadline, ok := parent.Deadline(); ok && time.Until(deadline) <= DatabaseRequestTimeout {
+ return context.WithValue(parent, databaseContextKey{}, parent), func() {}
+ }
+ databaseContext, cancel := context.WithTimeout(parent, DatabaseRequestTimeout)
+ return context.WithValue(databaseContext, databaseContextKey{}, databaseContext), cancel
+}
+
+func IsDatabaseUnavailable(err error) bool {
+ return errors.Is(err, ErrDatabaseUnavailable) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)
+}
+
+func DatabaseUnavailable(err error) error {
+ if errors.Is(err, ErrDatabaseUnavailable) {
+ return err
+ }
+ return errors.Join(ErrDatabaseUnavailable, err)
+}
+
+func ClassifyDatabaseError(err error) error {
+ if err == nil || errors.Is(err, ErrDatabaseUnavailable) {
+ return err
+ }
+ if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+ return DatabaseUnavailable(err)
+ }
+
+ var postgresError *pgconn.PgError
+ if errors.As(err, &postgresError) {
+ return err
+ }
+ var connectError *pgconn.ConnectError
+ if errors.As(err, &connectError) {
+ return DatabaseUnavailable(err)
+ }
+ var networkError net.Error
+ if errors.As(err, &networkError) || errors.Is(err, net.ErrClosed) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+ return DatabaseUnavailable(err)
+ }
+ return err
+}
+
var githubTrustResolver GitHubTrustResolver
func SetGitHubTrustResolver(resolver GitHubTrustResolver) {
diff --git a/libs/fleet/backend/auth/github_oidc_test.go b/libs/fleet/backend/auth/github_oidc_test.go
index fe7b59fb7e..e77c9a3675 100644
--- a/libs/fleet/backend/auth/github_oidc_test.go
+++ b/libs/fleet/backend/auth/github_oidc_test.go
@@ -6,15 +6,19 @@ import (
"crypto/rsa"
"encoding/base64"
"encoding/json"
+ "errors"
+ "net"
"net/http"
"net/http/httptest"
"strings"
+ "syscall"
"testing"
"time"
"cyclops-cs-backend/config"
jwt "github.com/golang-jwt/jwt/v5"
+ "github.com/jackc/pgx/v5/pgconn"
)
const (
@@ -242,3 +246,199 @@ func bigEndianBytes(v int) []byte {
}
return out
}
+
+func TestTokenAuthMiddlewareDoesNotExtendGitHubResolverDeadline(t *testing.T) {
+ signingKey, jwksURL := newGitHubJWKS(t)
+ t.Cleanup(func() {
+ SetGitHubTrustResolver(nil)
+ })
+ if err := Init(&config.AuthConfiguration{
+ Issuer: "https://issuer.example.test/realms/cyclops-cs",
+ JWKSUri: jwksURL,
+ SigningAlgs: []string{"RS256"},
+ SPAClientID: "cyclops-cs-spa",
+ KeyClientPfx: "key-",
+ UserKeyClientPfx: "ukey-",
+ GitHubOIDCIssuer: testGitHubIssuer,
+ GitHubOIDCJWKSUri: jwksURL,
+ GitHubOIDCAudience: testGitHubAudience,
+ GitHubOIDCEnabled: true,
+ GitHubOIDCAlgs: []string{"RS256"},
+ }); err != nil {
+ t.Fatalf("Init err = %v", err)
+ }
+ resolverDeadline := make(chan time.Time, 1)
+ SetGitHubTrustResolver(githubTrustResolverFunc(func(ctx context.Context, _ string) ([]GitHubTrustPolicy, error) {
+ deadline, ok := ctx.Deadline()
+ if !ok {
+ t.Fatal("resolver context has no deadline")
+ }
+ resolverDeadline <- deadline
+ return []GitHubTrustPolicy{{ID: "p1", OwnerSub: "user-123", Repository: "trycua/cloud", AllowedNamespaces: []string{"ns-a"}, Enabled: true}}, nil
+ }))
+ raw := signGitHubToken(t, signingKey, jwt.MapClaims{
+ "iss": testGitHubIssuer, "aud": testGitHubAudience, "sub": "repo:trycua/cloud:ref:refs/heads/main",
+ "repository": "trycua/cloud", "exp": time.Now().Add(time.Hour).Unix(), "iat": time.Now().Add(-time.Minute).Unix(),
+ })
+ parent, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second))
+ defer cancel()
+ wantDeadline, ok := parent.Deadline()
+ if !ok {
+ t.Fatal("request context has no deadline")
+ }
+ request := httptest.NewRequest(http.MethodGet, "/", nil).WithContext(parent)
+ request.Header.Set("Authorization", "Bearer "+raw)
+ response := httptest.NewRecorder()
+
+ TokenAuthMiddleware(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})).ServeHTTP(response, request)
+
+ if response.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
+ }
+ if gotDeadline := <-resolverDeadline; !gotDeadline.Equal(wantDeadline) {
+ t.Fatalf("resolver deadline = %s, want request deadline %s", gotDeadline, wantDeadline)
+ }
+}
+
+func TestTokenAuthMiddlewareReturnsServiceUnavailableForGitHubTrustDatabaseOutage(t *testing.T) {
+ signingKey, jwksURL := newGitHubJWKS(t)
+ t.Cleanup(func() {
+ SetGitHubTrustResolver(nil)
+ })
+ if err := Init(&config.AuthConfiguration{
+ Issuer: "https://issuer.example.test/realms/cyclops-cs",
+ JWKSUri: jwksURL,
+ SigningAlgs: []string{"RS256"},
+ SPAClientID: "cyclops-cs-spa",
+ KeyClientPfx: "key-",
+ UserKeyClientPfx: "ukey-",
+ GitHubOIDCIssuer: testGitHubIssuer,
+ GitHubOIDCJWKSUri: jwksURL,
+ GitHubOIDCAudience: testGitHubAudience,
+ GitHubOIDCEnabled: true,
+ GitHubOIDCAlgs: []string{"RS256"},
+ }); err != nil {
+ t.Fatalf("Init err = %v", err)
+ }
+ SetGitHubTrustResolver(githubTrustResolverFunc(func(context.Context, string) ([]GitHubTrustPolicy, error) {
+ return nil, ErrDatabaseUnavailable
+ }))
+ raw := signGitHubToken(t, signingKey, jwt.MapClaims{
+ "iss": testGitHubIssuer, "aud": testGitHubAudience, "sub": "repo:trycua/cloud:ref:refs/heads/main",
+ "repository": "trycua/cloud", "exp": time.Now().Add(time.Hour).Unix(), "iat": time.Now().Add(-time.Minute).Unix(),
+ })
+ request := httptest.NewRequest(http.MethodGet, "/", nil)
+ request.Header.Set("Authorization", "Bearer "+raw)
+ response := httptest.NewRecorder()
+
+ TokenAuthMiddleware(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+ t.Fatal("next handler must not run")
+ })).ServeHTTP(response, request)
+
+ if response.Code != http.StatusServiceUnavailable {
+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
+ }
+}
+
+func TestClassifyDatabaseErrorRecognizesTransportButNotPostgresErrors(t *testing.T) {
+ connectionRefused := &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}
+ classified := ClassifyDatabaseError(errors.New("resolve policies: " + connectionRefused.Error()))
+ if errors.Is(classified, ErrDatabaseUnavailable) {
+ t.Fatal("string-only network error must not be classified as database unavailable")
+ }
+
+ classified = ClassifyDatabaseError(errors.Join(errors.New("resolve policies"), connectionRefused))
+ if !errors.Is(classified, ErrDatabaseUnavailable) {
+ t.Fatalf("classified error = %v, want database unavailable sentinel", classified)
+ }
+ if !errors.Is(classified, connectionRefused) {
+ t.Fatalf("classified error = %v, want original transport error", classified)
+ }
+
+ postgresError := &pgconn.PgError{Code: "42501", Message: "permission denied"}
+ classified = ClassifyDatabaseError(errors.Join(errors.New("resolve policies"), postgresError))
+ if errors.Is(classified, ErrDatabaseUnavailable) {
+ t.Fatalf("postgres authorization error = %v, must not be classified as unavailable", classified)
+ }
+}
+
+func TestTokenAuthMiddlewareReturnsServiceUnavailableForImmediateGitHubTrustConnectionFailure(t *testing.T) {
+ signingKey, jwksURL := newGitHubJWKS(t)
+ t.Cleanup(func() {
+ SetGitHubTrustResolver(nil)
+ })
+ if err := Init(&config.AuthConfiguration{
+ Issuer: "https://issuer.example.test/realms/cyclops-cs",
+ JWKSUri: jwksURL,
+ SigningAlgs: []string{"RS256"},
+ SPAClientID: "cyclops-cs-spa",
+ KeyClientPfx: "key-",
+ UserKeyClientPfx: "ukey-",
+ GitHubOIDCIssuer: testGitHubIssuer,
+ GitHubOIDCJWKSUri: jwksURL,
+ GitHubOIDCAudience: testGitHubAudience,
+ GitHubOIDCEnabled: true,
+ GitHubOIDCAlgs: []string{"RS256"},
+ }); err != nil {
+ t.Fatalf("Init err = %v", err)
+ }
+ connectionRefused := &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}
+ SetGitHubTrustResolver(githubTrustResolverFunc(func(context.Context, string) ([]GitHubTrustPolicy, error) {
+ return nil, ClassifyDatabaseError(connectionRefused)
+ }))
+ raw := signGitHubToken(t, signingKey, jwt.MapClaims{
+ "iss": testGitHubIssuer, "aud": testGitHubAudience, "sub": "repo:trycua/cloud:ref:refs/heads/main",
+ "repository": "trycua/cloud", "exp": time.Now().Add(time.Hour).Unix(), "iat": time.Now().Add(-time.Minute).Unix(),
+ })
+ request := httptest.NewRequest(http.MethodGet, "/", nil)
+ request.Header.Set("Authorization", "Bearer "+raw)
+ response := httptest.NewRecorder()
+
+ TokenAuthMiddleware(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+ t.Fatal("next handler must not run")
+ })).ServeHTTP(response, request)
+
+ if response.Code != http.StatusServiceUnavailable {
+ t.Fatalf("status = %d, want 503; body = %s", response.Code, response.Body.String())
+ }
+}
+
+func TestTokenAuthMiddlewareKeepsGitHubTrustPostgresErrorsUnauthorized(t *testing.T) {
+ signingKey, jwksURL := newGitHubJWKS(t)
+ t.Cleanup(func() {
+ SetGitHubTrustResolver(nil)
+ })
+ if err := Init(&config.AuthConfiguration{
+ Issuer: "https://issuer.example.test/realms/cyclops-cs",
+ JWKSUri: jwksURL,
+ SigningAlgs: []string{"RS256"},
+ SPAClientID: "cyclops-cs-spa",
+ KeyClientPfx: "key-",
+ UserKeyClientPfx: "ukey-",
+ GitHubOIDCIssuer: testGitHubIssuer,
+ GitHubOIDCJWKSUri: jwksURL,
+ GitHubOIDCAudience: testGitHubAudience,
+ GitHubOIDCEnabled: true,
+ GitHubOIDCAlgs: []string{"RS256"},
+ }); err != nil {
+ t.Fatalf("Init err = %v", err)
+ }
+ SetGitHubTrustResolver(githubTrustResolverFunc(func(context.Context, string) ([]GitHubTrustPolicy, error) {
+ return nil, &pgconn.PgError{Code: "42501", Message: "permission denied"}
+ }))
+ raw := signGitHubToken(t, signingKey, jwt.MapClaims{
+ "iss": testGitHubIssuer, "aud": testGitHubAudience, "sub": "repo:trycua/cloud:ref:refs/heads/main",
+ "repository": "trycua/cloud", "exp": time.Now().Add(time.Hour).Unix(), "iat": time.Now().Add(-time.Minute).Unix(),
+ })
+ request := httptest.NewRequest(http.MethodGet, "/", nil)
+ request.Header.Set("Authorization", "Bearer "+raw)
+ response := httptest.NewRecorder()
+
+ TokenAuthMiddleware(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+ t.Fatal("next handler must not run")
+ })).ServeHTTP(response, request)
+
+ if response.Code != http.StatusUnauthorized {
+ t.Fatalf("status = %d, want 401; body = %s", response.Code, response.Body.String())
+ }
+}
diff --git a/libs/fleet/backend/auth/middlewares.go b/libs/fleet/backend/auth/middlewares.go
index 7285439cb3..ef31231e33 100644
--- a/libs/fleet/backend/auth/middlewares.go
+++ b/libs/fleet/backend/auth/middlewares.go
@@ -362,6 +362,10 @@ func TokenAuthMiddleware(next http.Handler) http.Handler {
user, err := validateWithContext(r.Context(), raw)
if err != nil {
result = err.Error()
+ if IsDatabaseUnavailable(err) {
+ writeJSONErr(w, http.StatusServiceUnavailable, "authentication unavailable")
+ return
+ }
writeJSONErr(w, http.StatusUnauthorized, "auth token is invalid")
return
}
diff --git a/libs/fleet/backend/cmd/manifest-contract/main.go b/libs/fleet/backend/cmd/manifest-contract/main.go
new file mode 100644
index 0000000000..353b63d644
--- /dev/null
+++ b/libs/fleet/backend/cmd/manifest-contract/main.go
@@ -0,0 +1,279 @@
+package main
+
+import (
+ "bufio"
+ "fmt"
+ "io"
+ "os"
+ "regexp"
+ "strings"
+
+ "sigs.k8s.io/yaml"
+)
+
+var (
+ servingNames = []string{"cyclops-cs", "cyclops-cs-backend"}
+ projectorImageTag = regexp.MustCompile(`^main-[0-9]+$`)
+)
+
+type manifestObject struct {
+ APIVersion string `json:"apiVersion"`
+ Kind string `json:"kind"`
+ Metadata struct {
+ Name string `json:"name"`
+ Namespace string `json:"namespace"`
+ } `json:"metadata"`
+ Spec struct {
+ Replicas int `json:"replicas"`
+ ProgressDeadlineSeconds int `json:"progressDeadlineSeconds"`
+ Strategy struct {
+ Type string `json:"type"`
+ RollingUpdate struct {
+ MaxUnavailable int `json:"maxUnavailable"`
+ MaxSurge int `json:"maxSurge"`
+ } `json:"rollingUpdate"`
+ } `json:"strategy"`
+ Selector labelSelector `json:"selector"`
+ Template struct {
+ Metadata struct {
+ Labels map[string]string `json:"labels"`
+ } `json:"metadata"`
+ Spec struct {
+ TopologySpreadConstraints []topologySpreadConstraint `json:"topologySpreadConstraints"`
+ Containers []container `json:"containers"`
+ } `json:"spec"`
+ } `json:"template"`
+ MinAvailable int `json:"minAvailable"`
+ } `json:"spec"`
+}
+
+type labelSelector struct {
+ MatchLabels map[string]string `json:"matchLabels"`
+}
+
+type topologySpreadConstraint struct {
+ MaxSkew int `json:"maxSkew"`
+ TopologyKey string `json:"topologyKey"`
+ WhenUnsatisfiable string `json:"whenUnsatisfiable"`
+ LabelSelector labelSelector `json:"labelSelector"`
+}
+
+type container struct {
+ Name string `json:"name"`
+ Image string `json:"image"`
+ ReadinessProbe probe `json:"readinessProbe"`
+ LivenessProbe probe `json:"livenessProbe"`
+}
+
+type probe struct {
+ HTTPGet struct {
+ Path string `json:"path"`
+ } `json:"httpGet"`
+}
+
+func main() {
+ if err := run(os.Args[1:], os.Stdin); err != nil {
+ fmt.Fprintln(os.Stderr, err)
+ os.Exit(1)
+ }
+}
+
+func run(args []string, stdin io.Reader) error {
+ if len(args) == 0 {
+ return verify(stdin)
+ }
+
+ mode := "serving"
+ path := args[0]
+ if args[0] == "projector" {
+ mode = "projector"
+ if len(args) == 1 {
+ return verifyProjector(stdin)
+ }
+ if len(args) != 2 {
+ return fmt.Errorf("usage: manifest-contract [manifest.yaml] | manifest-contract projector [manifest.yaml]")
+ }
+ path = args[1]
+ } else if len(args) != 1 {
+ return fmt.Errorf("usage: manifest-contract [manifest.yaml] | manifest-contract projector [manifest.yaml]")
+ }
+
+ file, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ if mode == "projector" {
+ return verifyProjector(file)
+ }
+ return verify(file)
+}
+
+func verify(reader io.Reader) error {
+ objects, err := parseDocuments(reader)
+ if err != nil {
+ return err
+ }
+
+ for _, name := range servingNames {
+ deployment, err := findObject(objects, "Deployment", name)
+ if err != nil {
+ return err
+ }
+ if err := verifyDeployment(deployment, name); err != nil {
+ return err
+ }
+
+ pdb, err := findObject(objects, "PodDisruptionBudget", name)
+ if err != nil {
+ return err
+ }
+ if err := verifyPDB(pdb, name); err != nil {
+ return err
+ }
+ }
+
+ for _, object := range objects {
+ if object.Metadata.Name == "k8s-state-projector" || object.Metadata.Name == "cyclops-cs-k8s-state-projector" {
+ return fmt.Errorf("serving manifest must not contain state projector")
+ }
+ }
+
+ backend, err := findObject(objects, "Deployment", "cyclops-cs-backend")
+ if err != nil {
+ return err
+ }
+ return verifyBackendProbes(backend)
+}
+
+func parseDocuments(reader io.Reader) ([]manifestObject, error) {
+ var documents []manifestObject
+ var current strings.Builder
+ scanner := bufio.NewScanner(reader)
+ scanner.Buffer(make([]byte, 64*1024), 10*1024*1024)
+ for scanner.Scan() {
+ line := scanner.Text()
+ if strings.TrimSpace(line) == "---" {
+ if err := appendDocument(&documents, current.String()); err != nil {
+ return nil, err
+ }
+ current.Reset()
+ continue
+ }
+ current.WriteString(line)
+ current.WriteByte('\n')
+ }
+ if err := scanner.Err(); err != nil {
+ return nil, err
+ }
+ if err := appendDocument(&documents, current.String()); err != nil {
+ return nil, err
+ }
+ return documents, nil
+}
+
+func appendDocument(documents *[]manifestObject, source string) error {
+ if strings.TrimSpace(source) == "" {
+ return nil
+ }
+
+ var object manifestObject
+ if err := yaml.Unmarshal([]byte(source), &object); err != nil {
+ return fmt.Errorf("parse manifest document: %w", err)
+ }
+ *documents = append(*documents, object)
+ return nil
+}
+
+func findObject(objects []manifestObject, kind, name string) (manifestObject, error) {
+ var matches []manifestObject
+ for _, object := range objects {
+ if object.Kind == kind && object.Metadata.Name == name {
+ matches = append(matches, object)
+ }
+ }
+ if len(matches) != 1 {
+ return manifestObject{}, fmt.Errorf("expected exactly one %s %q, found %d", kind, name, len(matches))
+ }
+ return matches[0], nil
+}
+
+func verifyDeployment(deployment manifestObject, name string) error {
+ if deployment.APIVersion != "apps/v1" || deployment.Metadata.Namespace != "cyclops-cs" {
+ return fmt.Errorf("Deployment %q has wrong apiVersion or namespace", name)
+ }
+ if deployment.Spec.Replicas != 2 || deployment.Spec.ProgressDeadlineSeconds != 600 {
+ return fmt.Errorf("Deployment %q must set replicas 2 and progressDeadlineSeconds 600", name)
+ }
+ if deployment.Spec.Strategy.Type != "RollingUpdate" || deployment.Spec.Strategy.RollingUpdate.MaxUnavailable != 0 || deployment.Spec.Strategy.RollingUpdate.MaxSurge != 1 {
+ return fmt.Errorf("Deployment %q must use RollingUpdate maxUnavailable 0 and maxSurge 1", name)
+ }
+ if deployment.Spec.Selector.MatchLabels["app"] != name {
+ return fmt.Errorf("Deployment %q selector app label must equal %q", name, name)
+ }
+ if deployment.Spec.Template.Metadata.Labels["app"] != name {
+ return fmt.Errorf("Deployment %q pod template app label must equal %q", name, name)
+ }
+ for _, constraint := range deployment.Spec.Template.Spec.TopologySpreadConstraints {
+ if constraint.MaxSkew == 1 && constraint.TopologyKey == "kubernetes.io/hostname" && constraint.WhenUnsatisfiable == "ScheduleAnyway" && constraint.LabelSelector.MatchLabels["app"] == name {
+ return nil
+ }
+ }
+ return fmt.Errorf("Deployment %q must spread pods by app label %q", name, name)
+}
+
+func verifyPDB(pdb manifestObject, name string) error {
+ if pdb.APIVersion != "policy/v1" || pdb.Metadata.Namespace != "cyclops-cs" {
+ return fmt.Errorf("PodDisruptionBudget %q has wrong apiVersion or namespace", name)
+ }
+ if pdb.Spec.MinAvailable != 1 || pdb.Spec.Selector.MatchLabels["app"] != name {
+ return fmt.Errorf("PodDisruptionBudget %q must select app %q with minAvailable 1", name, name)
+ }
+ return nil
+}
+
+func verifyBackendProbes(backend manifestObject) error {
+ for _, container := range backend.Spec.Template.Spec.Containers {
+ if container.Name == "cyclops-cs-backend" && container.ReadinessProbe.HTTPGet.Path == "/healthz" && container.LivenessProbe.HTTPGet.Path == "/healthz" {
+ return nil
+ }
+ }
+ return fmt.Errorf("Deployment %q backend container must use /healthz readiness and liveness probes", backend.Metadata.Name)
+}
+
+func verifyProjector(reader io.Reader) error {
+ objects, err := parseDocuments(reader)
+ if err != nil {
+ return err
+ }
+
+ projector, err := findObject(objects, "Deployment", "k8s-state-projector")
+ if err != nil {
+ return err
+ }
+ if projector.APIVersion != "apps/v1" || projector.Metadata.Namespace != "cyclops-cs" {
+ return fmt.Errorf("projector Deployment has wrong apiVersion or namespace")
+ }
+
+ for _, container := range projector.Spec.Template.Spec.Containers {
+ if container.Name != "projector" {
+ continue
+ }
+ return verifyProjectorImage(container.Image)
+ }
+ return fmt.Errorf("projector Deployment must contain projector container")
+}
+
+func verifyProjectorImage(image string) error {
+ const repository = "296062593712.dkr.ecr.us-west-2.amazonaws.com/cyclops-cs-backend"
+
+ imageRepository, tag, found := strings.Cut(image, ":")
+ if !found || imageRepository != repository {
+ return fmt.Errorf("projector container image repository must equal %q", repository)
+ }
+ if !projectorImageTag.MatchString(tag) {
+ return fmt.Errorf("projector container image tag must match ^main-[0-9]+$")
+ }
+ return nil
+}
diff --git a/libs/fleet/backend/cmd/manifest-contract/main_test.go b/libs/fleet/backend/cmd/manifest-contract/main_test.go
new file mode 100644
index 0000000000..256afd1f13
--- /dev/null
+++ b/libs/fleet/backend/cmd/manifest-contract/main_test.go
@@ -0,0 +1,218 @@
+package main
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestVerifyAcceptsServingObjects(t *testing.T) {
+ t.Parallel()
+
+ if err := verify(strings.NewReader(validManifest())); err != nil {
+ t.Fatalf("verify() error = %v, want nil", err)
+ }
+}
+
+func TestVerifyRejectsPodTemplateLabelMismatch(t *testing.T) {
+ t.Parallel()
+
+ manifest := strings.Replace(
+ validManifest(),
+ " app: cyclops-cs\n spec:\n topologySpreadConstraints:",
+ " app: wrong-app\n spec:\n topologySpreadConstraints:",
+ 1,
+ )
+
+ err := verify(strings.NewReader(manifest))
+ if err == nil {
+ t.Fatal("verify() error = nil, want pod template label mismatch")
+ }
+ if !strings.Contains(err.Error(), "pod template app label") {
+ t.Fatalf("verify() error = %q, want pod template app label failure", err)
+ }
+}
+
+func TestVerifyRejectsStateProjector(t *testing.T) {
+ t.Parallel()
+
+ err := verify(strings.NewReader(validManifest() + `
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: k8s-state-projector
+ namespace: cyclops-cs
+`))
+ if err == nil {
+ t.Fatal("verify() error = nil, want state projector rejection")
+ }
+ if !strings.Contains(err.Error(), "state projector") {
+ t.Fatalf("verify() error = %q, want state projector failure", err)
+ }
+}
+
+func TestVerifyProjectorAcceptsExpectedImage(t *testing.T) {
+ t.Parallel()
+
+ manifest := strings.Replace(validProjectorManifest(), "PROJECTOR_IMAGE", "296062593712.dkr.ecr.us-west-2.amazonaws.com/cyclops-cs-backend:main-1786507556", 1)
+ if err := verifyProjector(strings.NewReader(manifest)); err != nil {
+ t.Fatalf("verifyProjector() error = %v, want nil", err)
+ }
+}
+
+func TestVerifyProjectorRejectsWrongImage(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ image string
+ want string
+ }{
+ {
+ name: "repository",
+ image: "296062593712.dkr.ecr.us-west-2.amazonaws.com/wrong-repository:main-1786507556",
+ want: "repository",
+ },
+ {
+ name: "tag",
+ image: "296062593712.dkr.ecr.us-west-2.amazonaws.com/cyclops-cs-backend:latest",
+ want: "tag",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ manifest := strings.Replace(validProjectorManifest(), "PROJECTOR_IMAGE", test.image, 1)
+
+ err := verifyProjector(strings.NewReader(manifest))
+ if err == nil {
+ t.Fatal("verifyProjector() error = nil, want image validation failure")
+ }
+ if !strings.Contains(err.Error(), test.want) {
+ t.Fatalf("verifyProjector() error = %q, want %q failure", err, test.want)
+ }
+ })
+ }
+}
+
+func validProjectorManifest() string {
+ return `apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: k8s-state-projector
+ namespace: cyclops-cs
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: cyclops-cs-k8s-state-projector
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: cyclops-cs-k8s-state-projector
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: k8s-state-projector
+ namespace: cyclops-cs
+spec:
+ template:
+ spec:
+ containers:
+ - name: projector
+ image: PROJECTOR_IMAGE
+`
+}
+
+func validManifest() string {
+ return `apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: cyclops-cs
+ namespace: cyclops-cs
+spec:
+ replicas: 2
+ progressDeadlineSeconds: 600
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 1
+ selector:
+ matchLabels:
+ app: cyclops-cs
+ template:
+ metadata:
+ labels:
+ app: cyclops-cs
+ spec:
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ labelSelector:
+ matchLabels:
+ app: cyclops-cs
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+ name: cyclops-cs
+ namespace: cyclops-cs
+spec:
+ minAvailable: 1
+ selector:
+ matchLabels:
+ app: cyclops-cs
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: cyclops-cs-backend
+ namespace: cyclops-cs
+spec:
+ replicas: 2
+ progressDeadlineSeconds: 600
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 1
+ selector:
+ matchLabels:
+ app: cyclops-cs-backend
+ template:
+ metadata:
+ labels:
+ app: cyclops-cs-backend
+ spec:
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ labelSelector:
+ matchLabels:
+ app: cyclops-cs-backend
+ containers:
+ - name: cyclops-cs-backend
+ readinessProbe:
+ httpGet:
+ path: /healthz
+ livenessProbe:
+ httpGet:
+ path: /healthz
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+ name: cyclops-cs-backend
+ namespace: cyclops-cs
+spec:
+ minAvailable: 1
+ selector:
+ matchLabels:
+ app: cyclops-cs-backend
+`
+}
diff --git a/libs/fleet/backend/database/migrator.go b/libs/fleet/backend/database/migrator.go
index dab6c57722..823e1aa008 100644
--- a/libs/fleet/backend/database/migrator.go
+++ b/libs/fleet/backend/database/migrator.go
@@ -772,6 +772,10 @@ func reconcileStaticRoleContracts(ctx context.Context, transaction pgx.Tx) ([]ro
return nil, nil, fmt.Errorf("read migration role: %w", err)
}
membershipContracts := staticMembershipContracts(migrationOwner)
+ membershipEvents, err := reconcileStaticMembershipContracts(ctx, transaction, migrationOwner, membershipContracts)
+ if err != nil {
+ return nil, nil, err
+ }
if err := validateStaticRoleMemberships(ctx, transaction, migrationOwner, membershipContracts); err != nil {
return nil, nil, err
}
@@ -789,45 +793,79 @@ func reconcileStaticRoleContracts(ctx context.Context, transaction pgx.Tx) ([]ro
roleEvents = append(roleEvents, roleReconciliationEvent{Role: contract.role, DurationMS: time.Since(started).Milliseconds()})
}
- membershipEvents := make([]membershipReconciliationEvent, 0, len(membershipContracts))
- for _, contract := range membershipContracts {
- started := time.Now()
+ if err := reconcileStaticRoleSettings(ctx, transaction); err != nil {
+ return nil, nil, err
+ }
+ return roleEvents, membershipEvents, nil
+}
+
+func reconcileStaticMembershipContracts(ctx context.Context, transaction pgx.Tx, migrationOwner string, membershipContracts []staticMembershipContract) ([]membershipReconciliationEvent, error) {
+ grantsByContract := make([][]staticMembershipGrant, len(membershipContracts))
+ for index, contract := range membershipContracts {
grants, err := readStaticMembershipGrants(ctx, transaction, contract)
if err != nil {
- return nil, nil, err
+ return nil, err
}
+ grantsByContract[index] = grants
+ }
+ repairs, err := staticMembershipReconciliationPlan(migrationOwner, membershipContracts, grantsByContract)
+ if err != nil {
+ return nil, err
+ }
- var migrationOwnerGrant *staticMembershipGrant
- for index := range grants {
- grant := &grants[index]
- if grant.grantor == migrationOwner {
- migrationOwnerGrant = grant
- continue
- }
- if allowsImplicitCreatorAdminMembership(contract.role, contract.member, migrationOwner, *grant) {
- continue
- }
- return nil, nil, foreignStaticMembershipGrantError(contract, grant.grantor, migrationOwner)
+ membershipEvents := make([]membershipReconciliationEvent, 0, len(membershipContracts))
+ for index, contract := range membershipContracts {
+ started := time.Now()
+ if !isStaticMembershipRepair(contract, repairs) {
+ membershipEvents = append(membershipEvents, membershipReconciliationEvent{Role: contract.role, DurationMS: time.Since(started).Milliseconds()})
+ continue
}
- if migrationOwnerGrant == nil || migrationOwnerGrant.admin != contract.admin || migrationOwnerGrant.inherit != contract.inherit || migrationOwnerGrant.set != contract.set {
- ddl := newRuntimeDDL(transaction)
- if migrationOwnerGrant != nil {
- if err := ddl.revokeStaticMembership(ctx, contract, migrationOwner); err != nil {
- return nil, nil, fmt.Errorf("remove drifted static role membership %s -> %s: %w", contract.role, contract.member, err)
- }
- }
- if err := ddl.grantStaticMembership(ctx, contract, migrationOwner); err != nil {
- return nil, nil, fmt.Errorf("reconcile static role membership %s -> %s: %w", contract.role, contract.member, err)
+ authoritativeGrant, err := authoritativeStaticMembershipGrant(contract, migrationOwner, grantsByContract[index])
+ if err != nil && !errors.Is(err, errStaticMembershipGrantMissing) && !errors.Is(err, errStaticMembershipGrantOwnerOptions) {
+ return nil, err
+ }
+
+ ddl := newRuntimeDDL(transaction)
+ if authoritativeGrant != nil {
+ if err := ddl.revokeStaticMembership(ctx, contract, migrationOwner); err != nil {
+ return nil, fmt.Errorf("remove drifted static role membership %s -> %s: %w", contract.role, contract.member, err)
}
}
+ if err := ddl.grantStaticMembership(ctx, contract, migrationOwner); err != nil {
+ return nil, fmt.Errorf("reconcile static role membership %s -> %s: %w", contract.role, contract.member, err)
+ }
membershipEvents = append(membershipEvents, membershipReconciliationEvent{Role: contract.role, DurationMS: time.Since(started).Milliseconds()})
}
+ return membershipEvents, nil
+}
- if err := reconcileStaticRoleSettings(ctx, transaction); err != nil {
- return nil, nil, err
+func staticMembershipReconciliationPlan(migrationOwner string, membershipContracts []staticMembershipContract, grantsByContract [][]staticMembershipGrant) ([]staticMembershipContract, error) {
+ if len(membershipContracts) != len(grantsByContract) {
+ return nil, fmt.Errorf("static membership reconciliation plan has %d contracts and %d grant sets", len(membershipContracts), len(grantsByContract))
}
- return roleEvents, membershipEvents, nil
+
+ repairs := make([]staticMembershipContract, 0, len(membershipContracts))
+ for index, contract := range membershipContracts {
+ _, err := authoritativeStaticMembershipGrant(contract, migrationOwner, grantsByContract[index])
+ if err == nil {
+ continue
+ }
+ if !errors.Is(err, errStaticMembershipGrantMissing) && !errors.Is(err, errStaticMembershipGrantOwnerOptions) {
+ return nil, err
+ }
+ repairs = append(repairs, contract)
+ }
+ return repairs, nil
+}
+
+func isStaticMembershipRepair(contract staticMembershipContract, repairs []staticMembershipContract) bool {
+ for _, repair := range repairs {
+ if repair == contract {
+ return true
+ }
+ }
+ return false
}
func foreignStaticMembershipGrantError(contract staticMembershipContract, grantor, migrationOwner string) error {
@@ -840,10 +878,9 @@ func validateStaticRoleMemberships(ctx context.Context, transaction pgx.Tx, migr
if err != nil {
return err
}
- for _, grant := range grants {
- if grant.grantor != migrationOwner && contract.member != migrationOwner && !allowsImplicitCreatorAdminMembership(contract.role, contract.member, migrationOwner, grant) {
- return foreignStaticMembershipGrantError(contract, grant.grantor, migrationOwner)
- }
+ _, err = authoritativeStaticMembershipGrant(contract, migrationOwner, grants)
+ if err != nil && !errors.Is(err, errStaticMembershipGrantMissing) && !errors.Is(err, errStaticMembershipGrantOwnerOptions) {
+ return err
}
}
@@ -873,8 +910,16 @@ func validateStaticRoleMembers(ctx context.Context, transaction pgx.Tx, migratio
if err != nil {
return err
}
- allowMigrationOwnerGrant := isDeclaredStaticMembership(role, migrationOwner, contracts)
- if !staticCreatorAdminMembershipsAreExact(migrationOwner, creatorGrants, allowMigrationOwnerGrant) {
+ declaredContract, allowMigrationOwnerGrant := declaredStaticMembershipContract(role, migrationOwner, contracts)
+ var declaredGrant *staticMembershipGrant
+ if allowMigrationOwnerGrant {
+ var err error
+ declaredGrant, err = authoritativeStaticMembershipGrant(declaredContract, migrationOwner, creatorGrants)
+ if err != nil {
+ return err
+ }
+ }
+ if !staticCreatorAdminMembershipsAreExact(migrationOwner, creatorGrants, allowMigrationOwnerGrant, declaredGrant) {
return staticCreatorAdminMembershipError(role, migrationOwner)
}
@@ -965,7 +1010,7 @@ func validateQueryTenantRoleMembers(ctx context.Context, transaction pgx.Tx, mig
if err != nil {
return nil, err
}
- if !staticCreatorAdminMembershipsAreExact(migrationOwner, creatorGrants, false) {
+ if !staticCreatorAdminMembershipsAreExact(migrationOwner, creatorGrants, false, nil) {
return nil, staticCreatorAdminMembershipError("k8s_query_tenant", migrationOwner)
}
for _, tenant := range dynamicTenants {
@@ -1038,10 +1083,14 @@ func validateRegisteredTenantRoleMemberships(ctx context.Context, transaction pg
return nil
}
-func staticCreatorAdminMembershipsAreExact(member string, grants []staticMembershipGrant, allowOwnerGrant bool) bool {
+func staticCreatorAdminMembershipsAreExact(member string, grants []staticMembershipGrant, allowOwnerGrant bool, declaredGrant *staticMembershipGrant) bool {
foreignCount := 0
ownerCount := 0
for _, grant := range grants {
+ if declaredGrant != nil && grant == *declaredGrant {
+ declaredGrant = nil
+ continue
+ }
if grant.grantor == member {
ownerCount++
continue
@@ -1094,13 +1143,18 @@ func allowsRegisteredTenantCreatorAdminMembership(tenant dynamicTenantRole, memb
grant.admin && !grant.inherit && !grant.set
}
-func isDeclaredStaticMembership(role, member string, contracts []staticMembershipContract) bool {
+func declaredStaticMembershipContract(role, member string, contracts []staticMembershipContract) (staticMembershipContract, bool) {
for _, contract := range contracts {
if contract.role == role && contract.member == member {
- return true
+ return contract, true
}
}
- return false
+ return staticMembershipContract{}, false
+}
+
+func isDeclaredStaticMembership(role, member string, contracts []staticMembershipContract) bool {
+ _, ok := declaredStaticMembershipContract(role, member, contracts)
+ return ok
}
func isLowerHex(value string) bool {
@@ -1255,6 +1309,40 @@ func allowsImplicitCreatorAdminMembership(role, member, migrationOwner string, g
!grant.set
}
+var (
+ errStaticMembershipGrantMissing = errors.New("static membership grant missing")
+ errStaticMembershipGrantOwnerOptions = errors.New("static membership grant owner options drifted")
+)
+
+func authoritativeStaticMembershipGrant(contract staticMembershipContract, migrationOwner string, grants []staticMembershipGrant) (*staticMembershipGrant, error) {
+ nonImplicitGrants := make([]staticMembershipGrant, 0, len(grants))
+ for _, grant := range grants {
+ if !allowsImplicitCreatorAdminMembership(contract.role, contract.member, migrationOwner, grant) {
+ nonImplicitGrants = append(nonImplicitGrants, grant)
+ }
+ }
+ if len(nonImplicitGrants) == 0 {
+ return nil, fmt.Errorf("static role membership %s -> %s: %w", contract.role, contract.member, errStaticMembershipGrantMissing)
+ }
+ for _, grant := range nonImplicitGrants {
+ if grant.grantor != migrationOwner && !grant.grantorSuperuser {
+ return nil, foreignStaticMembershipGrantError(contract, grant.grantor, migrationOwner)
+ }
+ }
+ if len(nonImplicitGrants) != 1 {
+ return nil, fmt.Errorf("static role membership %s -> %s has %d non-implicit grants; revoke duplicate grants without CASCADE before rerunning the migration", contract.role, contract.member, len(nonImplicitGrants))
+ }
+
+ grant := nonImplicitGrants[0]
+ if grant.admin != contract.admin || grant.inherit != contract.inherit || grant.set != contract.set {
+ if grant.grantor == migrationOwner {
+ return &grant, fmt.Errorf("static role membership %s -> %s: %w", contract.role, contract.member, errStaticMembershipGrantOwnerOptions)
+ }
+ return nil, fmt.Errorf("static role membership %s -> %s has options that do not match its contract", contract.role, contract.member)
+ }
+ return &grant, nil
+}
+
func readStaticMembershipGrants(ctx context.Context, transaction pgx.Tx, contract staticMembershipContract) ([]staticMembershipGrant, error) {
rows, err := transaction.Query(ctx, `
select grantor_role.rolname, grantor_role.rolsuper, membership.admin_option, membership.inherit_option, membership.set_option
diff --git a/libs/fleet/backend/database/migrator_pg_test.go b/libs/fleet/backend/database/migrator_pg_test.go
index d0e5d4289f..9be1ac17d7 100644
--- a/libs/fleet/backend/database/migrator_pg_test.go
+++ b/libs/fleet/backend/database/migrator_pg_test.go
@@ -570,6 +570,38 @@ func TestRunFailsClosedForStaticRoleCreateDBDrift(t *testing.T) {
}
}
+func TestRunPreservesMaintenanceSuperuserStaticMembershipGrant(t *testing.T) {
+ maintenanceURL := requireMigratorIntegration(t)
+ ctx := context.Background()
+ migrationURL, credentials := isolatedMigrationDatabase(t, ctx, maintenanceURL)
+ if err := Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials}); err != nil {
+ t.Fatal(err)
+ }
+
+ migrationOwner := currentRole(t, ctx, migrationURL)
+ maintenance := connect(t, ctx, maintenanceURLForDatabase(t, maintenanceURL, migrationURL))
+ defer maintenance.Close(ctx)
+ migrationOwnerIdentifier := pgx.Identifier{migrationOwner}.Sanitize()
+ if _, err := maintenance.Exec(ctx, "revoke k8s_query_tenant from k8s_role_admin granted by "+migrationOwnerIdentifier+" restrict"); err != nil {
+ t.Fatal("remove migration-owner static membership")
+ }
+ if _, err := maintenance.Exec(ctx, `grant k8s_query_tenant to k8s_role_admin with admin true, inherit false, set false`); err != nil {
+ t.Fatal("create maintenance-superuser static membership")
+ }
+
+ before := staticMembershipRows(t, ctx, maintenance, "k8s_query_tenant", "k8s_role_admin")
+ if len(before) != 1 || before[0].grantor == migrationOwner || !before[0].grantorSuperuser || !before[0].admin || before[0].inherit || before[0].set {
+ t.Fatalf("maintenance-superuser static membership = %+v, want one exact grant from a superuser other than %s", before, migrationOwner)
+ }
+
+ if err := Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials}); err != nil {
+ t.Fatal(err)
+ }
+ if after := staticMembershipRows(t, ctx, maintenance, "k8s_query_tenant", "k8s_role_admin"); !reflect.DeepEqual(after, before) {
+ t.Fatalf("maintenance-superuser static membership changed during rerun: before=%+v after=%+v", before, after)
+ }
+}
+
func TestRunFailsClosedForForeignGrantorMembership(t *testing.T) {
maintenanceURL := os.Getenv("CYCLOPS_TEST_DATABASE_URL")
if maintenanceURL == "" {
@@ -632,6 +664,134 @@ func TestRunFailsClosedForForeignGrantorMembership(t *testing.T) {
}
}
+func TestRunPreflightLeavesEarlierRepairableStaticMembershipUnchanged(t *testing.T) {
+ maintenanceURL := requireMigratorIntegration(t)
+
+ ctx := context.Background()
+
+ for _, testCase := range []struct {
+ name string
+ introduceFirst func(t *testing.T, maintenance *pgx.Conn, migrationOwner string)
+ introduceLater func(t *testing.T, maintenance *pgx.Conn, migrationURL, migrationOwner string)
+ wantError string
+ }{
+ {
+ name: "missing before foreign membership",
+ introduceFirst: func(t *testing.T, maintenance *pgx.Conn, migrationOwner string) {
+ t.Helper()
+ identifier := pgx.Identifier{migrationOwner}.Sanitize()
+ if _, err := maintenance.Exec(ctx, "revoke k8s_state_owner from "+identifier+" granted by "+identifier+" restrict"); err != nil {
+ t.Fatal("remove repairable state-owner membership")
+ }
+ },
+ introduceLater: func(t *testing.T, maintenance *pgx.Conn, migrationURL, migrationOwner string) {
+ t.Helper()
+ foreignGrantor := "foreign_grantor_" + testToken(t)[:12]
+ foreignIdentifier := pgx.Identifier{foreignGrantor}.Sanitize()
+ t.Cleanup(func() {
+ cleanupCtx := context.Background()
+ connection := connect(t, cleanupCtx, maintenanceURLForDatabase(t, maintenanceURL, migrationURL))
+ defer connection.Close(cleanupCtx)
+ if _, err := connection.Exec(cleanupCtx, "drop owned by "+foreignIdentifier); err != nil {
+ t.Errorf("drop foreign grantor-owned objects: %v", err)
+ }
+ dropRole(t, cleanupCtx, maintenanceURL, foreignGrantor)
+ })
+ if _, err := maintenance.Exec(ctx, "create role "+foreignIdentifier); err != nil {
+ t.Fatal("create foreign grantor")
+ }
+ if _, err := maintenance.Exec(ctx, "grant k8s_query_tenant to "+foreignIdentifier+" with admin true, inherit false, set false"); err != nil {
+ t.Fatal("grant query tenant admin to foreign grantor")
+ }
+ identifier := pgx.Identifier{migrationOwner}.Sanitize()
+ if _, err := maintenance.Exec(ctx, "revoke k8s_query_tenant from k8s_role_admin granted by "+identifier+" restrict"); err != nil {
+ t.Fatal("remove migration-owner query tenant membership")
+ }
+ if _, err := maintenance.Exec(ctx, "set role "+foreignIdentifier); err != nil {
+ t.Fatal("set foreign grantor role")
+ }
+ if _, err := maintenance.Exec(ctx, `grant k8s_query_tenant to k8s_role_admin with admin true, inherit false, set false`); err != nil {
+ t.Fatal("create foreign static membership")
+ }
+ if _, err := maintenance.Exec(ctx, `reset role`); err != nil {
+ t.Fatal("reset foreign grantor role")
+ }
+ },
+ wantError: "has grantor foreign_grantor_",
+ },
+ {
+ name: "SET drift before duplicate membership",
+ introduceFirst: func(t *testing.T, maintenance *pgx.Conn, migrationOwner string) {
+ t.Helper()
+ identifier := pgx.Identifier{migrationOwner}.Sanitize()
+ if _, err := maintenance.Exec(ctx, "revoke k8s_state_owner from "+identifier+" granted by "+identifier+" restrict"); err != nil {
+ t.Fatal("remove state-owner membership before SET drift")
+ }
+ if _, err := maintenance.Exec(ctx, "grant k8s_state_owner to "+identifier+" with admin false, inherit false, set false granted by "+identifier); err != nil {
+ t.Fatal("introduce repairable state-owner SET drift")
+ }
+ },
+ introduceLater: func(t *testing.T, maintenance *pgx.Conn, migrationURL, migrationOwner string) {
+ t.Helper()
+ if _, err := maintenance.Exec(ctx, `grant k8s_query_tenant to k8s_role_admin with admin true, inherit false, set false`); err != nil {
+ t.Fatal("create postgres-grantor duplicate static membership")
+ }
+ },
+ wantError: "has 2 non-implicit grants",
+ },
+ } {
+ t.Run(testCase.name, func(t *testing.T) {
+ migrationURL, credentials := isolatedMigrationDatabase(t, ctx, maintenanceURL)
+ if err := Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials}); err != nil {
+ t.Fatal("initial Run()", err)
+ }
+ migrationOwner := currentRole(t, ctx, migrationURL)
+ maintenance := connect(t, ctx, maintenanceURL)
+ defer maintenance.Close(ctx)
+ testCase.introduceFirst(t, maintenance, migrationOwner)
+ testCase.introduceLater(t, maintenance, migrationURL, migrationOwner)
+
+ connection := connect(t, ctx, migrationURL)
+ defer connection.Close(ctx)
+ before := staticMembershipRows(t, ctx, connection, "k8s_state_owner", migrationOwner)
+ err := Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials})
+ if err == nil || !strings.Contains(err.Error(), testCase.wantError) {
+ t.Fatalf("Run() error = %v, want fail-closed error containing %q", err, testCase.wantError)
+ }
+ if after := staticMembershipRows(t, ctx, connection, "k8s_state_owner", migrationOwner); !reflect.DeepEqual(after, before) {
+ t.Fatalf("repairable state-owner membership changed after fail-closed preflight: before=%+v after=%+v", before, after)
+ }
+ })
+ }
+}
+
+func staticMembershipRows(t *testing.T, ctx context.Context, connection *pgx.Conn, role, member string) []staticMembershipGrant {
+ t.Helper()
+ rows, err := connection.Query(ctx, `
+ select grantor_role.rolname, grantor_role.rolsuper, membership.admin_option, membership.inherit_option, membership.set_option
+ from pg_auth_members as membership
+ join pg_roles as grantor_role on grantor_role.oid = membership.grantor
+ where membership.roleid = $1::regrole and membership.member = $2::regrole
+ order by grantor_role.rolname`, role, member)
+ if err != nil {
+ t.Fatalf("read static membership %s -> %s: %v", role, member, err)
+ }
+ defer rows.Close()
+
+ var grants []staticMembershipGrant
+ for rows.Next() {
+ var grant staticMembershipGrant
+ if err := rows.Scan(&grant.grantor, &grant.grantorSuperuser, &grant.admin, &grant.inherit, &grant.set); err != nil {
+ t.Fatalf("scan static membership %s -> %s: %v", role, member, err)
+ }
+ grants = append(grants, grant)
+ }
+ if err := rows.Err(); err != nil {
+ t.Fatalf("iterate static membership %s -> %s: %v", role, member, err)
+ }
+ return grants
+}
+
func TestRunFailsClosedForNonSuperuserStaticCreatorAdminGrantor(t *testing.T) {
maintenanceURL := requireMigratorIntegration(t)
ctx := context.Background()
@@ -667,7 +827,7 @@ func TestRunFailsClosedForNonSuperuserStaticCreatorAdminGrantor(t *testing.T) {
t.Fatal("introduce static role repair sentinel")
}
- assertRunFailsBeforeStaticRoleRepair(t, ctx, migrationURL, credentials, "true PostgreSQL superuser")
+ assertRunFailsBeforeStaticRoleRepair(t, ctx, migrationURL, credentials, "has grantor")
}
func TestRunFailsClosedForRegisteredTenantMembershipDrift(t *testing.T) {
@@ -1248,7 +1408,7 @@ func assertStaticCreatorAdminMemberships(t *testing.T, ctx context.Context, migr
}
rows.Close()
allowOwnerGrant := role == "k8s_state_owner" || role == "k8s_reporting_owner"
- if !staticCreatorAdminMembershipsAreExact(migrationOwner, grants, allowOwnerGrant) {
+ if !staticCreatorAdminMembershipsAreExact(migrationOwner, grants, allowOwnerGrant, nil) {
t.Fatalf("static creator memberships for %s = %+v", role, grants)
}
}
diff --git a/libs/fleet/backend/database/migrator_test.go b/libs/fleet/backend/database/migrator_test.go
index e6d3b9543d..08813676b7 100644
--- a/libs/fleet/backend/database/migrator_test.go
+++ b/libs/fleet/backend/database/migrator_test.go
@@ -4,6 +4,7 @@ import (
"bytes"
"crypto/sha256"
"encoding/hex"
+ "errors"
"log/slog"
"os"
"reflect"
@@ -175,12 +176,19 @@ func TestAllowsRegisteredTenantCreatorAdminMembership(t *testing.T) {
func TestStaticCreatorAdminMembershipsRequireOneForeignSuperuserGrant(t *testing.T) {
foreign := staticMembershipGrant{grantor: "bootstrap_owner", grantorSuperuser: true, admin: true, inherit: false, set: false}
ownerDrift := staticMembershipGrant{grantor: "migration_owner", admin: true, inherit: true, set: true}
- if !staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{foreign}, false) {
+ if !staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{foreign}, false, nil) {
t.Fatal("expected one foreign superuser creator row to be allowed")
}
- if !staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{foreign, ownerDrift}, true) {
+ if !staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{foreign, ownerDrift}, true, nil) {
t.Fatal("expected one repairable migration-owner row to be allowed")
}
+ delegatedContractGrant := staticMembershipGrant{grantor: "rdsadmin", grantorSuperuser: true, admin: false, inherit: false, set: true}
+ if !staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{foreign, delegatedContractGrant}, true, &delegatedContractGrant) {
+ t.Fatal("expected an authoritative delegated contract grant to be ignored")
+ }
+ if staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{foreign, delegatedContractGrant, {grantor: "other_bootstrap_owner", grantorSuperuser: true, admin: true, inherit: false, set: false}}, true, &delegatedContractGrant) {
+ t.Fatal("unexpectedly allowed an unrelated foreign creator grant")
+ }
for _, testCase := range []struct {
name string
@@ -195,13 +203,85 @@ func TestStaticCreatorAdminMembershipsRequireOneForeignSuperuserGrant(t *testing
{name: "duplicate owner row", grants: []staticMembershipGrant{foreign, ownerDrift, {grantor: "migration_owner", admin: false, inherit: false, set: true}}, allowOwnerGrant: true},
} {
t.Run(testCase.name, func(t *testing.T) {
- if staticCreatorAdminMembershipsAreExact("migration_owner", testCase.grants, testCase.allowOwnerGrant) {
+ if staticCreatorAdminMembershipsAreExact("migration_owner", testCase.grants, testCase.allowOwnerGrant, nil) {
t.Fatal("unexpectedly allowed static creator-admin memberships")
}
})
}
}
+func TestStaticMembershipReconciliationPlanRepairsOnlyAfterFailClosedPreflight(t *testing.T) {
+ stateOwner := staticMembershipContract{role: "k8s_state_owner", member: "migration_owner", admin: false, inherit: false, set: true}
+ reportingOwner := staticMembershipContract{role: "k8s_reporting_owner", member: "migration_owner", admin: false, inherit: false, set: true}
+ healthyGrant := staticMembershipGrant{grantor: "migration_owner", admin: false, inherit: false, set: true}
+ driftedStateOwnerGrant := staticMembershipGrant{grantor: "migration_owner", admin: false, inherit: false, set: false}
+
+ for _, testCase := range []struct {
+ name string
+ grants [][]staticMembershipGrant
+ wantRepairs []staticMembershipContract
+ wantError string
+ }{
+ {
+ name: "missing state-owner membership",
+ grants: [][]staticMembershipGrant{nil, {healthyGrant}},
+ wantRepairs: []staticMembershipContract{stateOwner},
+ },
+ {
+ name: "set-drifted state-owner membership",
+ grants: [][]staticMembershipGrant{{driftedStateOwnerGrant}, {healthyGrant}},
+ wantRepairs: []staticMembershipContract{stateOwner},
+ },
+ {
+ name: "foreign grant blocks earlier state-owner repair",
+ grants: [][]staticMembershipGrant{
+ nil,
+ {{grantor: "foreign_owner", admin: false, inherit: false, set: true}},
+ },
+ wantError: "grantor foreign_owner",
+ },
+ {
+ name: "duplicate grant blocks earlier state-owner repair",
+ grants: [][]staticMembershipGrant{
+ nil,
+ {healthyGrant, {grantor: "rdsadmin", grantorSuperuser: true, admin: false, inherit: false, set: true}},
+ },
+ wantError: "2 non-implicit grants",
+ },
+ } {
+ t.Run(testCase.name, func(t *testing.T) {
+ repairs, err := staticMembershipReconciliationPlan("migration_owner", []staticMembershipContract{stateOwner, reportingOwner}, testCase.grants)
+ if testCase.wantError != "" {
+ if err == nil || !strings.Contains(err.Error(), testCase.wantError) {
+ t.Fatalf("staticMembershipReconciliationPlan() error = %v, want containing %q", err, testCase.wantError)
+ }
+ if repairs != nil {
+ t.Fatalf("staticMembershipReconciliationPlan() repairs = %#v, want nil before any mutation", repairs)
+ }
+ return
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(repairs, testCase.wantRepairs) {
+ t.Fatalf("staticMembershipReconciliationPlan() repairs = %#v, want %#v", repairs, testCase.wantRepairs)
+ }
+ })
+ }
+}
+
+func TestStaticCreatorAdminMembershipsAllowDelegatedContractGrantAlongsideImplicitCreatorGrant(t *testing.T) {
+ implicitCreatorGrant := staticMembershipGrant{grantor: "bootstrap_owner", grantorSuperuser: true, admin: true, inherit: false, set: false}
+ delegatedContractGrant := staticMembershipGrant{grantor: "rdsadmin", grantorSuperuser: true, admin: false, inherit: false, set: true}
+
+ if !staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{implicitCreatorGrant, delegatedContractGrant}, true, &delegatedContractGrant) {
+ t.Fatal("expected exact delegated contract and implicit creator grants to coexist")
+ }
+ foreignCreatorGrant := staticMembershipGrant{grantor: "foreign_creator", grantorSuperuser: true, admin: true, inherit: false, set: false}
+ if staticCreatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{implicitCreatorGrant, delegatedContractGrant, foreignCreatorGrant}, true, &delegatedContractGrant) {
+ t.Fatal("unexpectedly allowed unrelated foreign creator grant")
+ }
+}
func TestCreatorAdminMembershipsRequireOneForeignSuperuserGrant(t *testing.T) {
valid := staticMembershipGrant{grantor: "bootstrap_owner", grantorSuperuser: true, admin: true, inherit: false, set: false}
if !creatorAdminMembershipsAreExact("migration_owner", []staticMembershipGrant{valid}) {
@@ -289,6 +369,130 @@ func TestAllowsImplicitCreatorAdminMembership(t *testing.T) {
}
}
+func TestAuthoritativeStaticMembershipGrant(t *testing.T) {
+ declaredContract := staticMembershipContract{
+ role: "k8s_query_tenant", member: "k8s_role_admin",
+ admin: true, inherit: false, set: false,
+ }
+ creatorContract := staticMembershipContract{
+ role: "k8s_state_owner", member: "migration_owner",
+ admin: false, inherit: false, set: true,
+ }
+ implicitCreatorGrant := staticMembershipGrant{
+ grantor: "bootstrap_owner", grantorSuperuser: true,
+ admin: true, inherit: false, set: false,
+ }
+
+ for _, testCase := range []struct {
+ name string
+ contract staticMembershipContract
+ grants []staticMembershipGrant
+ wantGrantor string
+ wantError bool
+ wantErrorIs error
+ wantReturnedGrant bool
+ }{
+ {
+ name: "migration owner grant",
+ contract: declaredContract,
+ grants: []staticMembershipGrant{{grantor: "migration_owner", admin: true, inherit: false, set: false}},
+ wantGrantor: "migration_owner",
+ },
+ {
+ name: "delegated superuser exact grant",
+ contract: declaredContract,
+ grants: []staticMembershipGrant{{grantor: "rdsadmin", grantorSuperuser: true, admin: true, inherit: false, set: false}},
+ wantGrantor: "rdsadmin",
+ },
+ {
+ name: "implicit creator grant is ignored beside owner grant",
+ contract: creatorContract,
+ grants: []staticMembershipGrant{implicitCreatorGrant, {grantor: "migration_owner", admin: false, inherit: false, set: true}},
+ wantGrantor: "migration_owner",
+ },
+ {
+ name: "missing grant",
+ contract: declaredContract,
+ wantError: true,
+ wantErrorIs: errStaticMembershipGrantMissing,
+ },
+ {
+ name: "implicit creator grant does not satisfy declared contract",
+ contract: creatorContract,
+ grants: []staticMembershipGrant{implicitCreatorGrant},
+ wantError: true,
+ wantErrorIs: errStaticMembershipGrantMissing,
+ },
+ {
+ name: "non-superuser grantor",
+ contract: declaredContract,
+ grants: []staticMembershipGrant{{grantor: "foreign_owner", admin: true, inherit: false, set: false}},
+ wantError: true,
+ },
+ {
+ name: "owner admin option drift",
+ contract: declaredContract,
+ grants: []staticMembershipGrant{{grantor: "migration_owner", admin: false, inherit: false, set: false}},
+ wantError: true,
+ wantErrorIs: errStaticMembershipGrantOwnerOptions,
+ wantReturnedGrant: true,
+ },
+ {
+ name: "owner inherit option drift",
+ contract: declaredContract,
+ grants: []staticMembershipGrant{{grantor: "migration_owner", admin: true, inherit: true, set: false}},
+ wantError: true,
+ wantErrorIs: errStaticMembershipGrantOwnerOptions,
+ wantReturnedGrant: true,
+ },
+ {
+ name: "owner set option drift",
+ contract: declaredContract,
+ grants: []staticMembershipGrant{{grantor: "migration_owner", admin: true, inherit: false, set: true}},
+ wantError: true,
+ wantErrorIs: errStaticMembershipGrantOwnerOptions,
+ wantReturnedGrant: true,
+ },
+ {
+ name: "delegated superuser option drift",
+ contract: declaredContract,
+ grants: []staticMembershipGrant{{grantor: "rdsadmin", grantorSuperuser: true, admin: true, inherit: true, set: false}},
+ wantError: true,
+ },
+ {
+ name: "owner and superuser duplicates",
+ contract: declaredContract,
+ grants: []staticMembershipGrant{
+ {grantor: "migration_owner", admin: true, inherit: false, set: false},
+ {grantor: "rdsadmin", grantorSuperuser: true, admin: true, inherit: false, set: false},
+ },
+ wantError: true,
+ },
+ } {
+ t.Run(testCase.name, func(t *testing.T) {
+ grant, err := authoritativeStaticMembershipGrant(testCase.contract, "migration_owner", testCase.grants)
+ if testCase.wantError {
+ if err == nil {
+ t.Fatal("expected an error")
+ }
+ if testCase.wantErrorIs != nil && !errors.Is(err, testCase.wantErrorIs) {
+ t.Fatalf("error = %v, want errors.Is(_, %v)", err, testCase.wantErrorIs)
+ }
+ if (grant != nil) != testCase.wantReturnedGrant {
+ t.Fatalf("returned grant = %#v, want present %t", grant, testCase.wantReturnedGrant)
+ }
+ return
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if grant.grantor != testCase.wantGrantor {
+ t.Fatalf("grantor = %q, want %q", grant.grantor, testCase.wantGrantor)
+ }
+ })
+ }
+}
+
func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {
files, err := embeddedMigrations()
if err != nil {
diff --git a/libs/fleet/backend/database/runtime_ddl.go b/libs/fleet/backend/database/runtime_ddl.go
index 6b830f1467..72016e6bcf 100644
--- a/libs/fleet/backend/database/runtime_ddl.go
+++ b/libs/fleet/backend/database/runtime_ddl.go
@@ -299,17 +299,22 @@ func (ddl runtimeDDL) revokeStaticMembership(ctx context.Context, contract stati
}
func (ddl runtimeDDL) grantStaticMembership(ctx context.Context, contract staticMembershipContract, grantor string) error {
- _, err := ddl.transaction.Exec(ctx,
- "grant "+pgx.Identifier{contract.role}.Sanitize()+
- " to "+pgx.Identifier{contract.member}.Sanitize()+
- " with admin "+postgresBoolean(contract.admin)+
- ", inherit "+postgresBoolean(contract.inherit)+
- ", set "+postgresBoolean(contract.set)+
- " granted by "+pgx.Identifier{grantor}.Sanitize(),
- )
+ _, err := ddl.transaction.Exec(ctx, staticMembershipGrantStatement(contract, grantor))
return err
}
+func staticMembershipGrantStatement(contract staticMembershipContract, grantor string) string {
+ statement := "grant " + pgx.Identifier{contract.role}.Sanitize() +
+ " to " + pgx.Identifier{contract.member}.Sanitize() +
+ " with admin " + postgresBoolean(contract.admin) +
+ ", inherit " + postgresBoolean(contract.inherit) +
+ ", set " + postgresBoolean(contract.set)
+ if !contract.admin {
+ statement += " granted by " + pgx.Identifier{grantor}.Sanitize()
+ }
+ return statement
+}
+
func (ddl runtimeDDL) reconcileStaticRoleSettings(ctx context.Context, role staticRoleContract, actual []staticRoleSetting, desired staticRoleSettingsContract) error {
if staticRoleSettingsMatch(actual, desired) {
return nil
diff --git a/libs/fleet/backend/database/runtime_ddl_test.go b/libs/fleet/backend/database/runtime_ddl_test.go
index b9b0052b92..f6691e00a1 100644
--- a/libs/fleet/backend/database/runtime_ddl_test.go
+++ b/libs/fleet/backend/database/runtime_ddl_test.go
@@ -16,6 +16,39 @@ import (
const expectedInsertAppliedMigrationStatement = `insert into cyclops_migrations.applied_migrations (version, filename, sha256) values ($1, $2, $3)`
+func TestRuntimeDDLStaticMembershipGrantorSelection(t *testing.T) {
+ tests := []struct {
+ name string
+ contract staticMembershipContract
+ want string
+ }{
+ {
+ name: "ADMIN TRUE lets PostgreSQL select the grantor",
+ contract: staticMembershipContract{
+ role: "k8s_query_tenant", member: "k8s_role_admin",
+ admin: true, inherit: false, set: false,
+ },
+ want: `grant "k8s_query_tenant" to "k8s_role_admin" with admin TRUE, inherit FALSE, set FALSE`,
+ },
+ {
+ name: "ADMIN FALSE retains the migration owner grantor",
+ contract: staticMembershipContract{
+ role: "k8s_query_tenant", member: "tenant_query_role",
+ admin: false, inherit: true, set: false,
+ },
+ want: `grant "k8s_query_tenant" to "tenant_query_role" with admin FALSE, inherit TRUE, set FALSE granted by "migration_owner"`,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ if got := staticMembershipGrantStatement(test.contract, "migration_owner"); got != test.want {
+ t.Fatalf("staticMembershipGrantStatement() = %q, want %q", got, test.want)
+ }
+ })
+ }
+}
+
func TestMigrationAuthorityExecArguments(t *testing.T) {
packageDir := databasePackageDir(t)
entries, err := os.ReadDir(packageDir)
diff --git a/libs/fleet/backend/go.mod b/libs/fleet/backend/go.mod
index 34701ae3a4..bb12006001 100644
--- a/libs/fleet/backend/go.mod
+++ b/libs/fleet/backend/go.mod
@@ -19,6 +19,7 @@ require (
github.com/stripe/stripe-go/v85 v85.2.0
github.com/swaggo/http-swagger/v2 v2.0.2
github.com/swaggo/swag v1.16.6
+ sigs.k8s.io/yaml v1.4.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0
golang.org/x/sync v0.17.0
@@ -112,7 +113,6 @@ require (
golang.org/x/tools v0.37.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
- sigs.k8s.io/yaml v1.4.0 // indirect
)
replace github.com/trycua/cloud/pkg/featureflags => ../../pkg/featureflags
diff --git a/libs/fleet/backend/handlers/database_context.go b/libs/fleet/backend/handlers/database_context.go
new file mode 100644
index 0000000000..ce96f9bbc9
--- /dev/null
+++ b/libs/fleet/backend/handlers/database_context.go
@@ -0,0 +1,13 @@
+package handlers
+
+import (
+ "context"
+
+ "cyclops-cs-backend/auth"
+)
+
+const databaseRequestTimeout = auth.DatabaseRequestTimeout
+
+func databaseContext(parent context.Context) (context.Context, context.CancelFunc) {
+ return auth.DatabaseContext(parent)
+}
diff --git a/libs/fleet/backend/handlers/github_trust_policies.go b/libs/fleet/backend/handlers/github_trust_policies.go
index a38eaebe8f..d393cc9871 100644
--- a/libs/fleet/backend/handlers/github_trust_policies.go
+++ b/libs/fleet/backend/handlers/github_trust_policies.go
@@ -47,9 +47,11 @@ func (h Handlers) ListGitHubTrustPolicies(w http.ResponseWriter, r *http.Request
writeErr(w, http.StatusServiceUnavailable, "github trust policies are not configured")
return
}
- policies, err := h.GitHubTrustPolicies.List(r.Context(), user.ID)
+ ctx, cancel := databaseContext(r.Context())
+ defer cancel()
+ policies, err := h.GitHubTrustPolicies.List(ctx, user.ID)
if err != nil {
- writeErr(w, http.StatusInternalServerError, "failed to list github trust policies")
+ writeGitHubTrustStoreErr(w, err, "failed to list github trust policies")
return
}
resp := GitHubTrustPolicyListResponse{
@@ -84,8 +86,10 @@ func (h Handlers) CreateGitHubTrustPolicy(w http.ResponseWriter, r *http.Request
return
}
policy.OwnerSub = user.ID
- if err := h.GitHubTrustPolicies.Create(r.Context(), policy); err != nil {
- writeErr(w, http.StatusInternalServerError, "failed to create github trust policy")
+ ctx, cancel := databaseContext(r.Context())
+ defer cancel()
+ if err := h.GitHubTrustPolicies.Create(ctx, policy); err != nil {
+ writeGitHubTrustStoreErr(w, err, "failed to create github trust policy")
return
}
writeJSON(w, http.StatusCreated, policyResponse(policy))
@@ -102,9 +106,11 @@ func (h Handlers) UpdateGitHubTrustPolicy(w http.ResponseWriter, r *http.Request
return
}
id := r.PathValue("id")
- current, err := h.GitHubTrustPolicies.Get(r.Context(), user.ID, id)
+ ctx, cancel := databaseContext(r.Context())
+ defer cancel()
+ current, err := h.GitHubTrustPolicies.Get(ctx, user.ID, id)
if err != nil {
- writeErr(w, http.StatusInternalServerError, "failed to load github trust policy")
+ writeGitHubTrustStoreErr(w, err, "failed to load github trust policy")
return
}
if current == nil {
@@ -142,8 +148,12 @@ func (h Handlers) UpdateGitHubTrustPolicy(w http.ResponseWriter, r *http.Request
policy.ID = current.ID
policy.OwnerSub = current.OwnerSub
policy.CreatedAt = current.CreatedAt
- if err := h.GitHubTrustPolicies.Update(r.Context(), policy); err != nil {
- writeErr(w, http.StatusInternalServerError, "failed to update github trust policy")
+ if err := h.GitHubTrustPolicies.Update(ctx, policy); err != nil {
+ if errors.Is(err, githubtrust.ErrNotFound) {
+ writeErr(w, http.StatusNotFound, "github trust policy not found")
+ return
+ }
+ writeGitHubTrustStoreErr(w, err, "failed to update github trust policy")
return
}
writeJSON(w, http.StatusOK, policyResponse(policy))
@@ -159,9 +169,11 @@ func (h Handlers) DeleteGitHubTrustPolicy(w http.ResponseWriter, r *http.Request
writeErr(w, http.StatusServiceUnavailable, "github trust policies are not configured")
return
}
- found, err := h.GitHubTrustPolicies.Delete(r.Context(), user.ID, r.PathValue("id"))
+ ctx, cancel := databaseContext(r.Context())
+ defer cancel()
+ found, err := h.GitHubTrustPolicies.Delete(ctx, user.ID, r.PathValue("id"))
if err != nil {
- writeErr(w, http.StatusInternalServerError, "failed to delete github trust policy")
+ writeGitHubTrustStoreErr(w, err, "failed to delete github trust policy")
return
}
if !found {
@@ -183,8 +195,14 @@ type githubTrustResolver struct {
}
func (r githubTrustResolver) ResolveGitHubTrustPolicies(ctx context.Context, repository string) ([]auth.GitHubTrustPolicy, error) {
- policies, err := r.store.ResolveByRepository(ctx, repository)
+ databaseCtx, cancel := databaseContext(ctx)
+ defer cancel()
+ policies, err := r.store.ResolveByRepository(databaseCtx, repository)
if err != nil {
+ err = auth.ClassifyDatabaseError(err)
+ if auth.IsDatabaseUnavailable(err) {
+ return nil, auth.DatabaseUnavailable(err)
+ }
return nil, err
}
out := make([]auth.GitHubTrustPolicy, 0, len(policies))
@@ -213,6 +231,15 @@ func policyResponse(policy *githubtrust.Policy) GitHubTrustPolicyResponse {
}
}
+func writeGitHubTrustStoreErr(w http.ResponseWriter, err error, fallback string) {
+ err = auth.ClassifyDatabaseError(err)
+ if auth.IsDatabaseUnavailable(err) {
+ writeErr(w, http.StatusServiceUnavailable, "github trust policies unavailable")
+ return
+ }
+ writeErr(w, http.StatusInternalServerError, fallback)
+}
+
func writeTrustPolicyValidationErr(w http.ResponseWriter, err error) {
switch {
case errors.Is(err, githubtrust.ErrInvalidRepository),
diff --git a/libs/fleet/backend/handlers/github_trust_policies_test.go b/libs/fleet/backend/handlers/github_trust_policies_test.go
index e529d449c5..a363cad456 100644
--- a/libs/fleet/backend/handlers/github_trust_policies_test.go
+++ b/libs/fleet/backend/handlers/github_trust_policies_test.go
@@ -5,17 +5,26 @@ import (
"context"
"encoding/json"
"errors"
+ "net"
"net/http"
"net/http/httptest"
+ "syscall"
"testing"
"time"
"cyclops-cs-backend/auth"
"cyclops-cs-backend/config"
"cyclops-cs-backend/githubtrust"
+
+ "github.com/jackc/pgx/v5/pgconn"
)
type fakeGitHubTrustStore struct {
+ listContext context.Context
+ createContext context.Context
+ getContext context.Context
+ updateContext context.Context
+ deleteContext context.Context
listByOwner []*githubtrust.Policy
createInput *githubtrust.Policy
updateInput *githubtrust.Policy
@@ -31,10 +40,11 @@ type fakeGitHubTrustStore struct {
resolveResult []*githubtrust.Policy
resolveErr error
resolvedRepoName string
+ resolveContext context.Context
}
func (f *fakeGitHubTrustStore) List(ctx context.Context, ownerSub string) ([]*githubtrust.Policy, error) {
- _ = ctx
+ f.listContext = ctx
if f.listErr != nil {
return nil, f.listErr
}
@@ -44,7 +54,7 @@ func (f *fakeGitHubTrustStore) List(ctx context.Context, ownerSub string) ([]*gi
}
func (f *fakeGitHubTrustStore) Create(ctx context.Context, policy *githubtrust.Policy) error {
- _ = ctx
+ f.createContext = ctx
if f.createErr != nil {
return f.createErr
}
@@ -54,7 +64,7 @@ func (f *fakeGitHubTrustStore) Create(ctx context.Context, policy *githubtrust.P
}
func (f *fakeGitHubTrustStore) Get(ctx context.Context, ownerSub, id string) (*githubtrust.Policy, error) {
- _ = ctx
+ f.getContext = ctx
_ = ownerSub
_ = id
if f.getErr != nil {
@@ -68,7 +78,7 @@ func (f *fakeGitHubTrustStore) Get(ctx context.Context, ownerSub, id string) (*g
}
func (f *fakeGitHubTrustStore) Update(ctx context.Context, policy *githubtrust.Policy) error {
- _ = ctx
+ f.updateContext = ctx
if f.updateErr != nil {
return f.updateErr
}
@@ -78,7 +88,7 @@ func (f *fakeGitHubTrustStore) Update(ctx context.Context, policy *githubtrust.P
}
func (f *fakeGitHubTrustStore) Delete(ctx context.Context, ownerSub, id string) (bool, error) {
- _ = ctx
+ f.deleteContext = ctx
f.deleteOwner = ownerSub
f.deleteID = id
if f.deleteErr != nil {
@@ -88,7 +98,7 @@ func (f *fakeGitHubTrustStore) Delete(ctx context.Context, ownerSub, id string)
}
func (f *fakeGitHubTrustStore) ResolveByRepository(ctx context.Context, repository string) ([]*githubtrust.Policy, error) {
- _ = ctx
+ f.resolveContext = ctx
f.resolvedRepoName = repository
if f.resolveErr != nil {
return nil, f.resolveErr
@@ -229,3 +239,204 @@ func authConfigForHandlers() config.AuthConfiguration {
GitHubOIDCLegacyAudiences: []string{"cyclops-cs"},
}
}
+
+func TestGitHubTrustPoliciesUseBoundedDatabaseContext(t *testing.T) {
+ policy := &githubtrust.Policy{
+ ID: "policy-1",
+ OwnerSub: "user-123",
+ Name: "ci",
+ Repository: "trycua/cloud",
+ AllowedNamespaces: []string{"ns-a"},
+ Enabled: true,
+ CreatedAt: time.Date(2026, 6, 26, 10, 0, 0, 0, time.UTC),
+ }
+ tests := []struct {
+ name string
+ request *http.Request
+ call func(Handlers, http.ResponseWriter, *http.Request)
+ context func(*fakeGitHubTrustStore) context.Context
+ }{
+ {"list", httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil), Handlers.ListGitHubTrustPolicies, func(store *fakeGitHubTrustStore) context.Context { return store.listContext }},
+ {"create", httptest.NewRequest(http.MethodPost, "/api/github-trust-policies", bytes.NewBufferString(`{"name":"ci","repository":"trycua/cloud","allowed_namespaces":["ns-a"],"enabled":true}`)), Handlers.CreateGitHubTrustPolicy, func(store *fakeGitHubTrustStore) context.Context { return store.createContext }},
+ {"update", httptest.NewRequest(http.MethodPatch, "/api/github-trust-policies/policy-1", bytes.NewBufferString(`{"enabled":false}`)), Handlers.UpdateGitHubTrustPolicy, func(store *fakeGitHubTrustStore) context.Context { return store.updateContext }},
+ {"delete", httptest.NewRequest(http.MethodDelete, "/api/github-trust-policies/policy-1", nil), Handlers.DeleteGitHubTrustPolicy, func(store *fakeGitHubTrustStore) context.Context { return store.deleteContext }},
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ store := &fakeGitHubTrustStore{getResult: policy, deleteFound: true}
+ request := withUser(test.request, &auth.User{ID: "user-123", AZP: "cyclops-cs-spa"})
+ if test.name == "update" || test.name == "delete" {
+ request.SetPathValue("id", "policy-1")
+ }
+ response := httptest.NewRecorder()
+ start := time.Now()
+
+ test.call(Handlers{GitHubTrustPolicies: store}, response, request)
+
+ assertBoundedDatabaseContext(t, test.context(store), start)
+ if test.name == "update" {
+ assertBoundedDatabaseContext(t, store.getContext, start)
+ }
+ })
+ }
+}
+
+func assertBoundedDatabaseContext(t *testing.T, ctx context.Context, start time.Time) {
+ t.Helper()
+ if ctx == nil {
+ t.Fatal("store context was not captured")
+ }
+ deadline, ok := ctx.Deadline()
+ if !ok {
+ t.Fatal("store context has no deadline")
+ }
+ if deadline.Before(start) {
+ t.Fatalf("deadline = %s, before test start %s", deadline, start)
+ }
+ if deadline.After(start.Add(databaseRequestTimeout + time.Second)) {
+ t.Fatalf("deadline = %s, want no later than %s", deadline, start.Add(databaseRequestTimeout+time.Second))
+ }
+ if !errors.Is(ctx.Err(), context.Canceled) {
+ t.Fatalf("context error after handler returned = %v, want context canceled", ctx.Err())
+ }
+}
+
+func TestGitHubTrustResolverUsesBoundedDatabaseContext(t *testing.T) {
+ store := &fakeGitHubTrustStore{}
+ resolver := NewGitHubTrustResolver(store)
+ start := time.Now()
+
+ if _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud"); err != nil {
+ t.Fatalf("resolve github trust policies: %v", err)
+ }
+
+ assertBoundedDatabaseContext(t, store.resolveContext, start)
+}
+
+func TestGitHubTrustResolverHonorsExistingDatabaseDeadline(t *testing.T) {
+ store := &fakeGitHubTrustStore{}
+ resolver := NewGitHubTrustResolver(store)
+ parent, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second))
+ defer cancel()
+ wantDeadline, ok := parent.Deadline()
+ if !ok {
+ t.Fatal("parent context has no deadline")
+ }
+
+ if _, err := resolver.ResolveGitHubTrustPolicies(parent, "trycua/cloud"); err != nil {
+ t.Fatalf("resolve github trust policies: %v", err)
+ }
+
+ gotDeadline, ok := store.resolveContext.Deadline()
+ if !ok {
+ t.Fatal("resolver context has no deadline")
+ }
+ if !gotDeadline.Equal(wantDeadline) {
+ t.Fatalf("resolver deadline = %s, want parent deadline %s", gotDeadline, wantDeadline)
+ }
+}
+
+func TestGitHubTrustResolverNormalizesDatabaseFailures(t *testing.T) {
+ store := &fakeGitHubTrustStore{resolveErr: context.DeadlineExceeded}
+ resolver := NewGitHubTrustResolver(store)
+
+ _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud")
+ if !errors.Is(err, auth.ErrDatabaseUnavailable) {
+ t.Fatalf("resolve error = %v, want database unavailable sentinel", err)
+ }
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("resolve error = %v, want original deadline error", err)
+ }
+}
+
+func TestGitHubTrustResolverClassifiesImmediateNetworkFailures(t *testing.T) {
+ connectionRefused := &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}
+ store := &fakeGitHubTrustStore{resolveErr: connectionRefused}
+ resolver := NewGitHubTrustResolver(store)
+
+ _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud")
+ if !errors.Is(err, auth.ErrDatabaseUnavailable) {
+ t.Fatalf("resolve error = %v, want database unavailable sentinel", err)
+ }
+ if !errors.Is(err, connectionRefused) {
+ t.Fatalf("resolve error = %v, want original connection error", err)
+ }
+}
+
+func TestGitHubTrustResolverDoesNotClassifyPostgresErrorsAsUnavailable(t *testing.T) {
+ postgresError := &pgconn.PgError{Code: "42501", Message: "permission denied"}
+ store := &fakeGitHubTrustStore{resolveErr: postgresError}
+ resolver := NewGitHubTrustResolver(store)
+
+ _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud")
+ if errors.Is(err, auth.ErrDatabaseUnavailable) {
+ t.Fatalf("resolve error = %v, must not be database unavailable", err)
+ }
+}
+
+func TestGitHubTrustPolicies_DatabaseFailuresReturnServiceUnavailable(t *testing.T) {
+ tests := []struct {
+ name string
+ err error
+ }{
+ {name: "unavailable", err: auth.ErrDatabaseUnavailable},
+ {name: "deadline", err: context.DeadlineExceeded},
+ {name: "canceled", err: context.Canceled},
+ {name: "connection refused", err: &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}},
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ store := &fakeGitHubTrustStore{listErr: test.err}
+ h := Handlers{GitHubTrustPolicies: store}
+ request := withUser(httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil), &auth.User{ID: "user-123"})
+ response := httptest.NewRecorder()
+
+ h.ListGitHubTrustPolicies(response, request)
+
+ if response.Code != http.StatusServiceUnavailable {
+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
+ }
+ })
+ }
+}
+
+func TestGitHubTrustPolicies_PostgresErrorsRemainInternalServerErrors(t *testing.T) {
+ store := &fakeGitHubTrustStore{listErr: &pgconn.PgError{Code: "42501", Message: "permission denied"}}
+ h := Handlers{GitHubTrustPolicies: store}
+ request := withUser(httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil), &auth.User{ID: "user-123"})
+ response := httptest.NewRecorder()
+
+ h.ListGitHubTrustPolicies(response, request)
+
+ if response.Code != http.StatusInternalServerError {
+ t.Fatalf("status = %d, want 500; body = %s", response.Code, response.Body.String())
+ }
+}
+
+func TestUpdateGitHubTrustPolicySharesDatabaseBudget(t *testing.T) {
+ policy := &githubtrust.Policy{
+ ID: "policy-1", OwnerSub: "user-123", Name: "ci", Repository: "trycua/cloud",
+ AllowedNamespaces: []string{"ns-a"}, Enabled: true,
+ }
+ store := &fakeGitHubTrustStore{getResult: policy}
+ h := Handlers{GitHubTrustPolicies: store}
+ request := withUser(httptest.NewRequest(http.MethodPatch, "/api/github-trust-policies/policy-1", bytes.NewBufferString(`{"enabled":false}`)), &auth.User{ID: "user-123"})
+ request.SetPathValue("id", "policy-1")
+ response := httptest.NewRecorder()
+
+ h.UpdateGitHubTrustPolicy(response, request)
+
+ if response.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
+ }
+ getDeadline, getOK := store.getContext.Deadline()
+ updateDeadline, updateOK := store.updateContext.Deadline()
+ if !getOK || !updateOK {
+ t.Fatalf("deadlines get=%t update=%t, want both", getOK, updateOK)
+ }
+ if !getDeadline.Equal(updateDeadline) {
+ t.Fatalf("deadlines get=%s update=%s, want one shared budget", getDeadline, updateDeadline)
+ }
+}
diff --git a/libs/fleet/backend/handlers/handlers.go b/libs/fleet/backend/handlers/handlers.go
index eecb06abc3..08f5f75542 100644
--- a/libs/fleet/backend/handlers/handlers.go
+++ b/libs/fleet/backend/handlers/handlers.go
@@ -32,8 +32,6 @@ type Handlers struct {
StateQueryExecutor StateQueryExecutor
- Readiness *Readiness
-
ChatEnabled bool
Conversations chat.ConversationStore
Model chat.ModelClient
@@ -51,11 +49,11 @@ type Handlers struct {
func New(admin *keycloak.Admin, cfg *config.Configuration) Handlers {
return Handlers{
- Admin: admin,
- GatewayCfg: cfg.Gateway,
- AuthCfg: cfg.Auth,
- KC: cfg.Keycloak,
- Stripe: cfg.Stripe,
+ Admin: admin,
+ GatewayCfg: cfg.Gateway,
+ AuthCfg: cfg.Auth,
+ KC: cfg.Keycloak,
+ Stripe: cfg.Stripe,
ChatEnabled: cfg.Chat.Enabled,
chatLocks: newConversationLockRegistry(),
}
diff --git a/libs/fleet/backend/handlers/health.go b/libs/fleet/backend/handlers/health.go
index d6c5d6481f..07b4f4d157 100644
--- a/libs/fleet/backend/handlers/health.go
+++ b/libs/fleet/backend/handlers/health.go
@@ -2,7 +2,6 @@ package handlers
import (
"net/http"
- "sync/atomic"
)
// HealthResponse is the body of GET /healthz and GET /readyz.
@@ -10,20 +9,6 @@ type HealthResponse struct {
OK bool `json:"ok"`
}
-// Readiness tracks whether startup dependencies required for serving traffic
-// have completed successfully. It starts fail-closed.
-type Readiness struct {
- databaseReady atomic.Bool
-}
-
-func NewReadiness() *Readiness {
- return &Readiness{}
-}
-
-func (r *Readiness) MarkReady() {
- r.databaseReady.Store(true)
-}
-
// GetHealth godoc
//
// @Summary Liveness probe
@@ -36,9 +21,5 @@ func (h Handlers) GetHealth(w http.ResponseWriter, _ *http.Request) {
}
func (h Handlers) GetReadiness(w http.ResponseWriter, _ *http.Request) {
- if h.Readiness == nil || !h.Readiness.databaseReady.Load() {
- writeJSON(w, http.StatusServiceUnavailable, HealthResponse{OK: false})
- return
- }
writeJSON(w, http.StatusOK, HealthResponse{OK: true})
}
diff --git a/libs/fleet/backend/handlers/health_test.go b/libs/fleet/backend/handlers/health_test.go
index f7f2adfe11..0cf991fb10 100644
--- a/libs/fleet/backend/handlers/health_test.go
+++ b/libs/fleet/backend/handlers/health_test.go
@@ -16,24 +16,10 @@ func TestHealthIsLiveWhenDatabaseIsUnavailable(t *testing.T) {
}
}
-func TestReadinessFailsClosedUntilDatabaseSchemaIsReady(t *testing.T) {
- readiness := NewReadiness()
- h := Handlers{Readiness: readiness}
-
- for _, name := range []string{"initial state", "schema check failure"} {
- t.Run(name, func(t *testing.T) {
- response := httptest.NewRecorder()
- h.GetReadiness(response, httptest.NewRequest(http.MethodGet, "/readyz", nil))
-
- if response.Code < http.StatusBadRequest {
- t.Fatalf("status = %d, want non-2xx", response.Code)
- }
- })
- }
-
- readiness.MarkReady()
+func TestReadinessIsReadyWhenDatabaseIsUnavailable(t *testing.T) {
response := httptest.NewRecorder()
- h.GetReadiness(response, httptest.NewRequest(http.MethodGet, "/readyz", nil))
+
+ Handlers{}.GetReadiness(response, httptest.NewRequest(http.MethodGet, "/readyz", nil))
if response.Code != http.StatusOK {
t.Fatalf("status = %d, want %d", response.Code, http.StatusOK)
diff --git a/libs/fleet/backend/handlers/state_query.go b/libs/fleet/backend/handlers/state_query.go
index efe3280ca8..1c8869c0c5 100644
--- a/libs/fleet/backend/handlers/state_query.go
+++ b/libs/fleet/backend/handlers/state_query.go
@@ -8,6 +8,7 @@ import (
"mime"
"net/http"
+ "cyclops-cs-backend/auth"
"cyclops-cs-backend/identity"
"cyclops-cs-backend/statequery"
"github.com/jackc/pgx/v5/pgconn"
@@ -63,6 +64,9 @@ func (writer *stateQueryStreamWriter) WriteRow(values []any) error {
func (writer *stateQueryStreamWriter) Finish(err error) error {
if err != nil {
+ if auth.IsDatabaseUnavailable(err) {
+ return writer.write(stateQueryErrorEvent{Type: "error", Error: "state query unavailable"})
+ }
return writer.write(stateQueryErrorEvent{Type: "error", Error: err.Error()})
}
return writer.write(stateQueryDoneEvent{Type: "done"})
@@ -106,9 +110,14 @@ func (h Handlers) QueryState(w http.ResponseWriter, r *http.Request) {
user := currentUser(r)
stream := newStateQueryStreamWriter(w)
- err = stream.Finish(h.StateQueryExecutor.Execute(
- r.Context(), identity.PersonalGroup(r.Context(), user.ID), string(body), stream,
- ))
+ ctx, cancel := databaseContext(r.Context())
+ defer cancel()
+ err = h.StateQueryExecutor.Execute(ctx, identity.PersonalGroup(r.Context(), user.ID), string(body), stream)
+ if err != nil && !stream.started {
+ writeErr(w, http.StatusServiceUnavailable, "state query unavailable")
+ return
+ }
+ err = stream.Finish(err)
if err != nil {
slog.Debug("state query stream failed", "err", err)
}
diff --git a/libs/fleet/backend/handlers/state_query_test.go b/libs/fleet/backend/handlers/state_query_test.go
index fbcd18f13d..7ba652cec0 100644
--- a/libs/fleet/backend/handlers/state_query_test.go
+++ b/libs/fleet/backend/handlers/state_query_test.go
@@ -7,6 +7,7 @@ import (
"net/http/httptest"
"strings"
"testing"
+ "time"
"cyclops-cs-backend/auth"
"cyclops-cs-backend/statequery"
@@ -21,6 +22,22 @@ type fakeStateQueryExecutor struct {
err error
}
+type deadlineStateQueryExecutor struct {
+ context context.Context
+ write bool
+ err error
+}
+
+func (fake *deadlineStateQueryExecutor) Execute(ctx context.Context, _ string, _ string, writer statequery.ResultWriter) error {
+ fake.context = ctx
+ if fake.write {
+ if err := writer.WriteFieldDescriptions([]pgconn.FieldDescription{{Name: "name", DataTypeOID: 25}}); err != nil {
+ return err
+ }
+ }
+ return fake.err
+}
+
func (fake *fakeStateQueryExecutor) Execute(_ context.Context, tenant, query string, writer statequery.ResultWriter) error {
fake.tenant, fake.query = tenant, query
if err := writer.WriteFieldDescriptions(fake.fields); err != nil {
@@ -110,3 +127,117 @@ func TestQueryStateWritesExecutorErrorAsTerminalEvent(t *testing.T) {
t.Fatalf("body = %q, want suffix %q", response.Body.String(), wantSuffix)
}
}
+
+func TestQueryStateUsesBoundedDatabaseContext(t *testing.T) {
+ tests := []struct {
+ name string
+ write bool
+ err error
+ wantStatus int
+ wantBody string
+ wantSuffix string
+ wantUnavailable bool
+ }{
+ {"deadline before stream start", false, context.DeadlineExceeded, http.StatusServiceUnavailable, "", "", true},
+ {"executor error before stream start", false, errors.New("stream failed"), http.StatusServiceUnavailable, "", "", true},
+ {"deadline after stream start", true, context.DeadlineExceeded, http.StatusOK, "", `{"type":"error","error":"state query unavailable"}` + "\n", true},
+ {"executor error after stream start", true, errors.New("stream failed"), http.StatusOK, "", `{"type":"error","error":"stream failed"}` + "\n", false},
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ executor := &deadlineStateQueryExecutor{write: test.write, err: test.err}
+ h := Handlers{StateQueryExecutor: executor}
+ request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})
+ request.Header.Set("Content-Type", "application/sql")
+ response := httptest.NewRecorder()
+ start := time.Now()
+
+ h.QueryState(response, request)
+
+ assertBoundedDatabaseContext(t, executor.context, start)
+ if response.Code != test.wantStatus {
+ t.Fatalf("status = %d, want %d; body = %s", response.Code, test.wantStatus, response.Body.String())
+ }
+ if test.wantBody != "" && response.Body.String() != test.wantBody {
+ t.Fatalf("body = %q, want %q", response.Body.String(), test.wantBody)
+ }
+ if test.wantSuffix != "" && !strings.HasSuffix(response.Body.String(), test.wantSuffix) {
+ t.Fatalf("body = %q, want suffix %q", response.Body.String(), test.wantSuffix)
+ }
+ if test.wantUnavailable && strings.Contains(response.Body.String(), test.err.Error()) {
+ t.Fatalf("body exposes internal error: %q", response.Body.String())
+ }
+ })
+ }
+}
+
+func TestQueryStateDatabaseFailureBeforeStreamReturnsServiceUnavailable(t *testing.T) {
+ executor := &deadlineStateQueryExecutor{err: auth.ErrDatabaseUnavailable}
+ h := Handlers{StateQueryExecutor: executor}
+ request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})
+ request.Header.Set("Content-Type", "application/sql")
+ response := httptest.NewRecorder()
+
+ h.QueryState(response, request)
+
+ if response.Code != http.StatusServiceUnavailable {
+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
+ }
+ if strings.Contains(response.Body.String(), auth.ErrDatabaseUnavailable.Error()) {
+ t.Fatalf("body exposes internal error: %s", response.Body.String())
+ }
+}
+
+func TestQueryStateDatabaseFailureAfterStreamWritesSanitizedTerminalEvent(t *testing.T) {
+ executor := &deadlineStateQueryExecutor{write: true, err: auth.ErrDatabaseUnavailable}
+ h := Handlers{StateQueryExecutor: executor}
+ request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})
+ request.Header.Set("Content-Type", "application/sql")
+ response := httptest.NewRecorder()
+
+ h.QueryState(response, request)
+
+ if response.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
+ }
+ if !strings.HasSuffix(response.Body.String(), `{"type":"error","error":"state query unavailable"}`+"\n") {
+ t.Fatalf("body = %q", response.Body.String())
+ }
+}
+
+type cancellationStateQueryExecutor struct {
+ started chan<- struct{}
+ done chan<- struct{}
+}
+
+func (executor cancellationStateQueryExecutor) Execute(ctx context.Context, _, _ string, _ statequery.ResultWriter) error {
+ close(executor.started)
+ <-ctx.Done()
+ close(executor.done)
+ return ctx.Err()
+}
+
+func TestQueryStatePropagatesRequestCancellation(t *testing.T) {
+ started := make(chan struct{})
+ done := make(chan struct{})
+ handlerDone := make(chan struct{})
+ h := Handlers{StateQueryExecutor: cancellationStateQueryExecutor{started: started, done: done}}
+ parent, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")).WithContext(parent), &auth.User{ID: "alice"})
+ request.Header.Set("Content-Type", "application/sql")
+ response := httptest.NewRecorder()
+
+ go func() {
+ h.QueryState(response, request)
+ close(handlerDone)
+ }()
+ <-started
+ cancel()
+ <-done
+ <-handlerDone
+ if response.Code != http.StatusServiceUnavailable {
+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
+ }
+}
diff --git a/libs/fleet/backend/main.go b/libs/fleet/backend/main.go
index aa37ea7cee..c07e2e1ab2 100644
--- a/libs/fleet/backend/main.go
+++ b/libs/fleet/backend/main.go
@@ -25,6 +25,7 @@ import (
"log/slog"
"net/http"
"os"
+ "time"
"github.com/spf13/cobra"
@@ -320,7 +321,6 @@ func run() error {
}
h := handlers.New(admin, cfg)
- h.Readiness = handlers.NewReadiness()
if cfg.Stripe.SecretKey != "" {
h.Billing = billing.NewService(billing.NewStripeGateway(cfg.Stripe.SecretKey))
slog.Info("stripe billing: hosted flows enabled")
@@ -331,45 +331,7 @@ func run() error {
h.WorkloadAdmin = workloadAdmin
h.WorkloadAudience = cfg.Keycloak.WorkloadAudience
h.WorkloadTokenURL = cfg.Keycloak.WorkloadTokenURL
- databaseReady := false
- if cfg.Database.URL != "" {
- if err := database.RequireVersion(ctx, cfg.Database.URL, 1); err != nil {
- slog.Error("postgres database schema unavailable", "err", err)
- } else {
- databaseReady = true
- h.Readiness.MarkReady()
- slog.Info("postgres database schema ready", "version", 1)
- }
- }
-
- stateSchemaReady := databaseReady && cfg.Database.StateQueryDSN != "" && cfg.Database.StateQueryTenantPassword != ""
- if stateSchemaReady {
- executor, err := statequery.NewExecutor(
- cfg.Database.StateQueryDSN,
- cfg.Database.StateQueryTenantPassword,
- )
- if err != nil {
- slog.Error("kubernetes state query: executor init failed; /api/state/query will return 503", "err", err)
- } else {
- h.StateQueryExecutor = executor
- slog.Info("kubernetes state query: query endpoint enabled")
- }
- }
- // Postgres-backed GitHub OIDC trust policies (CUA-675). A connection
- // failure is non-fatal — the routes stay registered and reply 503 until
- // the database is reachable, mirroring how the other optional surfaces
- // degrade gracefully.
- if databaseReady {
- if store, err := githubtrust.New(ctx, cfg.Database.URL); err != nil {
- slog.Error("github trust policies: init failed; /api/github-trust-policies will return 503", "err", err)
- } else if store != nil {
- h.GitHubTrustPolicies = store
- auth.SetGitHubTrustResolver(handlers.NewGitHubTrustResolver(store))
- slog.Info("github trust policies: enabled")
- }
- } else {
- slog.Info("github trust policies: disabled (database schema unavailable)")
- }
+ initializeDatabaseFeatures(ctx, cfg.Database, &h, defaultDatabaseFeatureDependencies())
if cfg.Chat.Enabled {
h.Conversations = chat.NewMemoryConversationStore()
@@ -385,3 +347,62 @@ func run() error {
}
return nil
}
+
+const databaseStartupTimeout = 5 * time.Second
+
+type databaseFeatureDependencies struct {
+ requireVersion func(context.Context, string, int64) error
+ newStateQueryExecutor func(string, string) (handlers.StateQueryExecutor, error)
+ newGitHubTrustStore func(context.Context, string) (githubtrust.Store, error)
+}
+
+func defaultDatabaseFeatureDependencies() databaseFeatureDependencies {
+ return databaseFeatureDependencies{
+ requireVersion: database.RequireVersion,
+ newStateQueryExecutor: func(dsn, tenantPassword string) (handlers.StateQueryExecutor, error) {
+ return statequery.NewExecutor(dsn, tenantPassword)
+ },
+ newGitHubTrustStore: githubtrust.New,
+ }
+}
+
+func initializeDatabaseFeatures(parent context.Context, cfg config.DatabaseConfiguration, h *handlers.Handlers, dependencies databaseFeatureDependencies) {
+ h.StateQueryExecutor = nil
+ h.GitHubTrustPolicies = nil
+ auth.SetGitHubTrustResolver(nil)
+
+ if cfg.StateQueryDSN != "" && cfg.StateQueryTenantPassword != "" {
+ executor, err := dependencies.newStateQueryExecutor(cfg.StateQueryDSN, cfg.StateQueryTenantPassword)
+ if err != nil {
+ slog.Error("kubernetes state query: executor init failed; /api/state/query will return 503", "err", err)
+ } else {
+ h.StateQueryExecutor = executor
+ slog.Info("kubernetes state query: query endpoint enabled")
+ }
+ }
+
+ if cfg.URL == "" {
+ slog.Info("postgres-backed features disabled (DATABASE_URL unset)")
+ return
+ }
+
+ ctx, cancel := context.WithTimeout(parent, databaseStartupTimeout)
+ defer cancel()
+
+ if err := dependencies.requireVersion(ctx, cfg.URL, 1); err != nil {
+ slog.Error("postgres database schema unavailable; database-backed routes will return 503", "err", err)
+ return
+ }
+ slog.Info("postgres database schema ready", "version", 1)
+
+ store, err := dependencies.newGitHubTrustStore(ctx, cfg.URL)
+ if err != nil {
+ slog.Error("github trust policies: init failed; /api/github-trust-policies will return 503", "err", err)
+ return
+ }
+ if store != nil {
+ h.GitHubTrustPolicies = store
+ auth.SetGitHubTrustResolver(handlers.NewGitHubTrustResolver(store))
+ slog.Info("github trust policies: enabled")
+ }
+}
diff --git a/libs/fleet/backend/main_test.go b/libs/fleet/backend/main_test.go
index 3cfd7f3a46..3570088880 100644
--- a/libs/fleet/backend/main_test.go
+++ b/libs/fleet/backend/main_test.go
@@ -23,7 +23,9 @@ import (
"cyclops-cs-backend/auth"
"cyclops-cs-backend/billing"
"cyclops-cs-backend/config"
+ "cyclops-cs-backend/githubtrust"
"cyclops-cs-backend/handlers"
+ "cyclops-cs-backend/statequery"
jwt "github.com/golang-jwt/jwt/v5"
"github.com/trycua/cloud/pkg/featureflags"
@@ -51,7 +53,7 @@ func TestHealthAndReadinessRoutes(t *testing.T) {
statusCode int
}{
{path: "/healthz", statusCode: http.StatusOK},
- {path: "/readyz", statusCode: http.StatusServiceUnavailable},
+ {path: "/readyz", statusCode: http.StatusOK},
} {
t.Run(test.path, func(t *testing.T) {
response := httptest.NewRecorder()
@@ -458,3 +460,164 @@ func TestNginxRoutesChatToBackend(t *testing.T) {
t.Fatal("nginx.conf backend matcher does not include chat")
}
}
+
+func TestInitializeDatabaseFeatures(t *testing.T) {
+ tests := []struct {
+ name string
+ config config.DatabaseConfiguration
+ requireVersionError error
+ newExecutorError error
+ newStoreError error
+ wantExecutor bool
+ wantStore bool
+ wantRequireVersion bool
+ wantNewExecutor bool
+ wantNewStore bool
+ }{
+ {
+ name: "state query initializes without application database",
+ config: config.DatabaseConfiguration{
+ StateQueryDSN: "postgres://state-query/state-query",
+ StateQueryTenantPassword: "tenant-password",
+ },
+ wantExecutor: true,
+ wantNewExecutor: true,
+ },
+ {
+ name: "state query initializes when application schema validation fails",
+ config: config.DatabaseConfiguration{
+ URL: "postgres://application/application",
+ StateQueryDSN: "postgres://state-query/state-query",
+ StateQueryTenantPassword: "tenant-password",
+ },
+ requireVersionError: fmt.Errorf("schema unavailable"),
+ wantExecutor: true,
+ wantRequireVersion: true,
+ wantNewExecutor: true,
+ },
+ {
+ name: "application database enables GitHub trust store",
+ config: config.DatabaseConfiguration{URL: "postgres://application/application"},
+ wantStore: true,
+ wantRequireVersion: true,
+ wantNewStore: true,
+ },
+ {
+ name: "constructor failures are non-fatal and clear stale fields",
+ config: config.DatabaseConfiguration{
+ URL: "postgres://application/application",
+ StateQueryDSN: "postgres://state-query/state-query",
+ StateQueryTenantPassword: "tenant-password",
+ },
+ newExecutorError: fmt.Errorf("invalid state query configuration"),
+ newStoreError: fmt.Errorf("store unavailable"),
+ wantRequireVersion: true,
+ wantNewExecutor: true,
+ wantNewStore: true,
+ },
+ {
+ name: "missing database config clears stale fields",
+ config: config.DatabaseConfiguration{},
+ wantRequireVersion: false,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ var requireVersionContext context.Context
+ var storeContext context.Context
+ newExecutorCalls := 0
+ newStoreCalls := 0
+ dependencies := databaseFeatureDependencies{
+ requireVersion: func(ctx context.Context, _ string, _ int64) error {
+ requireVersionContext = ctx
+ return test.requireVersionError
+ },
+ newStateQueryExecutor: func(_ string, _ string) (handlers.StateQueryExecutor, error) {
+ newExecutorCalls++
+ if test.newExecutorError != nil {
+ return nil, test.newExecutorError
+ }
+ return stateQueryExecutorStub{}, nil
+ },
+ newGitHubTrustStore: func(ctx context.Context, _ string) (githubtrust.Store, error) {
+ newStoreCalls++
+ storeContext = ctx
+ if test.newStoreError != nil {
+ return nil, test.newStoreError
+ }
+ return githubTrustStoreStub{}, nil
+ },
+ }
+ h := handlers.Handlers{
+ StateQueryExecutor: stateQueryExecutorStub{},
+ GitHubTrustPolicies: githubTrustStoreStub{},
+ }
+ auth.SetGitHubTrustResolver(handlers.NewGitHubTrustResolver(githubTrustStoreStub{}))
+ t.Cleanup(func() { auth.SetGitHubTrustResolver(nil) })
+
+ initializeDatabaseFeatures(context.Background(), test.config, &h, dependencies)
+
+ if got := newExecutorCalls == 1; got != test.wantNewExecutor {
+ t.Fatalf("state-query constructor called = %t, want %t", got, test.wantNewExecutor)
+ }
+ if got := newStoreCalls == 1; got != test.wantNewStore {
+ t.Fatalf("GitHub trust store constructor called = %t, want %t", got, test.wantNewStore)
+ }
+ if got := h.StateQueryExecutor != nil; got != test.wantExecutor {
+ t.Fatalf("StateQueryExecutor present = %t, want %t", got, test.wantExecutor)
+ }
+ if got := h.GitHubTrustPolicies != nil; got != test.wantStore {
+ t.Fatalf("GitHubTrustPolicies present = %t, want %t", got, test.wantStore)
+ }
+ assertStartupContext(t, requireVersionContext, test.wantRequireVersion)
+ assertStartupContext(t, storeContext, test.wantNewStore)
+ })
+ }
+}
+
+func assertStartupContext(t *testing.T, ctx context.Context, wantCalled bool) {
+ t.Helper()
+ if !wantCalled {
+ if ctx != nil {
+ t.Fatal("initializer received a context unexpectedly")
+ }
+ return
+ }
+ if ctx == nil {
+ t.Fatal("initializer did not receive a context")
+ }
+ deadline, ok := ctx.Deadline()
+ if !ok {
+ t.Fatal("initializer context has no deadline")
+ }
+ if remaining := time.Until(deadline); remaining > databaseStartupTimeout+100*time.Millisecond {
+ t.Fatalf("context deadline is too far away: %s", remaining)
+ }
+ if err := ctx.Err(); err != context.Canceled {
+ t.Fatalf("initializer context error = %v, want %v", err, context.Canceled)
+ }
+}
+
+type stateQueryExecutorStub struct{}
+
+func (stateQueryExecutorStub) Execute(context.Context, string, string, statequery.ResultWriter) error {
+ return nil
+}
+
+type githubTrustStoreStub struct{}
+
+func (githubTrustStoreStub) List(context.Context, string) ([]*githubtrust.Policy, error) {
+ return nil, nil
+}
+func (githubTrustStoreStub) Create(context.Context, *githubtrust.Policy) error { return nil }
+func (githubTrustStoreStub) Get(context.Context, string, string) (*githubtrust.Policy, error) {
+ return nil, nil
+}
+func (githubTrustStoreStub) Update(context.Context, *githubtrust.Policy) error { return nil }
+func (githubTrustStoreStub) Delete(context.Context, string, string) (bool, error) {
+ return false, nil
+}
+func (githubTrustStoreStub) ResolveByRepository(context.Context, string) ([]*githubtrust.Policy, error) {
+ return nil, nil
+}
diff --git a/libs/fleet/backend/startup_migration_contract_test.go b/libs/fleet/backend/startup_migration_contract_test.go
index 0d885cf544..504582a0a6 100644
--- a/libs/fleet/backend/startup_migration_contract_test.go
+++ b/libs/fleet/backend/startup_migration_contract_test.go
@@ -738,23 +738,6 @@ func TestActiveBackendSourcesDoNotOwnMigrations(t *testing.T) {
}
}
-func TestMainRequiresMigrationVersionWithoutRunningMigrations(t *testing.T) {
- source := readSource(t, "main.go")
- if !strings.Contains(source, "database.RequireVersion(ctx, cfg.Database.URL, 1)") {
- t.Fatal("backend must require migration version 1")
- }
-}
-
-func TestBackendReadinessStartsFailClosedAndRequiresSchema(t *testing.T) {
- source := readSource(t, "main.go")
- if !strings.Contains(source, "h.Readiness = handlers.NewReadiness()") {
- t.Fatal("backend readiness must start fail-closed before the database check")
- }
- if !strings.Contains(source, "h.Readiness.MarkReady()") {
- t.Fatal("backend readiness must become ready after the database schema check")
- }
-}
-
func TestBackendDeploymentKeepsCoreAPIReadyWithoutDatabaseSchema(t *testing.T) {
source := readSource(t, filepath.Join("..", "..", "clusters", "kopf-k3s", "cyclops-cs", "backend-deployment.yaml"))
for _, probe := range []string{"readinessProbe", "livenessProbe"} {