Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cyclops): query usage allocations via DataFusion (#7121)
r33drichardsabfaae44a452
diff --git a/libs/fleet/backend/config/config.go b/libs/fleet/backend/config/config.goindex 20c0368532..768faab82d 100644--- a/libs/fleet/backend/config/config.go+++ b/libs/fleet/backend/config/config.go@@ -14,6 +14,7 @@ package configimport ("fmt""net/url"+ "regexp""slices""strconv""strings"@@ -124,20 +125,30 @@ type ChatConfiguration struct {}// UsageConfiguration enables the usage provider only when DatabaseURL is set.-// OpenCostBaseURL may be configured before the usage credential arrives; in-// that state the provider remains disabled so serving readiness is unaffected.+// Query transport settings may be present before the usage credential arrives;+// in that state the provider remains disabled so serving readiness is unaffected.type UsageConfiguration struct {- DatabaseURL string- OpenCostBaseURL string- QueryTimeout time.Duration- MaxResponseBytes int64+ DatabaseURL string+ QueryWebhookURL string+ QueryHMACSecret string+ QueryResultBucket string+ QueryResultPrefix string+ QueryCluster string+ QueryEnvironment string+ QueryTimeout time.Duration+ QueryPollInterval time.Duration+ MaxResponseBytes int64}+var usageQueryPartitionValue = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)+const (- minUsageQueryTimeout = time.Second- maxUsageQueryTimeout = 2 * time.Minute- minUsageResponseSize = 64 * 1024- maxUsageResponseSize = 32 * 1024 * 1024+ clusterUsageQueryWebhookURL = "http://cua-temporal-webhook.temporal.svc.cluster.local/hooks/opencost-query"+ externalUsageQueryWebhookURL = "https://cua-temporal-webhook.tail204509.ts.net/hooks/opencost-query"+ minUsageQueryTimeout = time.Second+ maxUsageQueryTimeout = 2 * time.Minute+ minUsageResponseSize = 64 * 1024+ maxUsageResponseSize = 32 * 1024 * 1024)type MetricsConfiguration struct {@@ -185,9 +196,15 @@ var specs = []flagSpec{{"database.state-query-dsn", "state-query-database-dsn", "STATE_QUERY_DATABASE_DSN", "", "Postgres connection options for tenant state queries"},{"database.state-query-tenant-password", "state-query-tenant-password", "STATE_QUERY_TENANT_PASSWORD", "", "Shared password for tenant query login roles"},{"usage.database-url", "usage-database-url", "USAGE_DATABASE_URL", "", "Postgres URL for usage events"},- {"usage.opencost-base-url", "opencost-base-url", "OPENCOST_BASE_URL", "", "OpenCost base URL for usage allocations"},- {"usage.query-timeout", "usage-query-timeout", "USAGE_QUERY_TIMEOUT", "20s", "Usage provider query timeout"},- {"usage.max-response-bytes", "usage-max-response-bytes", "USAGE_MAX_RESPONSE_BYTES", "8388608", "Maximum OpenCost response bytes"},+ {"usage.query-webhook-url", "usage-query-webhook-url", "USAGE_QUERY_WEBHOOK_URL", clusterUsageQueryWebhookURL, "DataFusion allocation query webhook URL"},+ {"usage.query-hmac-secret", "usage-query-hmac-secret", "USAGE_QUERY_HMAC_SECRET", "", "DataFusion allocation query webhook HMAC secret"},+ {"usage.query-result-bucket", "usage-query-result-bucket", "USAGE_QUERY_RESULT_BUCKET", "nanoclaw-telemetry-files", "S3 bucket for transient allocation query results"},+ {"usage.query-result-prefix", "usage-query-result-prefix", "USAGE_QUERY_RESULT_PREFIX", "cyclops/usage-query", "S3 prefix for transient allocation query results"},+ {"usage.query-cluster", "usage-query-cluster", "USAGE_QUERY_CLUSTER", "kopf-k3s", "OpenCost allocation cluster partition"},+ {"usage.query-environment", "usage-query-environment", "USAGE_QUERY_ENVIRONMENT", "production", "OpenCost allocation environment partition"},+ {"usage.query-timeout", "usage-query-timeout", "USAGE_QUERY_TIMEOUT", "45s", "Usage provider query timeout"},+ {"usage.query-poll-interval", "usage-query-poll-interval", "USAGE_QUERY_POLL_INTERVAL", "1s", "Allocation query status poll interval"},+ {"usage.max-response-bytes", "usage-max-response-bytes", "USAGE_MAX_RESPONSE_BYTES", "8388608", "Maximum allocation query response bytes"},{"stripe.secret-key", "stripe-secret-key", "STRIPE_SECRET_KEY", "", "Stripe secret key (server-only)"},{"stripe.webhook-secret", "stripe-webhook-secret", "STRIPE_WEBHOOK_SECRET", "", "Stripe webhook signing secret"},{"stripe.checkout-success-url", "stripe-checkout-success-url", "STRIPE_CHECKOUT_SUCCESS_URL", "", "Stripe Checkout success redirect URL"},@@ -235,6 +252,10 @@ func LoadConfig() (*Configuration, error) {if err != nil {return nil, err}+ usagePollInterval, err := parseUsagePollInterval(viper.GetString("usage.query-poll-interval"))+ if err != nil {+ return nil, err+ }usageMaxResponseBytes, err := parseUsageMaxResponseBytes(viper.GetString("usage.max-response-bytes"))if err != nil {return nil, err@@ -306,10 +327,16 @@ func LoadConfig() (*Configuration, error) {Model: viper.GetString("chat.model"),},Usage: UsageConfiguration{- DatabaseURL: strings.TrimSpace(viper.GetString("usage.database-url")),- OpenCostBaseURL: strings.TrimSpace(viper.GetString("usage.opencost-base-url")),- QueryTimeout: usageQueryTimeout,- MaxResponseBytes: usageMaxResponseBytes,+ DatabaseURL: strings.TrimSpace(viper.GetString("usage.database-url")),+ QueryWebhookURL: strings.TrimSpace(viper.GetString("usage.query-webhook-url")),+ QueryHMACSecret: strings.TrimSpace(viper.GetString("usage.query-hmac-secret")),+ QueryResultBucket: strings.TrimSpace(viper.GetString("usage.query-result-bucket")),+ QueryResultPrefix: strings.Trim(strings.TrimSpace(viper.GetString("usage.query-result-prefix")), "/"),+ QueryCluster: strings.TrimSpace(viper.GetString("usage.query-cluster")),+ QueryEnvironment: strings.TrimSpace(viper.GetString("usage.query-environment")),+ QueryTimeout: usageQueryTimeout,+ QueryPollInterval: usagePollInterval,+ MaxResponseBytes: usageMaxResponseBytes,},Metrics: MetricsConfiguration{Addr: viper.GetString("metrics.addr")},Telemetry: TelemetryConfiguration{@@ -353,17 +380,25 @@ func validateUsageConfiguration(cfg UsageConfiguration) error {if cfg.QueryTimeout < minUsageQueryTimeout || cfg.QueryTimeout > maxUsageQueryTimeout {return fmt.Errorf("USAGE_QUERY_TIMEOUT must be between 1s and 2m")}+ if cfg.QueryPollInterval < 250*time.Millisecond || cfg.QueryPollInterval > 5*time.Second {+ return fmt.Errorf("USAGE_QUERY_POLL_INTERVAL must be between 250ms and 5s")+ }if cfg.MaxResponseBytes < minUsageResponseSize || cfg.MaxResponseBytes > maxUsageResponseSize {return fmt.Errorf("USAGE_MAX_RESPONSE_BYTES must be between 65536 and 33554432")}- // Deployments set OPENCOST_BASE_URL before the usage ExternalSecret is- // available. An absent database URL therefore disables usage, while a- // database URL without OpenCost remains an unsafe configuration error.if cfg.DatabaseURL == "" {return nil}- if cfg.OpenCostBaseURL == "" {- return fmt.Errorf("USAGE_DATABASE_URL requires OPENCOST_BASE_URL")+ for name, value := range map[string]string{+ "USAGE_QUERY_HMAC_SECRET": cfg.QueryHMACSecret,+ "USAGE_QUERY_RESULT_BUCKET": cfg.QueryResultBucket,+ "USAGE_QUERY_RESULT_PREFIX": cfg.QueryResultPrefix,+ "USAGE_QUERY_CLUSTER": cfg.QueryCluster,+ "USAGE_QUERY_ENVIRONMENT": cfg.QueryEnvironment,+ } {+ if value == "" {+ return fmt.Errorf("USAGE_DATABASE_URL requires %s", name)+ }}databaseURL, err := url.Parse(cfg.DatabaseURL)if err != nil {@@ -379,12 +414,19 @@ func validateUsageConfiguration(cfg UsageConfiguration) error {if postgresConfig.ConnConfig.User != "cyclops_usage_reader" {return fmt.Errorf("usage database URL must use cyclops_usage_reader")}- openCostURL, err := url.Parse(cfg.OpenCostBaseURL)- if err != nil {- return newSanitizedError("invalid OpenCost URL", err)+ if cfg.QueryWebhookURL != clusterUsageQueryWebhookURL && cfg.QueryWebhookURL != externalUsageQueryWebhookURL {+ return fmt.Errorf("invalid allocation query webhook URL")+ }+ if cfg.QueryResultBucket != "nanoclaw-telemetry-files" {+ return fmt.Errorf("invalid allocation query result bucket")}- if (openCostURL.Scheme != "http" && openCostURL.Scheme != "https") || openCostURL.Host == "" || openCostURL.RawQuery != "" || openCostURL.Fragment != "" || openCostURL.User != nil {- return fmt.Errorf("invalid OpenCost URL")+ if cfg.QueryResultPrefix != "cyclops/usage-query" {+ return fmt.Errorf("invalid allocation query result prefix")+ }+ for _, value := range []string{cfg.QueryCluster, cfg.QueryEnvironment} {+ if !usageQueryPartitionValue.MatchString(value) {+ return fmt.Errorf("invalid allocation query partition")+ }}return nil}@@ -397,6 +439,14 @@ func parseUsageQueryTimeout(raw string) (time.Duration, error) {return value, nil}+func parseUsagePollInterval(raw string) (time.Duration, error) {+ value, err := time.ParseDuration(strings.TrimSpace(raw))+ if err != nil {+ return 0, fmt.Errorf("invalid USAGE_QUERY_POLL_INTERVAL: %w", err)+ }+ return value, nil+}+func parseUsageMaxResponseBytes(raw string) (int64, error) {value, err := strconv.ParseInt(strings.TrimSpace(raw), 10, 64)if err != nil {diff --git a/libs/fleet/backend/config/config_test.go b/libs/fleet/backend/config/config_test.goindex 0070304d9f..488476a023 100644--- a/libs/fleet/backend/config/config_test.go+++ b/libs/fleet/backend/config/config_test.go@@ -277,41 +277,72 @@ func TestLoadConfig_ChatDisabledDoesNotRequireCredentials(t *testing.T) {func TestLoadConfig_UsageConfiguration(t *testing.T) {validDatabaseURL := "postgres://cyclops_usage_reader:[email protected]/cyclops?sslmode=require"- validOpenCostURL := "https://opencost.example/api"+ defaults := UsageConfiguration{+ QueryWebhookURL: "http://cua-temporal-webhook.temporal.svc.cluster.local/hooks/opencost-query",+ QueryResultBucket: "nanoclaw-telemetry-files",+ QueryResultPrefix: "cyclops/usage-query",+ QueryCluster: "kopf-k3s",+ QueryEnvironment: "production",+ QueryTimeout: 45 * time.Second,+ QueryPollInterval: time.Second,+ MaxResponseBytes: 8388608,+ }tests := []struct {name stringenv map[string]stringwantError stringwant UsageConfiguration}{- {name: "absent disables usage", want: UsageConfiguration{QueryTimeout: 20 * time.Second, MaxResponseBytes: 8388608}},- {name: "database only is rejected", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL}, wantError: "USAGE_DATABASE_URL requires OPENCOST_BASE_URL"},- {name: "OpenCost only disables usage", env: map[string]string{"OPENCOST_BASE_URL": validOpenCostURL}, want: UsageConfiguration{OpenCostBaseURL: validOpenCostURL, QueryTimeout: 20 * time.Second, MaxResponseBytes: 8388608}},+ {name: "absent disables usage", want: defaults},+ {name: "query settings only disable usage", env: map[string]string{"USAGE_QUERY_HMAC_SECRET": "secret"}, want: func() UsageConfiguration { value := defaults; value.QueryHMACSecret = "secret"; return value }()},+ {name: "database requires HMAC secret", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL}, wantError: "USAGE_DATABASE_URL requires USAGE_QUERY_HMAC_SECRET"},{name: "valid values",env: map[string]string{- "USAGE_DATABASE_URL": validDatabaseURL, "OPENCOST_BASE_URL": validOpenCostURL,- "USAGE_QUERY_TIMEOUT": "45s", "USAGE_MAX_RESPONSE_BYTES": "1048576",+ "USAGE_DATABASE_URL": validDatabaseURL, "USAGE_QUERY_HMAC_SECRET": "secret",+ "USAGE_QUERY_TIMEOUT": "60s", "USAGE_QUERY_POLL_INTERVAL": "500ms", "USAGE_MAX_RESPONSE_BYTES": "1048576",},- want: UsageConfiguration{DatabaseURL: validDatabaseURL, OpenCostBaseURL: validOpenCostURL, QueryTimeout: 45 * time.Second, MaxResponseBytes: 1048576},+ want: func() UsageConfiguration {+ value := defaults+ value.DatabaseURL = validDatabaseURL+ value.QueryHMACSecret = "secret"+ value.QueryTimeout = time.Minute+ value.QueryPollInterval = 500 * time.Millisecond+ value.MaxResponseBytes = 1048576+ return value+ }(),},- {name: "libpq database DSN is rejected", env: map[string]string{"USAGE_DATABASE_URL": "user=cyclops_usage_reader host=db.example dbname=cyclops", "OPENCOST_BASE_URL": validOpenCostURL}, wantError: "invalid usage database URL"},- {name: "database reader role is required", env: map[string]string{"USAGE_DATABASE_URL": "postgres://[email protected]/cyclops", "OPENCOST_BASE_URL": validOpenCostURL}, wantError: "usage database URL must use cyclops_usage_reader"},- {name: "OpenCost URL cannot have a query", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL, "OPENCOST_BASE_URL": "https://opencost.example?token=secret"}, wantError: "invalid OpenCost URL"},+ {name: "libpq database DSN is rejected", env: map[string]string{"USAGE_DATABASE_URL": "user=cyclops_usage_reader host=db.example dbname=cyclops", "USAGE_QUERY_HMAC_SECRET": "secret"}, wantError: "invalid usage database URL"},+ {name: "database reader role is required", env: map[string]string{"USAGE_DATABASE_URL": "postgres://[email protected]/cyclops", "USAGE_QUERY_HMAC_SECRET": "secret"}, wantError: "usage database URL must use cyclops_usage_reader"},+ {name: "external webhook remains allowed", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL, "USAGE_QUERY_HMAC_SECRET": "secret", "USAGE_QUERY_WEBHOOK_URL": externalUsageQueryWebhookURL}, want: func() UsageConfiguration {+ value := defaults+ value.DatabaseURL = validDatabaseURL+ value.QueryHMACSecret = "secret"+ value.QueryWebhookURL = externalUsageQueryWebhookURL+ return value+ }()},+ {name: "webhook is allow-listed", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL, "USAGE_QUERY_HMAC_SECRET": "secret", "USAGE_QUERY_WEBHOOK_URL": "https://example.test/hooks/opencost-query"}, wantError: "invalid allocation query webhook URL"},+ {name: "result bucket is allow-listed", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL, "USAGE_QUERY_HMAC_SECRET": "secret", "USAGE_QUERY_RESULT_BUCKET": "other"}, wantError: "invalid allocation query result bucket"},{name: "malformed timeout is rejected while disabled", env: map[string]string{"USAGE_QUERY_TIMEOUT": "not-a-duration"}, wantError: "invalid USAGE_QUERY_TIMEOUT"},+ {name: "malformed poll interval is rejected while disabled", env: map[string]string{"USAGE_QUERY_POLL_INTERVAL": "not-a-duration"}, wantError: "invalid USAGE_QUERY_POLL_INTERVAL"},{name: "malformed response limit is rejected while disabled", env: map[string]string{"USAGE_MAX_RESPONSE_BYTES": "not-a-number"}, wantError: "invalid USAGE_MAX_RESPONSE_BYTES"},{name: "short timeout is rejected while disabled", env: map[string]string{"USAGE_QUERY_TIMEOUT": "500ms"}, wantError: "USAGE_QUERY_TIMEOUT must be between 1s and 2m"},+ {name: "short poll interval is rejected while disabled", env: map[string]string{"USAGE_QUERY_POLL_INTERVAL": "10ms"}, wantError: "USAGE_QUERY_POLL_INTERVAL must be between 250ms and 5s"},{name: "small response limit is rejected while disabled", env: map[string]string{"USAGE_MAX_RESPONSE_BYTES": "1"}, wantError: "USAGE_MAX_RESPONSE_BYTES must be between 65536 and 33554432"},- {name: "timeout is bounded", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL, "OPENCOST_BASE_URL": validOpenCostURL, "USAGE_QUERY_TIMEOUT": "500ms"}, wantError: "USAGE_QUERY_TIMEOUT must be between 1s and 2m"},- {name: "response limit is bounded", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL, "OPENCOST_BASE_URL": validOpenCostURL, "USAGE_MAX_RESPONSE_BYTES": "1"}, wantError: "USAGE_MAX_RESPONSE_BYTES must be between 65536 and 33554432"},}+ keys := []string{+ "USAGE_DATABASE_URL", "USAGE_QUERY_WEBHOOK_URL", "USAGE_QUERY_HMAC_SECRET",+ "USAGE_QUERY_RESULT_BUCKET", "USAGE_QUERY_RESULT_PREFIX", "USAGE_QUERY_CLUSTER",+ "USAGE_QUERY_ENVIRONMENT", "USAGE_QUERY_TIMEOUT", "USAGE_QUERY_POLL_INTERVAL",+ "USAGE_MAX_RESPONSE_BYTES",+ }for _, test := range tests {t.Run(test.name, func(t *testing.T) {viper.Reset()t.Cleanup(viper.Reset)t.Setenv("KC_ADMIN_CLIENT_SECRET", "secret")- for _, key := range []string{"USAGE_DATABASE_URL", "OPENCOST_BASE_URL", "USAGE_QUERY_TIMEOUT", "USAGE_MAX_RESPONSE_BYTES"} {+ for _, key := range keys {t.Setenv(key, "")}for key, value := range test.env {diff --git a/libs/fleet/backend/go.mod b/libs/fleet/backend/go.modindex 386253b04b..7fd5fcefb1 100644--- a/libs/fleet/backend/go.mod+++ b/libs/fleet/backend/go.mod@@ -7,6 +7,8 @@ toolchain go1.24.7require (github.com/MicahParks/keyfunc/v2 v2.1.0github.com/Nerzal/gocloak/v13 v13.9.0+ github.com/aws/aws-sdk-go-v2/config v1.32.17+ github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2github.com/golang-jwt/jwt/v5 v5.2.1github.com/google/uuid v1.6.0github.com/jackc/pgx/v5 v5.5.5@@ -20,15 +22,17 @@ require (github.com/stripe/stripe-go/v85 v85.2.0github.com/swaggo/http-swagger/v2 v2.0.2github.com/swaggo/swag v1.16.6- sigs.k8s.io/yaml v1.4.0go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0golang.org/x/sync v0.17.0+ sigs.k8s.io/yaml v1.4.0)require (github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect- github.com/aws/aws-sdk-go-v2/config v1.32.17 // indirect+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirectgithub.com/cenkalti/backoff/v4 v4.3.0 // indirectgithub.com/felixge/httpsnoop v1.0.4 // indirectgithub.com/fsnotify/fsnotify v1.9.0 // indirect@@ -73,7 +77,7 @@ require (github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirectgithub.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirectgithub.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect- github.com/aws/smithy-go v1.25.1 // indirect+ github.com/aws/smithy-go v1.25.1github.com/beorn7/perks v1.0.1 // indirectgithub.com/cespare/xxhash/v2 v2.3.0 // indirectgithub.com/go-ini/ini v1.67.0 // indirectdiff --git a/libs/fleet/backend/go.sum b/libs/fleet/backend/go.sumindex 1a87303bba..2806be8909 100644--- a/libs/fleet/backend/go.sum+++ b/libs/fleet/backend/go.sum@@ -12,6 +12,8 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Iggithub.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU=github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE=github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=@@ -26,8 +28,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIugithub.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2 h1:U3ygWUhCpiSPYSHOrRhb3gOl9T5Y3kB8k5Vjs//57bE=+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=github.com/aws/aws-sdk-go-v2/service/ssm v1.68.6 h1:0LPJjbSNEDHidGOXa0LfvSVbdn9/GdlJUQTgE0kFpso=diff --git a/libs/fleet/backend/handlers/config.go b/libs/fleet/backend/handlers/config.goindex 960039edf0..8b68b9a7b0 100644--- a/libs/fleet/backend/handlers/config.go+++ b/libs/fleet/backend/handlers/config.go@@ -53,6 +53,7 @@ func (h Handlers) GetConfig(w http.ResponseWriter, r *http.Request) {slog.WarnContext(ctx, "usage access eval failed; defaulting off", "err", err)usageEnabled = false}+ usageEnabled = usageEnabled && h.Usage != nilchatEnabled, err := h.chatEnabled(ctx, user)if err != nil {diff --git a/libs/fleet/backend/handlers/config_test.go b/libs/fleet/backend/handlers/config_test.goindex c3d6dd94b9..7e5c43c276 100644--- a/libs/fleet/backend/handlers/config_test.go+++ b/libs/fleet/backend/handlers/config_test.go@@ -53,15 +53,36 @@ func TestGetConfigReturnsEffectiveChatAccess(t *testing.T) {}func TestGetConfigReturnsEffectiveUsageAccess(t *testing.T) {- h := Handlers{usageAccessEvaluator: func(context.Context, *auth.User) (bool, error) { return true, nil }, adminAccessEvaluator: func(context.Context, *auth.User) (bool, error) { return false, nil }}- w := httptest.NewRecorder()- h.GetConfig(w, withUser(httptest.NewRequest(http.MethodGet, "/api/config", nil), &auth.User{ID: "user", AZP: "cyclops-cs-spa"}))- var response ConfigResponse- if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {- t.Fatal(err)+ tests := []struct {+ name string+ provider UsageProvider+ allowed bool+ want bool+ }{+ {name: "available and allowed", provider: &fakeUsageProvider{}, allowed: true, want: true},+ {name: "unavailable", allowed: true, want: false},+ {name: "not allowed", provider: &fakeUsageProvider{}, allowed: false, want: false},}- if !response.Usage {- t.Fatal("usage false")++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ h := Handlers{+ Usage: test.provider,+ usageAccessEvaluator: func(context.Context, *auth.User) (bool, error) {+ return test.allowed, nil+ },+ adminAccessEvaluator: func(context.Context, *auth.User) (bool, error) { return false, nil },+ }+ w := httptest.NewRecorder()+ h.GetConfig(w, withUser(httptest.NewRequest(http.MethodGet, "/api/config", nil), &auth.User{ID: "user", AZP: "cyclops-cs-spa"}))+ var response ConfigResponse+ if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {+ t.Fatal(err)+ }+ if response.Usage != test.want {+ t.Fatalf("usage = %v, want %v", response.Usage, test.want)+ }+ })}}diff --git a/libs/fleet/backend/main.go b/libs/fleet/backend/main.goindex b6cb16624d..a7429e2c9b 100644--- a/libs/fleet/backend/main.go+++ b/libs/fleet/backend/main.go@@ -586,22 +586,30 @@ func retryDatabaseFeatures(ctx context.Context, cfg config.DatabaseConfigurationfunc initializeUsageProvider(ctx context.Context, cfg config.UsageConfiguration) (usage.Provider, func(), error) {noop := func() {}- // OpenCost is deployed independently from the usage credential. Keep the- // provider disabled until the database URL arrives so serving stays ready.+ // Query transport settings are deployed independently from the read-only+ // usage credential. Keep the provider disabled until the database URL arrives.if cfg.DatabaseURL == "" {return nil, noop, nil}- if cfg.OpenCostBaseURL == "" {- return nil, noop, fmt.Errorf("usage provider requires OpenCost when the database URL is configured")- }events, err := usage.NewPostgresEventStore(ctx, cfg.DatabaseURL)if err != nil {return nil, noop, fmt.Errorf("create usage event store: %w", err)}- allocations, err := usage.NewOpenCostClient(cfg.OpenCostBaseURL, cfg.QueryTimeout, cfg.MaxResponseBytes)+ allocations, err := usage.NewDataFusionAllocationClient(+ cfg.QueryWebhookURL,+ cfg.QueryHMACSecret,+ cfg.QueryResultBucket,+ cfg.QueryResultPrefix,+ cfg.QueryCluster,+ cfg.QueryEnvironment,+ cfg.QueryTimeout,+ cfg.QueryPollInterval,+ cfg.MaxResponseBytes,+ usage.NewS3QueryObjectStore(),+ )if err != nil {events.Close()- return nil, noop, fmt.Errorf("create OpenCost client: %w", err)+ return nil, noop, fmt.Errorf("create DataFusion allocation client: %w", err)}return usage.NewProvider(events, allocations, nil), events.Close, nil}diff --git a/libs/fleet/backend/main_test.go b/libs/fleet/backend/main_test.goindex a12a4a2e5a..d7e739e4e6 100644--- a/libs/fleet/backend/main_test.go+++ b/libs/fleet/backend/main_test.go@@ -1170,10 +1170,16 @@ func TestRetryDatabaseFeaturesStopsOnceReady(t *testing.T) {func TestInitializeUsageProviderDoesNotGateReadiness(t *testing.T) {provider, closeProvider, err := initializeUsageProvider(context.Background(), config.UsageConfiguration{- DatabaseURL: "postgres://cyclops_usage_reader:[email protected]:1/cyclops?sslmode=disable",- OpenCostBaseURL: "http://127.0.0.1:1",- QueryTimeout: time.Second,- MaxResponseBytes: 64 * 1024,+ DatabaseURL: "postgres://cyclops_usage_reader:[email protected]:1/cyclops?sslmode=disable",+ QueryWebhookURL: "https://cua-temporal-webhook.tail204509.ts.net/hooks/opencost-query",+ QueryHMACSecret: "secret",+ QueryResultBucket: "nanoclaw-telemetry-files",+ QueryResultPrefix: "cyclops/usage-query",+ QueryCluster: "kopf-k3s",+ QueryEnvironment: "production",+ QueryTimeout: time.Second,+ QueryPollInterval: time.Second,+ MaxResponseBytes: 64 * 1024,})if err != nil {t.Fatalf("initializeUsageProvider() error = %v", err)@@ -1196,11 +1202,17 @@ func TestInitializeUsageProviderDoesNotGateReadiness(t *testing.T) {}}-func TestInitializeUsageProviderOpenCostOnlyIsDisabled(t *testing.T) {+func TestInitializeUsageProviderQueryTransportOnlyIsDisabled(t *testing.T) {provider, closeProvider, err := initializeUsageProvider(context.Background(), config.UsageConfiguration{- OpenCostBaseURL: "http://127.0.0.1:1",- QueryTimeout: time.Second,- MaxResponseBytes: 64 * 1024,+ QueryWebhookURL: "https://cua-temporal-webhook.tail204509.ts.net/hooks/opencost-query",+ QueryHMACSecret: "secret",+ QueryResultBucket: "nanoclaw-telemetry-files",+ QueryResultPrefix: "cyclops/usage-query",+ QueryCluster: "kopf-k3s",+ QueryEnvironment: "production",+ QueryTimeout: time.Second,+ QueryPollInterval: time.Second,+ MaxResponseBytes: 64 * 1024,})if err != nil {t.Fatalf("initializeUsageProvider() error = %v", err)diff --git a/libs/fleet/backend/usage/datafusion.go b/libs/fleet/backend/usage/datafusion.gonew file mode 100644index 0000000000..43e33a2788--- /dev/null+++ b/libs/fleet/backend/usage/datafusion.go@@ -0,0 +1,440 @@+package usage++import (+ "bytes"+ "context"+ "crypto/hmac"+ "crypto/sha256"+ "encoding/csv"+ "encoding/hex"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "math"+ "net/http"+ "net/url"+ "regexp"+ "sort"+ "strconv"+ "strings"+ "time"++ "github.com/google/uuid"+)++const (+ maxDataFusionAllocationRows = 100000+ maxAllocationPartitions = 2000+ maxAllocationNamespaces = 500+ maxAllocationQueryBytes = 65536+ queryPresignGrace = 10 * time.Minute+ queryCleanupTimeout = 5 * time.Second+)++var (+ ErrQueryObjectNotFound = errors.New("query object not found")+ queryPartitionValue = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)+ queryPodName = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9.]*[a-z0-9])?$`)+)++type QueryObjectStore interface {+ PresignPut(context.Context, string, string, string, time.Duration) (string, error)+ Get(context.Context, string, string, int64) ([]byte, error)+ Delete(context.Context, string, string) error+}++type dataFusionAllocationClient struct {+ webhookURL *url.URL+ hmacSecret string+ resultBucket string+ resultPrefix string+ cluster string+ environment string+ timeout time.Duration+ pollInterval time.Duration+ maxResponseSize int64+ store QueryObjectStore+ httpClient *http.Client+}++type dataFusionWebhookPayload struct {+ Dataset string `json:"dataset"`+ Cluster string `json:"cluster"`+ Environment string `json:"environment"`+ Window dataFusionQueryWindow `json:"window"`+ Query string `json:"query"`+ OutputURL string `json:"output_url"`+ OutputFormat string `json:"output_format"`+ StatusURL string `json:"status_url"`+}++type dataFusionQueryWindow struct {+ Start time.Time `json:"start"`+ End time.Time `json:"end"`+}++type dataFusionQueryStatus struct {+ Status string `json:"status"`+ Rows int64 `json:"rows"`+ PartitionsExpanded int `json:"partitions_expanded"`+ PartitionsMatched int `json:"partitions_matched"`+ PartitionsMissing int `json:"partitions_missing"`+}++func NewDataFusionAllocationClient(rawURL, hmacSecret, resultBucket, resultPrefix, cluster, environment string, timeout, pollInterval time.Duration, maxResponseBytes int64, store QueryObjectStore) (*dataFusionAllocationClient, error) {+ webhookURL, err := url.Parse(strings.TrimSpace(rawURL))+ if err != nil {+ return nil, newSanitizedError("invalid allocation query webhook URL", err)+ }+ if (webhookURL.Scheme != "http" && webhookURL.Scheme != "https") || webhookURL.Host == "" || webhookURL.User != nil || webhookURL.RawQuery != "" || webhookURL.Fragment != "" {+ return nil, fmt.Errorf("invalid allocation query webhook URL")+ }+ resultPrefix = strings.Trim(strings.TrimSpace(resultPrefix), "/")+ if strings.TrimSpace(hmacSecret) == "" || strings.TrimSpace(resultBucket) == "" || !safeResultPrefix(resultPrefix) || !queryPartitionValue.MatchString(cluster) || !queryPartitionValue.MatchString(environment) || timeout <= 0 || pollInterval <= 0 || maxResponseBytes <= 0 || maxResponseBytes == math.MaxInt64 || store == nil {+ return nil, fmt.Errorf("invalid DataFusion allocation client configuration")+ }+ return &dataFusionAllocationClient{+ webhookURL: webhookURL,+ hmacSecret: hmacSecret,+ resultBucket: strings.TrimSpace(resultBucket),+ resultPrefix: resultPrefix,+ cluster: cluster,+ environment: environment,+ timeout: timeout,+ pollInterval: pollInterval,+ maxResponseSize: maxResponseBytes,+ store: store,+ httpClient: &http.Client{+ CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse },+ },+ }, nil+}++func safeResultPrefix(prefix string) bool {+ if prefix == "" || strings.Contains(prefix, "\\") {+ return false+ }+ for _, part := range strings.Split(prefix, "/") {+ if part == "" || part == "." || part == ".." || !queryPartitionValue.MatchString(part) {+ return false+ }+ }+ return true+}++func (client *dataFusionAllocationClient) Allocations(ctx context.Context, start, end time.Time, step time.Duration, namespaces []string) ([]Allocation, time.Time, bool, error) {+ if err := ctx.Err(); err != nil {+ return nil, time.Time{}, false, err+ }+ if start.IsZero() || end.IsZero() || !end.After(start) || start.UTC() != start || end.UTC() != end || !start.Equal(start.Truncate(time.Hour)) || !end.Equal(end.Truncate(time.Hour)) || len(namespaces) == 0 {+ return nil, time.Time{}, false, fmt.Errorf("invalid DataFusion allocation query")+ }+ if step != time.Hour && step != 24*time.Hour {+ return nil, time.Time{}, false, fmt.Errorf("unsupported allocation step")+ }+ if end.Sub(start) > maxAllocationPartitions*time.Hour {+ return nil, time.Time{}, false, fmt.Errorf("allocation query window exceeds limit")+ }+ namespaces, err := safeNamespaces(namespaces)+ if err != nil {+ return nil, time.Time{}, false, err+ }++ if len(namespaces) > maxAllocationNamespaces {+ return nil, time.Time{}, false, fmt.Errorf("allocation namespace limit exceeded")+ }+ query := allocationSQL(namespaces)+ if len(query) > maxAllocationQueryBytes {+ return nil, time.Time{}, false, fmt.Errorf("allocation query exceeds limit")+ }+ requestCtx, cancel := context.WithTimeout(ctx, client.timeout)+ defer cancel()++ prefix := fmt.Sprintf("%s/%s/%s", client.resultPrefix, time.Now().UTC().Format("2006-01-02"), uuid.NewString())+ resultKey := prefix + "/result.csv"+ statusKey := prefix + "/status.json"+ defer func() {+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), queryCleanupTimeout)+ defer cleanupCancel()+ _ = client.store.Delete(cleanupCtx, client.resultBucket, resultKey)+ _ = client.store.Delete(cleanupCtx, client.resultBucket, statusKey)+ }()++ expires := client.timeout + queryPresignGrace+ resultURL, err := client.store.PresignPut(requestCtx, client.resultBucket, resultKey, "text/csv", expires)+ if err != nil {+ return nil, time.Time{}, false, newSanitizedError("prepare allocation query result", err)+ }+ statusURL, err := client.store.PresignPut(requestCtx, client.resultBucket, statusKey, "application/json", expires)+ if err != nil {+ return nil, time.Time{}, false, newSanitizedError("prepare allocation query status", err)+ }+ payload, err := json.Marshal(dataFusionWebhookPayload{+ Dataset: "allocation",+ Cluster: client.cluster,+ Environment: client.environment,+ Window: dataFusionQueryWindow{Start: start, End: end},+ Query: query,+ OutputURL: resultURL,+ OutputFormat: "csv",+ StatusURL: statusURL,+ })+ if err != nil {+ return nil, time.Time{}, false, newSanitizedError("encode allocation query request", err)+ }+ request, err := http.NewRequestWithContext(requestCtx, http.MethodPost, client.webhookURL.String(), bytes.NewReader(payload))+ if err != nil {+ return nil, time.Time{}, false, newSanitizedError("build allocation query request", err)+ }+ request.Header.Set("Content-Type", "application/json")+ request.Header.Set("X-Hub-Signature-256", signQueryPayload(payload, client.hmacSecret))+ response, err := client.httpClient.Do(request)+ if err != nil {+ safeErr := newSanitizedError("send allocation query request", err)+ if errors.Is(requestCtx.Err(), context.DeadlineExceeded) {+ return nil, time.Time{}, false, errors.Join(errors.New("allocation query timed out"), context.DeadlineExceeded, safeErr)+ }+ if contextErr := requestCtx.Err(); contextErr != nil {+ return nil, time.Time{}, false, errors.Join(contextErr, safeErr)+ }+ return nil, time.Time{}, false, safeErr+ }+ if response == nil {+ return nil, time.Time{}, false, fmt.Errorf("allocation query webhook returned no response")+ }+ drainAndClose(response.Body)+ if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {+ return nil, time.Time{}, false, fmt.Errorf("allocation query webhook returned HTTP %d", response.StatusCode)+ }++ status, err := client.waitForQuery(requestCtx, statusKey)+ if err != nil {+ return nil, time.Time{}, false, err+ }+ if status.Rows > maxDataFusionAllocationRows {+ return nil, time.Time{}, false, fmt.Errorf("allocation query row limit exceeded")+ }+ contents, err := client.store.Get(requestCtx, client.resultBucket, resultKey, client.maxResponseSize)+ if err != nil {+ return nil, time.Time{}, false, newSanitizedError("read allocation query result", err)+ }+ allocations, asOf, parsedRows, legacyPartial, err := parseDataFusionAllocations(contents, start, end, namespaces)+ if err != nil {+ return nil, time.Time{}, false, err+ }+ if int64(parsedRows) != status.Rows {+ return nil, time.Time{}, false, fmt.Errorf("allocation query row count mismatch")+ }+ partial := legacyPartial || status.PartitionsMissing > 0 || status.PartitionsMatched != status.PartitionsExpanded+ if !partial {+ asOf = end+ } else if asOf.IsZero() {+ asOf = start+ }+ return allocations, asOf, partial, nil+}++func safeNamespaces(input []string) ([]string, error) {+ seen := make(map[string]struct{}, len(input))+ result := make([]string, 0, len(input))+ for _, namespace := range input {+ if len(namespace) > 63 || !dns1123Label.MatchString(namespace) {+ return nil, fmt.Errorf("invalid namespace")+ }+ if _, ok := seen[namespace]; ok {+ continue+ }+ seen[namespace] = struct{}{}+ result = append(result, namespace)+ }+ sort.Strings(result)+ return result, nil+}++func allocationSQL(namespaces []string) string {+ literals := make([]string, len(namespaces))+ for index, namespace := range namespaces {+ literals[index] = "'" + namespace + "'"+ }+ return "SELECT window_start, window_end, namespace, pod, cpu_usage_core_hours, cpu_request_core_hours, ram_usage_byte_hours, ram_request_byte_hours " ++ "FROM allocation WHERE namespace IN (" + strings.Join(literals, ",") + ") " ++ "ORDER BY window_start, namespace, pod LIMIT 100001"+}++func signQueryPayload(body []byte, secret string) string {+ mac := hmac.New(sha256.New, []byte(secret))+ _, _ = mac.Write(body)+ return "sha256=" + hex.EncodeToString(mac.Sum(nil))+}++func drainAndClose(body io.ReadCloser) {+ if body == nil {+ return+ }+ _, _ = io.Copy(io.Discard, io.LimitReader(body, 64<<10))+ _ = body.Close()+}++func (client *dataFusionAllocationClient) waitForQuery(ctx context.Context, statusKey string) (dataFusionQueryStatus, error) {+ ticker := time.NewTicker(client.pollInterval)+ defer ticker.Stop()+ for {+ status, ready, err := client.readQueryStatus(ctx, statusKey)+ if err != nil {+ return dataFusionQueryStatus{}, err+ }+ if ready {+ return status, nil+ }+ select {+ case <-ctx.Done():+ if errors.Is(ctx.Err(), context.DeadlineExceeded) {+ return dataFusionQueryStatus{}, errors.Join(errors.New("allocation query timed out"), context.DeadlineExceeded)+ }+ return dataFusionQueryStatus{}, ctx.Err()+ case <-ticker.C:+ }+ }+}++func (client *dataFusionAllocationClient) readQueryStatus(ctx context.Context, statusKey string) (dataFusionQueryStatus, bool, error) {+ body, err := client.store.Get(ctx, client.resultBucket, statusKey, client.maxResponseSize)+ if err != nil {+ if errors.Is(err, ErrQueryObjectNotFound) {+ return dataFusionQueryStatus{}, false, nil+ }+ return dataFusionQueryStatus{}, false, newSanitizedError("read allocation query status", err)+ }+ var status dataFusionQueryStatus+ if decodeErr := json.Unmarshal(body, &status); decodeErr != nil {+ return dataFusionQueryStatus{}, false, newSanitizedError("decode allocation query status", decodeErr)+ }+ switch status.Status {+ case "succeeded":+ if status.Rows < 0 || status.PartitionsExpanded < 0 || status.PartitionsMatched < 0 || status.PartitionsMissing < 0 || status.PartitionsMatched+status.PartitionsMissing != status.PartitionsExpanded {+ return dataFusionQueryStatus{}, false, fmt.Errorf("invalid allocation query status")+ }+ return status, true, nil+ case "failed":+ return dataFusionQueryStatus{}, false, fmt.Errorf("allocation query failed")+ case "running", "":+ return dataFusionQueryStatus{}, false, nil+ default:+ return dataFusionQueryStatus{}, false, fmt.Errorf("invalid allocation query status")+ }+}++func parseDataFusionAllocations(contents []byte, requestedStart, requestedEnd time.Time, namespaces []string) ([]Allocation, time.Time, int, bool, error) {+ reader := csv.NewReader(bytes.NewReader(contents))+ reader.FieldsPerRecord = -1+ header, err := reader.Read()+ if err != nil {+ return nil, time.Time{}, 0, false, newSanitizedError("decode allocation query result", err)+ }+ wantHeader := []string{"window_start", "window_end", "namespace", "pod", "cpu_usage_core_hours", "cpu_request_core_hours", "ram_usage_byte_hours", "ram_request_byte_hours"}+ if len(header) != len(wantHeader) {+ return nil, time.Time{}, 0, false, fmt.Errorf("decode allocation query result: unexpected columns")+ }+ for index := range header {+ if header[index] != wantHeader[index] {+ return nil, time.Time{}, 0, false, fmt.Errorf("decode allocation query result: unexpected columns")+ }+ }+ allowed := make(map[string]struct{}, len(namespaces))+ for _, namespace := range namespaces {+ allowed[namespace] = struct{}{}+ }+ allocations := make([]Allocation, 0)+ var asOf time.Time+ parsedRows := 0+ legacyPartial := false+ for rowNumber := 2; ; rowNumber++ {+ record, readErr := reader.Read()+ if readErr == io.EOF {+ break+ }+ if readErr != nil {+ return nil, time.Time{}, 0, false, newSanitizedError("decode allocation query result: malformed row", readErr)+ }+ if len(record) != len(wantHeader) {+ return nil, time.Time{}, 0, false, fmt.Errorf("decode allocation query result: malformed row")+ }+ parsedRows+++ if parsedRows > maxDataFusionAllocationRows {+ return nil, time.Time{}, 0, false, fmt.Errorf("allocation query row limit exceeded")+ }+ if record[3] == "NULL" || record[4] == "NULL" || record[5] == "NULL" || record[6] == "NULL" || record[7] == "NULL" {+ legacyPartial = true+ continue+ }+ allocation, parseErr := parseDataFusionAllocation(record, requestedStart, requestedEnd, allowed)+ if parseErr != nil {+ return nil, time.Time{}, 0, false, newSanitizedError(fmt.Sprintf("decode allocation query result: malformed row %d", rowNumber), parseErr)+ }+ allocations = append(allocations, allocation)+ if allocation.End.After(asOf) {+ asOf = allocation.End+ }+ }+ return allocations, asOf, parsedRows, legacyPartial, nil+}++func parseDataFusionAllocation(record []string, requestedStart, requestedEnd time.Time, allowed map[string]struct{}) (Allocation, error) {+ start, err := parseDataFusionTimestamp(record[0])+ if err != nil {+ return Allocation{}, err+ }+ end, err := parseDataFusionTimestamp(record[1])+ if err != nil {+ return Allocation{}, newSanitizedError("invalid allocation window", err)+ }+ if !end.After(start) || end.Sub(start) != time.Hour || start.Before(requestedStart) || end.After(requestedEnd) {+ return Allocation{}, fmt.Errorf("invalid allocation window")+ }+ namespace := record[2]+ if _, ok := allowed[namespace]; !ok {+ return Allocation{}, fmt.Errorf("unexpected namespace")+ }+ pod := record[3]+ if len(pod) > 253 || !queryPodName.MatchString(pod) {+ return Allocation{}, fmt.Errorf("invalid pod")+ }+ values := make([]float64, 4)+ for index := range values {+ value, parseErr := strconv.ParseFloat(record[index+4], 64)+ if parseErr != nil {+ return Allocation{}, newSanitizedError("invalid allocation value", parseErr)+ }+ if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value > 1e18 {+ return Allocation{}, fmt.Errorf("invalid allocation value")+ }+ values[index] = value+ }+ hours := end.Sub(start).Hours()+ return Allocation{+ Start: start.UTC(),+ End: end.UTC(),+ Namespace: namespace,+ Pod: pod,+ Minutes: end.Sub(start).Minutes(),+ CPUUsageAverage: values[0] / hours,+ CPURequestAverage: values[1] / hours,+ RAMUsageAverageBytes: values[2] / hours,+ RAMRequestAverageBytes: values[3] / hours,+ }, nil+}++func parseDataFusionTimestamp(raw string) (time.Time, error) {+ parsed, err := time.Parse(time.RFC3339Nano, raw)+ if err == nil {+ return parsed.UTC(), nil+ }+ parsed, fallbackErr := time.ParseInLocation("2006-01-02 15:04:05.999999999", raw, time.UTC)+ if fallbackErr != nil {+ return time.Time{}, errors.Join(err, fallbackErr)+ }+ return parsed, nil+}++var _ AllocationClient = (*dataFusionAllocationClient)(nil)diff --git a/libs/fleet/backend/usage/datafusion_s3.go b/libs/fleet/backend/usage/datafusion_s3.gonew file mode 100644index 0000000000..7e5b1b5ffb--- /dev/null+++ b/libs/fleet/backend/usage/datafusion_s3.go@@ -0,0 +1,86 @@+package usage++import (+ "context"+ "errors"+ "fmt"+ "io"+ "sync"+ "time"++ awsconfig "github.com/aws/aws-sdk-go-v2/config"+ "github.com/aws/aws-sdk-go-v2/service/s3"+ "github.com/aws/smithy-go"+)++type s3QueryObjectStore struct {+ once sync.Once+ client *s3.Client+ presigner *s3.PresignClient+ err error+}++func NewS3QueryObjectStore() QueryObjectStore {+ return &s3QueryObjectStore{}+}++func (store *s3QueryObjectStore) load(ctx context.Context) error {+ store.once.Do(func() {+ configuration, err := awsconfig.LoadDefaultConfig(ctx)+ if err != nil {+ store.err = err+ return+ }+ store.client = s3.NewFromConfig(configuration)+ store.presigner = s3.NewPresignClient(store.client)+ })+ return store.err+}++func (store *s3QueryObjectStore) PresignPut(ctx context.Context, bucket, key, contentType string, expires time.Duration) (string, error) {+ if err := store.load(ctx); err != nil {+ return "", err+ }+ result, err := store.presigner.PresignPutObject(ctx, &s3.PutObjectInput{+ Bucket: &bucket,+ Key: &key,+ ContentType: &contentType,+ }, func(options *s3.PresignOptions) {+ options.Expires = expires+ })+ if err != nil {+ return "", err+ }+ return result.URL, nil+}++func (store *s3QueryObjectStore) Get(ctx context.Context, bucket, key string, maxBytes int64) ([]byte, error) {+ if err := store.load(ctx); err != nil {+ return nil, err+ }+ result, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &key})+ if err != nil {+ var apiError smithy.APIError+ if errors.As(err, &apiError) && (apiError.ErrorCode() == "NoSuchKey" || apiError.ErrorCode() == "NotFound") {+ return nil, errors.Join(ErrQueryObjectNotFound, newSanitizedError("query object not found", err))+ }+ return nil, err+ }+ defer result.Body.Close()+ body, err := io.ReadAll(io.LimitReader(result.Body, maxBytes+1))+ if err != nil {+ return nil, err+ }+ if int64(len(body)) > maxBytes {+ return nil, fmt.Errorf("query object exceeds limit")+ }+ return body, nil+}++func (store *s3QueryObjectStore) Delete(ctx context.Context, bucket, key string) error {+ if err := store.load(ctx); err != nil {+ return err+ }+ _, err := store.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &key})+ return err+}diff --git a/libs/fleet/backend/usage/datafusion_test.go b/libs/fleet/backend/usage/datafusion_test.gonew file mode 100644index 0000000000..d3e6cb5938--- /dev/null+++ b/libs/fleet/backend/usage/datafusion_test.go@@ -0,0 +1,222 @@+package usage++import (+ "context"+ "crypto/hmac"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "testing"+ "time"+)++type fakeQueryObjectStore struct {+ mu sync.Mutex+ status [][]byte+ result []byte+ deletedKeys []string+}++func (store *fakeQueryObjectStore) PresignPut(_ context.Context, _ string, key, contentType string, _ time.Duration) (string, error) {+ return "https://uploads.example.test/" + key + "?X-Amz-Signature=secret-" + contentType, nil+}++func (store *fakeQueryObjectStore) Get(_ context.Context, _ string, key string, _ int64) ([]byte, error) {+ store.mu.Lock()+ defer store.mu.Unlock()+ if strings.HasSuffix(key, "/status.json") {+ if len(store.status) == 0 {+ return nil, ErrQueryObjectNotFound+ }+ body := store.status[0]+ store.status = store.status[1:]+ return body, nil+ }+ if strings.HasSuffix(key, "/result.csv") {+ return store.result, nil+ }+ return nil, ErrQueryObjectNotFound+}++func (store *fakeQueryObjectStore) Delete(_ context.Context, _ string, key string) error {+ store.mu.Lock()+ defer store.mu.Unlock()+ store.deletedKeys = append(store.deletedKeys, key)+ return nil+}++func TestDataFusionAllocationClientQueriesBoundedParquetAndMapsSemantics(t *testing.T) {+ t.Parallel()+ secret := "webhook-secret"+ var payload dataFusionWebhookPayload+ server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {+ body, err := io.ReadAll(request.Body)+ if err != nil {+ t.Fatal(err)+ }+ mac := hmac.New(sha256.New, []byte(secret))+ _, _ = mac.Write(body)+ wantSignature := "sha256=" + hex.EncodeToString(mac.Sum(nil))+ if request.Header.Get("X-Hub-Signature-256") != wantSignature {+ t.Fatalf("signature = %q, want %q", request.Header.Get("X-Hub-Signature-256"), wantSignature)+ }+ if err := json.Unmarshal(body, &payload); err != nil {+ t.Fatal(err)+ }+ writer.WriteHeader(http.StatusAccepted)+ }))+ defer server.Close()++ start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)+ store := &fakeQueryObjectStore{+ status: [][]byte{[]byte(`{"status":"succeeded","rows":1,"partitions_expanded":1,"partitions_matched":1,"partitions_missing":0}`)},+ result: []byte("window_start,window_end,namespace,pod,cpu_usage_core_hours,cpu_request_core_hours,ram_usage_byte_hours,ram_request_byte_hours\n" ++ "2026-08-19 09:00:00,2026-08-19 10:00:00,ns-a,virt-launcher-vm-a-x,2,4,3221225472,6442450944\n"),+ }+ client, err := NewDataFusionAllocationClient(server.URL, secret, "results-bucket", "cyclops/usage-query", "kopf-k3s", "production", time.Second, time.Millisecond, 1<<20, store)+ if err != nil {+ t.Fatal(err)+ }+ allocations, asOf, partial, err := client.Allocations(context.Background(), start, start.Add(time.Hour), time.Hour, []string{"ns-b", "ns-a", "ns-a"})+ if err != nil || partial || !asOf.Equal(start.Add(time.Hour)) || len(allocations) != 1 {+ t.Fatalf("allocations=%#v asOf=%s partial=%t err=%v", allocations, asOf, partial, err)+ }+ allocation := allocations[0]+ if allocation.Minutes != 60 || allocation.CPUUsageAverage != 2 || allocation.CPURequestAverage != 4 || allocation.RAMUsageAverageBytes != 3*gibibyte || allocation.RAMRequestAverageBytes != 6*gibibyte {+ t.Fatalf("allocation = %#v", allocation)+ }+ if payload.Dataset != "allocation" || payload.Cluster != "kopf-k3s" || payload.Environment != "production" || payload.OutputFormat != "csv" {+ t.Fatalf("payload = %#v", payload)+ }+ for _, fragment := range []string{"namespace IN ('ns-a','ns-b')", "LIMIT 100001", "cpu_usage_core_hours", "cpu_request_core_hours", "ram_usage_byte_hours", "ram_request_byte_hours"} {+ if !strings.Contains(payload.Query, fragment) {+ t.Fatalf("query %q does not contain %q", payload.Query, fragment)+ }+ }+ if len(store.deletedKeys) != 2 {+ t.Fatalf("deleted keys = %v", store.deletedKeys)+ }+}++func TestDataFusionAllocationClientRejectsUnsafeNamespaceBeforeWebhook(t *testing.T) {+ t.Parallel()+ called := false+ server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true }))+ defer server.Close()+ client, err := NewDataFusionAllocationClient(server.URL, "secret", "bucket", "cyclops/usage-query", "kopf-k3s", "production", time.Second, time.Millisecond, 1<<20, &fakeQueryObjectStore{})+ if err != nil {+ t.Fatal(err)+ }+ start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)+ _, _, _, err = client.Allocations(context.Background(), start, start.Add(time.Hour), time.Hour, []string{"ns-a') OR true --"})+ if err == nil || !strings.Contains(err.Error(), "invalid namespace") || called {+ t.Fatalf("called=%t err=%v", called, err)+ }+}++func TestDataFusionAllocationClientHandlesMalformedNon2xxAndTimeoutSafely(t *testing.T) {+ t.Parallel()+ start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)+ for name, test := range map[string]struct {+ handler http.HandlerFunc+ store *fakeQueryObjectStore+ want string+ }{+ "non_2xx": {+ handler: func(writer http.ResponseWriter, _ *http.Request) {+ http.Error(writer, "SELECT secret FROM tenant", http.StatusBadGateway)+ },+ store: &fakeQueryObjectStore{},+ want: "query webhook returned HTTP 502",+ },+ "malformed_csv": {+ handler: func(writer http.ResponseWriter, _ *http.Request) { writer.WriteHeader(http.StatusAccepted) },+ store: &fakeQueryObjectStore{+ status: [][]byte{[]byte(`{"status":"succeeded","rows":1,"partitions_expanded":1,"partitions_matched":1}`)},+ result: []byte("window_start,namespace\nhttps://bucket.example/?X-Amz-Signature=leak,ns-a\n"),+ },+ want: "decode allocation query result",+ },+ "timeout": {+ handler: func(writer http.ResponseWriter, _ *http.Request) { writer.WriteHeader(http.StatusAccepted) },+ store: &fakeQueryObjectStore{},+ want: "allocation query timed out",+ },+ } {+ t.Run(name, func(t *testing.T) {+ server := httptest.NewServer(test.handler)+ defer server.Close()+ timeout := time.Second+ if name == "timeout" {+ timeout = 5 * time.Millisecond+ }+ client, err := NewDataFusionAllocationClient(server.URL, "secret", "bucket", "cyclops/usage-query", "kopf-k3s", "production", timeout, time.Millisecond, 1<<20, test.store)+ if err != nil {+ t.Fatal(err)+ }+ _, _, _, err = client.Allocations(context.Background(), start, start.Add(time.Hour), time.Hour, []string{"ns-a"})+ if err == nil || !strings.Contains(err.Error(), test.want) {+ t.Fatalf("error = %v, want containing %q", err, test.want)+ }+ for _, forbidden := range []string{"SELECT secret", "X-Amz-Signature", "tenant"} {+ if strings.Contains(err.Error(), forbidden) {+ t.Fatalf("error leaked %q: %v", forbidden, err)+ }+ }+ })+ }+}++func TestDataFusionAllocationClientPreservesCancellationAndMissingPartitions(t *testing.T) {+ t.Parallel()+ start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)+ server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { writer.WriteHeader(http.StatusAccepted) }))+ defer server.Close()+ store := &fakeQueryObjectStore{+ status: [][]byte{[]byte(`{"status":"succeeded","rows":0,"partitions_expanded":2,"partitions_matched":1,"partitions_missing":1}`)},+ result: []byte("window_start,window_end,namespace,pod,cpu_usage_core_hours,cpu_request_core_hours,ram_usage_byte_hours,ram_request_byte_hours\n"),+ }+ client, err := NewDataFusionAllocationClient(server.URL, "secret", "bucket", "cyclops/usage-query", "kopf-k3s", "production", time.Second, time.Millisecond, 1<<20, store)+ if err != nil {+ t.Fatal(err)+ }+ allocations, asOf, partial, err := client.Allocations(context.Background(), start, start.Add(2*time.Hour), time.Hour, []string{"ns-a"})+ if err != nil || len(allocations) != 0 || !partial || !asOf.Equal(start) {+ t.Fatalf("allocations=%v asOf=%s partial=%t err=%v", allocations, asOf, partial, err)+ }++ cancelled, cancel := context.WithCancel(context.Background())+ cancel()+ _, _, _, err = client.Allocations(cancelled, start, start.Add(time.Hour), time.Hour, []string{"ns-a"})+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("error = %v, want context.Canceled", err)+ }+}++func TestDataFusionAllocationClientMarksLegacyParquetRowsPartial(t *testing.T) {+ t.Parallel()+ start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)+ server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {+ writer.WriteHeader(http.StatusAccepted)+ }))+ defer server.Close()+ store := &fakeQueryObjectStore{+ status: [][]byte{[]byte(`{"status":"succeeded","table":"allocation","rows":1,"partitions_expanded":1,"partitions_matched":1,"partitions_missing":0}`)},+ result: []byte("window_start,window_end,namespace,pod,cpu_usage_core_hours,cpu_request_core_hours,ram_usage_byte_hours,ram_request_byte_hours\n" ++ "2026-08-19T09:00:00Z,2026-08-19T10:00:00Z,ns-a,NULL,NULL,NULL,NULL,NULL\n"),+ }+ client, err := NewDataFusionAllocationClient(server.URL, "secret", "bucket", "cyclops/usage-query", "kopf-k3s", "production", time.Second, time.Millisecond, 1<<20, store)+ if err != nil {+ t.Fatal(err)+ }+ allocations, asOf, partial, err := client.Allocations(context.Background(), start, start.Add(time.Hour), time.Hour, []string{"ns-a"})+ if err != nil || len(allocations) != 0 || !partial || !asOf.Equal(start) {+ t.Fatalf("allocations=%v asOf=%s partial=%t err=%v", allocations, asOf, partial, err)+ }+}diff --git a/libs/fleet/backend/usage/opencost.go b/libs/fleet/backend/usage/opencost.godeleted file mode 100644index b409d63f21..0000000000--- a/libs/fleet/backend/usage/opencost.go+++ /dev/null@@ -1,328 +0,0 @@-package usage--import (- "context"- "encoding/json"- "fmt"- "io"- "math"- "net/http"- "net/url"- "regexp"- "sort"- "strings"- "time"-)--const maxAllocationRows = 100000--var dns1123Label = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)--type openCostClient struct {- baseURL *url.URL- httpClient *http.Client- timeout time.Duration- maxResponseBytes int64-}--func NewOpenCostClient(rawURL string, timeout time.Duration, maxResponseBytes int64) (*openCostClient, error) {- baseURL, err := url.Parse(rawURL)- if err != nil {- return nil, newSanitizedError("invalid OpenCost URL", err)- }- if (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" || baseURL.RawQuery != "" || baseURL.Fragment != "" || baseURL.User != nil {- return nil, fmt.Errorf("invalid OpenCost URL")- }- if timeout <= 0 || maxResponseBytes <= 0 || maxResponseBytes == math.MaxInt64 {- return nil, fmt.Errorf("invalid OpenCost client limits")- }- return &openCostClient{- baseURL: baseURL,- timeout: timeout,- maxResponseBytes: maxResponseBytes,- httpClient: &http.Client{- Timeout: timeout,- CheckRedirect: func(_ *http.Request, _ []*http.Request) error {- return http.ErrUseLastResponse- },- },- }, nil-}--func (c *openCostClient) Allocations(ctx context.Context, start, end time.Time, step time.Duration, namespaces []string) ([]Allocation, time.Time, bool, error) {- if start.IsZero() || end.IsZero() || !end.After(start) || len(namespaces) == 0 {- return nil, time.Time{}, false, fmt.Errorf("invalid OpenCost allocation query")- }- stepText, err := openCostStep(step)- if err != nil {- return nil, time.Time{}, false, err- }- filter, namespaces, err := namespaceFilter(namespaces)- if err != nil {- return nil, time.Time{}, false, err- }-- endpoint := *c.baseURL- endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/allocation/compute"- values := make(url.Values)- values.Set("window", start.UTC().Format(time.RFC3339)+","+end.UTC().Format(time.RFC3339))- values.Set("aggregate", "cluster,namespace,pod")- values.Set("includeIdle", "false")- values.Set("includeSharedCostBreakdown", "false")- values.Set("accumulate", "false")- values.Set("step", stepText)- values.Set("filter", filter)- endpoint.RawQuery = values.Encode()-- requestCtx, cancel := context.WithTimeout(ctx, c.timeout)- defer cancel()- request, err := http.NewRequestWithContext(requestCtx, http.MethodGet, endpoint.String(), nil)- if err != nil {- return nil, time.Time{}, false, fmt.Errorf("build OpenCost request: %w", err)- }- response, err := c.httpClient.Do(request)- if err != nil {- return nil, time.Time{}, false, fmt.Errorf("request OpenCost allocations: %w", err)- }- defer response.Body.Close()- if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {- return nil, time.Time{}, false, fmt.Errorf("OpenCost allocation request failed with status %d", response.StatusCode)- }- body, err := io.ReadAll(io.LimitReader(response.Body, c.maxResponseBytes+1))- if err != nil {- return nil, time.Time{}, false, fmt.Errorf("read OpenCost allocation response: %w", err)- }- if int64(len(body)) > c.maxResponseBytes {- return nil, time.Time{}, false, fmt.Errorf("OpenCost allocation response exceeds limit")- }- allocations, asOf, incomplete, err := parseOpenCostAllocations(body, start, end, step, namespaces)- if err != nil {- return nil, time.Time{}, false, err- }- return allocations, asOf, incomplete, nil-}--func openCostStep(step time.Duration) (string, error) {- switch step {- case time.Hour:- return "1h", nil- case 24 * time.Hour:- return "1d", nil- default:- return "", fmt.Errorf("unsupported OpenCost allocation step")- }-}--func namespaceFilter(input []string) (string, []string, error) {- seen := make(map[string]struct{}, len(input))- namespaces := make([]string, 0, len(input))- for _, namespace := range input {- if len(namespace) > 63 || !dns1123Label.MatchString(namespace) {- return "", nil, fmt.Errorf("invalid namespace")- }- if _, ok := seen[namespace]; !ok {- seen[namespace] = struct{}{}- namespaces = append(namespaces, namespace)- }- }- sort.Strings(namespaces)- parts := make([]string, len(namespaces))- for index, namespace := range namespaces {- parts[index] = `namespace:"` + namespace + `"`- }- return strings.Join(parts, "|"), namespaces, nil-}--type openCostResponse struct {- Code *int `json:"code"`- Data json.RawMessage `json:"data"`- Error json.RawMessage `json:"error"`- Warnings json.RawMessage `json:"warnings"`-}--type openCostAllocation struct {- Properties struct {- Namespace string `json:"namespace"`- Pod string `json:"pod"`- } `json:"properties"`- Window json.RawMessage `json:"window"`- Minutes *float64 `json:"minutes"`- CPUUsageAverage *float64 `json:"cpuCoreUsageAverage"`- CPURequestAverage *float64 `json:"cpuCoreRequestAverage"`- RAMUsageAverageBytes *float64 `json:"ramByteUsageAverage"`- RAMRequestAverageBytes *float64 `json:"ramByteRequestAverage"`-}--type allocationFrame struct {- start time.Time- end time.Time-}--func parseOpenCostAllocations(body []byte, requestedStart, requestedEnd time.Time, step time.Duration, namespaces []string) ([]Allocation, time.Time, bool, error) {- var payload openCostResponse- if err := json.Unmarshal(body, &payload); err != nil {- return nil, time.Time{}, false, fmt.Errorf("parse OpenCost allocation response: %w", err)- }- if payload.Code == nil || *payload.Code != http.StatusOK || hasOpenCostError(payload.Error) {- return nil, time.Time{}, false, fmt.Errorf("OpenCost allocation response reports an error")- }- if len(payload.Data) == 0 || string(payload.Data) == "null" {- return nil, time.Time{}, false, fmt.Errorf("OpenCost allocation response has no data")- }- var frames []map[string]json.RawMessage- if err := json.Unmarshal(payload.Data, &frames); err != nil {- return nil, time.Time{}, false, newSanitizedError("OpenCost allocation response has invalid data", err)- }- if frames == nil {- return nil, time.Time{}, false, fmt.Errorf("OpenCost allocation response has invalid data")- }-- allowed := make(map[string]struct{}, len(namespaces))- for _, namespace := range namespaces {- allowed[namespace] = struct{}{}- }- incomplete := hasOpenCostWarnings(payload.Warnings)- coverage := make([]allocationFrame, 0, len(frames))- for _, frame := range frames {- window, ok := openCostFrameWindow(frame, requestedStart, requestedEnd, step)- if !ok {- incomplete = true- continue- }- coverage = append(coverage, window)- }- asOf, coverageComplete := completeCoverageBoundary(coverage, requestedStart, requestedEnd)- incomplete = incomplete || !coverageComplete-- allocations := make([]Allocation, 0)- rowCount := 0- for _, group := range frames {- for _, raw := range group {- rowCount++- if rowCount > maxAllocationRows {- return nil, time.Time{}, false, fmt.Errorf("OpenCost allocation response exceeds row limit")- }- var row openCostAllocation- if err := json.Unmarshal(raw, &row); err != nil {- return nil, time.Time{}, false, newSanitizedError("OpenCost allocation response contains malformed allocation data", err)- }- allocation, ok := validateOpenCostAllocation(row, requestedStart, requestedEnd, step, allowed)- if !ok {- incomplete = true- continue- }- if allocation.End.After(asOf) {- continue- }- allocations = append(allocations, allocation)- }- }- return allocations, asOf, incomplete, nil-}--func hasOpenCostError(raw json.RawMessage) bool {- return len(raw) > 0 && string(raw) != "null" && string(raw) != `""`-}--func hasOpenCostWarnings(raw json.RawMessage) bool {- return len(raw) > 0 && string(raw) != "null" && string(raw) != "[]"-}--func openCostFrameWindow(frame map[string]json.RawMessage, requestedStart, requestedEnd time.Time, step time.Duration) (allocationFrame, bool) {- if len(frame) == 0 {- return allocationFrame{}, false- }- var result allocationFrame- for _, raw := range frame {- var header struct {- Window json.RawMessage `json:"window"`- }- if err := json.Unmarshal(raw, &header); err != nil {- return allocationFrame{}, false- }- start, end, ok := parseOpenCostWindow(header.Window)- if !ok || start.Before(requestedStart) || end.After(requestedEnd) || !end.After(start) || end.Sub(start) != step {- return allocationFrame{}, false- }- if result.start.IsZero() {- result = allocationFrame{start: start, end: end}- continue- }- if !result.start.Equal(start) || !result.end.Equal(end) {- return allocationFrame{}, false- }- }- return result, !result.start.IsZero()-}--func completeCoverageBoundary(frames []allocationFrame, requestedStart, requestedEnd time.Time) (time.Time, bool) {- if len(frames) == 0 {- return requestedStart, false- }- sort.Slice(frames, func(i, j int) bool {- return frames[i].start.Before(frames[j].start)- })- latest := requestedStart- expected := requestedStart- complete := true- for _, frame := range frames {- if !frame.start.Equal(expected) {- complete = false- }- if frame.end.After(expected) {- expected = frame.end- }- if frame.end.After(latest) {- latest = frame.end- }- }- if latest.After(requestedEnd) {- latest = requestedEnd- }- return latest, complete && expected.Equal(requestedEnd)-}--func validateOpenCostAllocation(row openCostAllocation, requestedStart, requestedEnd time.Time, step time.Duration, allowed map[string]struct{}) (Allocation, bool) {- if _, ok := allowed[row.Properties.Namespace]; !ok || strings.TrimSpace(row.Properties.Pod) == "" || len(row.Properties.Pod) > 253 || row.Minutes == nil || row.CPUUsageAverage == nil || row.CPURequestAverage == nil || row.RAMUsageAverageBytes == nil || row.RAMRequestAverageBytes == nil {- return Allocation{}, false- }- start, end, ok := parseOpenCostWindow(row.Window)- if !ok || start.Before(requestedStart) || end.After(requestedEnd) || !end.After(start) || end.Sub(start) != step {- return Allocation{}, false- }- values := []float64{*row.Minutes, *row.CPUUsageAverage, *row.CPURequestAverage, *row.RAMUsageAverageBytes, *row.RAMRequestAverageBytes}- for _, value := range values {- if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value > 1e15 {- return Allocation{}, false- }- }- if *row.Minutes > end.Sub(start).Minutes()+0.000001 {- return Allocation{}, false- }- return Allocation{Start: start, End: end, Namespace: row.Properties.Namespace, Pod: row.Properties.Pod, Minutes: *row.Minutes, CPUUsageAverage: *row.CPUUsageAverage, CPURequestAverage: *row.CPURequestAverage, RAMUsageAverageBytes: *row.RAMUsageAverageBytes, RAMRequestAverageBytes: *row.RAMRequestAverageBytes}, true-}--func parseOpenCostWindow(raw json.RawMessage) (time.Time, time.Time, bool) {- var object struct {- Start string `json:"start"`- End string `json:"end"`- }- if json.Unmarshal(raw, &object) == nil && object.Start != "" && object.End != "" {- start, startErr := time.Parse(time.RFC3339, object.Start)- end, endErr := time.Parse(time.RFC3339, object.End)- return start.UTC(), end.UTC(), startErr == nil && endErr == nil- }- var legacy string- if err := json.Unmarshal(raw, &legacy); err != nil || len(legacy) < 5 || legacy[0] != '[' || legacy[len(legacy)-1] != ')' {- return time.Time{}, time.Time{}, false- }- parts := strings.Split(strings.TrimSuffix(strings.TrimPrefix(legacy, "["), ")"), ",")- if len(parts) != 2 {- return time.Time{}, time.Time{}, false- }- start, startErr := time.Parse(time.RFC3339, strings.TrimSpace(parts[0]))- end, endErr := time.Parse(time.RFC3339, strings.TrimSpace(parts[1]))- return start.UTC(), end.UTC(), startErr == nil && endErr == nil-}--var _ AllocationClient = (*openCostClient)(nil)diff --git a/libs/fleet/backend/usage/opencost_test.go b/libs/fleet/backend/usage/opencost_test.godeleted file mode 100644index e2e88f6b39..0000000000--- a/libs/fleet/backend/usage/opencost_test.go+++ /dev/null@@ -1,161 +0,0 @@-package usage--import (- "context"- "fmt"- "math"- "net/http"- "net/http/httptest"- "testing"- "time"-)--func TestOpenCostClientUsesFixedScopedAllocationQuery(t *testing.T) {- t.Parallel()- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- if r.URL.Path != "/allocation/compute" {- t.Fatalf("path = %q", r.URL.Path)- }- q := r.URL.Query()- if q.Get("aggregate") != "cluster,namespace,pod" || q.Get("includeIdle") != "false" || q.Get("includeSharedCostBreakdown") != "false" || q.Get("accumulate") != "false" || q.Get("step") != "1h" || q.Get("filter") != `namespace:"ns-a"|namespace:"ns-b"` || q.Has("filterNamespaces") {- t.Fatalf("query = %#v", q)- }- _, _ = w.Write([]byte(`{"code":200,"data":[{"cluster/ns-a/virt-launcher-vm-a-x":{"properties":{"namespace":"ns-a","pod":"virt-launcher-vm-a-x"},"window":{"start":"2026-08-19T09:00:00Z","end":"2026-08-19T10:00:00Z"},"minutes":60,"cpuCoreUsageAverage":2,"cpuCoreRequestAverage":4,"ramByteUsageAverage":3221225472,"ramByteRequestAverage":6442450944}}],"meta":{"timeGenerated":"2026-08-19T10:01:00Z"}}`))- }))- defer server.Close()- client, err := NewOpenCostClient(server.URL, time.Second, 1<<20)- if err != nil {- t.Fatal(err)- }- start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)- allocations, asOf, partial, err := client.Allocations(context.Background(), start, start.Add(time.Hour), time.Hour, []string{"ns-b", "ns-a"})- if err != nil || partial || !asOf.Equal(start.Add(time.Hour)) || len(allocations) != 1 {- t.Fatalf("allocations=%#v asOf=%s partial=%t err=%v", allocations, asOf, partial, err)- }- if allocations[0].CPUUsageAverage != 2 || allocations[0].CPURequestAverage != 4 || allocations[0].RAMUsageAverageBytes != 3*gibibyte || allocations[0].RAMRequestAverageBytes != 6*gibibyte {- t.Fatalf("allocation = %#v", allocations[0])- }-}--func TestParseOpenCostAllocationsUsesProductionWindowShapeAndCoverage(t *testing.T) {- t.Parallel()- start := time.Date(2026, 7, 13, 3, 0, 0, 0, time.UTC)- body := []byte(`{"code":200,"data":[{"default/api":{"properties":{"namespace":"default","pod":"virt-launcher-api-x"},"window":{"start":"2026-07-13T03:00:00Z","end":"2026-07-13T04:00:00Z"},"minutes":60,"cpuCoreUsageAverage":0,"cpuCoreRequestAverage":2,"ramByteUsageAverage":0,"ramByteRequestAverage":1073741824}}]}`)- allocations, asOf, partial, err := parseOpenCostAllocations(body, start, start.Add(time.Hour), time.Hour, []string{"default"})- if err != nil || partial || !asOf.Equal(start.Add(time.Hour)) || len(allocations) != 1 {- t.Fatalf("allocations=%#v asOf=%s partial=%t err=%v", allocations, asOf, partial, err)- }- if allocations[0].CPUUsageAverage != 0 || allocations[0].RAMUsageAverageBytes != 0 {- t.Fatalf("zero averages were not retained: %#v", allocations[0])- }-}--func TestParseOpenCostAllocationsMarksIncompleteCoverage(t *testing.T) {- t.Parallel()- start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)- row := func(windowStart, windowEnd time.Time) string {- return fmt.Sprintf(`{"properties":{"namespace":"ns-a","pod":"virt-launcher-vm-a-x"},"window":{"start":"%s","end":"%s"},"minutes":60,"cpuCoreUsageAverage":1,"cpuCoreRequestAverage":2,"ramByteUsageAverage":1073741824,"ramByteRequestAverage":2147483648}`, windowStart.Format(time.RFC3339), windowEnd.Format(time.RFC3339))- }- for name, test := range map[string]struct {- body []byte- wantCutoff time.Time- wantRows int- }{- "empty": {[]byte(`{"code":200,"data":[]}`), start, 0},- "missing_frame": {[]byte(`{"code":200,"data":[{"first":` + row(start, start.Add(time.Hour)) + `},{}]}`), start.Add(time.Hour), 1},- "gap": {[]byte(`{"code":200,"data":[{"first":` + row(start, start.Add(time.Hour)) + `},{"third":` + row(start.Add(2*time.Hour), start.Add(3*time.Hour)) + `}]}`), start.Add(3 * time.Hour), 2},- "missing_prefix": {[]byte(`{"code":200,"data":[{"third":` + row(start.Add(2*time.Hour), start.Add(3*time.Hour)) + `}]}`), start.Add(3 * time.Hour), 1},- } {- t.Run(name, func(t *testing.T) {- allocations, asOf, partial, err := parseOpenCostAllocations(test.body, start, start.Add(3*time.Hour), time.Hour, []string{"ns-a"})- if err != nil || !partial || !asOf.Equal(test.wantCutoff) || len(allocations) != test.wantRows {- t.Fatalf("allocations=%#v asOf=%s partial=%t err=%v", allocations, asOf, partial, err)- }- })- }-}--func TestParseOpenCostAllocationsRejectsSourceErrorsAndMalformedRequiredValues(t *testing.T) {- t.Parallel()- start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)- for name, body := range map[string][]byte{- "missing_code": []byte(`{"data":[]}`),- "error_code": []byte(`{"code":500,"error":"upstream failed"}`),- "success_error": []byte(`{"code":200,"error":"unexpected","data":[]}`),- } {- t.Run(name, func(t *testing.T) {- if _, _, _, err := parseOpenCostAllocations(body, start, start.Add(time.Hour), time.Hour, []string{"ns-a"}); err == nil {- t.Fatal("expected source envelope error")- }- })- }- body := []byte(`{"code":200,"data":[{"missing":{"properties":{"namespace":"ns-a","pod":"virt-launcher-vm-a-x"},"window":{"start":"2026-08-19T09:00:00Z","end":"2026-08-19T10:00:00Z"},"minutes":60,"cpuCoreUsageAverage":null,"cpuCoreRequestAverage":2,"ramByteUsageAverage":1073741824,"ramByteRequestAverage":2147483648}}]}`)- allocations, _, partial, err := parseOpenCostAllocations(body, start, start.Add(time.Hour), time.Hour, []string{"ns-a"})- if err != nil || !partial || len(allocations) != 0 {- t.Fatalf("allocations=%#v partial=%t err=%v", allocations, partial, err)- }-}--func TestOpenCostClientRejectsUnsafeNamespace(t *testing.T) {- t.Parallel()- client, err := NewOpenCostClient("https://opencost.invalid", time.Second, 1<<20)- if err != nil {- t.Fatal(err)- }- _, _, _, err = client.Allocations(context.Background(), time.Now().Add(-time.Hour), time.Now(), time.Hour, []string{"ns-a\"|cluster:\"all"})- if err == nil {- t.Fatal("expected namespace validation error")- }-}--func TestParseOpenCostAllocationsExcludesBadRowsAsIncomplete(t *testing.T) {- t.Parallel()- start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)- body := []byte(`{"code":200,"data":[{"valid":{"properties":{"namespace":"ns-a","pod":"virt-launcher-vm-a-x"},"window":"[2026-08-19T09:00:00Z,2026-08-19T10:00:00Z)","minutes":60,"cpuCoreUsageAverage":1,"cpuCoreRequestAverage":2,"ramByteUsageAverage":1073741824,"ramByteRequestAverage":2147483648},"out-of-scope":{"properties":{"namespace":"other","pod":"pod"},"window":"[2026-08-19T09:00:00Z,2026-08-19T10:00:00Z)","minutes":60,"cpuCoreUsageAverage":1,"cpuCoreRequestAverage":2,"ramByteUsageAverage":1073741824,"ramByteRequestAverage":2147483648},"oversized":{"properties":{"namespace":"ns-a","pod":"pod"},"window":"[2026-08-19T09:00:00Z,2026-08-19T10:00:00Z)","minutes":60,"cpuCoreUsageAverage":1000000000000001,"cpuCoreRequestAverage":2,"ramByteUsageAverage":1073741824,"ramByteRequestAverage":2147483648}}],"warnings":["partial"]}`)- allocations, _, partial, err := parseOpenCostAllocations(body, start, start.Add(time.Hour), time.Hour, []string{"ns-a"})- if err != nil || !partial || len(allocations) != 1 {- t.Fatalf("allocations=%#v partial=%t err=%v", allocations, partial, err)- }-}--func TestOpenCostClientRejectsUnsafeBaseURLAndResponseLimitOverflow(t *testing.T) {- t.Parallel()- for _, rawURL := range []string{- "https://opencost.invalid?filterNamespaces=legacy",- "https://opencost.invalid#fragment",- "https://user:[email protected]",- "ftp://opencost.invalid",- } {- if _, err := NewOpenCostClient(rawURL, time.Second, 1<<20); err == nil {- t.Fatalf("expected unsafe URL rejection for %q", rawURL)- }- }- if _, err := NewOpenCostClient("https://opencost.invalid", time.Second, math.MaxInt64); err == nil {- t.Fatal("expected response limit overflow rejection")- }-}--func TestOpenCostClientRejectsRedirectAndResponseOverLimit(t *testing.T) {- t.Parallel()- for name, handler := range map[string]http.HandlerFunc{- "redirect": func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/elsewhere", http.StatusFound) },- "limit": func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(make([]byte, 128)) },- } {- t.Run(name, func(t *testing.T) {- server := httptest.NewServer(handler)- defer server.Close()- limit := int64(1 << 20)- if name == "limit" {- limit = 64- }- client, err := NewOpenCostClient(server.URL, time.Second, limit)- if err != nil {- t.Fatal(err)- }- _, _, _, err = client.Allocations(context.Background(), time.Now().UTC().Truncate(time.Hour), time.Now().UTC().Truncate(time.Hour).Add(time.Hour), time.Hour, []string{"ns-a"})- if err == nil {- t.Fatal("expected request error")- }- })- }-}diff --git a/libs/fleet/backend/usage/provider.go b/libs/fleet/backend/usage/provider.goindex d48333d636..4992d64edc 100644--- a/libs/fleet/backend/usage/provider.go+++ b/libs/fleet/backend/usage/provider.go@@ -4,6 +4,7 @@ import ("context""errors""fmt"+ "regexp""sort""strings""time"@@ -13,7 +14,10 @@ import ("go.opentelemetry.io/otel/trace")-var ErrPoolNotFound = errors.New("usage pool not found")+var (+ ErrPoolNotFound = errors.New("usage pool not found")+ dns1123Label = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)+)type provider struct {events EventStore@@ -163,10 +167,10 @@ func (p *provider) load(ctx context.Context, query Query) (usageData, error) {))allocations, asOf, sourcePartial, err := p.allocations.Allocations(allocationsCtx, start, end, step, namespaces)if err != nil {- markUsageSpanError(allocationsSpan, "OpenCost allocation query failed")+ markUsageSpanError(allocationsSpan, "allocation query failed")allocationsSpan.End()- markUsageSpanError(span, "OpenCost allocation query failed")- return usageData{}, fmt.Errorf("read OpenCost allocations: %w", err)+ markUsageSpanError(span, "allocation query failed")+ return usageData{}, fmt.Errorf("read allocation data: %w", err)}allocationsSpan.SetAttributes(attribute.Int("usage.allocation_count", len(allocations)),diff --git a/libs/fleet/backend/usage/safe_error_test.go b/libs/fleet/backend/usage/safe_error_test.goindex 2b944197e2..e8f33cab57 100644--- a/libs/fleet/backend/usage/safe_error_test.go+++ b/libs/fleet/backend/usage/safe_error_test.go@@ -7,8 +7,8 @@ import (func TestNewSanitizedErrorRedactsAndPreservesCause(t *testing.T) {cause := errors.New("secret upstream detail")- err := newSanitizedError("invalid OpenCost URL", cause)- if err.Error() != "invalid OpenCost URL" {+ err := newSanitizedError("invalid allocation query webhook URL", cause)+ if err.Error() != "invalid allocation query webhook URL" {t.Fatalf("Error() = %q", err.Error())}if !errors.Is(err, cause) {