Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cyclops): query usage allocations via DataFusion (#7121)

r33drichardsabfaae44a452
diff --git a/libs/fleet/backend/config/config.go b/libs/fleet/backend/config/config.go
index 20c0368532..768faab82d 100644
--- a/libs/fleet/backend/config/config.go
+++ b/libs/fleet/backend/config/config.go
@@ -14,6 +14,7 @@ package config
import (
"fmt"
"net/url"
+ "regexp"
"slices"
"strconv"
"strings"
@@ -124,20 +125,30 @@ type ChatConfiguration struct {
}
// UsageConfiguration enables the usage provider only when DatabaseURL is set.
-// OpenCostBaseURL may be configured before the usage credential arrives; in
-// that state the provider remains disabled so serving readiness is unaffected.
+// Query transport settings may be present before the usage credential arrives;
+// in that state the provider remains disabled so serving readiness is unaffected.
type UsageConfiguration struct {
- DatabaseURL string
- OpenCostBaseURL string
- QueryTimeout time.Duration
- MaxResponseBytes int64
+ DatabaseURL string
+ QueryWebhookURL string
+ QueryHMACSecret string
+ QueryResultBucket string
+ QueryResultPrefix string
+ QueryCluster string
+ QueryEnvironment string
+ QueryTimeout time.Duration
+ QueryPollInterval time.Duration
+ MaxResponseBytes int64
}
+var usageQueryPartitionValue = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
+
const (
- minUsageQueryTimeout = time.Second
- maxUsageQueryTimeout = 2 * time.Minute
- minUsageResponseSize = 64 * 1024
- maxUsageResponseSize = 32 * 1024 * 1024
+ clusterUsageQueryWebhookURL = "http://cua-temporal-webhook.temporal.svc.cluster.local/hooks/opencost-query"
+ externalUsageQueryWebhookURL = "https://cua-temporal-webhook.tail204509.ts.net/hooks/opencost-query"
+ minUsageQueryTimeout = time.Second
+ maxUsageQueryTimeout = 2 * time.Minute
+ minUsageResponseSize = 64 * 1024
+ maxUsageResponseSize = 32 * 1024 * 1024
)
type MetricsConfiguration struct {
@@ -185,9 +196,15 @@ var specs = []flagSpec{
{"database.state-query-dsn", "state-query-database-dsn", "STATE_QUERY_DATABASE_DSN", "", "Postgres connection options for tenant state queries"},
{"database.state-query-tenant-password", "state-query-tenant-password", "STATE_QUERY_TENANT_PASSWORD", "", "Shared password for tenant query login roles"},
{"usage.database-url", "usage-database-url", "USAGE_DATABASE_URL", "", "Postgres URL for usage events"},
- {"usage.opencost-base-url", "opencost-base-url", "OPENCOST_BASE_URL", "", "OpenCost base URL for usage allocations"},
- {"usage.query-timeout", "usage-query-timeout", "USAGE_QUERY_TIMEOUT", "20s", "Usage provider query timeout"},
- {"usage.max-response-bytes", "usage-max-response-bytes", "USAGE_MAX_RESPONSE_BYTES", "8388608", "Maximum OpenCost response bytes"},
+ {"usage.query-webhook-url", "usage-query-webhook-url", "USAGE_QUERY_WEBHOOK_URL", clusterUsageQueryWebhookURL, "DataFusion allocation query webhook URL"},
+ {"usage.query-hmac-secret", "usage-query-hmac-secret", "USAGE_QUERY_HMAC_SECRET", "", "DataFusion allocation query webhook HMAC secret"},
+ {"usage.query-result-bucket", "usage-query-result-bucket", "USAGE_QUERY_RESULT_BUCKET", "nanoclaw-telemetry-files", "S3 bucket for transient allocation query results"},
+ {"usage.query-result-prefix", "usage-query-result-prefix", "USAGE_QUERY_RESULT_PREFIX", "cyclops/usage-query", "S3 prefix for transient allocation query results"},
+ {"usage.query-cluster", "usage-query-cluster", "USAGE_QUERY_CLUSTER", "kopf-k3s", "OpenCost allocation cluster partition"},
+ {"usage.query-environment", "usage-query-environment", "USAGE_QUERY_ENVIRONMENT", "production", "OpenCost allocation environment partition"},
+ {"usage.query-timeout", "usage-query-timeout", "USAGE_QUERY_TIMEOUT", "45s", "Usage provider query timeout"},
+ {"usage.query-poll-interval", "usage-query-poll-interval", "USAGE_QUERY_POLL_INTERVAL", "1s", "Allocation query status poll interval"},
+ {"usage.max-response-bytes", "usage-max-response-bytes", "USAGE_MAX_RESPONSE_BYTES", "8388608", "Maximum allocation query response bytes"},
{"stripe.secret-key", "stripe-secret-key", "STRIPE_SECRET_KEY", "", "Stripe secret key (server-only)"},
{"stripe.webhook-secret", "stripe-webhook-secret", "STRIPE_WEBHOOK_SECRET", "", "Stripe webhook signing secret"},
{"stripe.checkout-success-url", "stripe-checkout-success-url", "STRIPE_CHECKOUT_SUCCESS_URL", "", "Stripe Checkout success redirect URL"},
@@ -235,6 +252,10 @@ func LoadConfig() (*Configuration, error) {
if err != nil {
return nil, err
}
+ usagePollInterval, err := parseUsagePollInterval(viper.GetString("usage.query-poll-interval"))
+ if err != nil {
+ return nil, err
+ }
usageMaxResponseBytes, err := parseUsageMaxResponseBytes(viper.GetString("usage.max-response-bytes"))
if err != nil {
return nil, err
@@ -306,10 +327,16 @@ func LoadConfig() (*Configuration, error) {
Model: viper.GetString("chat.model"),
},
Usage: UsageConfiguration{
- DatabaseURL: strings.TrimSpace(viper.GetString("usage.database-url")),
- OpenCostBaseURL: strings.TrimSpace(viper.GetString("usage.opencost-base-url")),
- QueryTimeout: usageQueryTimeout,
- MaxResponseBytes: usageMaxResponseBytes,
+ DatabaseURL: strings.TrimSpace(viper.GetString("usage.database-url")),
+ QueryWebhookURL: strings.TrimSpace(viper.GetString("usage.query-webhook-url")),
+ QueryHMACSecret: strings.TrimSpace(viper.GetString("usage.query-hmac-secret")),
+ QueryResultBucket: strings.TrimSpace(viper.GetString("usage.query-result-bucket")),
+ QueryResultPrefix: strings.Trim(strings.TrimSpace(viper.GetString("usage.query-result-prefix")), "/"),
+ QueryCluster: strings.TrimSpace(viper.GetString("usage.query-cluster")),
+ QueryEnvironment: strings.TrimSpace(viper.GetString("usage.query-environment")),
+ QueryTimeout: usageQueryTimeout,
+ QueryPollInterval: usagePollInterval,
+ MaxResponseBytes: usageMaxResponseBytes,
},
Metrics: MetricsConfiguration{Addr: viper.GetString("metrics.addr")},
Telemetry: TelemetryConfiguration{
@@ -353,17 +380,25 @@ func validateUsageConfiguration(cfg UsageConfiguration) error {
if cfg.QueryTimeout < minUsageQueryTimeout || cfg.QueryTimeout > maxUsageQueryTimeout {
return fmt.Errorf("USAGE_QUERY_TIMEOUT must be between 1s and 2m")
}
+ if cfg.QueryPollInterval < 250*time.Millisecond || cfg.QueryPollInterval > 5*time.Second {
+ return fmt.Errorf("USAGE_QUERY_POLL_INTERVAL must be between 250ms and 5s")
+ }
if cfg.MaxResponseBytes < minUsageResponseSize || cfg.MaxResponseBytes > maxUsageResponseSize {
return fmt.Errorf("USAGE_MAX_RESPONSE_BYTES must be between 65536 and 33554432")
}
- // Deployments set OPENCOST_BASE_URL before the usage ExternalSecret is
- // available. An absent database URL therefore disables usage, while a
- // database URL without OpenCost remains an unsafe configuration error.
if cfg.DatabaseURL == "" {
return nil
}
- if cfg.OpenCostBaseURL == "" {
- return fmt.Errorf("USAGE_DATABASE_URL requires OPENCOST_BASE_URL")
+ for name, value := range map[string]string{
+ "USAGE_QUERY_HMAC_SECRET": cfg.QueryHMACSecret,
+ "USAGE_QUERY_RESULT_BUCKET": cfg.QueryResultBucket,
+ "USAGE_QUERY_RESULT_PREFIX": cfg.QueryResultPrefix,
+ "USAGE_QUERY_CLUSTER": cfg.QueryCluster,
+ "USAGE_QUERY_ENVIRONMENT": cfg.QueryEnvironment,
+ } {
+ if value == "" {
+ return fmt.Errorf("USAGE_DATABASE_URL requires %s", name)
+ }
}
databaseURL, err := url.Parse(cfg.DatabaseURL)
if err != nil {
@@ -379,12 +414,19 @@ func validateUsageConfiguration(cfg UsageConfiguration) error {
if postgresConfig.ConnConfig.User != "cyclops_usage_reader" {
return fmt.Errorf("usage database URL must use cyclops_usage_reader")
}
- openCostURL, err := url.Parse(cfg.OpenCostBaseURL)
- if err != nil {
- return newSanitizedError("invalid OpenCost URL", err)
+ if cfg.QueryWebhookURL != clusterUsageQueryWebhookURL && cfg.QueryWebhookURL != externalUsageQueryWebhookURL {
+ return fmt.Errorf("invalid allocation query webhook URL")
+ }
+ if cfg.QueryResultBucket != "nanoclaw-telemetry-files" {
+ return fmt.Errorf("invalid allocation query result bucket")
}
- if (openCostURL.Scheme != "http" && openCostURL.Scheme != "https") || openCostURL.Host == "" || openCostURL.RawQuery != "" || openCostURL.Fragment != "" || openCostURL.User != nil {
- return fmt.Errorf("invalid OpenCost URL")
+ if cfg.QueryResultPrefix != "cyclops/usage-query" {
+ return fmt.Errorf("invalid allocation query result prefix")
+ }
+ for _, value := range []string{cfg.QueryCluster, cfg.QueryEnvironment} {
+ if !usageQueryPartitionValue.MatchString(value) {
+ return fmt.Errorf("invalid allocation query partition")
+ }
}
return nil
}
@@ -397,6 +439,14 @@ func parseUsageQueryTimeout(raw string) (time.Duration, error) {
return value, nil
}
+func parseUsagePollInterval(raw string) (time.Duration, error) {
+ value, err := time.ParseDuration(strings.TrimSpace(raw))
+ if err != nil {
+ return 0, fmt.Errorf("invalid USAGE_QUERY_POLL_INTERVAL: %w", err)
+ }
+ return value, nil
+}
+
func parseUsageMaxResponseBytes(raw string) (int64, error) {
value, err := strconv.ParseInt(strings.TrimSpace(raw), 10, 64)
if err != nil {
diff --git a/libs/fleet/backend/config/config_test.go b/libs/fleet/backend/config/config_test.go
index 0070304d9f..488476a023 100644
--- a/libs/fleet/backend/config/config_test.go
+++ b/libs/fleet/backend/config/config_test.go
@@ -277,41 +277,72 @@ func TestLoadConfig_ChatDisabledDoesNotRequireCredentials(t *testing.T) {
func TestLoadConfig_UsageConfiguration(t *testing.T) {
validDatabaseURL := "postgres://cyclops_usage_reader:[email protected]/cyclops?sslmode=require"
- validOpenCostURL := "https://opencost.example/api"
+ defaults := UsageConfiguration{
+ QueryWebhookURL: "http://cua-temporal-webhook.temporal.svc.cluster.local/hooks/opencost-query",
+ QueryResultBucket: "nanoclaw-telemetry-files",
+ QueryResultPrefix: "cyclops/usage-query",
+ QueryCluster: "kopf-k3s",
+ QueryEnvironment: "production",
+ QueryTimeout: 45 * time.Second,
+ QueryPollInterval: time.Second,
+ MaxResponseBytes: 8388608,
+ }
tests := []struct {
name string
env map[string]string
wantError string
want UsageConfiguration
}{
- {name: "absent disables usage", want: UsageConfiguration{QueryTimeout: 20 * time.Second, MaxResponseBytes: 8388608}},
- {name: "database only is rejected", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL}, wantError: "USAGE_DATABASE_URL requires OPENCOST_BASE_URL"},
- {name: "OpenCost only disables usage", env: map[string]string{"OPENCOST_BASE_URL": validOpenCostURL}, want: UsageConfiguration{OpenCostBaseURL: validOpenCostURL, QueryTimeout: 20 * time.Second, MaxResponseBytes: 8388608}},
+ {name: "absent disables usage", want: defaults},
+ {name: "query settings only disable usage", env: map[string]string{"USAGE_QUERY_HMAC_SECRET": "secret"}, want: func() UsageConfiguration { value := defaults; value.QueryHMACSecret = "secret"; return value }()},
+ {name: "database requires HMAC secret", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL}, wantError: "USAGE_DATABASE_URL requires USAGE_QUERY_HMAC_SECRET"},
{
name: "valid values",
env: map[string]string{
- "USAGE_DATABASE_URL": validDatabaseURL, "OPENCOST_BASE_URL": validOpenCostURL,
- "USAGE_QUERY_TIMEOUT": "45s", "USAGE_MAX_RESPONSE_BYTES": "1048576",
+ "USAGE_DATABASE_URL": validDatabaseURL, "USAGE_QUERY_HMAC_SECRET": "secret",
+ "USAGE_QUERY_TIMEOUT": "60s", "USAGE_QUERY_POLL_INTERVAL": "500ms", "USAGE_MAX_RESPONSE_BYTES": "1048576",
},
- want: UsageConfiguration{DatabaseURL: validDatabaseURL, OpenCostBaseURL: validOpenCostURL, QueryTimeout: 45 * time.Second, MaxResponseBytes: 1048576},
+ want: func() UsageConfiguration {
+ value := defaults
+ value.DatabaseURL = validDatabaseURL
+ value.QueryHMACSecret = "secret"
+ value.QueryTimeout = time.Minute
+ value.QueryPollInterval = 500 * time.Millisecond
+ value.MaxResponseBytes = 1048576
+ return value
+ }(),
},
- {name: "libpq database DSN is rejected", env: map[string]string{"USAGE_DATABASE_URL": "user=cyclops_usage_reader host=db.example dbname=cyclops", "OPENCOST_BASE_URL": validOpenCostURL}, wantError: "invalid usage database URL"},
- {name: "database reader role is required", env: map[string]string{"USAGE_DATABASE_URL": "postgres://[email protected]/cyclops", "OPENCOST_BASE_URL": validOpenCostURL}, wantError: "usage database URL must use cyclops_usage_reader"},
- {name: "OpenCost URL cannot have a query", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL, "OPENCOST_BASE_URL": "https://opencost.example?token=secret"}, wantError: "invalid OpenCost URL"},
+ {name: "libpq database DSN is rejected", env: map[string]string{"USAGE_DATABASE_URL": "user=cyclops_usage_reader host=db.example dbname=cyclops", "USAGE_QUERY_HMAC_SECRET": "secret"}, wantError: "invalid usage database URL"},
+ {name: "database reader role is required", env: map[string]string{"USAGE_DATABASE_URL": "postgres://[email protected]/cyclops", "USAGE_QUERY_HMAC_SECRET": "secret"}, wantError: "usage database URL must use cyclops_usage_reader"},
+ {name: "external webhook remains allowed", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL, "USAGE_QUERY_HMAC_SECRET": "secret", "USAGE_QUERY_WEBHOOK_URL": externalUsageQueryWebhookURL}, want: func() UsageConfiguration {
+ value := defaults
+ value.DatabaseURL = validDatabaseURL
+ value.QueryHMACSecret = "secret"
+ value.QueryWebhookURL = externalUsageQueryWebhookURL
+ return value
+ }()},
+ {name: "webhook is allow-listed", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL, "USAGE_QUERY_HMAC_SECRET": "secret", "USAGE_QUERY_WEBHOOK_URL": "https://example.test/hooks/opencost-query"}, wantError: "invalid allocation query webhook URL"},
+ {name: "result bucket is allow-listed", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL, "USAGE_QUERY_HMAC_SECRET": "secret", "USAGE_QUERY_RESULT_BUCKET": "other"}, wantError: "invalid allocation query result bucket"},
{name: "malformed timeout is rejected while disabled", env: map[string]string{"USAGE_QUERY_TIMEOUT": "not-a-duration"}, wantError: "invalid USAGE_QUERY_TIMEOUT"},
+ {name: "malformed poll interval is rejected while disabled", env: map[string]string{"USAGE_QUERY_POLL_INTERVAL": "not-a-duration"}, wantError: "invalid USAGE_QUERY_POLL_INTERVAL"},
{name: "malformed response limit is rejected while disabled", env: map[string]string{"USAGE_MAX_RESPONSE_BYTES": "not-a-number"}, wantError: "invalid USAGE_MAX_RESPONSE_BYTES"},
{name: "short timeout is rejected while disabled", env: map[string]string{"USAGE_QUERY_TIMEOUT": "500ms"}, wantError: "USAGE_QUERY_TIMEOUT must be between 1s and 2m"},
+ {name: "short poll interval is rejected while disabled", env: map[string]string{"USAGE_QUERY_POLL_INTERVAL": "10ms"}, wantError: "USAGE_QUERY_POLL_INTERVAL must be between 250ms and 5s"},
{name: "small response limit is rejected while disabled", env: map[string]string{"USAGE_MAX_RESPONSE_BYTES": "1"}, wantError: "USAGE_MAX_RESPONSE_BYTES must be between 65536 and 33554432"},
- {name: "timeout is bounded", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL, "OPENCOST_BASE_URL": validOpenCostURL, "USAGE_QUERY_TIMEOUT": "500ms"}, wantError: "USAGE_QUERY_TIMEOUT must be between 1s and 2m"},
- {name: "response limit is bounded", env: map[string]string{"USAGE_DATABASE_URL": validDatabaseURL, "OPENCOST_BASE_URL": validOpenCostURL, "USAGE_MAX_RESPONSE_BYTES": "1"}, wantError: "USAGE_MAX_RESPONSE_BYTES must be between 65536 and 33554432"},
}
+ keys := []string{
+ "USAGE_DATABASE_URL", "USAGE_QUERY_WEBHOOK_URL", "USAGE_QUERY_HMAC_SECRET",
+ "USAGE_QUERY_RESULT_BUCKET", "USAGE_QUERY_RESULT_PREFIX", "USAGE_QUERY_CLUSTER",
+ "USAGE_QUERY_ENVIRONMENT", "USAGE_QUERY_TIMEOUT", "USAGE_QUERY_POLL_INTERVAL",
+ "USAGE_MAX_RESPONSE_BYTES",
+ }
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
viper.Reset()
t.Cleanup(viper.Reset)
t.Setenv("KC_ADMIN_CLIENT_SECRET", "secret")
- for _, key := range []string{"USAGE_DATABASE_URL", "OPENCOST_BASE_URL", "USAGE_QUERY_TIMEOUT", "USAGE_MAX_RESPONSE_BYTES"} {
+ for _, key := range keys {
t.Setenv(key, "")
}
for key, value := range test.env {
diff --git a/libs/fleet/backend/go.mod b/libs/fleet/backend/go.mod
index 386253b04b..7fd5fcefb1 100644
--- a/libs/fleet/backend/go.mod
+++ b/libs/fleet/backend/go.mod
@@ -7,6 +7,8 @@ toolchain go1.24.7
require (
github.com/MicahParks/keyfunc/v2 v2.1.0
github.com/Nerzal/gocloak/v13 v13.9.0
+ github.com/aws/aws-sdk-go-v2/config v1.32.17
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2
github.com/golang-jwt/jwt/v5 v5.2.1
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.5.5
@@ -20,15 +22,17 @@ require (
github.com/stripe/stripe-go/v85 v85.2.0
github.com/swaggo/http-swagger/v2 v2.0.2
github.com/swaggo/swag v1.16.6
- sigs.k8s.io/yaml v1.4.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0
golang.org/x/sync v0.17.0
+ sigs.k8s.io/yaml v1.4.0
)
require (
github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect
- github.com/aws/aws-sdk-go-v2/config v1.32.17 // indirect
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
@@ -73,7 +77,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
- github.com/aws/smithy-go v1.25.1 // indirect
+ github.com/aws/smithy-go v1.25.1
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
diff --git a/libs/fleet/backend/go.sum b/libs/fleet/backend/go.sum
index 1a87303bba..2806be8909 100644
--- a/libs/fleet/backend/go.sum
+++ b/libs/fleet/backend/go.sum
@@ -12,6 +12,8 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU=
github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE=
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
@@ -26,8 +28,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2 h1:U3ygWUhCpiSPYSHOrRhb3gOl9T5Y3kB8k5Vjs//57bE=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
github.com/aws/aws-sdk-go-v2/service/ssm v1.68.6 h1:0LPJjbSNEDHidGOXa0LfvSVbdn9/GdlJUQTgE0kFpso=
diff --git a/libs/fleet/backend/handlers/config.go b/libs/fleet/backend/handlers/config.go
index 960039edf0..8b68b9a7b0 100644
--- a/libs/fleet/backend/handlers/config.go
+++ b/libs/fleet/backend/handlers/config.go
@@ -53,6 +53,7 @@ func (h Handlers) GetConfig(w http.ResponseWriter, r *http.Request) {
slog.WarnContext(ctx, "usage access eval failed; defaulting off", "err", err)
usageEnabled = false
}
+ usageEnabled = usageEnabled && h.Usage != nil
chatEnabled, err := h.chatEnabled(ctx, user)
if err != nil {
diff --git a/libs/fleet/backend/handlers/config_test.go b/libs/fleet/backend/handlers/config_test.go
index c3d6dd94b9..7e5c43c276 100644
--- a/libs/fleet/backend/handlers/config_test.go
+++ b/libs/fleet/backend/handlers/config_test.go
@@ -53,15 +53,36 @@ func TestGetConfigReturnsEffectiveChatAccess(t *testing.T) {
}
func TestGetConfigReturnsEffectiveUsageAccess(t *testing.T) {
- h := Handlers{usageAccessEvaluator: func(context.Context, *auth.User) (bool, error) { return true, nil }, adminAccessEvaluator: func(context.Context, *auth.User) (bool, error) { return false, nil }}
- w := httptest.NewRecorder()
- h.GetConfig(w, withUser(httptest.NewRequest(http.MethodGet, "/api/config", nil), &auth.User{ID: "user", AZP: "cyclops-cs-spa"}))
- var response ConfigResponse
- if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
- t.Fatal(err)
+ tests := []struct {
+ name string
+ provider UsageProvider
+ allowed bool
+ want bool
+ }{
+ {name: "available and allowed", provider: &fakeUsageProvider{}, allowed: true, want: true},
+ {name: "unavailable", allowed: true, want: false},
+ {name: "not allowed", provider: &fakeUsageProvider{}, allowed: false, want: false},
}
- if !response.Usage {
- t.Fatal("usage false")
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ h := Handlers{
+ Usage: test.provider,
+ usageAccessEvaluator: func(context.Context, *auth.User) (bool, error) {
+ return test.allowed, nil
+ },
+ adminAccessEvaluator: func(context.Context, *auth.User) (bool, error) { return false, nil },
+ }
+ w := httptest.NewRecorder()
+ h.GetConfig(w, withUser(httptest.NewRequest(http.MethodGet, "/api/config", nil), &auth.User{ID: "user", AZP: "cyclops-cs-spa"}))
+ var response ConfigResponse
+ if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+ t.Fatal(err)
+ }
+ if response.Usage != test.want {
+ t.Fatalf("usage = %v, want %v", response.Usage, test.want)
+ }
+ })
}
}
diff --git a/libs/fleet/backend/main.go b/libs/fleet/backend/main.go
index b6cb16624d..a7429e2c9b 100644
--- a/libs/fleet/backend/main.go
+++ b/libs/fleet/backend/main.go
@@ -586,22 +586,30 @@ func retryDatabaseFeatures(ctx context.Context, cfg config.DatabaseConfiguration
func initializeUsageProvider(ctx context.Context, cfg config.UsageConfiguration) (usage.Provider, func(), error) {
noop := func() {}
- // OpenCost is deployed independently from the usage credential. Keep the
- // provider disabled until the database URL arrives so serving stays ready.
+ // Query transport settings are deployed independently from the read-only
+ // usage credential. Keep the provider disabled until the database URL arrives.
if cfg.DatabaseURL == "" {
return nil, noop, nil
}
- if cfg.OpenCostBaseURL == "" {
- return nil, noop, fmt.Errorf("usage provider requires OpenCost when the database URL is configured")
- }
events, err := usage.NewPostgresEventStore(ctx, cfg.DatabaseURL)
if err != nil {
return nil, noop, fmt.Errorf("create usage event store: %w", err)
}
- allocations, err := usage.NewOpenCostClient(cfg.OpenCostBaseURL, cfg.QueryTimeout, cfg.MaxResponseBytes)
+ allocations, err := usage.NewDataFusionAllocationClient(
+ cfg.QueryWebhookURL,
+ cfg.QueryHMACSecret,
+ cfg.QueryResultBucket,
+ cfg.QueryResultPrefix,
+ cfg.QueryCluster,
+ cfg.QueryEnvironment,
+ cfg.QueryTimeout,
+ cfg.QueryPollInterval,
+ cfg.MaxResponseBytes,
+ usage.NewS3QueryObjectStore(),
+ )
if err != nil {
events.Close()
- return nil, noop, fmt.Errorf("create OpenCost client: %w", err)
+ return nil, noop, fmt.Errorf("create DataFusion allocation client: %w", err)
}
return usage.NewProvider(events, allocations, nil), events.Close, nil
}
diff --git a/libs/fleet/backend/main_test.go b/libs/fleet/backend/main_test.go
index a12a4a2e5a..d7e739e4e6 100644
--- a/libs/fleet/backend/main_test.go
+++ b/libs/fleet/backend/main_test.go
@@ -1170,10 +1170,16 @@ func TestRetryDatabaseFeaturesStopsOnceReady(t *testing.T) {
func TestInitializeUsageProviderDoesNotGateReadiness(t *testing.T) {
provider, closeProvider, err := initializeUsageProvider(context.Background(), config.UsageConfiguration{
- DatabaseURL: "postgres://cyclops_usage_reader:[email protected]:1/cyclops?sslmode=disable",
- OpenCostBaseURL: "http://127.0.0.1:1",
- QueryTimeout: time.Second,
- MaxResponseBytes: 64 * 1024,
+ DatabaseURL: "postgres://cyclops_usage_reader:[email protected]:1/cyclops?sslmode=disable",
+ QueryWebhookURL: "https://cua-temporal-webhook.tail204509.ts.net/hooks/opencost-query",
+ QueryHMACSecret: "secret",
+ QueryResultBucket: "nanoclaw-telemetry-files",
+ QueryResultPrefix: "cyclops/usage-query",
+ QueryCluster: "kopf-k3s",
+ QueryEnvironment: "production",
+ QueryTimeout: time.Second,
+ QueryPollInterval: time.Second,
+ MaxResponseBytes: 64 * 1024,
})
if err != nil {
t.Fatalf("initializeUsageProvider() error = %v", err)
@@ -1196,11 +1202,17 @@ func TestInitializeUsageProviderDoesNotGateReadiness(t *testing.T) {
}
}
-func TestInitializeUsageProviderOpenCostOnlyIsDisabled(t *testing.T) {
+func TestInitializeUsageProviderQueryTransportOnlyIsDisabled(t *testing.T) {
provider, closeProvider, err := initializeUsageProvider(context.Background(), config.UsageConfiguration{
- OpenCostBaseURL: "http://127.0.0.1:1",
- QueryTimeout: time.Second,
- MaxResponseBytes: 64 * 1024,
+ QueryWebhookURL: "https://cua-temporal-webhook.tail204509.ts.net/hooks/opencost-query",
+ QueryHMACSecret: "secret",
+ QueryResultBucket: "nanoclaw-telemetry-files",
+ QueryResultPrefix: "cyclops/usage-query",
+ QueryCluster: "kopf-k3s",
+ QueryEnvironment: "production",
+ QueryTimeout: time.Second,
+ QueryPollInterval: time.Second,
+ MaxResponseBytes: 64 * 1024,
})
if err != nil {
t.Fatalf("initializeUsageProvider() error = %v", err)
diff --git a/libs/fleet/backend/usage/datafusion.go b/libs/fleet/backend/usage/datafusion.go
new file mode 100644
index 0000000000..43e33a2788
--- /dev/null
+++ b/libs/fleet/backend/usage/datafusion.go
@@ -0,0 +1,440 @@
+package usage
+
+import (
+ "bytes"
+ "context"
+ "crypto/hmac"
+ "crypto/sha256"
+ "encoding/csv"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "net/http"
+ "net/url"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/google/uuid"
+)
+
+const (
+ maxDataFusionAllocationRows = 100000
+ maxAllocationPartitions = 2000
+ maxAllocationNamespaces = 500
+ maxAllocationQueryBytes = 65536
+ queryPresignGrace = 10 * time.Minute
+ queryCleanupTimeout = 5 * time.Second
+)
+
+var (
+ ErrQueryObjectNotFound = errors.New("query object not found")
+ queryPartitionValue = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
+ queryPodName = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9.]*[a-z0-9])?$`)
+)
+
+type QueryObjectStore interface {
+ PresignPut(context.Context, string, string, string, time.Duration) (string, error)
+ Get(context.Context, string, string, int64) ([]byte, error)
+ Delete(context.Context, string, string) error
+}
+
+type dataFusionAllocationClient struct {
+ webhookURL *url.URL
+ hmacSecret string
+ resultBucket string
+ resultPrefix string
+ cluster string
+ environment string
+ timeout time.Duration
+ pollInterval time.Duration
+ maxResponseSize int64
+ store QueryObjectStore
+ httpClient *http.Client
+}
+
+type dataFusionWebhookPayload struct {
+ Dataset string `json:"dataset"`
+ Cluster string `json:"cluster"`
+ Environment string `json:"environment"`
+ Window dataFusionQueryWindow `json:"window"`
+ Query string `json:"query"`
+ OutputURL string `json:"output_url"`
+ OutputFormat string `json:"output_format"`
+ StatusURL string `json:"status_url"`
+}
+
+type dataFusionQueryWindow struct {
+ Start time.Time `json:"start"`
+ End time.Time `json:"end"`
+}
+
+type dataFusionQueryStatus struct {
+ Status string `json:"status"`
+ Rows int64 `json:"rows"`
+ PartitionsExpanded int `json:"partitions_expanded"`
+ PartitionsMatched int `json:"partitions_matched"`
+ PartitionsMissing int `json:"partitions_missing"`
+}
+
+func NewDataFusionAllocationClient(rawURL, hmacSecret, resultBucket, resultPrefix, cluster, environment string, timeout, pollInterval time.Duration, maxResponseBytes int64, store QueryObjectStore) (*dataFusionAllocationClient, error) {
+ webhookURL, err := url.Parse(strings.TrimSpace(rawURL))
+ if err != nil {
+ return nil, newSanitizedError("invalid allocation query webhook URL", err)
+ }
+ if (webhookURL.Scheme != "http" && webhookURL.Scheme != "https") || webhookURL.Host == "" || webhookURL.User != nil || webhookURL.RawQuery != "" || webhookURL.Fragment != "" {
+ return nil, fmt.Errorf("invalid allocation query webhook URL")
+ }
+ resultPrefix = strings.Trim(strings.TrimSpace(resultPrefix), "/")
+ if strings.TrimSpace(hmacSecret) == "" || strings.TrimSpace(resultBucket) == "" || !safeResultPrefix(resultPrefix) || !queryPartitionValue.MatchString(cluster) || !queryPartitionValue.MatchString(environment) || timeout <= 0 || pollInterval <= 0 || maxResponseBytes <= 0 || maxResponseBytes == math.MaxInt64 || store == nil {
+ return nil, fmt.Errorf("invalid DataFusion allocation client configuration")
+ }
+ return &dataFusionAllocationClient{
+ webhookURL: webhookURL,
+ hmacSecret: hmacSecret,
+ resultBucket: strings.TrimSpace(resultBucket),
+ resultPrefix: resultPrefix,
+ cluster: cluster,
+ environment: environment,
+ timeout: timeout,
+ pollInterval: pollInterval,
+ maxResponseSize: maxResponseBytes,
+ store: store,
+ httpClient: &http.Client{
+ CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse },
+ },
+ }, nil
+}
+
+func safeResultPrefix(prefix string) bool {
+ if prefix == "" || strings.Contains(prefix, "\\") {
+ return false
+ }
+ for _, part := range strings.Split(prefix, "/") {
+ if part == "" || part == "." || part == ".." || !queryPartitionValue.MatchString(part) {
+ return false
+ }
+ }
+ return true
+}
+
+func (client *dataFusionAllocationClient) Allocations(ctx context.Context, start, end time.Time, step time.Duration, namespaces []string) ([]Allocation, time.Time, bool, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, time.Time{}, false, err
+ }
+ if start.IsZero() || end.IsZero() || !end.After(start) || start.UTC() != start || end.UTC() != end || !start.Equal(start.Truncate(time.Hour)) || !end.Equal(end.Truncate(time.Hour)) || len(namespaces) == 0 {
+ return nil, time.Time{}, false, fmt.Errorf("invalid DataFusion allocation query")
+ }
+ if step != time.Hour && step != 24*time.Hour {
+ return nil, time.Time{}, false, fmt.Errorf("unsupported allocation step")
+ }
+ if end.Sub(start) > maxAllocationPartitions*time.Hour {
+ return nil, time.Time{}, false, fmt.Errorf("allocation query window exceeds limit")
+ }
+ namespaces, err := safeNamespaces(namespaces)
+ if err != nil {
+ return nil, time.Time{}, false, err
+ }
+
+ if len(namespaces) > maxAllocationNamespaces {
+ return nil, time.Time{}, false, fmt.Errorf("allocation namespace limit exceeded")
+ }
+ query := allocationSQL(namespaces)
+ if len(query) > maxAllocationQueryBytes {
+ return nil, time.Time{}, false, fmt.Errorf("allocation query exceeds limit")
+ }
+ requestCtx, cancel := context.WithTimeout(ctx, client.timeout)
+ defer cancel()
+
+ prefix := fmt.Sprintf("%s/%s/%s", client.resultPrefix, time.Now().UTC().Format("2006-01-02"), uuid.NewString())
+ resultKey := prefix + "/result.csv"
+ statusKey := prefix + "/status.json"
+ defer func() {
+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), queryCleanupTimeout)
+ defer cleanupCancel()
+ _ = client.store.Delete(cleanupCtx, client.resultBucket, resultKey)
+ _ = client.store.Delete(cleanupCtx, client.resultBucket, statusKey)
+ }()
+
+ expires := client.timeout + queryPresignGrace
+ resultURL, err := client.store.PresignPut(requestCtx, client.resultBucket, resultKey, "text/csv", expires)
+ if err != nil {
+ return nil, time.Time{}, false, newSanitizedError("prepare allocation query result", err)
+ }
+ statusURL, err := client.store.PresignPut(requestCtx, client.resultBucket, statusKey, "application/json", expires)
+ if err != nil {
+ return nil, time.Time{}, false, newSanitizedError("prepare allocation query status", err)
+ }
+ payload, err := json.Marshal(dataFusionWebhookPayload{
+ Dataset: "allocation",
+ Cluster: client.cluster,
+ Environment: client.environment,
+ Window: dataFusionQueryWindow{Start: start, End: end},
+ Query: query,
+ OutputURL: resultURL,
+ OutputFormat: "csv",
+ StatusURL: statusURL,
+ })
+ if err != nil {
+ return nil, time.Time{}, false, newSanitizedError("encode allocation query request", err)
+ }
+ request, err := http.NewRequestWithContext(requestCtx, http.MethodPost, client.webhookURL.String(), bytes.NewReader(payload))
+ if err != nil {
+ return nil, time.Time{}, false, newSanitizedError("build allocation query request", err)
+ }
+ request.Header.Set("Content-Type", "application/json")
+ request.Header.Set("X-Hub-Signature-256", signQueryPayload(payload, client.hmacSecret))
+ response, err := client.httpClient.Do(request)
+ if err != nil {
+ safeErr := newSanitizedError("send allocation query request", err)
+ if errors.Is(requestCtx.Err(), context.DeadlineExceeded) {
+ return nil, time.Time{}, false, errors.Join(errors.New("allocation query timed out"), context.DeadlineExceeded, safeErr)
+ }
+ if contextErr := requestCtx.Err(); contextErr != nil {
+ return nil, time.Time{}, false, errors.Join(contextErr, safeErr)
+ }
+ return nil, time.Time{}, false, safeErr
+ }
+ if response == nil {
+ return nil, time.Time{}, false, fmt.Errorf("allocation query webhook returned no response")
+ }
+ drainAndClose(response.Body)
+ if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
+ return nil, time.Time{}, false, fmt.Errorf("allocation query webhook returned HTTP %d", response.StatusCode)
+ }
+
+ status, err := client.waitForQuery(requestCtx, statusKey)
+ if err != nil {
+ return nil, time.Time{}, false, err
+ }
+ if status.Rows > maxDataFusionAllocationRows {
+ return nil, time.Time{}, false, fmt.Errorf("allocation query row limit exceeded")
+ }
+ contents, err := client.store.Get(requestCtx, client.resultBucket, resultKey, client.maxResponseSize)
+ if err != nil {
+ return nil, time.Time{}, false, newSanitizedError("read allocation query result", err)
+ }
+ allocations, asOf, parsedRows, legacyPartial, err := parseDataFusionAllocations(contents, start, end, namespaces)
+ if err != nil {
+ return nil, time.Time{}, false, err
+ }
+ if int64(parsedRows) != status.Rows {
+ return nil, time.Time{}, false, fmt.Errorf("allocation query row count mismatch")
+ }
+ partial := legacyPartial || status.PartitionsMissing > 0 || status.PartitionsMatched != status.PartitionsExpanded
+ if !partial {
+ asOf = end
+ } else if asOf.IsZero() {
+ asOf = start
+ }
+ return allocations, asOf, partial, nil
+}
+
+func safeNamespaces(input []string) ([]string, error) {
+ seen := make(map[string]struct{}, len(input))
+ result := make([]string, 0, len(input))
+ for _, namespace := range input {
+ if len(namespace) > 63 || !dns1123Label.MatchString(namespace) {
+ return nil, fmt.Errorf("invalid namespace")
+ }
+ if _, ok := seen[namespace]; ok {
+ continue
+ }
+ seen[namespace] = struct{}{}
+ result = append(result, namespace)
+ }
+ sort.Strings(result)
+ return result, nil
+}
+
+func allocationSQL(namespaces []string) string {
+ literals := make([]string, len(namespaces))
+ for index, namespace := range namespaces {
+ literals[index] = "'" + namespace + "'"
+ }
+ return "SELECT window_start, window_end, namespace, pod, cpu_usage_core_hours, cpu_request_core_hours, ram_usage_byte_hours, ram_request_byte_hours " +
+ "FROM allocation WHERE namespace IN (" + strings.Join(literals, ",") + ") " +
+ "ORDER BY window_start, namespace, pod LIMIT 100001"
+}
+
+func signQueryPayload(body []byte, secret string) string {
+ mac := hmac.New(sha256.New, []byte(secret))
+ _, _ = mac.Write(body)
+ return "sha256=" + hex.EncodeToString(mac.Sum(nil))
+}
+
+func drainAndClose(body io.ReadCloser) {
+ if body == nil {
+ return
+ }
+ _, _ = io.Copy(io.Discard, io.LimitReader(body, 64<<10))
+ _ = body.Close()
+}
+
+func (client *dataFusionAllocationClient) waitForQuery(ctx context.Context, statusKey string) (dataFusionQueryStatus, error) {
+ ticker := time.NewTicker(client.pollInterval)
+ defer ticker.Stop()
+ for {
+ status, ready, err := client.readQueryStatus(ctx, statusKey)
+ if err != nil {
+ return dataFusionQueryStatus{}, err
+ }
+ if ready {
+ return status, nil
+ }
+ select {
+ case <-ctx.Done():
+ if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+ return dataFusionQueryStatus{}, errors.Join(errors.New("allocation query timed out"), context.DeadlineExceeded)
+ }
+ return dataFusionQueryStatus{}, ctx.Err()
+ case <-ticker.C:
+ }
+ }
+}
+
+func (client *dataFusionAllocationClient) readQueryStatus(ctx context.Context, statusKey string) (dataFusionQueryStatus, bool, error) {
+ body, err := client.store.Get(ctx, client.resultBucket, statusKey, client.maxResponseSize)
+ if err != nil {
+ if errors.Is(err, ErrQueryObjectNotFound) {
+ return dataFusionQueryStatus{}, false, nil
+ }
+ return dataFusionQueryStatus{}, false, newSanitizedError("read allocation query status", err)
+ }
+ var status dataFusionQueryStatus
+ if decodeErr := json.Unmarshal(body, &status); decodeErr != nil {
+ return dataFusionQueryStatus{}, false, newSanitizedError("decode allocation query status", decodeErr)
+ }
+ switch status.Status {
+ case "succeeded":
+ if status.Rows < 0 || status.PartitionsExpanded < 0 || status.PartitionsMatched < 0 || status.PartitionsMissing < 0 || status.PartitionsMatched+status.PartitionsMissing != status.PartitionsExpanded {
+ return dataFusionQueryStatus{}, false, fmt.Errorf("invalid allocation query status")
+ }
+ return status, true, nil
+ case "failed":
+ return dataFusionQueryStatus{}, false, fmt.Errorf("allocation query failed")
+ case "running", "":
+ return dataFusionQueryStatus{}, false, nil
+ default:
+ return dataFusionQueryStatus{}, false, fmt.Errorf("invalid allocation query status")
+ }
+}
+
+func parseDataFusionAllocations(contents []byte, requestedStart, requestedEnd time.Time, namespaces []string) ([]Allocation, time.Time, int, bool, error) {
+ reader := csv.NewReader(bytes.NewReader(contents))
+ reader.FieldsPerRecord = -1
+ header, err := reader.Read()
+ if err != nil {
+ return nil, time.Time{}, 0, false, newSanitizedError("decode allocation query result", err)
+ }
+ wantHeader := []string{"window_start", "window_end", "namespace", "pod", "cpu_usage_core_hours", "cpu_request_core_hours", "ram_usage_byte_hours", "ram_request_byte_hours"}
+ if len(header) != len(wantHeader) {
+ return nil, time.Time{}, 0, false, fmt.Errorf("decode allocation query result: unexpected columns")
+ }
+ for index := range header {
+ if header[index] != wantHeader[index] {
+ return nil, time.Time{}, 0, false, fmt.Errorf("decode allocation query result: unexpected columns")
+ }
+ }
+ allowed := make(map[string]struct{}, len(namespaces))
+ for _, namespace := range namespaces {
+ allowed[namespace] = struct{}{}
+ }
+ allocations := make([]Allocation, 0)
+ var asOf time.Time
+ parsedRows := 0
+ legacyPartial := false
+ for rowNumber := 2; ; rowNumber++ {
+ record, readErr := reader.Read()
+ if readErr == io.EOF {
+ break
+ }
+ if readErr != nil {
+ return nil, time.Time{}, 0, false, newSanitizedError("decode allocation query result: malformed row", readErr)
+ }
+ if len(record) != len(wantHeader) {
+ return nil, time.Time{}, 0, false, fmt.Errorf("decode allocation query result: malformed row")
+ }
+ parsedRows++
+ if parsedRows > maxDataFusionAllocationRows {
+ return nil, time.Time{}, 0, false, fmt.Errorf("allocation query row limit exceeded")
+ }
+ if record[3] == "NULL" || record[4] == "NULL" || record[5] == "NULL" || record[6] == "NULL" || record[7] == "NULL" {
+ legacyPartial = true
+ continue
+ }
+ allocation, parseErr := parseDataFusionAllocation(record, requestedStart, requestedEnd, allowed)
+ if parseErr != nil {
+ return nil, time.Time{}, 0, false, newSanitizedError(fmt.Sprintf("decode allocation query result: malformed row %d", rowNumber), parseErr)
+ }
+ allocations = append(allocations, allocation)
+ if allocation.End.After(asOf) {
+ asOf = allocation.End
+ }
+ }
+ return allocations, asOf, parsedRows, legacyPartial, nil
+}
+
+func parseDataFusionAllocation(record []string, requestedStart, requestedEnd time.Time, allowed map[string]struct{}) (Allocation, error) {
+ start, err := parseDataFusionTimestamp(record[0])
+ if err != nil {
+ return Allocation{}, err
+ }
+ end, err := parseDataFusionTimestamp(record[1])
+ if err != nil {
+ return Allocation{}, newSanitizedError("invalid allocation window", err)
+ }
+ if !end.After(start) || end.Sub(start) != time.Hour || start.Before(requestedStart) || end.After(requestedEnd) {
+ return Allocation{}, fmt.Errorf("invalid allocation window")
+ }
+ namespace := record[2]
+ if _, ok := allowed[namespace]; !ok {
+ return Allocation{}, fmt.Errorf("unexpected namespace")
+ }
+ pod := record[3]
+ if len(pod) > 253 || !queryPodName.MatchString(pod) {
+ return Allocation{}, fmt.Errorf("invalid pod")
+ }
+ values := make([]float64, 4)
+ for index := range values {
+ value, parseErr := strconv.ParseFloat(record[index+4], 64)
+ if parseErr != nil {
+ return Allocation{}, newSanitizedError("invalid allocation value", parseErr)
+ }
+ if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value > 1e18 {
+ return Allocation{}, fmt.Errorf("invalid allocation value")
+ }
+ values[index] = value
+ }
+ hours := end.Sub(start).Hours()
+ return Allocation{
+ Start: start.UTC(),
+ End: end.UTC(),
+ Namespace: namespace,
+ Pod: pod,
+ Minutes: end.Sub(start).Minutes(),
+ CPUUsageAverage: values[0] / hours,
+ CPURequestAverage: values[1] / hours,
+ RAMUsageAverageBytes: values[2] / hours,
+ RAMRequestAverageBytes: values[3] / hours,
+ }, nil
+}
+
+func parseDataFusionTimestamp(raw string) (time.Time, error) {
+ parsed, err := time.Parse(time.RFC3339Nano, raw)
+ if err == nil {
+ return parsed.UTC(), nil
+ }
+ parsed, fallbackErr := time.ParseInLocation("2006-01-02 15:04:05.999999999", raw, time.UTC)
+ if fallbackErr != nil {
+ return time.Time{}, errors.Join(err, fallbackErr)
+ }
+ return parsed, nil
+}
+
+var _ AllocationClient = (*dataFusionAllocationClient)(nil)
diff --git a/libs/fleet/backend/usage/datafusion_s3.go b/libs/fleet/backend/usage/datafusion_s3.go
new file mode 100644
index 0000000000..7e5b1b5ffb
--- /dev/null
+++ b/libs/fleet/backend/usage/datafusion_s3.go
@@ -0,0 +1,86 @@
+package usage
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "sync"
+ "time"
+
+ awsconfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/smithy-go"
+)
+
+type s3QueryObjectStore struct {
+ once sync.Once
+ client *s3.Client
+ presigner *s3.PresignClient
+ err error
+}
+
+func NewS3QueryObjectStore() QueryObjectStore {
+ return &s3QueryObjectStore{}
+}
+
+func (store *s3QueryObjectStore) load(ctx context.Context) error {
+ store.once.Do(func() {
+ configuration, err := awsconfig.LoadDefaultConfig(ctx)
+ if err != nil {
+ store.err = err
+ return
+ }
+ store.client = s3.NewFromConfig(configuration)
+ store.presigner = s3.NewPresignClient(store.client)
+ })
+ return store.err
+}
+
+func (store *s3QueryObjectStore) PresignPut(ctx context.Context, bucket, key, contentType string, expires time.Duration) (string, error) {
+ if err := store.load(ctx); err != nil {
+ return "", err
+ }
+ result, err := store.presigner.PresignPutObject(ctx, &s3.PutObjectInput{
+ Bucket: &bucket,
+ Key: &key,
+ ContentType: &contentType,
+ }, func(options *s3.PresignOptions) {
+ options.Expires = expires
+ })
+ if err != nil {
+ return "", err
+ }
+ return result.URL, nil
+}
+
+func (store *s3QueryObjectStore) Get(ctx context.Context, bucket, key string, maxBytes int64) ([]byte, error) {
+ if err := store.load(ctx); err != nil {
+ return nil, err
+ }
+ result, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &key})
+ if err != nil {
+ var apiError smithy.APIError
+ if errors.As(err, &apiError) && (apiError.ErrorCode() == "NoSuchKey" || apiError.ErrorCode() == "NotFound") {
+ return nil, errors.Join(ErrQueryObjectNotFound, newSanitizedError("query object not found", err))
+ }
+ return nil, err
+ }
+ defer result.Body.Close()
+ body, err := io.ReadAll(io.LimitReader(result.Body, maxBytes+1))
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(body)) > maxBytes {
+ return nil, fmt.Errorf("query object exceeds limit")
+ }
+ return body, nil
+}
+
+func (store *s3QueryObjectStore) Delete(ctx context.Context, bucket, key string) error {
+ if err := store.load(ctx); err != nil {
+ return err
+ }
+ _, err := store.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &key})
+ return err
+}
diff --git a/libs/fleet/backend/usage/datafusion_test.go b/libs/fleet/backend/usage/datafusion_test.go
new file mode 100644
index 0000000000..d3e6cb5938
--- /dev/null
+++ b/libs/fleet/backend/usage/datafusion_test.go
@@ -0,0 +1,222 @@
+package usage
+
+import (
+ "context"
+ "crypto/hmac"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+type fakeQueryObjectStore struct {
+ mu sync.Mutex
+ status [][]byte
+ result []byte
+ deletedKeys []string
+}
+
+func (store *fakeQueryObjectStore) PresignPut(_ context.Context, _ string, key, contentType string, _ time.Duration) (string, error) {
+ return "https://uploads.example.test/" + key + "?X-Amz-Signature=secret-" + contentType, nil
+}
+
+func (store *fakeQueryObjectStore) Get(_ context.Context, _ string, key string, _ int64) ([]byte, error) {
+ store.mu.Lock()
+ defer store.mu.Unlock()
+ if strings.HasSuffix(key, "/status.json") {
+ if len(store.status) == 0 {
+ return nil, ErrQueryObjectNotFound
+ }
+ body := store.status[0]
+ store.status = store.status[1:]
+ return body, nil
+ }
+ if strings.HasSuffix(key, "/result.csv") {
+ return store.result, nil
+ }
+ return nil, ErrQueryObjectNotFound
+}
+
+func (store *fakeQueryObjectStore) Delete(_ context.Context, _ string, key string) error {
+ store.mu.Lock()
+ defer store.mu.Unlock()
+ store.deletedKeys = append(store.deletedKeys, key)
+ return nil
+}
+
+func TestDataFusionAllocationClientQueriesBoundedParquetAndMapsSemantics(t *testing.T) {
+ t.Parallel()
+ secret := "webhook-secret"
+ var payload dataFusionWebhookPayload
+ server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+ body, err := io.ReadAll(request.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ mac := hmac.New(sha256.New, []byte(secret))
+ _, _ = mac.Write(body)
+ wantSignature := "sha256=" + hex.EncodeToString(mac.Sum(nil))
+ if request.Header.Get("X-Hub-Signature-256") != wantSignature {
+ t.Fatalf("signature = %q, want %q", request.Header.Get("X-Hub-Signature-256"), wantSignature)
+ }
+ if err := json.Unmarshal(body, &payload); err != nil {
+ t.Fatal(err)
+ }
+ writer.WriteHeader(http.StatusAccepted)
+ }))
+ defer server.Close()
+
+ start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)
+ store := &fakeQueryObjectStore{
+ status: [][]byte{[]byte(`{"status":"succeeded","rows":1,"partitions_expanded":1,"partitions_matched":1,"partitions_missing":0}`)},
+ result: []byte("window_start,window_end,namespace,pod,cpu_usage_core_hours,cpu_request_core_hours,ram_usage_byte_hours,ram_request_byte_hours\n" +
+ "2026-08-19 09:00:00,2026-08-19 10:00:00,ns-a,virt-launcher-vm-a-x,2,4,3221225472,6442450944\n"),
+ }
+ client, err := NewDataFusionAllocationClient(server.URL, secret, "results-bucket", "cyclops/usage-query", "kopf-k3s", "production", time.Second, time.Millisecond, 1<<20, store)
+ if err != nil {
+ t.Fatal(err)
+ }
+ allocations, asOf, partial, err := client.Allocations(context.Background(), start, start.Add(time.Hour), time.Hour, []string{"ns-b", "ns-a", "ns-a"})
+ if err != nil || partial || !asOf.Equal(start.Add(time.Hour)) || len(allocations) != 1 {
+ t.Fatalf("allocations=%#v asOf=%s partial=%t err=%v", allocations, asOf, partial, err)
+ }
+ allocation := allocations[0]
+ if allocation.Minutes != 60 || allocation.CPUUsageAverage != 2 || allocation.CPURequestAverage != 4 || allocation.RAMUsageAverageBytes != 3*gibibyte || allocation.RAMRequestAverageBytes != 6*gibibyte {
+ t.Fatalf("allocation = %#v", allocation)
+ }
+ if payload.Dataset != "allocation" || payload.Cluster != "kopf-k3s" || payload.Environment != "production" || payload.OutputFormat != "csv" {
+ t.Fatalf("payload = %#v", payload)
+ }
+ for _, fragment := range []string{"namespace IN ('ns-a','ns-b')", "LIMIT 100001", "cpu_usage_core_hours", "cpu_request_core_hours", "ram_usage_byte_hours", "ram_request_byte_hours"} {
+ if !strings.Contains(payload.Query, fragment) {
+ t.Fatalf("query %q does not contain %q", payload.Query, fragment)
+ }
+ }
+ if len(store.deletedKeys) != 2 {
+ t.Fatalf("deleted keys = %v", store.deletedKeys)
+ }
+}
+
+func TestDataFusionAllocationClientRejectsUnsafeNamespaceBeforeWebhook(t *testing.T) {
+ t.Parallel()
+ called := false
+ server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true }))
+ defer server.Close()
+ client, err := NewDataFusionAllocationClient(server.URL, "secret", "bucket", "cyclops/usage-query", "kopf-k3s", "production", time.Second, time.Millisecond, 1<<20, &fakeQueryObjectStore{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)
+ _, _, _, err = client.Allocations(context.Background(), start, start.Add(time.Hour), time.Hour, []string{"ns-a') OR true --"})
+ if err == nil || !strings.Contains(err.Error(), "invalid namespace") || called {
+ t.Fatalf("called=%t err=%v", called, err)
+ }
+}
+
+func TestDataFusionAllocationClientHandlesMalformedNon2xxAndTimeoutSafely(t *testing.T) {
+ t.Parallel()
+ start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)
+ for name, test := range map[string]struct {
+ handler http.HandlerFunc
+ store *fakeQueryObjectStore
+ want string
+ }{
+ "non_2xx": {
+ handler: func(writer http.ResponseWriter, _ *http.Request) {
+ http.Error(writer, "SELECT secret FROM tenant", http.StatusBadGateway)
+ },
+ store: &fakeQueryObjectStore{},
+ want: "query webhook returned HTTP 502",
+ },
+ "malformed_csv": {
+ handler: func(writer http.ResponseWriter, _ *http.Request) { writer.WriteHeader(http.StatusAccepted) },
+ store: &fakeQueryObjectStore{
+ status: [][]byte{[]byte(`{"status":"succeeded","rows":1,"partitions_expanded":1,"partitions_matched":1}`)},
+ result: []byte("window_start,namespace\nhttps://bucket.example/?X-Amz-Signature=leak,ns-a\n"),
+ },
+ want: "decode allocation query result",
+ },
+ "timeout": {
+ handler: func(writer http.ResponseWriter, _ *http.Request) { writer.WriteHeader(http.StatusAccepted) },
+ store: &fakeQueryObjectStore{},
+ want: "allocation query timed out",
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ server := httptest.NewServer(test.handler)
+ defer server.Close()
+ timeout := time.Second
+ if name == "timeout" {
+ timeout = 5 * time.Millisecond
+ }
+ client, err := NewDataFusionAllocationClient(server.URL, "secret", "bucket", "cyclops/usage-query", "kopf-k3s", "production", timeout, time.Millisecond, 1<<20, test.store)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, _, _, err = client.Allocations(context.Background(), start, start.Add(time.Hour), time.Hour, []string{"ns-a"})
+ if err == nil || !strings.Contains(err.Error(), test.want) {
+ t.Fatalf("error = %v, want containing %q", err, test.want)
+ }
+ for _, forbidden := range []string{"SELECT secret", "X-Amz-Signature", "tenant"} {
+ if strings.Contains(err.Error(), forbidden) {
+ t.Fatalf("error leaked %q: %v", forbidden, err)
+ }
+ }
+ })
+ }
+}
+
+func TestDataFusionAllocationClientPreservesCancellationAndMissingPartitions(t *testing.T) {
+ t.Parallel()
+ start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)
+ server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { writer.WriteHeader(http.StatusAccepted) }))
+ defer server.Close()
+ store := &fakeQueryObjectStore{
+ status: [][]byte{[]byte(`{"status":"succeeded","rows":0,"partitions_expanded":2,"partitions_matched":1,"partitions_missing":1}`)},
+ result: []byte("window_start,window_end,namespace,pod,cpu_usage_core_hours,cpu_request_core_hours,ram_usage_byte_hours,ram_request_byte_hours\n"),
+ }
+ client, err := NewDataFusionAllocationClient(server.URL, "secret", "bucket", "cyclops/usage-query", "kopf-k3s", "production", time.Second, time.Millisecond, 1<<20, store)
+ if err != nil {
+ t.Fatal(err)
+ }
+ allocations, asOf, partial, err := client.Allocations(context.Background(), start, start.Add(2*time.Hour), time.Hour, []string{"ns-a"})
+ if err != nil || len(allocations) != 0 || !partial || !asOf.Equal(start) {
+ t.Fatalf("allocations=%v asOf=%s partial=%t err=%v", allocations, asOf, partial, err)
+ }
+
+ cancelled, cancel := context.WithCancel(context.Background())
+ cancel()
+ _, _, _, err = client.Allocations(cancelled, start, start.Add(time.Hour), time.Hour, []string{"ns-a"})
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("error = %v, want context.Canceled", err)
+ }
+}
+
+func TestDataFusionAllocationClientMarksLegacyParquetRowsPartial(t *testing.T) {
+ t.Parallel()
+ start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)
+ server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
+ writer.WriteHeader(http.StatusAccepted)
+ }))
+ defer server.Close()
+ store := &fakeQueryObjectStore{
+ status: [][]byte{[]byte(`{"status":"succeeded","table":"allocation","rows":1,"partitions_expanded":1,"partitions_matched":1,"partitions_missing":0}`)},
+ result: []byte("window_start,window_end,namespace,pod,cpu_usage_core_hours,cpu_request_core_hours,ram_usage_byte_hours,ram_request_byte_hours\n" +
+ "2026-08-19T09:00:00Z,2026-08-19T10:00:00Z,ns-a,NULL,NULL,NULL,NULL,NULL\n"),
+ }
+ client, err := NewDataFusionAllocationClient(server.URL, "secret", "bucket", "cyclops/usage-query", "kopf-k3s", "production", time.Second, time.Millisecond, 1<<20, store)
+ if err != nil {
+ t.Fatal(err)
+ }
+ allocations, asOf, partial, err := client.Allocations(context.Background(), start, start.Add(time.Hour), time.Hour, []string{"ns-a"})
+ if err != nil || len(allocations) != 0 || !partial || !asOf.Equal(start) {
+ t.Fatalf("allocations=%v asOf=%s partial=%t err=%v", allocations, asOf, partial, err)
+ }
+}
diff --git a/libs/fleet/backend/usage/opencost.go b/libs/fleet/backend/usage/opencost.go
deleted file mode 100644
index b409d63f21..0000000000
--- a/libs/fleet/backend/usage/opencost.go
+++ /dev/null
@@ -1,328 +0,0 @@
-package usage
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "io"
- "math"
- "net/http"
- "net/url"
- "regexp"
- "sort"
- "strings"
- "time"
-)
-
-const maxAllocationRows = 100000
-
-var dns1123Label = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)
-
-type openCostClient struct {
- baseURL *url.URL
- httpClient *http.Client
- timeout time.Duration
- maxResponseBytes int64
-}
-
-func NewOpenCostClient(rawURL string, timeout time.Duration, maxResponseBytes int64) (*openCostClient, error) {
- baseURL, err := url.Parse(rawURL)
- if err != nil {
- return nil, newSanitizedError("invalid OpenCost URL", err)
- }
- if (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" || baseURL.RawQuery != "" || baseURL.Fragment != "" || baseURL.User != nil {
- return nil, fmt.Errorf("invalid OpenCost URL")
- }
- if timeout <= 0 || maxResponseBytes <= 0 || maxResponseBytes == math.MaxInt64 {
- return nil, fmt.Errorf("invalid OpenCost client limits")
- }
- return &openCostClient{
- baseURL: baseURL,
- timeout: timeout,
- maxResponseBytes: maxResponseBytes,
- httpClient: &http.Client{
- Timeout: timeout,
- CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
- return http.ErrUseLastResponse
- },
- },
- }, nil
-}
-
-func (c *openCostClient) Allocations(ctx context.Context, start, end time.Time, step time.Duration, namespaces []string) ([]Allocation, time.Time, bool, error) {
- if start.IsZero() || end.IsZero() || !end.After(start) || len(namespaces) == 0 {
- return nil, time.Time{}, false, fmt.Errorf("invalid OpenCost allocation query")
- }
- stepText, err := openCostStep(step)
- if err != nil {
- return nil, time.Time{}, false, err
- }
- filter, namespaces, err := namespaceFilter(namespaces)
- if err != nil {
- return nil, time.Time{}, false, err
- }
-
- endpoint := *c.baseURL
- endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/allocation/compute"
- values := make(url.Values)
- values.Set("window", start.UTC().Format(time.RFC3339)+","+end.UTC().Format(time.RFC3339))
- values.Set("aggregate", "cluster,namespace,pod")
- values.Set("includeIdle", "false")
- values.Set("includeSharedCostBreakdown", "false")
- values.Set("accumulate", "false")
- values.Set("step", stepText)
- values.Set("filter", filter)
- endpoint.RawQuery = values.Encode()
-
- requestCtx, cancel := context.WithTimeout(ctx, c.timeout)
- defer cancel()
- request, err := http.NewRequestWithContext(requestCtx, http.MethodGet, endpoint.String(), nil)
- if err != nil {
- return nil, time.Time{}, false, fmt.Errorf("build OpenCost request: %w", err)
- }
- response, err := c.httpClient.Do(request)
- if err != nil {
- return nil, time.Time{}, false, fmt.Errorf("request OpenCost allocations: %w", err)
- }
- defer response.Body.Close()
- if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
- return nil, time.Time{}, false, fmt.Errorf("OpenCost allocation request failed with status %d", response.StatusCode)
- }
- body, err := io.ReadAll(io.LimitReader(response.Body, c.maxResponseBytes+1))
- if err != nil {
- return nil, time.Time{}, false, fmt.Errorf("read OpenCost allocation response: %w", err)
- }
- if int64(len(body)) > c.maxResponseBytes {
- return nil, time.Time{}, false, fmt.Errorf("OpenCost allocation response exceeds limit")
- }
- allocations, asOf, incomplete, err := parseOpenCostAllocations(body, start, end, step, namespaces)
- if err != nil {
- return nil, time.Time{}, false, err
- }
- return allocations, asOf, incomplete, nil
-}
-
-func openCostStep(step time.Duration) (string, error) {
- switch step {
- case time.Hour:
- return "1h", nil
- case 24 * time.Hour:
- return "1d", nil
- default:
- return "", fmt.Errorf("unsupported OpenCost allocation step")
- }
-}
-
-func namespaceFilter(input []string) (string, []string, error) {
- seen := make(map[string]struct{}, len(input))
- namespaces := make([]string, 0, len(input))
- for _, namespace := range input {
- if len(namespace) > 63 || !dns1123Label.MatchString(namespace) {
- return "", nil, fmt.Errorf("invalid namespace")
- }
- if _, ok := seen[namespace]; !ok {
- seen[namespace] = struct{}{}
- namespaces = append(namespaces, namespace)
- }
- }
- sort.Strings(namespaces)
- parts := make([]string, len(namespaces))
- for index, namespace := range namespaces {
- parts[index] = `namespace:"` + namespace + `"`
- }
- return strings.Join(parts, "|"), namespaces, nil
-}
-
-type openCostResponse struct {
- Code *int `json:"code"`
- Data json.RawMessage `json:"data"`
- Error json.RawMessage `json:"error"`
- Warnings json.RawMessage `json:"warnings"`
-}
-
-type openCostAllocation struct {
- Properties struct {
- Namespace string `json:"namespace"`
- Pod string `json:"pod"`
- } `json:"properties"`
- Window json.RawMessage `json:"window"`
- Minutes *float64 `json:"minutes"`
- CPUUsageAverage *float64 `json:"cpuCoreUsageAverage"`
- CPURequestAverage *float64 `json:"cpuCoreRequestAverage"`
- RAMUsageAverageBytes *float64 `json:"ramByteUsageAverage"`
- RAMRequestAverageBytes *float64 `json:"ramByteRequestAverage"`
-}
-
-type allocationFrame struct {
- start time.Time
- end time.Time
-}
-
-func parseOpenCostAllocations(body []byte, requestedStart, requestedEnd time.Time, step time.Duration, namespaces []string) ([]Allocation, time.Time, bool, error) {
- var payload openCostResponse
- if err := json.Unmarshal(body, &payload); err != nil {
- return nil, time.Time{}, false, fmt.Errorf("parse OpenCost allocation response: %w", err)
- }
- if payload.Code == nil || *payload.Code != http.StatusOK || hasOpenCostError(payload.Error) {
- return nil, time.Time{}, false, fmt.Errorf("OpenCost allocation response reports an error")
- }
- if len(payload.Data) == 0 || string(payload.Data) == "null" {
- return nil, time.Time{}, false, fmt.Errorf("OpenCost allocation response has no data")
- }
- var frames []map[string]json.RawMessage
- if err := json.Unmarshal(payload.Data, &frames); err != nil {
- return nil, time.Time{}, false, newSanitizedError("OpenCost allocation response has invalid data", err)
- }
- if frames == nil {
- return nil, time.Time{}, false, fmt.Errorf("OpenCost allocation response has invalid data")
- }
-
- allowed := make(map[string]struct{}, len(namespaces))
- for _, namespace := range namespaces {
- allowed[namespace] = struct{}{}
- }
- incomplete := hasOpenCostWarnings(payload.Warnings)
- coverage := make([]allocationFrame, 0, len(frames))
- for _, frame := range frames {
- window, ok := openCostFrameWindow(frame, requestedStart, requestedEnd, step)
- if !ok {
- incomplete = true
- continue
- }
- coverage = append(coverage, window)
- }
- asOf, coverageComplete := completeCoverageBoundary(coverage, requestedStart, requestedEnd)
- incomplete = incomplete || !coverageComplete
-
- allocations := make([]Allocation, 0)
- rowCount := 0
- for _, group := range frames {
- for _, raw := range group {
- rowCount++
- if rowCount > maxAllocationRows {
- return nil, time.Time{}, false, fmt.Errorf("OpenCost allocation response exceeds row limit")
- }
- var row openCostAllocation
- if err := json.Unmarshal(raw, &row); err != nil {
- return nil, time.Time{}, false, newSanitizedError("OpenCost allocation response contains malformed allocation data", err)
- }
- allocation, ok := validateOpenCostAllocation(row, requestedStart, requestedEnd, step, allowed)
- if !ok {
- incomplete = true
- continue
- }
- if allocation.End.After(asOf) {
- continue
- }
- allocations = append(allocations, allocation)
- }
- }
- return allocations, asOf, incomplete, nil
-}
-
-func hasOpenCostError(raw json.RawMessage) bool {
- return len(raw) > 0 && string(raw) != "null" && string(raw) != `""`
-}
-
-func hasOpenCostWarnings(raw json.RawMessage) bool {
- return len(raw) > 0 && string(raw) != "null" && string(raw) != "[]"
-}
-
-func openCostFrameWindow(frame map[string]json.RawMessage, requestedStart, requestedEnd time.Time, step time.Duration) (allocationFrame, bool) {
- if len(frame) == 0 {
- return allocationFrame{}, false
- }
- var result allocationFrame
- for _, raw := range frame {
- var header struct {
- Window json.RawMessage `json:"window"`
- }
- if err := json.Unmarshal(raw, &header); err != nil {
- return allocationFrame{}, false
- }
- start, end, ok := parseOpenCostWindow(header.Window)
- if !ok || start.Before(requestedStart) || end.After(requestedEnd) || !end.After(start) || end.Sub(start) != step {
- return allocationFrame{}, false
- }
- if result.start.IsZero() {
- result = allocationFrame{start: start, end: end}
- continue
- }
- if !result.start.Equal(start) || !result.end.Equal(end) {
- return allocationFrame{}, false
- }
- }
- return result, !result.start.IsZero()
-}
-
-func completeCoverageBoundary(frames []allocationFrame, requestedStart, requestedEnd time.Time) (time.Time, bool) {
- if len(frames) == 0 {
- return requestedStart, false
- }
- sort.Slice(frames, func(i, j int) bool {
- return frames[i].start.Before(frames[j].start)
- })
- latest := requestedStart
- expected := requestedStart
- complete := true
- for _, frame := range frames {
- if !frame.start.Equal(expected) {
- complete = false
- }
- if frame.end.After(expected) {
- expected = frame.end
- }
- if frame.end.After(latest) {
- latest = frame.end
- }
- }
- if latest.After(requestedEnd) {
- latest = requestedEnd
- }
- return latest, complete && expected.Equal(requestedEnd)
-}
-
-func validateOpenCostAllocation(row openCostAllocation, requestedStart, requestedEnd time.Time, step time.Duration, allowed map[string]struct{}) (Allocation, bool) {
- if _, ok := allowed[row.Properties.Namespace]; !ok || strings.TrimSpace(row.Properties.Pod) == "" || len(row.Properties.Pod) > 253 || row.Minutes == nil || row.CPUUsageAverage == nil || row.CPURequestAverage == nil || row.RAMUsageAverageBytes == nil || row.RAMRequestAverageBytes == nil {
- return Allocation{}, false
- }
- start, end, ok := parseOpenCostWindow(row.Window)
- if !ok || start.Before(requestedStart) || end.After(requestedEnd) || !end.After(start) || end.Sub(start) != step {
- return Allocation{}, false
- }
- values := []float64{*row.Minutes, *row.CPUUsageAverage, *row.CPURequestAverage, *row.RAMUsageAverageBytes, *row.RAMRequestAverageBytes}
- for _, value := range values {
- if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value > 1e15 {
- return Allocation{}, false
- }
- }
- if *row.Minutes > end.Sub(start).Minutes()+0.000001 {
- return Allocation{}, false
- }
- return Allocation{Start: start, End: end, Namespace: row.Properties.Namespace, Pod: row.Properties.Pod, Minutes: *row.Minutes, CPUUsageAverage: *row.CPUUsageAverage, CPURequestAverage: *row.CPURequestAverage, RAMUsageAverageBytes: *row.RAMUsageAverageBytes, RAMRequestAverageBytes: *row.RAMRequestAverageBytes}, true
-}
-
-func parseOpenCostWindow(raw json.RawMessage) (time.Time, time.Time, bool) {
- var object struct {
- Start string `json:"start"`
- End string `json:"end"`
- }
- if json.Unmarshal(raw, &object) == nil && object.Start != "" && object.End != "" {
- start, startErr := time.Parse(time.RFC3339, object.Start)
- end, endErr := time.Parse(time.RFC3339, object.End)
- return start.UTC(), end.UTC(), startErr == nil && endErr == nil
- }
- var legacy string
- if err := json.Unmarshal(raw, &legacy); err != nil || len(legacy) < 5 || legacy[0] != '[' || legacy[len(legacy)-1] != ')' {
- return time.Time{}, time.Time{}, false
- }
- parts := strings.Split(strings.TrimSuffix(strings.TrimPrefix(legacy, "["), ")"), ",")
- if len(parts) != 2 {
- return time.Time{}, time.Time{}, false
- }
- start, startErr := time.Parse(time.RFC3339, strings.TrimSpace(parts[0]))
- end, endErr := time.Parse(time.RFC3339, strings.TrimSpace(parts[1]))
- return start.UTC(), end.UTC(), startErr == nil && endErr == nil
-}
-
-var _ AllocationClient = (*openCostClient)(nil)
diff --git a/libs/fleet/backend/usage/opencost_test.go b/libs/fleet/backend/usage/opencost_test.go
deleted file mode 100644
index e2e88f6b39..0000000000
--- a/libs/fleet/backend/usage/opencost_test.go
+++ /dev/null
@@ -1,161 +0,0 @@
-package usage
-
-import (
- "context"
- "fmt"
- "math"
- "net/http"
- "net/http/httptest"
- "testing"
- "time"
-)
-
-func TestOpenCostClientUsesFixedScopedAllocationQuery(t *testing.T) {
- t.Parallel()
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.URL.Path != "/allocation/compute" {
- t.Fatalf("path = %q", r.URL.Path)
- }
- q := r.URL.Query()
- if q.Get("aggregate") != "cluster,namespace,pod" || q.Get("includeIdle") != "false" || q.Get("includeSharedCostBreakdown") != "false" || q.Get("accumulate") != "false" || q.Get("step") != "1h" || q.Get("filter") != `namespace:"ns-a"|namespace:"ns-b"` || q.Has("filterNamespaces") {
- t.Fatalf("query = %#v", q)
- }
- _, _ = w.Write([]byte(`{"code":200,"data":[{"cluster/ns-a/virt-launcher-vm-a-x":{"properties":{"namespace":"ns-a","pod":"virt-launcher-vm-a-x"},"window":{"start":"2026-08-19T09:00:00Z","end":"2026-08-19T10:00:00Z"},"minutes":60,"cpuCoreUsageAverage":2,"cpuCoreRequestAverage":4,"ramByteUsageAverage":3221225472,"ramByteRequestAverage":6442450944}}],"meta":{"timeGenerated":"2026-08-19T10:01:00Z"}}`))
- }))
- defer server.Close()
- client, err := NewOpenCostClient(server.URL, time.Second, 1<<20)
- if err != nil {
- t.Fatal(err)
- }
- start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)
- allocations, asOf, partial, err := client.Allocations(context.Background(), start, start.Add(time.Hour), time.Hour, []string{"ns-b", "ns-a"})
- if err != nil || partial || !asOf.Equal(start.Add(time.Hour)) || len(allocations) != 1 {
- t.Fatalf("allocations=%#v asOf=%s partial=%t err=%v", allocations, asOf, partial, err)
- }
- if allocations[0].CPUUsageAverage != 2 || allocations[0].CPURequestAverage != 4 || allocations[0].RAMUsageAverageBytes != 3*gibibyte || allocations[0].RAMRequestAverageBytes != 6*gibibyte {
- t.Fatalf("allocation = %#v", allocations[0])
- }
-}
-
-func TestParseOpenCostAllocationsUsesProductionWindowShapeAndCoverage(t *testing.T) {
- t.Parallel()
- start := time.Date(2026, 7, 13, 3, 0, 0, 0, time.UTC)
- body := []byte(`{"code":200,"data":[{"default/api":{"properties":{"namespace":"default","pod":"virt-launcher-api-x"},"window":{"start":"2026-07-13T03:00:00Z","end":"2026-07-13T04:00:00Z"},"minutes":60,"cpuCoreUsageAverage":0,"cpuCoreRequestAverage":2,"ramByteUsageAverage":0,"ramByteRequestAverage":1073741824}}]}`)
- allocations, asOf, partial, err := parseOpenCostAllocations(body, start, start.Add(time.Hour), time.Hour, []string{"default"})
- if err != nil || partial || !asOf.Equal(start.Add(time.Hour)) || len(allocations) != 1 {
- t.Fatalf("allocations=%#v asOf=%s partial=%t err=%v", allocations, asOf, partial, err)
- }
- if allocations[0].CPUUsageAverage != 0 || allocations[0].RAMUsageAverageBytes != 0 {
- t.Fatalf("zero averages were not retained: %#v", allocations[0])
- }
-}
-
-func TestParseOpenCostAllocationsMarksIncompleteCoverage(t *testing.T) {
- t.Parallel()
- start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)
- row := func(windowStart, windowEnd time.Time) string {
- return fmt.Sprintf(`{"properties":{"namespace":"ns-a","pod":"virt-launcher-vm-a-x"},"window":{"start":"%s","end":"%s"},"minutes":60,"cpuCoreUsageAverage":1,"cpuCoreRequestAverage":2,"ramByteUsageAverage":1073741824,"ramByteRequestAverage":2147483648}`, windowStart.Format(time.RFC3339), windowEnd.Format(time.RFC3339))
- }
- for name, test := range map[string]struct {
- body []byte
- wantCutoff time.Time
- wantRows int
- }{
- "empty": {[]byte(`{"code":200,"data":[]}`), start, 0},
- "missing_frame": {[]byte(`{"code":200,"data":[{"first":` + row(start, start.Add(time.Hour)) + `},{}]}`), start.Add(time.Hour), 1},
- "gap": {[]byte(`{"code":200,"data":[{"first":` + row(start, start.Add(time.Hour)) + `},{"third":` + row(start.Add(2*time.Hour), start.Add(3*time.Hour)) + `}]}`), start.Add(3 * time.Hour), 2},
- "missing_prefix": {[]byte(`{"code":200,"data":[{"third":` + row(start.Add(2*time.Hour), start.Add(3*time.Hour)) + `}]}`), start.Add(3 * time.Hour), 1},
- } {
- t.Run(name, func(t *testing.T) {
- allocations, asOf, partial, err := parseOpenCostAllocations(test.body, start, start.Add(3*time.Hour), time.Hour, []string{"ns-a"})
- if err != nil || !partial || !asOf.Equal(test.wantCutoff) || len(allocations) != test.wantRows {
- t.Fatalf("allocations=%#v asOf=%s partial=%t err=%v", allocations, asOf, partial, err)
- }
- })
- }
-}
-
-func TestParseOpenCostAllocationsRejectsSourceErrorsAndMalformedRequiredValues(t *testing.T) {
- t.Parallel()
- start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)
- for name, body := range map[string][]byte{
- "missing_code": []byte(`{"data":[]}`),
- "error_code": []byte(`{"code":500,"error":"upstream failed"}`),
- "success_error": []byte(`{"code":200,"error":"unexpected","data":[]}`),
- } {
- t.Run(name, func(t *testing.T) {
- if _, _, _, err := parseOpenCostAllocations(body, start, start.Add(time.Hour), time.Hour, []string{"ns-a"}); err == nil {
- t.Fatal("expected source envelope error")
- }
- })
- }
- body := []byte(`{"code":200,"data":[{"missing":{"properties":{"namespace":"ns-a","pod":"virt-launcher-vm-a-x"},"window":{"start":"2026-08-19T09:00:00Z","end":"2026-08-19T10:00:00Z"},"minutes":60,"cpuCoreUsageAverage":null,"cpuCoreRequestAverage":2,"ramByteUsageAverage":1073741824,"ramByteRequestAverage":2147483648}}]}`)
- allocations, _, partial, err := parseOpenCostAllocations(body, start, start.Add(time.Hour), time.Hour, []string{"ns-a"})
- if err != nil || !partial || len(allocations) != 0 {
- t.Fatalf("allocations=%#v partial=%t err=%v", allocations, partial, err)
- }
-}
-
-func TestOpenCostClientRejectsUnsafeNamespace(t *testing.T) {
- t.Parallel()
- client, err := NewOpenCostClient("https://opencost.invalid", time.Second, 1<<20)
- if err != nil {
- t.Fatal(err)
- }
- _, _, _, err = client.Allocations(context.Background(), time.Now().Add(-time.Hour), time.Now(), time.Hour, []string{"ns-a\"|cluster:\"all"})
- if err == nil {
- t.Fatal("expected namespace validation error")
- }
-}
-
-func TestParseOpenCostAllocationsExcludesBadRowsAsIncomplete(t *testing.T) {
- t.Parallel()
- start := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)
- body := []byte(`{"code":200,"data":[{"valid":{"properties":{"namespace":"ns-a","pod":"virt-launcher-vm-a-x"},"window":"[2026-08-19T09:00:00Z,2026-08-19T10:00:00Z)","minutes":60,"cpuCoreUsageAverage":1,"cpuCoreRequestAverage":2,"ramByteUsageAverage":1073741824,"ramByteRequestAverage":2147483648},"out-of-scope":{"properties":{"namespace":"other","pod":"pod"},"window":"[2026-08-19T09:00:00Z,2026-08-19T10:00:00Z)","minutes":60,"cpuCoreUsageAverage":1,"cpuCoreRequestAverage":2,"ramByteUsageAverage":1073741824,"ramByteRequestAverage":2147483648},"oversized":{"properties":{"namespace":"ns-a","pod":"pod"},"window":"[2026-08-19T09:00:00Z,2026-08-19T10:00:00Z)","minutes":60,"cpuCoreUsageAverage":1000000000000001,"cpuCoreRequestAverage":2,"ramByteUsageAverage":1073741824,"ramByteRequestAverage":2147483648}}],"warnings":["partial"]}`)
- allocations, _, partial, err := parseOpenCostAllocations(body, start, start.Add(time.Hour), time.Hour, []string{"ns-a"})
- if err != nil || !partial || len(allocations) != 1 {
- t.Fatalf("allocations=%#v partial=%t err=%v", allocations, partial, err)
- }
-}
-
-func TestOpenCostClientRejectsUnsafeBaseURLAndResponseLimitOverflow(t *testing.T) {
- t.Parallel()
- for _, rawURL := range []string{
- "https://opencost.invalid?filterNamespaces=legacy",
- "https://opencost.invalid#fragment",
- "https://user:[email protected]",
- "ftp://opencost.invalid",
- } {
- if _, err := NewOpenCostClient(rawURL, time.Second, 1<<20); err == nil {
- t.Fatalf("expected unsafe URL rejection for %q", rawURL)
- }
- }
- if _, err := NewOpenCostClient("https://opencost.invalid", time.Second, math.MaxInt64); err == nil {
- t.Fatal("expected response limit overflow rejection")
- }
-}
-
-func TestOpenCostClientRejectsRedirectAndResponseOverLimit(t *testing.T) {
- t.Parallel()
- for name, handler := range map[string]http.HandlerFunc{
- "redirect": func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/elsewhere", http.StatusFound) },
- "limit": func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(make([]byte, 128)) },
- } {
- t.Run(name, func(t *testing.T) {
- server := httptest.NewServer(handler)
- defer server.Close()
- limit := int64(1 << 20)
- if name == "limit" {
- limit = 64
- }
- client, err := NewOpenCostClient(server.URL, time.Second, limit)
- if err != nil {
- t.Fatal(err)
- }
- _, _, _, err = client.Allocations(context.Background(), time.Now().UTC().Truncate(time.Hour), time.Now().UTC().Truncate(time.Hour).Add(time.Hour), time.Hour, []string{"ns-a"})
- if err == nil {
- t.Fatal("expected request error")
- }
- })
- }
-}
diff --git a/libs/fleet/backend/usage/provider.go b/libs/fleet/backend/usage/provider.go
index d48333d636..4992d64edc 100644
--- a/libs/fleet/backend/usage/provider.go
+++ b/libs/fleet/backend/usage/provider.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "regexp"
"sort"
"strings"
"time"
@@ -13,7 +14,10 @@ import (
"go.opentelemetry.io/otel/trace"
)
-var ErrPoolNotFound = errors.New("usage pool not found")
+var (
+ ErrPoolNotFound = errors.New("usage pool not found")
+ dns1123Label = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)
+)
type provider struct {
events EventStore
@@ -163,10 +167,10 @@ func (p *provider) load(ctx context.Context, query Query) (usageData, error) {
))
allocations, asOf, sourcePartial, err := p.allocations.Allocations(allocationsCtx, start, end, step, namespaces)
if err != nil {
- markUsageSpanError(allocationsSpan, "OpenCost allocation query failed")
+ markUsageSpanError(allocationsSpan, "allocation query failed")
allocationsSpan.End()
- markUsageSpanError(span, "OpenCost allocation query failed")
- return usageData{}, fmt.Errorf("read OpenCost allocations: %w", err)
+ markUsageSpanError(span, "allocation query failed")
+ return usageData{}, fmt.Errorf("read allocation data: %w", err)
}
allocationsSpan.SetAttributes(
attribute.Int("usage.allocation_count", len(allocations)),
diff --git a/libs/fleet/backend/usage/safe_error_test.go b/libs/fleet/backend/usage/safe_error_test.go
index 2b944197e2..e8f33cab57 100644
--- a/libs/fleet/backend/usage/safe_error_test.go
+++ b/libs/fleet/backend/usage/safe_error_test.go
@@ -7,8 +7,8 @@ import (
func TestNewSanitizedErrorRedactsAndPreservesCause(t *testing.T) {
cause := errors.New("secret upstream detail")
- err := newSanitizedError("invalid OpenCost URL", cause)
- if err.Error() != "invalid OpenCost URL" {
+ err := newSanitizedError("invalid allocation query webhook URL", cause)
+ if err.Error() != "invalid allocation query webhook URL" {
t.Fatalf("Error() = %q", err.Error())
}
if !errors.Is(err, cause) {