Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
feat: Filesystem type filters for the otel collector (#1395)
Chris Birkholddb5dcc3eb1f7
diff --git a/docs/latest/gpu-collector/configuration.mdx b/docs/latest/gpu-collector/configuration.mdxindex 4a41c1c1..a56d6ed9 100644--- a/docs/latest/gpu-collector/configuration.mdx+++ b/docs/latest/gpu-collector/configuration.mdx@@ -18,6 +18,7 @@ The collector is configured entirely via environment variables. All variables fo| `OTEL_METRIC_EXPORT_INTERVAL` | `60000` | Metric polling interval in **milliseconds**. For self-hosted LLM hosts, `15000` is recommended || `OTEL_GPU_EBPF_ENABLED` | `true` on Linux; `false` elsewhere | eBPF CUDA activity tracing + stream-sync occupancy (Linux/NVIDIA only). Discovers `libcudart` via FS + `/proc` maps (fleet-friendly with host PID). Soft-fails without caps; set `false` to disable || `OPENLIT_HOST_METRICS` | `true` | Collect system + collector-process host metrics. Set `false` for a GPU-only light footprint |+| `OTEL_GPU_FS_TYPES_EXCLUDE` | `squashfs,erofs,iso9660,cramfs,romfs,cd9660,CDFS,UDF` | Filesystem types excluded from `system.filesystem.*` metrics (case-sensitive). Default skips image-based and optical filesystems that are 100% full by construction (e.g. snap mounts). Set to an empty string to report all types || `OTEL_GPU_PROCESS_CMDLINE` | `true` | Export truncated `process.command_line` on GPU process metrics || `OTEL_GPU_PROCESS_CMDLINE_MAX_LEN` | `512` | Max characters for `process.command_line` || `OTEL_GPU_ALLOCATED_UTIL_THRESHOLD` | `0.05` | Util threshold (0–1) used with process memory for `hw.gpu.allocated` |diff --git a/opentelemetry-gpu-collector/README.md b/opentelemetry-gpu-collector/README.mdindex 3c567501..f549dbc5 100644--- a/opentelemetry-gpu-collector/README.md+++ b/opentelemetry-gpu-collector/README.md@@ -155,6 +155,7 @@ All configuration uses standard OpenTelemetry environment variables.| `OTEL_RESOURCE_ATTRIBUTES` | `deployment.environment=default` | Prefer `host.name`, `k8s.*`, `cloud.provider`, `host.type`, `cloud.region` here; overrides auto identity || `OTEL_METRIC_EXPORT_INTERVAL` | `60000` | Metric polling interval in **milliseconds** || `OTEL_GPU_EBPF_ENABLED` | `true` on Linux; `false` elsewhere | eBPF CUDA kernel tracing (Linux only). Soft-fails without caps/CUDA; set `false` to disable |+| `OTEL_GPU_FS_TYPES_EXCLUDE` | `squashfs,erofs,iso9660,cramfs,romfs,cd9660,CDFS,UDF` | Filesystem types excluded from `system.filesystem.*` metrics (case-sensitive). The default skips image-based and optical filesystems that are 100% full by construction (e.g. snap mounts) on Linux, macOS, and Windows. Set to an empty string to report all types || `K8S_NODE_NAME` | | Downward API node name → also accepts Operator `OTEL_RESOURCE_ATTRIBUTES_NODE_NAME` or legacy `NODE_NAME` || `K8S_CLUSTER_NAME` | | Explicit cluster name (K8s only; on-prem when cloud detect fails) || `OPENLIT_K8S_NODE_LOOKUP` | `true` | Set `false` to skip Node API lookup for instance type / provider |diff --git a/opentelemetry-gpu-collector/cmd/collector/main.go b/opentelemetry-gpu-collector/cmd/collector/main.goindex a195b386..e2909535 100644--- a/opentelemetry-gpu-collector/cmd/collector/main.go+++ b/opentelemetry-gpu-collector/cmd/collector/main.go@@ -58,7 +58,7 @@ func run(logger *slog.Logger) error {var procColl *hostmetrics.ProcessCollectorif cfg.HostMetricsEnabled {var err error- sysColl, err = hostmetrics.NewSystemCollector(provider, logger)+ sysColl, err = hostmetrics.NewSystemCollector(provider, logger, cfg.FSTypesExclude)if err != nil {logger.Warn("system metrics unavailable", "error", err)}diff --git a/opentelemetry-gpu-collector/internal/config/config.go b/opentelemetry-gpu-collector/internal/config/config.goindex 78abf130..6f40f25c 100644--- a/opentelemetry-gpu-collector/internal/config/config.go+++ b/opentelemetry-gpu-collector/internal/config/config.go@@ -14,6 +14,7 @@ type Config struct {CollectionInterval time.DurationEBPFEnabled boolHostMetricsEnabled bool+ FSTypesExclude []stringProcessCmdline boolProcessCmdlineMaxLen int@@ -42,6 +43,16 @@ func Load() *Config {CollectionInterval: parseIntervalMS(os.Getenv("OTEL_METRIC_EXPORT_INTERVAL"), 60*time.Second),EBPFEnabled: parseBool(envOrDefault("OTEL_GPU_EBPF_ENABLED", strconv.FormatBool(defaultEBPF))),HostMetricsEnabled: parseBool(envOrDefault("OPENLIT_HOST_METRICS", "true")),+ // Filesystem types excluded from system.filesystem.* (comma-separated).+ // The defaults are image-based filesystems packed at creation: they+ // report zero free blocks by definition, so their utilization is 100%+ // forever and carries no signal. Snap squashfs mounts additionally get+ // a fresh mountpoint (= fresh time series) on every refresh. cd9660 is+ // the macOS/BSD ISO 9660 driver; CDFS and UDF are the Windows optical+ // media drivers (Windows reports filesystem types in uppercase). Set+ // the variable to an empty string to report every mounted filesystem;+ // the default applies only when it is unset.+ FSTypesExclude: parseList(lookupEnvOrDefault("OTEL_GPU_FS_TYPES_EXCLUDE", "squashfs,erofs,iso9660,cramfs,romfs,cd9660,CDFS,UDF")),ProcessCmdline: parseBool(envOrDefault("OTEL_GPU_PROCESS_CMDLINE", "true")),ProcessCmdlineMaxLen: parseInt(envOrDefault("OTEL_GPU_PROCESS_CMDLINE_MAX_LEN", "512"), 512),@@ -64,6 +75,15 @@ func envOrDefault(key, fallback string) string {return fallback}+// lookupEnvOrDefault returns the value of key if it is set — even if set to+// the empty string — and fallback only when the variable is unset.+func lookupEnvOrDefault(key, fallback string) string {+ if v, ok := os.LookupEnv(key); ok {+ return v+ }+ return fallback+}+// parseIntervalMS parses OTEL_METRIC_EXPORT_INTERVAL which is specified in milliseconds.func parseIntervalMS(s string, fallback time.Duration) time.Duration {if s == "" {@@ -76,6 +96,17 @@ func parseIntervalMS(s string, fallback time.Duration) time.Duration {return time.Duration(ms) * time.Millisecond}+// parseList parses a comma-separated string into its non-empty trimmed items.+func parseList(s string) []string {+ var out []string+ for _, item := range strings.Split(s, ",") {+ if item = strings.TrimSpace(item); item != "" {+ out = append(out, item)+ }+ }+ return out+}+func parseBool(s string) bool {b, _ := strconv.ParseBool(s)return bdiff --git a/opentelemetry-gpu-collector/internal/config/config_test.go b/opentelemetry-gpu-collector/internal/config/config_test.goindex 26611627..3678768a 100644--- a/opentelemetry-gpu-collector/internal/config/config_test.go+++ b/opentelemetry-gpu-collector/internal/config/config_test.go@@ -1,7 +1,9 @@package configimport (+ "os""runtime"+ "slices""testing""time")@@ -28,7 +30,6 @@ func TestParseIntervalMS(t *testing.T) {}}-func TestParseResourceAttr(t *testing.T) {tests := []struct {raw string@@ -52,12 +53,34 @@ func TestParseResourceAttr(t *testing.T) {}}+func TestParseList(t *testing.T) {+ tests := []struct {+ input string+ want []string+ }{+ {"ext4,xfs", []string{"ext4", "xfs"}},+ {" ext4 , xfs ,", []string{"ext4", "xfs"}},+ {"squashfs", []string{"squashfs"}},+ {"", nil},+ }++ for _, tt := range tests {+ if got := parseList(tt.input); !slices.Equal(got, tt.want) {+ t.Errorf("parseList(%q) = %v, want %v", tt.input, got, tt.want)+ }+ }+}+func TestLoadDefaults(t *testing.T) {// Ensure no env vars interfere.t.Setenv("OTEL_METRIC_EXPORT_INTERVAL", "")t.Setenv("OTEL_GPU_EBPF_ENABLED", "")t.Setenv("OTEL_SERVICE_NAME", "")t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "")+ // The exclude default applies only when the variable is UNSET; t.Setenv+ // registers the restore, os.Unsetenv makes it unset for this test.+ t.Setenv("OTEL_GPU_FS_TYPES_EXCLUDE", "")+ os.Unsetenv("OTEL_GPU_FS_TYPES_EXCLUDE")cfg := Load()@@ -74,6 +97,20 @@ func TestLoadDefaults(t *testing.T) {if cfg.Environment != "default" {t.Errorf("Environment = %q, want %q", cfg.Environment, "default")}+ if want := []string{"squashfs", "erofs", "iso9660", "cramfs", "romfs", "cd9660", "CDFS", "UDF"}; !slices.Equal(cfg.FSTypesExclude, want) {+ t.Errorf("FSTypesExclude = %v, want %v", cfg.FSTypesExclude, want)+ }+}++func TestLoadFSTypesExcludeSetButEmpty(t *testing.T) {+ // Explicitly set to empty means "exclude nothing", not "use the default".+ t.Setenv("OTEL_GPU_FS_TYPES_EXCLUDE", "")++ cfg := Load()++ if len(cfg.FSTypesExclude) != 0 {+ t.Errorf("FSTypesExclude = %v, want empty", cfg.FSTypesExclude)+ }}func TestLoadFromEnv(t *testing.T) {@@ -81,6 +118,7 @@ func TestLoadFromEnv(t *testing.T) {t.Setenv("OTEL_GPU_EBPF_ENABLED", "true")t.Setenv("OTEL_SERVICE_NAME", "gpu-collector")t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=production,team=ml")+ t.Setenv("OTEL_GPU_FS_TYPES_EXCLUDE", "squashfs,erofs")cfg := Load()@@ -96,6 +134,9 @@ func TestLoadFromEnv(t *testing.T) {if cfg.Environment != "production" {t.Errorf("Environment = %q, want %q", cfg.Environment, "production")}+ if want := []string{"squashfs", "erofs"}; !slices.Equal(cfg.FSTypesExclude, want) {+ t.Errorf("FSTypesExclude = %v, want %v", cfg.FSTypesExclude, want)+ }}func TestLoadEBPFDisabled(t *testing.T) {diff --git a/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go b/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.goindex 78e8cc36..3ec04ccf 100644--- a/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go+++ b/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go@@ -2,8 +2,10 @@ package hostmetricsimport ("log/slog"+ "slices""testing"+ "go.opentelemetry.io/otel/attribute""go.opentelemetry.io/otel/sdk/metric""go.opentelemetry.io/otel/sdk/metric/metricdata")@@ -13,7 +15,7 @@ func TestNewSystemCollectorInitializes(t *testing.T) {provider := metric.NewMeterProvider(metric.WithReader(reader))defer provider.Shutdown(t.Context())- sc, err := NewSystemCollector(provider, slog.Default())+ sc, err := NewSystemCollector(provider, slog.Default(), nil)if err != nil {t.Fatalf("NewSystemCollector() error = %v", err)}@@ -41,7 +43,7 @@ func TestSystemCollectorInstrumentNames(t *testing.T) {provider := metric.NewMeterProvider(metric.WithReader(reader))defer provider.Shutdown(t.Context())- sc, err := NewSystemCollector(provider, slog.Default())+ sc, err := NewSystemCollector(provider, slog.Default(), nil)if err != nil {t.Fatalf("NewSystemCollector() error = %v", err)}@@ -80,11 +82,87 @@ func TestSystemCollectorInstrumentNames(t *testing.T) {}}+func TestSystemCollectorFSTypeExclude(t *testing.T) {+ // reportedFSTypes collects the system.filesystem.type attribute values+ // present in the system.filesystem.* metrics of a collection.+ reportedFSTypes := func(rm *metricdata.ResourceMetrics) []string {+ seen := map[string]bool{}+ for _, sm := range rm.ScopeMetrics {+ for _, m := range sm.Metrics {+ switch m.Name {+ case "system.filesystem.usage", "system.filesystem.utilization":+ default:+ continue+ }+ var attrSets []attribute.Set+ switch data := m.Data.(type) {+ case metricdata.Sum[int64]:+ for _, dp := range data.DataPoints {+ attrSets = append(attrSets, dp.Attributes)+ }+ case metricdata.Gauge[float64]:+ for _, dp := range data.DataPoints {+ attrSets = append(attrSets, dp.Attributes)+ }+ }+ for _, attrs := range attrSets {+ if v, ok := attrs.Value(attribute.Key("system.filesystem.type")); ok {+ seen[v.AsString()] = true+ }+ }+ }+ }+ types := make([]string, 0, len(seen))+ for fsType := range seen {+ types = append(types, fsType)+ }+ return types+ }++ collect := func(t *testing.T, exclude []string) []string {+ t.Helper()+ reader := metric.NewManualReader()+ provider := metric.NewMeterProvider(metric.WithReader(reader))+ defer provider.Shutdown(t.Context())++ sc, err := NewSystemCollector(provider, slog.Default(), exclude)+ if err != nil {+ t.Fatalf("NewSystemCollector() error = %v", err)+ }+ defer sc.Close()++ var rm metricdata.ResourceMetrics+ if err := reader.Collect(t.Context(), &rm); err != nil {+ t.Fatalf("reader.Collect() error = %v", err)+ }+ return reportedFSTypes(&rm)+ }++ // With no exclusions, note which filesystem types this host reports.+ all := collect(t, nil)+ if len(all) == 0 {+ t.Skip("host reports no system.filesystem.* metrics")+ }+ slices.Sort(all)++ // Excluding every reported type must suppress all system.filesystem.* metrics.+ if got := collect(t, all); len(got) != 0 {+ t.Errorf("expected no filesystem types with all types excluded, got %v", got)+ }++ // Excluding an unknown type must not affect the reported set.+ got := collect(t, []string{"no-such-fs"})+ slices.Sort(got)+ if !slices.Equal(got, all) {+ t.Errorf("reported types changed by irrelevant exclusion: got %v, want %v", got, all)+ }+}+func TestSystemCollectorCloseIdempotent(t *testing.T) {provider := metric.NewMeterProvider()defer provider.Shutdown(t.Context())- sc, err := NewSystemCollector(provider, slog.Default())+ sc, err := NewSystemCollector(provider, slog.Default(), nil)if err != nil {t.Fatalf("NewSystemCollector() error = %v", err)}diff --git a/opentelemetry-gpu-collector/internal/hostmetrics/snapshot_test.go b/opentelemetry-gpu-collector/internal/hostmetrics/snapshot_test.gonew file mode 100644index 00000000..f26e837b--- /dev/null+++ b/opentelemetry-gpu-collector/internal/hostmetrics/snapshot_test.go@@ -0,0 +1,88 @@+package hostmetrics++import (+ "flag"+ "fmt"+ "log/slog"+ "strings"+ "testing"++ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/sdk/metric"+ "go.opentelemetry.io/otel/sdk/metric/metricdata"++ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/config"+)++var snapshotConfig = flag.Bool("snapshot.config", false,+ "TestSystemCollectorSnapshot: build the collector from config.Load() (honoring env vars such as OTEL_GPU_FS_TYPES_EXCLUDE) instead of unfiltered")++// TestSystemCollectorSnapshot logs a single snapshot of every metric the+// system collector reports on this host, one line per datapoint. Run with+// -v to see the output:+//+// go test ./internal/hostmetrics/ -run TestSystemCollectorSnapshot -v+//+// By default the collector is unfiltered (ground truth of what the host+// exposes). Pass -snapshot.config after -args to build it from the+// environment the way cmd/collector does, filesystem filter included:+//+// OTEL_GPU_FS_TYPES_EXCLUDE=apfs go test ./internal/hostmetrics/ -run TestSystemCollectorSnapshot -v -args -snapshot.config+func TestSystemCollectorSnapshot(t *testing.T) {+ var fsTypesExclude []string+ if *snapshotConfig {+ fsTypesExclude = config.Load().FSTypesExclude+ t.Logf("config loaded: filesystem exclude list %v", fsTypesExclude)+ }++ reader := metric.NewManualReader()+ provider := metric.NewMeterProvider(metric.WithReader(reader))+ defer provider.Shutdown(t.Context())++ sc, err := NewSystemCollector(provider, slog.Default(), fsTypesExclude)+ if err != nil {+ t.Fatalf("NewSystemCollector() error = %v", err)+ }+ defer sc.Close()++ var rm metricdata.ResourceMetrics+ if err := reader.Collect(t.Context(), &rm); err != nil {+ t.Fatalf("reader.Collect() error = %v", err)+ }++ attrString := func(set attribute.Set) string {+ items := make([]string, 0, set.Len())+ for iter := set.Iter(); iter.Next(); {+ kv := iter.Attribute()+ items = append(items, fmt.Sprintf("%s=%s", kv.Key, kv.Value.Emit()))+ }+ return strings.Join(items, " ")+ }++ for _, sm := range rm.ScopeMetrics {+ t.Logf("scope %s@%s", sm.Scope.Name, sm.Scope.Version)+ for _, m := range sm.Metrics {+ t.Logf(" %s (%s): %s", m.Name, m.Unit, m.Description)+ switch data := m.Data.(type) {+ case metricdata.Sum[int64]:+ for _, dp := range data.DataPoints {+ t.Logf(" {%s} %d", attrString(dp.Attributes), dp.Value)+ }+ case metricdata.Sum[float64]:+ for _, dp := range data.DataPoints {+ t.Logf(" {%s} %g", attrString(dp.Attributes), dp.Value)+ }+ case metricdata.Gauge[int64]:+ for _, dp := range data.DataPoints {+ t.Logf(" {%s} %d", attrString(dp.Attributes), dp.Value)+ }+ case metricdata.Gauge[float64]:+ for _, dp := range data.DataPoints {+ t.Logf(" {%s} %g", attrString(dp.Attributes), dp.Value)+ }+ default:+ t.Logf(" (unhandled data type %T)", m.Data)+ }+ }+ }+}diff --git a/opentelemetry-gpu-collector/internal/hostmetrics/system.go b/opentelemetry-gpu-collector/internal/hostmetrics/system.goindex a80a5cb3..bd63c35f 100644--- a/opentelemetry-gpu-collector/internal/hostmetrics/system.go+++ b/opentelemetry-gpu-collector/internal/hostmetrics/system.go@@ -21,13 +21,22 @@ import (// following the OpenTelemetry semantic conventions for system metrics.// https://opentelemetry.io/docs/specs/semconv/system/system-metrics/type SystemCollector struct {- logger *slog.Logger- reg []metric.Registration+ logger *slog.Logger+ fsExclude map[string]bool // filesystem types excluded from system.filesystem.*+ reg []metric.Registration}// NewSystemCollector creates system-level metric instruments and registers callbacks.-func NewSystemCollector(provider *sdkmetric.MeterProvider, logger *slog.Logger) (*SystemCollector, error) {+// fsTypesExclude lists filesystem types excluded from system.filesystem.*;+// an empty list reports every mounted filesystem.+func NewSystemCollector(provider *sdkmetric.MeterProvider, logger *slog.Logger, fsTypesExclude []string) (*SystemCollector, error) {sc := &SystemCollector{logger: logger}+ if len(fsTypesExclude) > 0 {+ sc.fsExclude = make(map[string]bool, len(fsTypesExclude))+ for _, t := range fsTypesExclude {+ sc.fsExclude[t] = true+ }+ }meter := provider.Meter("otelcol.system",metric.WithInstrumentationVersion("1.0.0"),@@ -249,6 +258,10 @@ func (sc *SystemCollector) collectFilesystem(_ context.Context, o metric.Observe}for _, p := range partitions {+ if sc.fsExclude[p.Fstype] {+ continue+ }+stat, err := disk.Usage(p.Mountpoint)if err != nil {continue