Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat: Filesystem type filters for the otel collector (#1395)

Chris Birkholddb5dcc3eb1f7
diff --git a/docs/latest/gpu-collector/configuration.mdx b/docs/latest/gpu-collector/configuration.mdx
index 4a41c1c1..a56d6ed9 100644
--- a/docs/latest/gpu-collector/configuration.mdx
+++ b/docs/latest/gpu-collector/configuration.mdx
@@ -18,6 +18,7 @@ The collector is configured entirely via environment variables. All variables fo
| `OTEL_METRIC_EXPORT_INTERVAL` | `60000` | Metric polling interval in **milliseconds**. For self-hosted LLM hosts, `15000` is recommended |
| `OTEL_GPU_EBPF_ENABLED` | `true` on Linux; `false` elsewhere | eBPF CUDA activity tracing + stream-sync occupancy (Linux/NVIDIA only). Discovers `libcudart` via FS + `/proc` maps (fleet-friendly with host PID). Soft-fails without caps; set `false` to disable |
| `OPENLIT_HOST_METRICS` | `true` | Collect system + collector-process host metrics. Set `false` for a GPU-only light footprint |
+| `OTEL_GPU_FS_TYPES_EXCLUDE` | `squashfs,erofs,iso9660,cramfs,romfs,cd9660,CDFS,UDF` | Filesystem types excluded from `system.filesystem.*` metrics (case-sensitive). Default skips image-based and optical filesystems that are 100% full by construction (e.g. snap mounts). Set to an empty string to report all types |
| `OTEL_GPU_PROCESS_CMDLINE` | `true` | Export truncated `process.command_line` on GPU process metrics |
| `OTEL_GPU_PROCESS_CMDLINE_MAX_LEN` | `512` | Max characters for `process.command_line` |
| `OTEL_GPU_ALLOCATED_UTIL_THRESHOLD` | `0.05` | Util threshold (0–1) used with process memory for `hw.gpu.allocated` |
diff --git a/opentelemetry-gpu-collector/README.md b/opentelemetry-gpu-collector/README.md
index 3c567501..f549dbc5 100644
--- a/opentelemetry-gpu-collector/README.md
+++ b/opentelemetry-gpu-collector/README.md
@@ -155,6 +155,7 @@ All configuration uses standard OpenTelemetry environment variables.
| `OTEL_RESOURCE_ATTRIBUTES` | `deployment.environment=default` | Prefer `host.name`, `k8s.*`, `cloud.provider`, `host.type`, `cloud.region` here; overrides auto identity |
| `OTEL_METRIC_EXPORT_INTERVAL` | `60000` | Metric polling interval in **milliseconds** |
| `OTEL_GPU_EBPF_ENABLED` | `true` on Linux; `false` elsewhere | eBPF CUDA kernel tracing (Linux only). Soft-fails without caps/CUDA; set `false` to disable |
+| `OTEL_GPU_FS_TYPES_EXCLUDE` | `squashfs,erofs,iso9660,cramfs,romfs,cd9660,CDFS,UDF` | Filesystem types excluded from `system.filesystem.*` metrics (case-sensitive). The default skips image-based and optical filesystems that are 100% full by construction (e.g. snap mounts) on Linux, macOS, and Windows. Set to an empty string to report all types |
| `K8S_NODE_NAME` | | Downward API node name → also accepts Operator `OTEL_RESOURCE_ATTRIBUTES_NODE_NAME` or legacy `NODE_NAME` |
| `K8S_CLUSTER_NAME` | | Explicit cluster name (K8s only; on-prem when cloud detect fails) |
| `OPENLIT_K8S_NODE_LOOKUP` | `true` | Set `false` to skip Node API lookup for instance type / provider |
diff --git a/opentelemetry-gpu-collector/cmd/collector/main.go b/opentelemetry-gpu-collector/cmd/collector/main.go
index a195b386..e2909535 100644
--- a/opentelemetry-gpu-collector/cmd/collector/main.go
+++ b/opentelemetry-gpu-collector/cmd/collector/main.go
@@ -58,7 +58,7 @@ func run(logger *slog.Logger) error {
var procColl *hostmetrics.ProcessCollector
if cfg.HostMetricsEnabled {
var err error
- sysColl, err = hostmetrics.NewSystemCollector(provider, logger)
+ sysColl, err = hostmetrics.NewSystemCollector(provider, logger, cfg.FSTypesExclude)
if err != nil {
logger.Warn("system metrics unavailable", "error", err)
}
diff --git a/opentelemetry-gpu-collector/internal/config/config.go b/opentelemetry-gpu-collector/internal/config/config.go
index 78abf130..6f40f25c 100644
--- a/opentelemetry-gpu-collector/internal/config/config.go
+++ b/opentelemetry-gpu-collector/internal/config/config.go
@@ -14,6 +14,7 @@ type Config struct {
CollectionInterval time.Duration
EBPFEnabled bool
HostMetricsEnabled bool
+ FSTypesExclude []string
ProcessCmdline bool
ProcessCmdlineMaxLen int
@@ -42,6 +43,16 @@ func Load() *Config {
CollectionInterval: parseIntervalMS(os.Getenv("OTEL_METRIC_EXPORT_INTERVAL"), 60*time.Second),
EBPFEnabled: parseBool(envOrDefault("OTEL_GPU_EBPF_ENABLED", strconv.FormatBool(defaultEBPF))),
HostMetricsEnabled: parseBool(envOrDefault("OPENLIT_HOST_METRICS", "true")),
+ // Filesystem types excluded from system.filesystem.* (comma-separated).
+ // The defaults are image-based filesystems packed at creation: they
+ // report zero free blocks by definition, so their utilization is 100%
+ // forever and carries no signal. Snap squashfs mounts additionally get
+ // a fresh mountpoint (= fresh time series) on every refresh. cd9660 is
+ // the macOS/BSD ISO 9660 driver; CDFS and UDF are the Windows optical
+ // media drivers (Windows reports filesystem types in uppercase). Set
+ // the variable to an empty string to report every mounted filesystem;
+ // the default applies only when it is unset.
+ FSTypesExclude: parseList(lookupEnvOrDefault("OTEL_GPU_FS_TYPES_EXCLUDE", "squashfs,erofs,iso9660,cramfs,romfs,cd9660,CDFS,UDF")),
ProcessCmdline: parseBool(envOrDefault("OTEL_GPU_PROCESS_CMDLINE", "true")),
ProcessCmdlineMaxLen: parseInt(envOrDefault("OTEL_GPU_PROCESS_CMDLINE_MAX_LEN", "512"), 512),
@@ -64,6 +75,15 @@ func envOrDefault(key, fallback string) string {
return fallback
}
+// lookupEnvOrDefault returns the value of key if it is set — even if set to
+// the empty string — and fallback only when the variable is unset.
+func lookupEnvOrDefault(key, fallback string) string {
+ if v, ok := os.LookupEnv(key); ok {
+ return v
+ }
+ return fallback
+}
+
// parseIntervalMS parses OTEL_METRIC_EXPORT_INTERVAL which is specified in milliseconds.
func parseIntervalMS(s string, fallback time.Duration) time.Duration {
if s == "" {
@@ -76,6 +96,17 @@ func parseIntervalMS(s string, fallback time.Duration) time.Duration {
return time.Duration(ms) * time.Millisecond
}
+// parseList parses a comma-separated string into its non-empty trimmed items.
+func parseList(s string) []string {
+ var out []string
+ for _, item := range strings.Split(s, ",") {
+ if item = strings.TrimSpace(item); item != "" {
+ out = append(out, item)
+ }
+ }
+ return out
+}
+
func parseBool(s string) bool {
b, _ := strconv.ParseBool(s)
return b
diff --git a/opentelemetry-gpu-collector/internal/config/config_test.go b/opentelemetry-gpu-collector/internal/config/config_test.go
index 26611627..3678768a 100644
--- a/opentelemetry-gpu-collector/internal/config/config_test.go
+++ b/opentelemetry-gpu-collector/internal/config/config_test.go
@@ -1,7 +1,9 @@
package config
import (
+ "os"
"runtime"
+ "slices"
"testing"
"time"
)
@@ -28,7 +30,6 @@ func TestParseIntervalMS(t *testing.T) {
}
}
-
func TestParseResourceAttr(t *testing.T) {
tests := []struct {
raw string
@@ -52,12 +53,34 @@ func TestParseResourceAttr(t *testing.T) {
}
}
+func TestParseList(t *testing.T) {
+ tests := []struct {
+ input string
+ want []string
+ }{
+ {"ext4,xfs", []string{"ext4", "xfs"}},
+ {" ext4 , xfs ,", []string{"ext4", "xfs"}},
+ {"squashfs", []string{"squashfs"}},
+ {"", nil},
+ }
+
+ for _, tt := range tests {
+ if got := parseList(tt.input); !slices.Equal(got, tt.want) {
+ t.Errorf("parseList(%q) = %v, want %v", tt.input, got, tt.want)
+ }
+ }
+}
+
func TestLoadDefaults(t *testing.T) {
// Ensure no env vars interfere.
t.Setenv("OTEL_METRIC_EXPORT_INTERVAL", "")
t.Setenv("OTEL_GPU_EBPF_ENABLED", "")
t.Setenv("OTEL_SERVICE_NAME", "")
t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "")
+ // The exclude default applies only when the variable is UNSET; t.Setenv
+ // registers the restore, os.Unsetenv makes it unset for this test.
+ t.Setenv("OTEL_GPU_FS_TYPES_EXCLUDE", "")
+ os.Unsetenv("OTEL_GPU_FS_TYPES_EXCLUDE")
cfg := Load()
@@ -74,6 +97,20 @@ func TestLoadDefaults(t *testing.T) {
if cfg.Environment != "default" {
t.Errorf("Environment = %q, want %q", cfg.Environment, "default")
}
+ if want := []string{"squashfs", "erofs", "iso9660", "cramfs", "romfs", "cd9660", "CDFS", "UDF"}; !slices.Equal(cfg.FSTypesExclude, want) {
+ t.Errorf("FSTypesExclude = %v, want %v", cfg.FSTypesExclude, want)
+ }
+}
+
+func TestLoadFSTypesExcludeSetButEmpty(t *testing.T) {
+ // Explicitly set to empty means "exclude nothing", not "use the default".
+ t.Setenv("OTEL_GPU_FS_TYPES_EXCLUDE", "")
+
+ cfg := Load()
+
+ if len(cfg.FSTypesExclude) != 0 {
+ t.Errorf("FSTypesExclude = %v, want empty", cfg.FSTypesExclude)
+ }
}
func TestLoadFromEnv(t *testing.T) {
@@ -81,6 +118,7 @@ func TestLoadFromEnv(t *testing.T) {
t.Setenv("OTEL_GPU_EBPF_ENABLED", "true")
t.Setenv("OTEL_SERVICE_NAME", "gpu-collector")
t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=production,team=ml")
+ t.Setenv("OTEL_GPU_FS_TYPES_EXCLUDE", "squashfs,erofs")
cfg := Load()
@@ -96,6 +134,9 @@ func TestLoadFromEnv(t *testing.T) {
if cfg.Environment != "production" {
t.Errorf("Environment = %q, want %q", cfg.Environment, "production")
}
+ if want := []string{"squashfs", "erofs"}; !slices.Equal(cfg.FSTypesExclude, want) {
+ t.Errorf("FSTypesExclude = %v, want %v", cfg.FSTypesExclude, want)
+ }
}
func TestLoadEBPFDisabled(t *testing.T) {
diff --git a/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go b/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go
index 78e8cc36..3ec04ccf 100644
--- a/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go
+++ b/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go
@@ -2,8 +2,10 @@ package hostmetrics
import (
"log/slog"
+ "slices"
"testing"
+ "go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
)
@@ -13,7 +15,7 @@ func TestNewSystemCollectorInitializes(t *testing.T) {
provider := metric.NewMeterProvider(metric.WithReader(reader))
defer provider.Shutdown(t.Context())
- sc, err := NewSystemCollector(provider, slog.Default())
+ sc, err := NewSystemCollector(provider, slog.Default(), nil)
if err != nil {
t.Fatalf("NewSystemCollector() error = %v", err)
}
@@ -41,7 +43,7 @@ func TestSystemCollectorInstrumentNames(t *testing.T) {
provider := metric.NewMeterProvider(metric.WithReader(reader))
defer provider.Shutdown(t.Context())
- sc, err := NewSystemCollector(provider, slog.Default())
+ sc, err := NewSystemCollector(provider, slog.Default(), nil)
if err != nil {
t.Fatalf("NewSystemCollector() error = %v", err)
}
@@ -80,11 +82,87 @@ func TestSystemCollectorInstrumentNames(t *testing.T) {
}
}
+func TestSystemCollectorFSTypeExclude(t *testing.T) {
+ // reportedFSTypes collects the system.filesystem.type attribute values
+ // present in the system.filesystem.* metrics of a collection.
+ reportedFSTypes := func(rm *metricdata.ResourceMetrics) []string {
+ seen := map[string]bool{}
+ for _, sm := range rm.ScopeMetrics {
+ for _, m := range sm.Metrics {
+ switch m.Name {
+ case "system.filesystem.usage", "system.filesystem.utilization":
+ default:
+ continue
+ }
+ var attrSets []attribute.Set
+ switch data := m.Data.(type) {
+ case metricdata.Sum[int64]:
+ for _, dp := range data.DataPoints {
+ attrSets = append(attrSets, dp.Attributes)
+ }
+ case metricdata.Gauge[float64]:
+ for _, dp := range data.DataPoints {
+ attrSets = append(attrSets, dp.Attributes)
+ }
+ }
+ for _, attrs := range attrSets {
+ if v, ok := attrs.Value(attribute.Key("system.filesystem.type")); ok {
+ seen[v.AsString()] = true
+ }
+ }
+ }
+ }
+ types := make([]string, 0, len(seen))
+ for fsType := range seen {
+ types = append(types, fsType)
+ }
+ return types
+ }
+
+ collect := func(t *testing.T, exclude []string) []string {
+ t.Helper()
+ reader := metric.NewManualReader()
+ provider := metric.NewMeterProvider(metric.WithReader(reader))
+ defer provider.Shutdown(t.Context())
+
+ sc, err := NewSystemCollector(provider, slog.Default(), exclude)
+ if err != nil {
+ t.Fatalf("NewSystemCollector() error = %v", err)
+ }
+ defer sc.Close()
+
+ var rm metricdata.ResourceMetrics
+ if err := reader.Collect(t.Context(), &rm); err != nil {
+ t.Fatalf("reader.Collect() error = %v", err)
+ }
+ return reportedFSTypes(&rm)
+ }
+
+ // With no exclusions, note which filesystem types this host reports.
+ all := collect(t, nil)
+ if len(all) == 0 {
+ t.Skip("host reports no system.filesystem.* metrics")
+ }
+ slices.Sort(all)
+
+ // Excluding every reported type must suppress all system.filesystem.* metrics.
+ if got := collect(t, all); len(got) != 0 {
+ t.Errorf("expected no filesystem types with all types excluded, got %v", got)
+ }
+
+ // Excluding an unknown type must not affect the reported set.
+ got := collect(t, []string{"no-such-fs"})
+ slices.Sort(got)
+ if !slices.Equal(got, all) {
+ t.Errorf("reported types changed by irrelevant exclusion: got %v, want %v", got, all)
+ }
+}
+
func TestSystemCollectorCloseIdempotent(t *testing.T) {
provider := metric.NewMeterProvider()
defer provider.Shutdown(t.Context())
- sc, err := NewSystemCollector(provider, slog.Default())
+ sc, err := NewSystemCollector(provider, slog.Default(), nil)
if err != nil {
t.Fatalf("NewSystemCollector() error = %v", err)
}
diff --git a/opentelemetry-gpu-collector/internal/hostmetrics/snapshot_test.go b/opentelemetry-gpu-collector/internal/hostmetrics/snapshot_test.go
new file mode 100644
index 00000000..f26e837b
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/hostmetrics/snapshot_test.go
@@ -0,0 +1,88 @@
+package hostmetrics
+
+import (
+ "flag"
+ "fmt"
+ "log/slog"
+ "strings"
+ "testing"
+
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/sdk/metric"
+ "go.opentelemetry.io/otel/sdk/metric/metricdata"
+
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/config"
+)
+
+var snapshotConfig = flag.Bool("snapshot.config", false,
+ "TestSystemCollectorSnapshot: build the collector from config.Load() (honoring env vars such as OTEL_GPU_FS_TYPES_EXCLUDE) instead of unfiltered")
+
+// TestSystemCollectorSnapshot logs a single snapshot of every metric the
+// system collector reports on this host, one line per datapoint. Run with
+// -v to see the output:
+//
+// go test ./internal/hostmetrics/ -run TestSystemCollectorSnapshot -v
+//
+// By default the collector is unfiltered (ground truth of what the host
+// exposes). Pass -snapshot.config after -args to build it from the
+// environment the way cmd/collector does, filesystem filter included:
+//
+// OTEL_GPU_FS_TYPES_EXCLUDE=apfs go test ./internal/hostmetrics/ -run TestSystemCollectorSnapshot -v -args -snapshot.config
+func TestSystemCollectorSnapshot(t *testing.T) {
+ var fsTypesExclude []string
+ if *snapshotConfig {
+ fsTypesExclude = config.Load().FSTypesExclude
+ t.Logf("config loaded: filesystem exclude list %v", fsTypesExclude)
+ }
+
+ reader := metric.NewManualReader()
+ provider := metric.NewMeterProvider(metric.WithReader(reader))
+ defer provider.Shutdown(t.Context())
+
+ sc, err := NewSystemCollector(provider, slog.Default(), fsTypesExclude)
+ if err != nil {
+ t.Fatalf("NewSystemCollector() error = %v", err)
+ }
+ defer sc.Close()
+
+ var rm metricdata.ResourceMetrics
+ if err := reader.Collect(t.Context(), &rm); err != nil {
+ t.Fatalf("reader.Collect() error = %v", err)
+ }
+
+ attrString := func(set attribute.Set) string {
+ items := make([]string, 0, set.Len())
+ for iter := set.Iter(); iter.Next(); {
+ kv := iter.Attribute()
+ items = append(items, fmt.Sprintf("%s=%s", kv.Key, kv.Value.Emit()))
+ }
+ return strings.Join(items, " ")
+ }
+
+ for _, sm := range rm.ScopeMetrics {
+ t.Logf("scope %s@%s", sm.Scope.Name, sm.Scope.Version)
+ for _, m := range sm.Metrics {
+ t.Logf(" %s (%s): %s", m.Name, m.Unit, m.Description)
+ switch data := m.Data.(type) {
+ case metricdata.Sum[int64]:
+ for _, dp := range data.DataPoints {
+ t.Logf(" {%s} %d", attrString(dp.Attributes), dp.Value)
+ }
+ case metricdata.Sum[float64]:
+ for _, dp := range data.DataPoints {
+ t.Logf(" {%s} %g", attrString(dp.Attributes), dp.Value)
+ }
+ case metricdata.Gauge[int64]:
+ for _, dp := range data.DataPoints {
+ t.Logf(" {%s} %d", attrString(dp.Attributes), dp.Value)
+ }
+ case metricdata.Gauge[float64]:
+ for _, dp := range data.DataPoints {
+ t.Logf(" {%s} %g", attrString(dp.Attributes), dp.Value)
+ }
+ default:
+ t.Logf(" (unhandled data type %T)", m.Data)
+ }
+ }
+ }
+}
diff --git a/opentelemetry-gpu-collector/internal/hostmetrics/system.go b/opentelemetry-gpu-collector/internal/hostmetrics/system.go
index a80a5cb3..bd63c35f 100644
--- a/opentelemetry-gpu-collector/internal/hostmetrics/system.go
+++ b/opentelemetry-gpu-collector/internal/hostmetrics/system.go
@@ -21,13 +21,22 @@ import (
// following the OpenTelemetry semantic conventions for system metrics.
// https://opentelemetry.io/docs/specs/semconv/system/system-metrics/
type SystemCollector struct {
- logger *slog.Logger
- reg []metric.Registration
+ logger *slog.Logger
+ fsExclude map[string]bool // filesystem types excluded from system.filesystem.*
+ reg []metric.Registration
}
// NewSystemCollector creates system-level metric instruments and registers callbacks.
-func NewSystemCollector(provider *sdkmetric.MeterProvider, logger *slog.Logger) (*SystemCollector, error) {
+// fsTypesExclude lists filesystem types excluded from system.filesystem.*;
+// an empty list reports every mounted filesystem.
+func NewSystemCollector(provider *sdkmetric.MeterProvider, logger *slog.Logger, fsTypesExclude []string) (*SystemCollector, error) {
sc := &SystemCollector{logger: logger}
+ if len(fsTypesExclude) > 0 {
+ sc.fsExclude = make(map[string]bool, len(fsTypesExclude))
+ for _, t := range fsTypesExclude {
+ sc.fsExclude[t] = true
+ }
+ }
meter := provider.Meter("otelcol.system",
metric.WithInstrumentationVersion("1.0.0"),
@@ -249,6 +258,10 @@ func (sc *SystemCollector) collectFilesystem(_ context.Context, o metric.Observe
}
for _, p := range partitions {
+ if sc.fsExclude[p.Fstype] {
+ continue
+ }
+
stat, err := disk.Usage(p.Mountpoint)
if err != nil {
continue