Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat: Don't collect write metrics for read-only drives. (#1397)

Chris Birkhold759f6a1a3105
diff --git a/opentelemetry-gpu-collector/README.md b/opentelemetry-gpu-collector/README.md
index c7e9dc46..8d6b8031 100644
--- a/opentelemetry-gpu-collector/README.md
+++ b/opentelemetry-gpu-collector/README.md
@@ -230,6 +230,8 @@ Follows the [OTel semantic conventions for system metrics](https://opentelemetry
| `system.network.io` | Counter | By | Network I/O bytes | `network.interface.name`, `network.io.direction`={receive,transmit} |
| `system.network.errors` | Counter | {error} | Network errors | `network.interface.name`, `network.io.direction`={receive,transmit} |
+On Linux, `disk.io.direction=write` series are omitted for read-only block devices (sysfs `ro` flag, e.g. loop devices backing snap images), since writes to them are impossible and the counters would stay zero. A device that becomes writable starts reporting on the next collection cycle.
+
### Process Metrics (all platforms)
Follows the [OTel semantic conventions for process metrics](https://opentelemetry.io/docs/specs/semconv/system/process-metrics/).
diff --git a/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go b/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go
index 47ab928e..78f7ff2b 100644
--- a/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go
+++ b/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go
@@ -2,6 +2,8 @@ package hostmetrics
import (
"log/slog"
+ "os"
+ "path/filepath"
"slices"
"testing"
@@ -12,6 +14,44 @@ import (
"github.com/openlit/openlit/opentelemetry-gpu-collector/internal/config"
)
+func TestIsReadOnlyDevice(t *testing.T) {
+ sysBlock := t.TempDir()
+ writeRoFlag := func(device, content string) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Join(sysBlock, device), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(sysBlock, device, "ro"), []byte(content), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ writeRoFlag("loop0", "1\n")
+ writeRoFlag("sda1", "0\n")
+ writeRoFlag("dm-0", "garbage")
+
+ tests := []struct {
+ name string
+ sysBlockPath string
+ device string
+ want bool
+ }{
+ {"ro flag set", sysBlock, "loop0", true},
+ {"ro flag unset", sysBlock, "sda1", false},
+ {"unparseable flag treated as writable", sysBlock, "dm-0", false},
+ {"missing sysfs entry treated as writable", sysBlock, "nvme0n1", false},
+ {"path into entry is not a device", sysBlock, "loop0/ro", false},
+ {"empty sysBlockPath (non-Linux) treated as writable", "", "loop0", false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ sc := &SystemCollector{sysBlockPath: tt.sysBlockPath}
+ if got := sc.isReadOnlyDevice(tt.device); got != tt.want {
+ t.Errorf("isReadOnlyDevice(%q) = %v, want %v", tt.device, got, tt.want)
+ }
+ })
+ }
+}
+
func TestNewSystemCollectorInitializes(t *testing.T) {
reader := metric.NewManualReader()
provider := metric.NewMeterProvider(metric.WithReader(reader))
diff --git a/opentelemetry-gpu-collector/internal/hostmetrics/system.go b/opentelemetry-gpu-collector/internal/hostmetrics/system.go
index 040bbacf..514018f2 100644
--- a/opentelemetry-gpu-collector/internal/hostmetrics/system.go
+++ b/opentelemetry-gpu-collector/internal/hostmetrics/system.go
@@ -1,9 +1,12 @@
package hostmetrics
import (
+ "bytes"
"context"
"fmt"
"log/slog"
+ "os"
+ "path/filepath"
"runtime"
"time"
@@ -25,12 +28,13 @@ import (
// following the OpenTelemetry semantic conventions for system metrics.
// https://opentelemetry.io/docs/specs/semconv/system/system-metrics/
type SystemCollector struct {
- logger *slog.Logger
- fsExclude map[string]bool // filesystem types excluded from system.filesystem.*
- netAllow map[string]bool // empty = all non-excluded
- netExclude map[string]bool
- skipNetwork bool // when NIC collector owns per-iface hw.network.*
- reg []metric.Registration
+ logger *slog.Logger
+ sysBlockPath string // sysfs block-device root, overridable in tests; empty on non-Linux
+ fsExclude map[string]bool // filesystem types excluded from system.filesystem.*
+ netAllow map[string]bool // empty = all non-excluded
+ netExclude map[string]bool
+ skipNetwork bool // when NIC collector owns per-iface hw.network.*
+ reg []metric.Registration
}
// NewSystemCollector creates system-level metric instruments and registers callbacks.
@@ -43,6 +47,9 @@ func NewSystemCollector(provider *sdkmetric.MeterProvider, logger *slog.Logger,
}
sc := &SystemCollector{logger: logger, skipNetwork: cfg.NICEnabled}
+ if runtime.GOOS == "linux" {
+ sc.sysBlockPath = "/sys/class/block"
+ }
if len(cfg.FSTypesExclude) > 0 {
sc.fsExclude = make(map[string]bool, len(cfg.FSTypesExclude))
for _, t := range cfg.FSTypesExclude {
@@ -472,18 +479,42 @@ func (sc *SystemCollector) collectDisk(_ context.Context, o metric.Observer,
attribute.String("system.device", device),
attribute.String("disk.io.direction", "read"),
)
+
+ o.ObserveInt64(ioBytes, int64(stat.ReadBytes), readAttrs)
+ o.ObserveInt64(ops, int64(stat.ReadCount), readAttrs)
+
+ // Writes to a read-only block device are impossible, so its write
+ // counters are zero for as long as it stays read-only (e.g. loop
+ // devices backing snap images). Skip the constant-zero series; the
+ // check runs every cycle, so a device turning writable starts
+ // reporting on the next collection.
+ if sc.isReadOnlyDevice(device) {
+ continue
+ }
+
writeAttrs := metric.WithAttributes(
attribute.String("system.device", device),
attribute.String("disk.io.direction", "write"),
)
- o.ObserveInt64(ioBytes, int64(stat.ReadBytes), readAttrs)
o.ObserveInt64(ioBytes, int64(stat.WriteBytes), writeAttrs)
- o.ObserveInt64(ops, int64(stat.ReadCount), readAttrs)
o.ObserveInt64(ops, int64(stat.WriteCount), writeAttrs)
}
}
+// isReadOnlyDevice reports whether a block device is read-only per its sysfs
+// ro flag. Device names from disk.IOCounters match the /sys/class/block
+// entries on Linux (sda1, nvme0n1p1, loop0, dm-0). sysBlockPath is empty on
+// non-Linux platforms (sysfs does not exist there), and devices without a
+// sysfs entry fail the read; both are treated as writable.
+func (sc *SystemCollector) isReadOnlyDevice(device string) bool {
+ if sc.sysBlockPath == "" {
+ return false
+ }
+ ro, err := os.ReadFile(filepath.Join(sc.sysBlockPath, device, "ro"))
+ return err == nil && bytes.Equal(bytes.TrimSpace(ro), []byte("1"))
+}
+
func (sc *SystemCollector) collectFilesystem(_ context.Context, o metric.Observer,
usage metric.Int64ObservableUpDownCounter,
utilization metric.Float64ObservableGauge,