Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

fix(agent): keep NVMe SMART data when a 128-bit counter exceeds uint64 (#2732)

Columbus Labs0f68d161b127
diff --git a/agent/smart_parse.go b/agent/smart_parse.go
index 9fa06b825..4c681d28a 100644
--- a/agent/smart_parse.go
+++ b/agent/smart_parse.go
@@ -40,7 +40,7 @@ func (sm *SmartManager) parseSmartctl(output []byte, dtype string) (*smart.Smart
}
smartData := newSmartData(data, dtype, data.ModelName, data.FirmwareVersion, log.Temperature)
if smartData.Capacity == 0 {
- smartData.Capacity = data.NVMeTotalCapacity
+ smartData.Capacity = data.NVMeTotalCapacity.Value
}
if smartData.Capacity == 0 && (runtime.GOOS == "darwin" || sm.darwinNvmeProvider != nil) {
smartData.Capacity = sm.lookupDarwinNvmeCapacity(data.SerialNumber)
@@ -271,21 +271,30 @@ func nvmeAttributes(log *smart.NVMeSmartHealthInformationLog, temperature uint8)
{Name: "AvailableSpare", RawValue: log.AvailableSpare},
{Name: "AvailableSpareThreshold", RawValue: log.AvailableSpareThreshold},
{Name: "PercentageUsed", RawValue: log.PercentageUsed},
- {Name: "DataUnitsRead", RawValue: log.DataUnitsRead},
- {Name: "DataUnitsWritten", RawValue: log.DataUnitsWritten},
- {Name: "HostReads", RawValue: log.HostReads},
- {Name: "HostWrites", RawValue: log.HostWrites},
- {Name: "ControllerBusyTime", RawValue: log.ControllerBusyTime},
- {Name: "PowerCycles", RawValue: log.PowerCycles},
- {Name: "PowerOnHours", RawValue: log.PowerOnHours},
- {Name: "UnsafeShutdowns", RawValue: log.UnsafeShutdowns},
- {Name: "MediaErrors", RawValue: log.MediaErrors},
- {Name: "NumErrLogEntries", RawValue: log.NumErrLogEntries},
+ nvmeCounterAttribute("DataUnitsRead", log.DataUnitsRead),
+ nvmeCounterAttribute("DataUnitsWritten", log.DataUnitsWritten),
+ nvmeCounterAttribute("HostReads", log.HostReads),
+ nvmeCounterAttribute("HostWrites", log.HostWrites),
+ nvmeCounterAttribute("ControllerBusyTime", log.ControllerBusyTime),
+ nvmeCounterAttribute("PowerCycles", log.PowerCycles),
+ nvmeCounterAttribute("PowerOnHours", log.PowerOnHours),
+ nvmeCounterAttribute("UnsafeShutdowns", log.UnsafeShutdowns),
+ nvmeCounterAttribute("MediaErrors", log.MediaErrors),
+ nvmeCounterAttribute("NumErrLogEntries", log.NumErrLogEntries),
{Name: "WarningTempTime", RawValue: log.WarningTempTime},
{Name: "CriticalCompTime", RawValue: log.CriticalCompTime},
}
}
+// nvmeCounterAttribute reports an overflowed 128-bit counter as invalid instead
+// of a made-up number, so neither the table nor the power-on hours show it.
+func nvmeCounterAttribute(name string, c smart.NVMeCounter) *smart.SmartAttribute {
+ if c.Overflow {
+ return &smart.SmartAttribute{Name: name, RawString: smart.NVMeCounterInvalid}
+ }
+ return &smart.SmartAttribute{Name: name, RawValue: c.Value}
+}
+
// lookupDarwinNvmeCapacity returns the capacity in bytes for a given NVMe serial number on Darwin.
// It uses system_profiler SPNVMeDataType to get capacity since Apple SSDs don't report user_capacity
// via smartctl. Results are cached after the first call via sync.Once.
diff --git a/agent/smart_test.go b/agent/smart_test.go
index 0bc5c213c..181dada4e 100644
--- a/agent/smart_test.go
+++ b/agent/smart_test.go
@@ -268,6 +268,86 @@ func TestParseSmartctlNvmeLargeCounters(t *testing.T) {
assertAttrValue(t, deviceData.Attributes, "PercentageUsed", 300)
}
+// NVMe health counters are 128-bit, so smartctl can print values past uint64.
+// They are reported as invalid instead of failing json.Unmarshal and dropping the device (#2731).
+func TestParseSmartctlNvmeCounterBeyondUint64(t *testing.T) {
+ deviceData := parseSmart(t, []byte(`{
+ "device": {"name": "/dev/nvme0", "type": "nvme"},
+ "serial_number": "NVME-OVERFLOW",
+ "smart_status": {"passed": true},
+ "nvme_smart_health_information_log": {
+ "temperature": 35,
+ "data_units_written": 340282366920938463463374607431768211455,
+ "power_on_hours": 1234,
+ "media_errors": 557325022338560884631142400
+ }
+ }`), "")
+ // An overflowed counter is reported as invalid, not as a made-up number.
+ for _, name := range []string{"MediaErrors", "DataUnitsWritten"} {
+ attr := findAttr(deviceData.Attributes, name)
+ require.NotNil(t, attr, name)
+ assert.Zero(t, attr.RawValue, name)
+ assert.Equal(t, smart.NVMeCounterInvalid, attr.RawString, name)
+ }
+ assertAttrValue(t, deviceData.Attributes, "PowerOnHours", 1234)
+ assert.Empty(t, findAttr(deviceData.Attributes, "PowerOnHours").RawString)
+ assert.Equal(t, uint8(35), deviceData.Temperature)
+}
+
+// Every 128-bit counter is reported as invalid when it overflows.
+func TestParseSmartctlNvmeEveryCounterOverflow(t *testing.T) {
+ fields := map[string]string{
+ "data_units_read": "DataUnitsRead",
+ "data_units_written": "DataUnitsWritten",
+ "host_reads": "HostReads",
+ "host_writes": "HostWrites",
+ "controller_busy_time": "ControllerBusyTime",
+ "power_cycles": "PowerCycles",
+ "power_on_hours": "PowerOnHours",
+ "unsafe_shutdowns": "UnsafeShutdowns",
+ "media_errors": "MediaErrors",
+ "num_err_log_entries": "NumErrLogEntries",
+ }
+ for field, name := range fields {
+ deviceData := parseSmart(t, []byte(`{
+ "device": {"name": "/dev/nvme0", "type": "nvme"},
+ "serial_number": "NVME-`+name+`",
+ "nvme_smart_health_information_log": {"temperature": 35, "`+field+`": 18446744073709551616}
+ }`), "")
+ attr := findAttr(deviceData.Attributes, name)
+ require.NotNil(t, attr, name)
+ assert.Zero(t, attr.RawValue, name)
+ assert.Equal(t, smart.NVMeCounterInvalid, attr.RawString, name)
+ }
+}
+
+// An overflowed TNVMCAP still parses the device; the capacity is left unknown
+// (0) rather than made up.
+func TestParseSmartctlNvmeCapacity(t *testing.T) {
+ deviceData := parseSmart(t, []byte(`{
+ "device": {"name": "/dev/nvme0", "type": "nvme"},
+ "serial_number": "NVME-OVERFLOW-CAP",
+ "smart_status": {"passed": true},
+ "nvme_total_capacity": 18446744073709551616,
+ "nvme_smart_health_information_log": {
+ "temperature": 35,
+ "power_cycles": 120,
+ "media_errors": null
+ }
+ }`), "")
+ assert.Zero(t, deviceData.Capacity)
+ assertAttrValue(t, deviceData.Attributes, "PowerCycles", 120)
+ assertAttrValue(t, deviceData.Attributes, "MediaErrors", 0)
+
+ deviceData = parseSmart(t, []byte(`{
+ "device": {"name": "/dev/nvme0", "type": "nvme"},
+ "serial_number": "NVME-CAP",
+ "nvme_total_capacity": 1000204886016,
+ "nvme_smart_health_information_log": {"temperature": 35}
+ }`), "")
+ assert.Equal(t, uint64(1000204886016), deviceData.Capacity)
+}
+
// Out-of-range temperatures are clamped instead of failing the whole parse.
func TestParseSmartctlTemperatureOutOfRange(t *testing.T) {
sata := parseSmart(t, []byte(`{
diff --git a/internal/entities/smart/smart.go b/internal/entities/smart/smart.go
index 87b760654..b61519300 100644
--- a/internal/entities/smart/smart.go
+++ b/internal/entities/smart/smart.go
@@ -2,6 +2,8 @@ package smart
import (
"encoding/json"
+ "errors"
+ "fmt"
"strconv"
"strings"
)
@@ -9,8 +11,9 @@ import (
// SmartctlOutput holds the fields we read from `smartctl -a --json` for any
// protocol. Sections that do not apply to a device are left nil or empty.
//
-// Counters use wide integer types on purpose: a value that overflows its Go type
-// makes json.Unmarshal fail and drops the whole device (see #2484, #2582).
+// A value that overflows its Go type makes json.Unmarshal fail and drops the
+// whole device (see #2484, #2582, #2731). Use a wide type, or NVMeCounter for
+// the 128-bit NVMe fields, rather than a narrow one.
type SmartctlOutput struct {
Smartctl struct {
ExitStatus int `json:"exit_status"`
@@ -32,7 +35,7 @@ type SmartctlOutput struct {
AtaDeviceStatistics json.RawMessage `json:"ata_device_statistics"` // decoded lazily, only when needed
// NVMe
- NVMeTotalCapacity uint64 `json:"nvme_total_capacity"`
+ NVMeTotalCapacity NVMeCounter `json:"nvme_total_capacity"`
NVMeSmartHealthInformationLog *NVMeSmartHealthInformationLog `json:"nvme_smart_health_information_log"`
// SCSI (scsi_vendor/scsi_product are also reported for SATA disks behind SAT)
@@ -87,23 +90,54 @@ type AtaDeviceStatisticsEntry struct {
}
type NVMeSmartHealthInformationLog struct {
- CriticalWarning uint64 `json:"critical_warning"`
- Temperature int64 `json:"temperature"`
- AvailableSpare uint64 `json:"available_spare"`
- AvailableSpareThreshold uint64 `json:"available_spare_threshold"`
- PercentageUsed uint64 `json:"percentage_used"`
- DataUnitsRead uint64 `json:"data_units_read"`
- DataUnitsWritten uint64 `json:"data_units_written"`
- HostReads uint64 `json:"host_reads"`
- HostWrites uint64 `json:"host_writes"`
- ControllerBusyTime uint64 `json:"controller_busy_time"`
- PowerCycles uint64 `json:"power_cycles"`
- PowerOnHours uint64 `json:"power_on_hours"`
- UnsafeShutdowns uint64 `json:"unsafe_shutdowns"`
- MediaErrors uint64 `json:"media_errors"`
- NumErrLogEntries uint64 `json:"num_err_log_entries"`
- WarningTempTime uint64 `json:"warning_temp_time"`
- CriticalCompTime uint64 `json:"critical_comp_time"`
+ CriticalWarning uint64 `json:"critical_warning"`
+ Temperature int64 `json:"temperature"`
+ AvailableSpare uint64 `json:"available_spare"`
+ AvailableSpareThreshold uint64 `json:"available_spare_threshold"`
+ PercentageUsed uint64 `json:"percentage_used"`
+ DataUnitsRead NVMeCounter `json:"data_units_read"`
+ DataUnitsWritten NVMeCounter `json:"data_units_written"`
+ HostReads NVMeCounter `json:"host_reads"`
+ HostWrites NVMeCounter `json:"host_writes"`
+ ControllerBusyTime NVMeCounter `json:"controller_busy_time"`
+ PowerCycles NVMeCounter `json:"power_cycles"`
+ PowerOnHours NVMeCounter `json:"power_on_hours"`
+ UnsafeShutdowns NVMeCounter `json:"unsafe_shutdowns"`
+ MediaErrors NVMeCounter `json:"media_errors"`
+ NumErrLogEntries NVMeCounter `json:"num_err_log_entries"`
+ WarningTempTime uint64 `json:"warning_temp_time"`
+ CriticalCompTime uint64 `json:"critical_comp_time"`
+}
+
+// NVMeCounter holds one of the 128-bit fields smartctl reports for NVMe
+// drives (health log counters, total capacity). smartctl prints the full value,
+// so faulty firmware can report a number past uint64 (#2731). Such a value
+// decodes as Overflow with Value 0 instead of failing json.Unmarshal and
+// dropping the device, so it is never shown as a real reading.
+type NVMeCounter struct {
+ Value uint64
+ Overflow bool
+}
+
+// NVMeCounterInvalid is the RawString of an attribute whose counter overflowed.
+const NVMeCounterInvalid = "invalid (out of range)"
+
+func (c *NVMeCounter) UnmarshalJSON(data []byte) error {
+ *c = NVMeCounter{}
+ s := string(data)
+ if s == "null" {
+ return nil
+ }
+ v, err := strconv.ParseUint(s, 10, 64)
+ if errors.Is(err, strconv.ErrRange) {
+ c.Overflow = true
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("invalid NVMe counter %q: %w", s, err)
+ }
+ c.Value = v
+ return nil
}
type ScsiErrorCounter struct {
diff --git a/internal/entities/smart/smart_test.go b/internal/entities/smart/smart_test.go
index f1b19bb83..722ba72c1 100644
--- a/internal/entities/smart/smart_test.go
+++ b/internal/entities/smart/smart_test.go
@@ -2,6 +2,7 @@ package smart
import (
"encoding/json"
+ "math"
"testing"
"github.com/stretchr/testify/assert"
@@ -60,3 +61,73 @@ func TestSmartRawValueUnmarshalDurationRawValue(t *testing.T) {
assert.EqualValues(t, 2748, raw.Value)
}
+
+func TestNVMeCounterUnmarshal(t *testing.T) {
+ for _, tc := range []struct {
+ input string
+ value uint64
+ overflow bool
+ }{
+ {`42`, 42, false},
+ {`0`, 0, false},
+ {`null`, 0, false},
+ {`18446744073709551615`, math.MaxUint64, false},
+ {`18446744073709551616`, 0, true},
+ {`557325022338560884631142400`, 0, true},
+ {`340282366920938463463374607431768211455`, 0, true},
+ } {
+ c := NVMeCounter{Value: 7, Overflow: !tc.overflow} // stale state must be reset
+ assert.NoError(t, json.Unmarshal([]byte(tc.input), &c), tc.input)
+ assert.Equal(t, NVMeCounter{Value: tc.value, Overflow: tc.overflow}, c, tc.input)
+ }
+
+ // smartctl prints these as plain integers; anything else is still an error.
+ for _, input := range []string{`-1`, `-0`, `1.5`, `10.0`, `1e3`, `"12"`, `true`, `{}`, `[]`} {
+ var c NVMeCounter
+ assert.Error(t, json.Unmarshal([]byte(input), &c), input)
+ }
+}
+
+func TestNVMeCounterEvery128BitField(t *testing.T) {
+ fields := map[string]func(*NVMeSmartHealthInformationLog) NVMeCounter{
+ "data_units_read": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.DataUnitsRead },
+ "data_units_written": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.DataUnitsWritten },
+ "host_reads": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.HostReads },
+ "host_writes": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.HostWrites },
+ "controller_busy_time": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.ControllerBusyTime },
+ "power_cycles": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.PowerCycles },
+ "power_on_hours": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.PowerOnHours },
+ "unsafe_shutdowns": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.UnsafeShutdowns },
+ "media_errors": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.MediaErrors },
+ "num_err_log_entries": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.NumErrLogEntries },
+ }
+ for name, get := range fields {
+ for _, tc := range []struct {
+ input string
+ want NVMeCounter
+ }{
+ {`18446744073709551616`, NVMeCounter{Overflow: true}},
+ {`12345`, NVMeCounter{Value: 12345}},
+ {`null`, NVMeCounter{}},
+ } {
+ var log NVMeSmartHealthInformationLog
+ err := json.Unmarshal([]byte(`{"`+name+`": `+tc.input+`}`), &log)
+ assert.NoError(t, err, name+" "+tc.input)
+ assert.Equal(t, tc.want, get(&log), name+" "+tc.input)
+ }
+ }
+
+ // TNVMCAP (nvme_total_capacity) is a 128-bit field too.
+ for _, tc := range []struct {
+ input string
+ want NVMeCounter
+ }{
+ {`18446744073709551616`, NVMeCounter{Overflow: true}},
+ {`1000204886016`, NVMeCounter{Value: 1000204886016}},
+ {`null`, NVMeCounter{}},
+ } {
+ var out SmartctlOutput
+ assert.NoError(t, json.Unmarshal([]byte(`{"nvme_total_capacity": `+tc.input+`}`), &out), tc.input)
+ assert.Equal(t, tc.want, out.NVMeTotalCapacity, tc.input)
+ }
+}
diff --git a/internal/site/src/components/routes/system/smart-table.tsx b/internal/site/src/components/routes/system/smart-table.tsx
index 98a4392f3..9800a147e 100644
--- a/internal/site/src/components/routes/system/smart-table.tsx
+++ b/internal/site/src/components/routes/system/smart-table.tsx
@@ -77,7 +77,7 @@ export const smartColumns: ColumnDef<SmartAttribute>[] = [
},
{
accessorFn: (row) => {
- if (row.n === "DataUnitsWritten" || row.n === "DataUnitsRead") {
+ if (!row.rs && (row.n === "DataUnitsWritten" || row.n === "DataUnitsRead")) {
return formatDataUnits(Number(row.rv ?? 0))
}
return row.rs || row.rv?.toString()