Lightweight server monitoring with historical data, docker stats, and alerts.
fix(agent): keep NVMe SMART data when a 128-bit counter exceeds uint64 (#2732)
Columbus Labs0f68d161b127
diff --git a/agent/smart_parse.go b/agent/smart_parse.goindex 9fa06b825..4c681d28a 100644--- a/agent/smart_parse.go+++ b/agent/smart_parse.go@@ -40,7 +40,7 @@ func (sm *SmartManager) parseSmartctl(output []byte, dtype string) (*smart.Smart}smartData := newSmartData(data, dtype, data.ModelName, data.FirmwareVersion, log.Temperature)if smartData.Capacity == 0 {- smartData.Capacity = data.NVMeTotalCapacity+ smartData.Capacity = data.NVMeTotalCapacity.Value}if smartData.Capacity == 0 && (runtime.GOOS == "darwin" || sm.darwinNvmeProvider != nil) {smartData.Capacity = sm.lookupDarwinNvmeCapacity(data.SerialNumber)@@ -271,21 +271,30 @@ func nvmeAttributes(log *smart.NVMeSmartHealthInformationLog, temperature uint8){Name: "AvailableSpare", RawValue: log.AvailableSpare},{Name: "AvailableSpareThreshold", RawValue: log.AvailableSpareThreshold},{Name: "PercentageUsed", RawValue: log.PercentageUsed},- {Name: "DataUnitsRead", RawValue: log.DataUnitsRead},- {Name: "DataUnitsWritten", RawValue: log.DataUnitsWritten},- {Name: "HostReads", RawValue: log.HostReads},- {Name: "HostWrites", RawValue: log.HostWrites},- {Name: "ControllerBusyTime", RawValue: log.ControllerBusyTime},- {Name: "PowerCycles", RawValue: log.PowerCycles},- {Name: "PowerOnHours", RawValue: log.PowerOnHours},- {Name: "UnsafeShutdowns", RawValue: log.UnsafeShutdowns},- {Name: "MediaErrors", RawValue: log.MediaErrors},- {Name: "NumErrLogEntries", RawValue: log.NumErrLogEntries},+ nvmeCounterAttribute("DataUnitsRead", log.DataUnitsRead),+ nvmeCounterAttribute("DataUnitsWritten", log.DataUnitsWritten),+ nvmeCounterAttribute("HostReads", log.HostReads),+ nvmeCounterAttribute("HostWrites", log.HostWrites),+ nvmeCounterAttribute("ControllerBusyTime", log.ControllerBusyTime),+ nvmeCounterAttribute("PowerCycles", log.PowerCycles),+ nvmeCounterAttribute("PowerOnHours", log.PowerOnHours),+ nvmeCounterAttribute("UnsafeShutdowns", log.UnsafeShutdowns),+ nvmeCounterAttribute("MediaErrors", log.MediaErrors),+ nvmeCounterAttribute("NumErrLogEntries", log.NumErrLogEntries),{Name: "WarningTempTime", RawValue: log.WarningTempTime},{Name: "CriticalCompTime", RawValue: log.CriticalCompTime},}}+// nvmeCounterAttribute reports an overflowed 128-bit counter as invalid instead+// of a made-up number, so neither the table nor the power-on hours show it.+func nvmeCounterAttribute(name string, c smart.NVMeCounter) *smart.SmartAttribute {+ if c.Overflow {+ return &smart.SmartAttribute{Name: name, RawString: smart.NVMeCounterInvalid}+ }+ return &smart.SmartAttribute{Name: name, RawValue: c.Value}+}+// lookupDarwinNvmeCapacity returns the capacity in bytes for a given NVMe serial number on Darwin.// It uses system_profiler SPNVMeDataType to get capacity since Apple SSDs don't report user_capacity// via smartctl. Results are cached after the first call via sync.Once.diff --git a/agent/smart_test.go b/agent/smart_test.goindex 0bc5c213c..181dada4e 100644--- a/agent/smart_test.go+++ b/agent/smart_test.go@@ -268,6 +268,86 @@ func TestParseSmartctlNvmeLargeCounters(t *testing.T) {assertAttrValue(t, deviceData.Attributes, "PercentageUsed", 300)}+// NVMe health counters are 128-bit, so smartctl can print values past uint64.+// They are reported as invalid instead of failing json.Unmarshal and dropping the device (#2731).+func TestParseSmartctlNvmeCounterBeyondUint64(t *testing.T) {+ deviceData := parseSmart(t, []byte(`{+ "device": {"name": "/dev/nvme0", "type": "nvme"},+ "serial_number": "NVME-OVERFLOW",+ "smart_status": {"passed": true},+ "nvme_smart_health_information_log": {+ "temperature": 35,+ "data_units_written": 340282366920938463463374607431768211455,+ "power_on_hours": 1234,+ "media_errors": 557325022338560884631142400+ }+ }`), "")+ // An overflowed counter is reported as invalid, not as a made-up number.+ for _, name := range []string{"MediaErrors", "DataUnitsWritten"} {+ attr := findAttr(deviceData.Attributes, name)+ require.NotNil(t, attr, name)+ assert.Zero(t, attr.RawValue, name)+ assert.Equal(t, smart.NVMeCounterInvalid, attr.RawString, name)+ }+ assertAttrValue(t, deviceData.Attributes, "PowerOnHours", 1234)+ assert.Empty(t, findAttr(deviceData.Attributes, "PowerOnHours").RawString)+ assert.Equal(t, uint8(35), deviceData.Temperature)+}++// Every 128-bit counter is reported as invalid when it overflows.+func TestParseSmartctlNvmeEveryCounterOverflow(t *testing.T) {+ fields := map[string]string{+ "data_units_read": "DataUnitsRead",+ "data_units_written": "DataUnitsWritten",+ "host_reads": "HostReads",+ "host_writes": "HostWrites",+ "controller_busy_time": "ControllerBusyTime",+ "power_cycles": "PowerCycles",+ "power_on_hours": "PowerOnHours",+ "unsafe_shutdowns": "UnsafeShutdowns",+ "media_errors": "MediaErrors",+ "num_err_log_entries": "NumErrLogEntries",+ }+ for field, name := range fields {+ deviceData := parseSmart(t, []byte(`{+ "device": {"name": "/dev/nvme0", "type": "nvme"},+ "serial_number": "NVME-`+name+`",+ "nvme_smart_health_information_log": {"temperature": 35, "`+field+`": 18446744073709551616}+ }`), "")+ attr := findAttr(deviceData.Attributes, name)+ require.NotNil(t, attr, name)+ assert.Zero(t, attr.RawValue, name)+ assert.Equal(t, smart.NVMeCounterInvalid, attr.RawString, name)+ }+}++// An overflowed TNVMCAP still parses the device; the capacity is left unknown+// (0) rather than made up.+func TestParseSmartctlNvmeCapacity(t *testing.T) {+ deviceData := parseSmart(t, []byte(`{+ "device": {"name": "/dev/nvme0", "type": "nvme"},+ "serial_number": "NVME-OVERFLOW-CAP",+ "smart_status": {"passed": true},+ "nvme_total_capacity": 18446744073709551616,+ "nvme_smart_health_information_log": {+ "temperature": 35,+ "power_cycles": 120,+ "media_errors": null+ }+ }`), "")+ assert.Zero(t, deviceData.Capacity)+ assertAttrValue(t, deviceData.Attributes, "PowerCycles", 120)+ assertAttrValue(t, deviceData.Attributes, "MediaErrors", 0)++ deviceData = parseSmart(t, []byte(`{+ "device": {"name": "/dev/nvme0", "type": "nvme"},+ "serial_number": "NVME-CAP",+ "nvme_total_capacity": 1000204886016,+ "nvme_smart_health_information_log": {"temperature": 35}+ }`), "")+ assert.Equal(t, uint64(1000204886016), deviceData.Capacity)+}+// Out-of-range temperatures are clamped instead of failing the whole parse.func TestParseSmartctlTemperatureOutOfRange(t *testing.T) {sata := parseSmart(t, []byte(`{diff --git a/internal/entities/smart/smart.go b/internal/entities/smart/smart.goindex 87b760654..b61519300 100644--- a/internal/entities/smart/smart.go+++ b/internal/entities/smart/smart.go@@ -2,6 +2,8 @@ package smartimport ("encoding/json"+ "errors"+ "fmt""strconv""strings")@@ -9,8 +11,9 @@ import (// SmartctlOutput holds the fields we read from `smartctl -a --json` for any// protocol. Sections that do not apply to a device are left nil or empty.//-// Counters use wide integer types on purpose: a value that overflows its Go type-// makes json.Unmarshal fail and drops the whole device (see #2484, #2582).+// A value that overflows its Go type makes json.Unmarshal fail and drops the+// whole device (see #2484, #2582, #2731). Use a wide type, or NVMeCounter for+// the 128-bit NVMe fields, rather than a narrow one.type SmartctlOutput struct {Smartctl struct {ExitStatus int `json:"exit_status"`@@ -32,7 +35,7 @@ type SmartctlOutput struct {AtaDeviceStatistics json.RawMessage `json:"ata_device_statistics"` // decoded lazily, only when needed// NVMe- NVMeTotalCapacity uint64 `json:"nvme_total_capacity"`+ NVMeTotalCapacity NVMeCounter `json:"nvme_total_capacity"`NVMeSmartHealthInformationLog *NVMeSmartHealthInformationLog `json:"nvme_smart_health_information_log"`// SCSI (scsi_vendor/scsi_product are also reported for SATA disks behind SAT)@@ -87,23 +90,54 @@ type AtaDeviceStatisticsEntry struct {}type NVMeSmartHealthInformationLog struct {- CriticalWarning uint64 `json:"critical_warning"`- Temperature int64 `json:"temperature"`- AvailableSpare uint64 `json:"available_spare"`- AvailableSpareThreshold uint64 `json:"available_spare_threshold"`- PercentageUsed uint64 `json:"percentage_used"`- DataUnitsRead uint64 `json:"data_units_read"`- DataUnitsWritten uint64 `json:"data_units_written"`- HostReads uint64 `json:"host_reads"`- HostWrites uint64 `json:"host_writes"`- ControllerBusyTime uint64 `json:"controller_busy_time"`- PowerCycles uint64 `json:"power_cycles"`- PowerOnHours uint64 `json:"power_on_hours"`- UnsafeShutdowns uint64 `json:"unsafe_shutdowns"`- MediaErrors uint64 `json:"media_errors"`- NumErrLogEntries uint64 `json:"num_err_log_entries"`- WarningTempTime uint64 `json:"warning_temp_time"`- CriticalCompTime uint64 `json:"critical_comp_time"`+ CriticalWarning uint64 `json:"critical_warning"`+ Temperature int64 `json:"temperature"`+ AvailableSpare uint64 `json:"available_spare"`+ AvailableSpareThreshold uint64 `json:"available_spare_threshold"`+ PercentageUsed uint64 `json:"percentage_used"`+ DataUnitsRead NVMeCounter `json:"data_units_read"`+ DataUnitsWritten NVMeCounter `json:"data_units_written"`+ HostReads NVMeCounter `json:"host_reads"`+ HostWrites NVMeCounter `json:"host_writes"`+ ControllerBusyTime NVMeCounter `json:"controller_busy_time"`+ PowerCycles NVMeCounter `json:"power_cycles"`+ PowerOnHours NVMeCounter `json:"power_on_hours"`+ UnsafeShutdowns NVMeCounter `json:"unsafe_shutdowns"`+ MediaErrors NVMeCounter `json:"media_errors"`+ NumErrLogEntries NVMeCounter `json:"num_err_log_entries"`+ WarningTempTime uint64 `json:"warning_temp_time"`+ CriticalCompTime uint64 `json:"critical_comp_time"`+}++// NVMeCounter holds one of the 128-bit fields smartctl reports for NVMe+// drives (health log counters, total capacity). smartctl prints the full value,+// so faulty firmware can report a number past uint64 (#2731). Such a value+// decodes as Overflow with Value 0 instead of failing json.Unmarshal and+// dropping the device, so it is never shown as a real reading.+type NVMeCounter struct {+ Value uint64+ Overflow bool+}++// NVMeCounterInvalid is the RawString of an attribute whose counter overflowed.+const NVMeCounterInvalid = "invalid (out of range)"++func (c *NVMeCounter) UnmarshalJSON(data []byte) error {+ *c = NVMeCounter{}+ s := string(data)+ if s == "null" {+ return nil+ }+ v, err := strconv.ParseUint(s, 10, 64)+ if errors.Is(err, strconv.ErrRange) {+ c.Overflow = true+ return nil+ }+ if err != nil {+ return fmt.Errorf("invalid NVMe counter %q: %w", s, err)+ }+ c.Value = v+ return nil}type ScsiErrorCounter struct {diff --git a/internal/entities/smart/smart_test.go b/internal/entities/smart/smart_test.goindex f1b19bb83..722ba72c1 100644--- a/internal/entities/smart/smart_test.go+++ b/internal/entities/smart/smart_test.go@@ -2,6 +2,7 @@ package smartimport ("encoding/json"+ "math""testing""github.com/stretchr/testify/assert"@@ -60,3 +61,73 @@ func TestSmartRawValueUnmarshalDurationRawValue(t *testing.T) {assert.EqualValues(t, 2748, raw.Value)}++func TestNVMeCounterUnmarshal(t *testing.T) {+ for _, tc := range []struct {+ input string+ value uint64+ overflow bool+ }{+ {`42`, 42, false},+ {`0`, 0, false},+ {`null`, 0, false},+ {`18446744073709551615`, math.MaxUint64, false},+ {`18446744073709551616`, 0, true},+ {`557325022338560884631142400`, 0, true},+ {`340282366920938463463374607431768211455`, 0, true},+ } {+ c := NVMeCounter{Value: 7, Overflow: !tc.overflow} // stale state must be reset+ assert.NoError(t, json.Unmarshal([]byte(tc.input), &c), tc.input)+ assert.Equal(t, NVMeCounter{Value: tc.value, Overflow: tc.overflow}, c, tc.input)+ }++ // smartctl prints these as plain integers; anything else is still an error.+ for _, input := range []string{`-1`, `-0`, `1.5`, `10.0`, `1e3`, `"12"`, `true`, `{}`, `[]`} {+ var c NVMeCounter+ assert.Error(t, json.Unmarshal([]byte(input), &c), input)+ }+}++func TestNVMeCounterEvery128BitField(t *testing.T) {+ fields := map[string]func(*NVMeSmartHealthInformationLog) NVMeCounter{+ "data_units_read": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.DataUnitsRead },+ "data_units_written": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.DataUnitsWritten },+ "host_reads": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.HostReads },+ "host_writes": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.HostWrites },+ "controller_busy_time": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.ControllerBusyTime },+ "power_cycles": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.PowerCycles },+ "power_on_hours": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.PowerOnHours },+ "unsafe_shutdowns": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.UnsafeShutdowns },+ "media_errors": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.MediaErrors },+ "num_err_log_entries": func(l *NVMeSmartHealthInformationLog) NVMeCounter { return l.NumErrLogEntries },+ }+ for name, get := range fields {+ for _, tc := range []struct {+ input string+ want NVMeCounter+ }{+ {`18446744073709551616`, NVMeCounter{Overflow: true}},+ {`12345`, NVMeCounter{Value: 12345}},+ {`null`, NVMeCounter{}},+ } {+ var log NVMeSmartHealthInformationLog+ err := json.Unmarshal([]byte(`{"`+name+`": `+tc.input+`}`), &log)+ assert.NoError(t, err, name+" "+tc.input)+ assert.Equal(t, tc.want, get(&log), name+" "+tc.input)+ }+ }++ // TNVMCAP (nvme_total_capacity) is a 128-bit field too.+ for _, tc := range []struct {+ input string+ want NVMeCounter+ }{+ {`18446744073709551616`, NVMeCounter{Overflow: true}},+ {`1000204886016`, NVMeCounter{Value: 1000204886016}},+ {`null`, NVMeCounter{}},+ } {+ var out SmartctlOutput+ assert.NoError(t, json.Unmarshal([]byte(`{"nvme_total_capacity": `+tc.input+`}`), &out), tc.input)+ assert.Equal(t, tc.want, out.NVMeTotalCapacity, tc.input)+ }+}diff --git a/internal/site/src/components/routes/system/smart-table.tsx b/internal/site/src/components/routes/system/smart-table.tsxindex 98a4392f3..9800a147e 100644--- a/internal/site/src/components/routes/system/smart-table.tsx+++ b/internal/site/src/components/routes/system/smart-table.tsx@@ -77,7 +77,7 @@ export const smartColumns: ColumnDef<SmartAttribute>[] = [},{accessorFn: (row) => {- if (row.n === "DataUnitsWritten" || row.n === "DataUnitsRead") {+ if (!row.rs && (row.n === "DataUnitsWritten" || row.n === "DataUnitsRead")) {return formatDataUnits(Number(row.rv ?? 0))}return row.rs || row.rv?.toString()