Lightweight server monitoring with historical data, docker stats, and alerts.
fix(agent): don't touch the health file when resolving its path (#2825)
Pavlo Sukhanovee043e72e38a
diff --git a/agent/health/health.go b/agent/health/health.goindex f6d00404f..cdf52ea85 100644--- a/agent/health/health.go+++ b/agent/health/health.go@@ -14,18 +14,19 @@ import ("time")-// healthFile is the path to the health file-var healthFile = getHealthFilePath()+// healthFiles are the candidate health file paths, most preferred first. On Linux+// /dev/shm comes first because /tmp is not always writable, e.g. for a non-root user+// in the agent image (#1455).+// Resolving them must not write: package init runs in the `agent health` process too.+var healthFiles = getHealthFilePaths("/dev/shm")-func getHealthFilePath() string {- filename := "beszel_health"+func getHealthFilePaths(shmDir string) []string {+ const filename = "beszel_health"+ tmpPath := filepath.Join(os.TempDir(), filename)if runtime.GOOS == "linux" {- fullPath := filepath.Join("/dev/shm", filename)- if err := updateHealthFile(fullPath); err == nil {- return fullPath- }+ return []string{filepath.Join(shmDir, filename), tmpPath}}- return filepath.Join(os.TempDir(), filename)+ return []string{tmpPath}}func updateHealthFile(path string) error {@@ -36,30 +37,52 @@ func updateHealthFile(path string) error {return file.Close()}-// Check checks if the agent is connected by checking the modification time of the health file+// Check checks if the agent is connected by checking the modification time of the health file.+// It runs in its own process (`agent health`), so it must only read: creating or touching the+// file here would refresh the timestamp it is about to check. The newest candidate counts, so a+// stale file left in another location (e.g. by an agent that ran as a different user) is ignored.func Check() error {- fileInfo, err := os.Stat(healthFile)- if err != nil {+ var newest time.Time+ var err error+ for _, path := range healthFiles {+ fileInfo, statErr := os.Stat(path)+ if statErr != nil {+ err = statErr+ continue+ }+ if fileInfo.ModTime().After(newest) {+ newest = fileInfo.ModTime()+ }+ }+ if newest.IsZero() {return err}- if time.Since(fileInfo.ModTime()) > 91*time.Second {+ if time.Since(newest) > 91*time.Second {log.Println("over 90 seconds since last connection")return errors.New("unhealthy")}return nil}-// Update updates the modification time of the health file+// Update updates the modification time of the health file, in the first location that is writablefunc Update() error {- return updateHealthFile(healthFile)+ var err error+ for _, path := range healthFiles {+ if err = updateHealthFile(path); err == nil {+ return nil+ }+ }+ return err}// CleanUp removes the health file. The path is shared by every agent on the host, so a file that// is already gone - another agent removed it, or a test ran two agents - is not a failure to// report: it would surface as the error from Start.func CleanUp() error {- if err := os.Remove(healthFile); err != nil && !errors.Is(err, fs.ErrNotExist) {- return err+ for _, path := range healthFiles {+ if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) {+ return err+ }}return nil}diff --git a/agent/health/health_test.go b/agent/health/health_test.goindex 1b5a86fb9..6b28e44db 100644--- a/agent/health/health_test.go+++ b/agent/health/health_test.go@@ -5,6 +5,7 @@ package healthimport ("os""path/filepath"+ "runtime""testing""time"@@ -15,11 +16,8 @@ import ()func TestHealth(t *testing.T) {- // Override healthFile to use a temporary directory for this test.- originalHealthFile := healthFile- tmpDir := t.TempDir()- healthFile = filepath.Join(tmpDir, "beszel_health_test")- defer func() { healthFile = originalHealthFile }()+ // Override healthFiles to use a temporary directory for this test.+ healthFile := useHealthFiles(t, filepath.Join(t.TempDir(), "beszel_health_test"))[0]t.Run("check with no health file", func(t *testing.T) {err := Check()@@ -67,10 +65,7 @@ func TestHealth(t *testing.T) {// The health file path is shared by every agent on the host, so a second removal must not be// reported as a failure: it reaches the caller as the error returned by Agent.Start.func TestCleanUpIsIdempotent(t *testing.T) {- dir := t.TempDir()- original := healthFile- healthFile = filepath.Join(dir, "beszel_health")- t.Cleanup(func() { healthFile = original })+ healthFile := useHealthFiles(t, filepath.Join(t.TempDir(), "beszel_health"))[0]require.NoError(t, Update())require.FileExists(t, healthFile)@@ -78,3 +73,68 @@ func TestCleanUpIsIdempotent(t *testing.T) {require.NoFileExists(t, healthFile)assert.NoError(t, CleanUp(), "removing a health file that is already gone is not an error")}++// useHealthFiles points the package at the given candidate paths for one test.+func useHealthFiles(t *testing.T, paths ...string) []string {+ t.Helper()+ original := healthFiles+ healthFiles = paths+ t.Cleanup(func() { healthFiles = original })+ return paths+}++// `agent health` runs package init and Check in its own process. If either wrote the file, a+// hung agent's stale timestamp would be refreshed by the very check meant to catch it.+func TestGetHealthFilePathsDoesNotWrite(t *testing.T) {+ shmDir := t.TempDir()+ paths := getHealthFilePaths(shmDir)+ if runtime.GOOS == "linux" {+ assert.Equal(t, filepath.Join(shmDir, "beszel_health"), paths[0])+ }+ assert.NoFileExists(t, filepath.Join(shmDir, "beszel_health"))+}++func TestCheckDoesNotTouchHealthFile(t *testing.T) {+ healthFile := useHealthFiles(t, filepath.Join(t.TempDir(), "beszel_health"))[0]+ require.NoError(t, Update())+ stale := time.Now().Add(-2 * time.Minute)+ require.NoError(t, os.Chtimes(healthFile, stale, stale))++ err := Check()+ require.Error(t, err)+ assert.Equal(t, "unhealthy", err.Error())++ info, err := os.Stat(healthFile)+ require.NoError(t, err)+ assert.True(t, info.ModTime().Equal(stale), "Check changed the health file's mtime")+}++// The first location is not always writable (#1455), so Update falls back to the next one and+// Check finds the file there.+func TestHealthFileFallback(t *testing.T) {+ dir := t.TempDir()+ unwritable := filepath.Join(dir, "missing-dir", "beszel_health")+ fallback := filepath.Join(dir, "beszel_health")+ useHealthFiles(t, unwritable, fallback)++ require.NoError(t, Update())+ assert.NoFileExists(t, unwritable)+ assert.FileExists(t, fallback)+ assert.NoError(t, Check())+}++// A file another agent left behind (say, root's, which a non-root agent cannot overwrite) must not+// hide the one the running agent keeps fresh.+func TestCheckIgnoresStaleFileInOtherLocation(t *testing.T) {+ dir := t.TempDir()+ stalePath := filepath.Join(dir, "shm_beszel_health")+ freshPath := filepath.Join(dir, "tmp_beszel_health")+ useHealthFiles(t, stalePath, freshPath)++ require.NoError(t, os.WriteFile(stalePath, nil, 0o644))+ stale := time.Now().Add(-time.Hour)+ require.NoError(t, os.Chtimes(stalePath, stale, stale))+ require.NoError(t, os.WriteFile(freshPath, nil, 0o644))++ assert.NoError(t, Check())+}