Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

fix(agent): don't touch the health file when resolving its path (#2825)

Pavlo Sukhanovee043e72e38a
diff --git a/agent/health/health.go b/agent/health/health.go
index f6d00404f..cdf52ea85 100644
--- a/agent/health/health.go
+++ b/agent/health/health.go
@@ -14,18 +14,19 @@ import (
"time"
)
-// healthFile is the path to the health file
-var healthFile = getHealthFilePath()
+// healthFiles are the candidate health file paths, most preferred first. On Linux
+// /dev/shm comes first because /tmp is not always writable, e.g. for a non-root user
+// in the agent image (#1455).
+// Resolving them must not write: package init runs in the `agent health` process too.
+var healthFiles = getHealthFilePaths("/dev/shm")
-func getHealthFilePath() string {
- filename := "beszel_health"
+func getHealthFilePaths(shmDir string) []string {
+ const filename = "beszel_health"
+ tmpPath := filepath.Join(os.TempDir(), filename)
if runtime.GOOS == "linux" {
- fullPath := filepath.Join("/dev/shm", filename)
- if err := updateHealthFile(fullPath); err == nil {
- return fullPath
- }
+ return []string{filepath.Join(shmDir, filename), tmpPath}
}
- return filepath.Join(os.TempDir(), filename)
+ return []string{tmpPath}
}
func updateHealthFile(path string) error {
@@ -36,30 +37,52 @@ func updateHealthFile(path string) error {
return file.Close()
}
-// Check checks if the agent is connected by checking the modification time of the health file
+// Check checks if the agent is connected by checking the modification time of the health file.
+// It runs in its own process (`agent health`), so it must only read: creating or touching the
+// file here would refresh the timestamp it is about to check. The newest candidate counts, so a
+// stale file left in another location (e.g. by an agent that ran as a different user) is ignored.
func Check() error {
- fileInfo, err := os.Stat(healthFile)
- if err != nil {
+ var newest time.Time
+ var err error
+ for _, path := range healthFiles {
+ fileInfo, statErr := os.Stat(path)
+ if statErr != nil {
+ err = statErr
+ continue
+ }
+ if fileInfo.ModTime().After(newest) {
+ newest = fileInfo.ModTime()
+ }
+ }
+ if newest.IsZero() {
return err
}
- if time.Since(fileInfo.ModTime()) > 91*time.Second {
+ if time.Since(newest) > 91*time.Second {
log.Println("over 90 seconds since last connection")
return errors.New("unhealthy")
}
return nil
}
-// Update updates the modification time of the health file
+// Update updates the modification time of the health file, in the first location that is writable
func Update() error {
- return updateHealthFile(healthFile)
+ var err error
+ for _, path := range healthFiles {
+ if err = updateHealthFile(path); err == nil {
+ return nil
+ }
+ }
+ return err
}
// CleanUp removes the health file. The path is shared by every agent on the host, so a file that
// is already gone - another agent removed it, or a test ran two agents - is not a failure to
// report: it would surface as the error from Start.
func CleanUp() error {
- if err := os.Remove(healthFile); err != nil && !errors.Is(err, fs.ErrNotExist) {
- return err
+ for _, path := range healthFiles {
+ if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) {
+ return err
+ }
}
return nil
}
diff --git a/agent/health/health_test.go b/agent/health/health_test.go
index 1b5a86fb9..6b28e44db 100644
--- a/agent/health/health_test.go
+++ b/agent/health/health_test.go
@@ -5,6 +5,7 @@ package health
import (
"os"
"path/filepath"
+ "runtime"
"testing"
"time"
@@ -15,11 +16,8 @@ import (
)
func TestHealth(t *testing.T) {
- // Override healthFile to use a temporary directory for this test.
- originalHealthFile := healthFile
- tmpDir := t.TempDir()
- healthFile = filepath.Join(tmpDir, "beszel_health_test")
- defer func() { healthFile = originalHealthFile }()
+ // Override healthFiles to use a temporary directory for this test.
+ healthFile := useHealthFiles(t, filepath.Join(t.TempDir(), "beszel_health_test"))[0]
t.Run("check with no health file", func(t *testing.T) {
err := Check()
@@ -67,10 +65,7 @@ func TestHealth(t *testing.T) {
// The health file path is shared by every agent on the host, so a second removal must not be
// reported as a failure: it reaches the caller as the error returned by Agent.Start.
func TestCleanUpIsIdempotent(t *testing.T) {
- dir := t.TempDir()
- original := healthFile
- healthFile = filepath.Join(dir, "beszel_health")
- t.Cleanup(func() { healthFile = original })
+ healthFile := useHealthFiles(t, filepath.Join(t.TempDir(), "beszel_health"))[0]
require.NoError(t, Update())
require.FileExists(t, healthFile)
@@ -78,3 +73,68 @@ func TestCleanUpIsIdempotent(t *testing.T) {
require.NoFileExists(t, healthFile)
assert.NoError(t, CleanUp(), "removing a health file that is already gone is not an error")
}
+
+// useHealthFiles points the package at the given candidate paths for one test.
+func useHealthFiles(t *testing.T, paths ...string) []string {
+ t.Helper()
+ original := healthFiles
+ healthFiles = paths
+ t.Cleanup(func() { healthFiles = original })
+ return paths
+}
+
+// `agent health` runs package init and Check in its own process. If either wrote the file, a
+// hung agent's stale timestamp would be refreshed by the very check meant to catch it.
+func TestGetHealthFilePathsDoesNotWrite(t *testing.T) {
+ shmDir := t.TempDir()
+ paths := getHealthFilePaths(shmDir)
+ if runtime.GOOS == "linux" {
+ assert.Equal(t, filepath.Join(shmDir, "beszel_health"), paths[0])
+ }
+ assert.NoFileExists(t, filepath.Join(shmDir, "beszel_health"))
+}
+
+func TestCheckDoesNotTouchHealthFile(t *testing.T) {
+ healthFile := useHealthFiles(t, filepath.Join(t.TempDir(), "beszel_health"))[0]
+ require.NoError(t, Update())
+ stale := time.Now().Add(-2 * time.Minute)
+ require.NoError(t, os.Chtimes(healthFile, stale, stale))
+
+ err := Check()
+ require.Error(t, err)
+ assert.Equal(t, "unhealthy", err.Error())
+
+ info, err := os.Stat(healthFile)
+ require.NoError(t, err)
+ assert.True(t, info.ModTime().Equal(stale), "Check changed the health file's mtime")
+}
+
+// The first location is not always writable (#1455), so Update falls back to the next one and
+// Check finds the file there.
+func TestHealthFileFallback(t *testing.T) {
+ dir := t.TempDir()
+ unwritable := filepath.Join(dir, "missing-dir", "beszel_health")
+ fallback := filepath.Join(dir, "beszel_health")
+ useHealthFiles(t, unwritable, fallback)
+
+ require.NoError(t, Update())
+ assert.NoFileExists(t, unwritable)
+ assert.FileExists(t, fallback)
+ assert.NoError(t, Check())
+}
+
+// A file another agent left behind (say, root's, which a non-root agent cannot overwrite) must not
+// hide the one the running agent keeps fresh.
+func TestCheckIgnoresStaleFileInOtherLocation(t *testing.T) {
+ dir := t.TempDir()
+ stalePath := filepath.Join(dir, "shm_beszel_health")
+ freshPath := filepath.Join(dir, "tmp_beszel_health")
+ useHealthFiles(t, stalePath, freshPath)
+
+ require.NoError(t, os.WriteFile(stalePath, nil, 0o644))
+ stale := time.Now().Add(-time.Hour)
+ require.NoError(t, os.Chtimes(stalePath, stale, stale))
+ require.NoError(t, os.WriteFile(freshPath, nil, 0o644))
+
+ assert.NoError(t, Check())
+}