Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (agent): Add lock file watcher to exit from process in case of lock file deletion

Rostislav Duginbcd13c27d3b2
diff --git a/agent/internal/features/start/lock_watcher.go b/agent/internal/features/start/lock_watcher.go
new file mode 100644
index 00000000..27e8ca46
--- /dev/null
+++ b/agent/internal/features/start/lock_watcher.go
@@ -0,0 +1,90 @@
+//go:build !windows
+
+package start
+
+import (
+ "context"
+ "log/slog"
+ "os"
+ "syscall"
+ "time"
+)
+
+const lockWatchInterval = 5 * time.Second
+
+type LockWatcher struct {
+ originalInode uint64
+ cancel context.CancelFunc
+ log *slog.Logger
+}
+
+func NewLockWatcher(lockFile *os.File, cancel context.CancelFunc, log *slog.Logger) (*LockWatcher, error) {
+ inode, err := getFileInode(lockFile)
+ if err != nil {
+ return nil, err
+ }
+
+ return &LockWatcher{
+ originalInode: inode,
+ cancel: cancel,
+ log: log,
+ }, nil
+}
+
+func (w *LockWatcher) Run(ctx context.Context) {
+ ticker := time.NewTicker(lockWatchInterval)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ w.check()
+ }
+ }
+}
+
+func (w *LockWatcher) check() {
+ info, err := os.Stat(lockFileName)
+ if err != nil {
+ w.log.Error("Lock file disappeared, shutting down", "file", lockFileName, "error", err)
+ w.cancel()
+
+ return
+ }
+
+ currentInode, err := getStatInode(info)
+ if err != nil {
+ w.log.Error("Failed to read lock file inode, shutting down", "error", err)
+ w.cancel()
+
+ return
+ }
+
+ if currentInode != w.originalInode {
+ w.log.Error("Lock file was replaced (inode changed), shutting down",
+ "originalInode", w.originalInode,
+ "currentInode", currentInode,
+ )
+ w.cancel()
+ }
+}
+
+func getFileInode(f *os.File) (uint64, error) {
+ info, err := f.Stat()
+ if err != nil {
+ return 0, err
+ }
+
+ return getStatInode(info)
+}
+
+func getStatInode(info os.FileInfo) (uint64, error) {
+ stat, ok := info.Sys().(*syscall.Stat_t)
+ if !ok {
+ return 0, os.ErrInvalid
+ }
+
+ return stat.Ino, nil
+}
diff --git a/agent/internal/features/start/lock_watcher_test.go b/agent/internal/features/start/lock_watcher_test.go
new file mode 100644
index 00000000..d1e3ab27
--- /dev/null
+++ b/agent/internal/features/start/lock_watcher_test.go
@@ -0,0 +1,110 @@
+//go:build !windows
+
+package start
+
+import (
+ "context"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "databasus-agent/internal/logger"
+)
+
+func Test_NewLockWatcher_CapturesInode(t *testing.T) {
+ setupTempDir(t)
+ log := logger.GetLogger()
+
+ lockFile, err := AcquireLock(log)
+ require.NoError(t, err)
+ defer ReleaseLock(lockFile)
+
+ _, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ watcher, err := NewLockWatcher(lockFile, cancel, log)
+ require.NoError(t, err)
+ assert.NotZero(t, watcher.originalInode)
+}
+
+func Test_LockWatcher_FileUnchanged_ContextNotCancelled(t *testing.T) {
+ setupTempDir(t)
+ log := logger.GetLogger()
+
+ lockFile, err := AcquireLock(log)
+ require.NoError(t, err)
+ defer ReleaseLock(lockFile)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ watcher, err := NewLockWatcher(lockFile, cancel, log)
+ require.NoError(t, err)
+
+ watcher.check()
+ watcher.check()
+ watcher.check()
+
+ select {
+ case <-ctx.Done():
+ t.Fatal("context should not be cancelled when lock file is unchanged")
+ default:
+ }
+}
+
+func Test_LockWatcher_FileDeleted_CancelsContext(t *testing.T) {
+ setupTempDir(t)
+ log := logger.GetLogger()
+
+ lockFile, err := AcquireLock(log)
+ require.NoError(t, err)
+ defer ReleaseLock(lockFile)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ watcher, err := NewLockWatcher(lockFile, cancel, log)
+ require.NoError(t, err)
+
+ err = os.Remove(lockFileName)
+ require.NoError(t, err)
+
+ watcher.check()
+
+ select {
+ case <-ctx.Done():
+ default:
+ t.Fatal("context should be cancelled when lock file is deleted")
+ }
+}
+
+func Test_LockWatcher_FileReplacedWithDifferentInode_CancelsContext(t *testing.T) {
+ setupTempDir(t)
+ log := logger.GetLogger()
+
+ lockFile, err := AcquireLock(log)
+ require.NoError(t, err)
+ defer ReleaseLock(lockFile)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ watcher, err := NewLockWatcher(lockFile, cancel, log)
+ require.NoError(t, err)
+
+ err = os.Remove(lockFileName)
+ require.NoError(t, err)
+
+ err = os.WriteFile(lockFileName, []byte("99999\n"), 0o644)
+ require.NoError(t, err)
+
+ watcher.check()
+
+ select {
+ case <-ctx.Done():
+ default:
+ t.Fatal("context should be cancelled when lock file inode changes")
+ }
+}
diff --git a/agent/internal/features/start/lock_watcher_windows.go b/agent/internal/features/start/lock_watcher_windows.go
new file mode 100644
index 00000000..3b41ba8e
--- /dev/null
+++ b/agent/internal/features/start/lock_watcher_windows.go
@@ -0,0 +1,17 @@
+//go:build windows
+
+package start
+
+import (
+ "context"
+ "log/slog"
+ "os"
+)
+
+type LockWatcher struct{}
+
+func NewLockWatcher(_ *os.File, _ context.CancelFunc, _ *slog.Logger) (*LockWatcher, error) {
+ return &LockWatcher{}, nil
+}
+
+func (w *LockWatcher) Run(_ context.Context) {}
diff --git a/agent/internal/features/start/start.go b/agent/internal/features/start/start.go
index 2e0ada6f..8cdd438d 100644
--- a/agent/internal/features/start/start.go
+++ b/agent/internal/features/start/start.go
@@ -66,6 +66,12 @@ func RunDaemon(cfg *config.Config, log *slog.Logger) error {
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
+ watcher, err := NewLockWatcher(lockFile, cancel, log)
+ if err != nil {
+ return fmt.Errorf("failed to initialize lock watcher: %w", err)
+ }
+ go watcher.Run(ctx)
+
apiClient := api.NewClient(cfg.DatabasusHost, cfg.Token, log)
fullBackuper := full_backup.NewFullBackuper(cfg, apiClient, log)