PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (agent): Add lock file watcher to exit from process in case of lock file deletion
Rostislav Duginbcd13c27d3b2
diff --git a/agent/internal/features/start/lock_watcher.go b/agent/internal/features/start/lock_watcher.gonew file mode 100644index 00000000..27e8ca46--- /dev/null+++ b/agent/internal/features/start/lock_watcher.go@@ -0,0 +1,90 @@+//go:build !windows++package start++import (+ "context"+ "log/slog"+ "os"+ "syscall"+ "time"+)++const lockWatchInterval = 5 * time.Second++type LockWatcher struct {+ originalInode uint64+ cancel context.CancelFunc+ log *slog.Logger+}++func NewLockWatcher(lockFile *os.File, cancel context.CancelFunc, log *slog.Logger) (*LockWatcher, error) {+ inode, err := getFileInode(lockFile)+ if err != nil {+ return nil, err+ }++ return &LockWatcher{+ originalInode: inode,+ cancel: cancel,+ log: log,+ }, nil+}++func (w *LockWatcher) Run(ctx context.Context) {+ ticker := time.NewTicker(lockWatchInterval)+ defer ticker.Stop()++ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ w.check()+ }+ }+}++func (w *LockWatcher) check() {+ info, err := os.Stat(lockFileName)+ if err != nil {+ w.log.Error("Lock file disappeared, shutting down", "file", lockFileName, "error", err)+ w.cancel()++ return+ }++ currentInode, err := getStatInode(info)+ if err != nil {+ w.log.Error("Failed to read lock file inode, shutting down", "error", err)+ w.cancel()++ return+ }++ if currentInode != w.originalInode {+ w.log.Error("Lock file was replaced (inode changed), shutting down",+ "originalInode", w.originalInode,+ "currentInode", currentInode,+ )+ w.cancel()+ }+}++func getFileInode(f *os.File) (uint64, error) {+ info, err := f.Stat()+ if err != nil {+ return 0, err+ }++ return getStatInode(info)+}++func getStatInode(info os.FileInfo) (uint64, error) {+ stat, ok := info.Sys().(*syscall.Stat_t)+ if !ok {+ return 0, os.ErrInvalid+ }++ return stat.Ino, nil+}diff --git a/agent/internal/features/start/lock_watcher_test.go b/agent/internal/features/start/lock_watcher_test.gonew file mode 100644index 00000000..d1e3ab27--- /dev/null+++ b/agent/internal/features/start/lock_watcher_test.go@@ -0,0 +1,110 @@+//go:build !windows++package start++import (+ "context"+ "os"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ "databasus-agent/internal/logger"+)++func Test_NewLockWatcher_CapturesInode(t *testing.T) {+ setupTempDir(t)+ log := logger.GetLogger()++ lockFile, err := AcquireLock(log)+ require.NoError(t, err)+ defer ReleaseLock(lockFile)++ _, cancel := context.WithCancel(context.Background())+ defer cancel()++ watcher, err := NewLockWatcher(lockFile, cancel, log)+ require.NoError(t, err)+ assert.NotZero(t, watcher.originalInode)+}++func Test_LockWatcher_FileUnchanged_ContextNotCancelled(t *testing.T) {+ setupTempDir(t)+ log := logger.GetLogger()++ lockFile, err := AcquireLock(log)+ require.NoError(t, err)+ defer ReleaseLock(lockFile)++ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()++ watcher, err := NewLockWatcher(lockFile, cancel, log)+ require.NoError(t, err)++ watcher.check()+ watcher.check()+ watcher.check()++ select {+ case <-ctx.Done():+ t.Fatal("context should not be cancelled when lock file is unchanged")+ default:+ }+}++func Test_LockWatcher_FileDeleted_CancelsContext(t *testing.T) {+ setupTempDir(t)+ log := logger.GetLogger()++ lockFile, err := AcquireLock(log)+ require.NoError(t, err)+ defer ReleaseLock(lockFile)++ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()++ watcher, err := NewLockWatcher(lockFile, cancel, log)+ require.NoError(t, err)++ err = os.Remove(lockFileName)+ require.NoError(t, err)++ watcher.check()++ select {+ case <-ctx.Done():+ default:+ t.Fatal("context should be cancelled when lock file is deleted")+ }+}++func Test_LockWatcher_FileReplacedWithDifferentInode_CancelsContext(t *testing.T) {+ setupTempDir(t)+ log := logger.GetLogger()++ lockFile, err := AcquireLock(log)+ require.NoError(t, err)+ defer ReleaseLock(lockFile)++ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()++ watcher, err := NewLockWatcher(lockFile, cancel, log)+ require.NoError(t, err)++ err = os.Remove(lockFileName)+ require.NoError(t, err)++ err = os.WriteFile(lockFileName, []byte("99999\n"), 0o644)+ require.NoError(t, err)++ watcher.check()++ select {+ case <-ctx.Done():+ default:+ t.Fatal("context should be cancelled when lock file inode changes")+ }+}diff --git a/agent/internal/features/start/lock_watcher_windows.go b/agent/internal/features/start/lock_watcher_windows.gonew file mode 100644index 00000000..3b41ba8e--- /dev/null+++ b/agent/internal/features/start/lock_watcher_windows.go@@ -0,0 +1,17 @@+//go:build windows++package start++import (+ "context"+ "log/slog"+ "os"+)++type LockWatcher struct{}++func NewLockWatcher(_ *os.File, _ context.CancelFunc, _ *slog.Logger) (*LockWatcher, error) {+ return &LockWatcher{}, nil+}++func (w *LockWatcher) Run(_ context.Context) {}diff --git a/agent/internal/features/start/start.go b/agent/internal/features/start/start.goindex 2e0ada6f..8cdd438d 100644--- a/agent/internal/features/start/start.go+++ b/agent/internal/features/start/start.go@@ -66,6 +66,12 @@ func RunDaemon(cfg *config.Config, log *slog.Logger) error {ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)defer cancel()+ watcher, err := NewLockWatcher(lockFile, cancel, log)+ if err != nil {+ return fmt.Errorf("failed to initialize lock watcher: %w", err)+ }+ go watcher.Run(ctx)+apiClient := api.NewClient(cfg.DatabasusHost, cfg.Token, log)fullBackuper := full_backup.NewFullBackuper(cfg, apiClient, log)