PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FEATURE (backups): Add backups cancelling
Rostislav Dugin9ea795b48f8e
diff --git a/backend/internal/features/backups/backups/backup_context_manager.go b/backend/internal/features/backups/backups/backup_context_manager.gonew file mode 100644index 00000000..dd2803f1--- /dev/null+++ b/backend/internal/features/backups/backups/backup_context_manager.go@@ -0,0 +1,47 @@+package backups++import (+ "context"+ "errors"+ "sync"++ "github.com/google/uuid"+)++type BackupContextManager struct {+ mu sync.RWMutex+ cancelFuncs map[uuid.UUID]context.CancelFunc+}++func NewBackupContextManager() *BackupContextManager {+ return &BackupContextManager{+ cancelFuncs: make(map[uuid.UUID]context.CancelFunc),+ }+}++func (m *BackupContextManager) RegisterBackup(backupID uuid.UUID, cancelFunc context.CancelFunc) {+ m.mu.Lock()+ defer m.mu.Unlock()+ m.cancelFuncs[backupID] = cancelFunc+}++func (m *BackupContextManager) CancelBackup(backupID uuid.UUID) error {+ m.mu.Lock()+ defer m.mu.Unlock()++ cancelFunc, exists := m.cancelFuncs[backupID]+ if !exists {+ return errors.New("backup is not in progress or already completed")+ }++ cancelFunc()+ delete(m.cancelFuncs, backupID)++ return nil+}++func (m *BackupContextManager) UnregisterBackup(backupID uuid.UUID) {+ m.mu.Lock()+ defer m.mu.Unlock()+ delete(m.cancelFuncs, backupID)+}diff --git a/backend/internal/features/backups/backups/controller.go b/backend/internal/features/backups/backups/controller.goindex 9718b992..c5285819 100644--- a/backend/internal/features/backups/backups/controller.go+++ b/backend/internal/features/backups/backups/controller.go@@ -19,6 +19,7 @@ func (c *BackupController) RegisterRoutes(router *gin.RouterGroup) {router.POST("/backups", c.MakeBackup)router.GET("/backups/:id/file", c.GetFile)router.DELETE("/backups/:id", c.DeleteBackup)+ router.POST("/backups/:id/cancel", c.CancelBackup)}// GetBackups@@ -126,6 +127,37 @@ func (c *BackupController) DeleteBackup(ctx *gin.Context) {ctx.Status(http.StatusNoContent)}+// CancelBackup+// @Summary Cancel an in-progress backup+// @Description Cancel a backup that is currently in progress+// @Tags backups+// @Param id path string true "Backup ID"+// @Success 204+// @Failure 400+// @Failure 401+// @Failure 500+// @Router /backups/{id}/cancel [post]+func (c *BackupController) CancelBackup(ctx *gin.Context) {+ user, ok := users_middleware.GetUserFromContext(ctx)+ if !ok {+ ctx.JSON(http.StatusUnauthorized, gin.H{"error": "User not authenticated"})+ return+ }++ id, err := uuid.Parse(ctx.Param("id"))+ if err != nil {+ ctx.JSON(http.StatusBadRequest, gin.H{"error": "invalid backup ID"})+ return+ }++ if err := c.backupService.CancelBackup(user, id); err != nil {+ ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})+ return+ }++ ctx.Status(http.StatusNoContent)+}+// GetFile// @Summary Download a backup file// @Description Download the backup file for the specified backupdiff --git a/backend/internal/features/backups/backups/controller_test.go b/backend/internal/features/backups/backups/controller_test.goindex 8c6c9c3a..ee76c77e 100644--- a/backend/internal/features/backups/backups/controller_test.go+++ b/backend/internal/features/backups/backups/controller_test.go@@ -492,6 +492,77 @@ func Test_DownloadBackup_AuditLogWritten(t *testing.T) {assert.True(t, found, "Audit log for backup download not found")}+func Test_CancelBackup_InProgressBackup_SuccessfullyCancelled(t *testing.T) {+ router := createTestRouter()+ owner := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)+ database := createTestDatabase("Test Database", workspace.ID, owner.Token, router)+ storage := createTestStorage(workspace.ID)++ configService := backups_config.GetBackupConfigService()+ config, err := configService.GetBackupConfigByDbId(database.ID)+ assert.NoError(t, err)++ config.IsBackupsEnabled = true+ config.StorageID = &storage.ID+ config.Storage = storage+ _, err = configService.SaveBackupConfig(config)+ assert.NoError(t, err)++ backup := &Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ Database: database,+ StorageID: storage.ID,+ Storage: storage,+ Status: BackupStatusInProgress,+ BackupSizeMb: 0,+ BackupDurationMs: 0,+ CreatedAt: time.Now().UTC(),+ }++ repo := &BackupRepository{}+ err = repo.Save(backup)+ assert.NoError(t, err)++ // Register a cancellable context for the backup+ GetBackupService().backupContextMgr.RegisterBackup(backup.ID, func() {})++ resp := test_utils.MakePostRequest(+ t,+ router,+ fmt.Sprintf("/api/v1/backups/%s/cancel", backup.ID.String()),+ "Bearer "+owner.Token,+ nil,+ http.StatusNoContent,+ )++ assert.Equal(t, http.StatusNoContent, resp.StatusCode)++ // Verify audit log was created+ admin := users_testing.CreateTestUser(users_enums.UserRoleAdmin)+ userService := users_services.GetUserService()+ adminUser, err := userService.GetUserFromToken(admin.Token)+ assert.NoError(t, err)++ auditLogService := audit_logs.GetAuditLogService()+ auditLogs, err := auditLogService.GetGlobalAuditLogs(+ adminUser,+ &audit_logs.GetAuditLogsRequest{Limit: 100, Offset: 0},+ )+ assert.NoError(t, err)++ foundCancelLog := false+ for _, log := range auditLogs.AuditLogs {+ if strings.Contains(log.Message, "Backup cancelled") &&+ strings.Contains(log.Message, database.Name) {+ foundCancelLog = true+ break+ }+ }+ assert.True(t, foundCancelLog, "Cancel audit log should be created")+}+func createTestRouter() *gin.Engine {return CreateTestRouter()}diff --git a/backend/internal/features/backups/backups/di.go b/backend/internal/features/backups/backups/di.goindex 29dd0ae0..f3a0c80f 100644--- a/backend/internal/features/backups/backups/di.go+++ b/backend/internal/features/backups/backups/di.go@@ -13,6 +13,9 @@ import ()var backupRepository = &BackupRepository{}++var backupContextManager = NewBackupContextManager()+var backupService = &BackupService{databases.GetDatabaseService(),storages.GetStorageService(),@@ -25,6 +28,7 @@ var backupService = &BackupService{[]BackupRemoveListener{},workspaces_services.GetWorkspaceService(),audit_logs.GetAuditLogService(),+ backupContextManager,}var backupBackgroundService = &BackupBackgroundService{diff --git a/backend/internal/features/backups/backups/enums.go b/backend/internal/features/backups/backups/enums.goindex ef06b19b..7335df0d 100644--- a/backend/internal/features/backups/backups/enums.go+++ b/backend/internal/features/backups/backups/enums.go@@ -6,4 +6,5 @@ const (BackupStatusInProgress BackupStatus = "IN_PROGRESS"BackupStatusCompleted BackupStatus = "COMPLETED"BackupStatusFailed BackupStatus = "FAILED"+ BackupStatusCanceled BackupStatus = "CANCELED")diff --git a/backend/internal/features/backups/backups/interfaces.go b/backend/internal/features/backups/backups/interfaces.goindex b4b7fb6e..072177b5 100644--- a/backend/internal/features/backups/backups/interfaces.go+++ b/backend/internal/features/backups/backups/interfaces.go@@ -1,6 +1,8 @@package backupsimport (+ "context"+backups_config "postgresus-backend/internal/features/backups/config""postgresus-backend/internal/features/databases""postgresus-backend/internal/features/notifiers"@@ -19,6 +21,7 @@ type NotificationSender interface {type CreateBackupUsecase interface {Execute(+ ctx context.Context,backupID uuid.UUID,backupConfig *backups_config.BackupConfig,database *databases.Database,diff --git a/backend/internal/features/backups/backups/service.go b/backend/internal/features/backups/backups/service.goindex 5a4db7e5..b89bbdec 100644--- a/backend/internal/features/backups/backups/service.go+++ b/backend/internal/features/backups/backups/service.go@@ -1,6 +1,7 @@package backupsimport (+ "context""errors""fmt""io"@@ -13,6 +14,7 @@ import (users_models "postgresus-backend/internal/features/users/models"workspaces_services "postgresus-backend/internal/features/workspaces/services""slices"+ "strings""time""github.com/google/uuid"@@ -34,6 +36,7 @@ type BackupService struct {workspaceService *workspaces_services.WorkspaceServiceauditLogService *audit_logs.AuditLogService+ backupContextMgr *BackupContextManager}func (s *BackupService) AddBackupRemoveListener(listener BackupRemoveListener) {@@ -247,7 +250,12 @@ func (s *BackupService) MakeBackup(databaseID uuid.UUID, isLastTry bool) {}}+ ctx, cancel := context.WithCancel(context.Background())+ s.backupContextMgr.RegisterBackup(backup.ID, cancel)+ defer s.backupContextMgr.UnregisterBackup(backup.ID)+err = s.createBackupUseCase.Execute(+ ctx,backup.ID,backupConfig,database,@@ -256,6 +264,34 @@ func (s *BackupService) MakeBackup(databaseID uuid.UUID, isLastTry bool) {)if err != nil {errMsg := err.Error()++ // Check if backup was cancelled (not due to shutdown)+ if strings.Contains(errMsg, "backup cancelled") && !strings.Contains(errMsg, "shutdown") {+ backup.Status = BackupStatusCanceled+ backup.BackupDurationMs = time.Since(start).Milliseconds()+ backup.BackupSizeMb = 0++ if err := s.backupRepository.Save(backup); err != nil {+ s.logger.Error("Failed to save cancelled backup", "error", err)+ }++ // Delete partial backup from storage+ storage, storageErr := s.storageService.GetStorageByID(backup.StorageID)+ if storageErr == nil {+ if deleteErr := storage.DeleteFile(backup.ID); deleteErr != nil {+ s.logger.Error(+ "Failed to delete partial backup file",+ "backupId",+ backup.ID,+ "error",+ deleteErr,+ )+ }+ }++ return+ }+backup.FailMessage = &errMsgbackup.Status = BackupStatusFailedbackup.BackupDurationMs = time.Since(start).Milliseconds()@@ -382,6 +418,48 @@ func (s *BackupService) GetBackup(backupID uuid.UUID) (*Backup, error) {return s.backupRepository.FindByID(backupID)}+func (s *BackupService) CancelBackup(+ user *users_models.User,+ backupID uuid.UUID,+) error {+ backup, err := s.backupRepository.FindByID(backupID)+ if err != nil {+ return err+ }++ if backup.Database.WorkspaceID == nil {+ return errors.New("cannot cancel backup for database without workspace")+ }++ canManage, err := s.workspaceService.CanUserManageDBs(*backup.Database.WorkspaceID, user)+ if err != nil {+ return err+ }+ if !canManage {+ return errors.New("insufficient permissions to cancel backup for this database")+ }++ if backup.Status != BackupStatusInProgress {+ return errors.New("backup is not in progress")+ }++ if err := s.backupContextMgr.CancelBackup(backupID); err != nil {+ return err+ }++ s.auditLogService.WriteAuditLog(+ fmt.Sprintf(+ "Backup cancelled for database: %s (ID: %s)",+ backup.Database.Name,+ backupID.String(),+ ),+ &user.ID,+ backup.Database.WorkspaceID,+ )++ return nil+}+func (s *BackupService) GetBackupFile(user *users_models.User,backupID uuid.UUID,diff --git a/backend/internal/features/backups/backups/service_test.go b/backend/internal/features/backups/backups/service_test.goindex 02f0cff1..f9ef3793 100644--- a/backend/internal/features/backups/backups/service_test.go+++ b/backend/internal/features/backups/backups/service_test.go@@ -1,6 +1,7 @@package backupsimport (+ "context""errors"backups_config "postgresus-backend/internal/features/backups/config""postgresus-backend/internal/features/databases"@@ -56,6 +57,7 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {[]BackupRemoveListener{},nil, // workspaceServicenil, // auditLogService+ NewBackupContextManager(),}// Set up expectations@@ -101,6 +103,7 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {[]BackupRemoveListener{},nil, // workspaceServicenil, // auditLogService+ NewBackupContextManager(),}backupService.MakeBackup(database.ID, true)@@ -123,6 +126,7 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {[]BackupRemoveListener{},nil, // workspaceServicenil, // auditLogService+ NewBackupContextManager(),}// capture arguments@@ -158,6 +162,7 @@ type CreateFailedBackupUsecase struct {}func (uc *CreateFailedBackupUsecase) Execute(+ ctx context.Context,backupID uuid.UUID,backupConfig *backups_config.BackupConfig,database *databases.Database,@@ -174,6 +179,7 @@ type CreateSuccessBackupUsecase struct {}func (uc *CreateSuccessBackupUsecase) Execute(+ ctx context.Context,backupID uuid.UUID,backupConfig *backups_config.BackupConfig,database *databases.Database,diff --git a/backend/internal/features/backups/backups/usecases/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/create_backup_uc.goindex 75d181db..61968f1f 100644--- a/backend/internal/features/backups/backups/usecases/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/create_backup_uc.go@@ -1,6 +1,7 @@package usecasesimport (+ "context""errors"usecases_postgresql "postgresus-backend/internal/features/backups/backups/usecases/postgresql"backups_config "postgresus-backend/internal/features/backups/config"@@ -16,6 +17,7 @@ type CreateBackupUsecase struct {// Execute creates a backup of the database and returns the backup size in MBfunc (uc *CreateBackupUsecase) Execute(+ ctx context.Context,backupID uuid.UUID,backupConfig *backups_config.BackupConfig,database *databases.Database,@@ -26,6 +28,7 @@ func (uc *CreateBackupUsecase) Execute() error {if database.Type == databases.DatabaseTypePostgres {return uc.CreatePostgresqlBackupUsecase.Execute(+ ctx,backupID,backupConfig,database,diff --git a/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.goindex 174a4dc3..4bcfd55a 100644--- a/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go@@ -29,6 +29,7 @@ type CreatePostgresqlBackupUsecase struct {// Execute creates a backup of the databasefunc (uc *CreatePostgresqlBackupUsecase) Execute(+ ctx context.Context,backupID uuid.UUID,backupConfig *backups_config.BackupConfig,db *databases.Database,@@ -81,6 +82,7 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(}return uc.streamToStorage(+ ctx,backupID,backupConfig,tools.GetPostgresqlExecutable(@@ -99,6 +101,7 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(// streamToStorage streams pg_dump output directly to storagefunc (uc *CreatePostgresqlBackupUsecase) streamToStorage(+ parentCtx context.Context,backupID uuid.UUID,backupConfig *backups_config.BackupConfig,pgBin string,@@ -112,7 +115,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(// if backup not fit into 23 hours, Postgresus// seems not to work for such database size- ctx, cancel := context.WithTimeout(context.Background(), 23*time.Hour)+ ctx, cancel := context.WithTimeout(parentCtx, 23*time.Hour)defer cancel()// Monitor for shutdown and cancel context if needed@@ -272,8 +275,9 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(bytesWritten := <-bytesWrittenChwaitErr := cmd.Wait()- // Check for shutdown before finalizing- if config.IsShouldShutdown() {+ // Check for shutdown or cancellation before finalizing+ select {+ case <-ctx.Done():if pipeWriter, ok := countingWriter.writer.(*io.PipeWriter); ok {if err := pipeWriter.Close(); err != nil {uc.logger.Error("Failed to close counting writer", "error", err)@@ -281,7 +285,12 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(}<-saveErrCh // Wait for storage to finish- return fmt.Errorf("backup cancelled due to shutdown")++ if config.IsShouldShutdown() {+ return fmt.Errorf("backup cancelled due to shutdown")+ }+ return fmt.Errorf("backup cancelled")+ default:}// Close the pipe writer to signal end of data@@ -303,8 +312,13 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(switch {case waitErr != nil:- if config.IsShouldShutdown() {- return fmt.Errorf("backup cancelled due to shutdown")+ select {+ case <-ctx.Done():+ if config.IsShouldShutdown() {+ return fmt.Errorf("backup cancelled due to shutdown")+ }+ return fmt.Errorf("backup cancelled")+ default:}// Enhanced error handling for PostgreSQL connection and SSL issues@@ -402,14 +416,24 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(return errors.New(errorMsg)case copyErr != nil:- if config.IsShouldShutdown() {- return fmt.Errorf("backup cancelled due to shutdown")+ select {+ case <-ctx.Done():+ if config.IsShouldShutdown() {+ return fmt.Errorf("backup cancelled due to shutdown")+ }+ return fmt.Errorf("backup cancelled")+ default:}return fmt.Errorf("copy to storage: %w", copyErr)case saveErr != nil:- if config.IsShouldShutdown() {- return fmt.Errorf("backup cancelled due to shutdown")+ select {+ case <-ctx.Done():+ if config.IsShouldShutdown() {+ return fmt.Errorf("backup cancelled due to shutdown")+ }+ return fmt.Errorf("backup cancelled")+ default:}return fmt.Errorf("save to storage: %w", saveErr)diff --git a/backend/internal/features/tests/postgresql_backup_restore_test.go b/backend/internal/features/tests/postgresql_backup_restore_test.goindex 06bc2a3c..3f203cc4 100644--- a/backend/internal/features/tests/postgresql_backup_restore_test.go+++ b/backend/internal/features/tests/postgresql_backup_restore_test.go@@ -1,6 +1,7 @@package testsimport (+ "context""fmt""os""path/filepath"@@ -138,6 +139,7 @@ func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {// Make backupprogressTracker := func(completedMBs float64) {}err = usecases_postgresql_backup.GetCreatePostgresqlBackupUsecase().Execute(+ context.Background(),backupID,backupConfig,backupDb,diff --git a/frontend/src/entity/backups/api/backupsApi.ts b/frontend/src/entity/backups/api/backupsApi.tsindex 957b5f0e..5c3da5f9 100644--- a/frontend/src/entity/backups/api/backupsApi.ts+++ b/frontend/src/entity/backups/api/backupsApi.ts@@ -32,4 +32,8 @@ export const backupsApi = {async downloadBackup(id: string): Promise<Blob> {return apiHelper.fetchGetBlob(`${getApplicationServer()}/api/v1/backups/${id}/file`);},++ async cancelBackup(id: string) {+ return apiHelper.fetchPostRaw(`${getApplicationServer()}/api/v1/backups/${id}/cancel`);+ },};diff --git a/frontend/src/entity/backups/model/BackupStatus.ts b/frontend/src/entity/backups/model/BackupStatus.tsindex dbe76513..3fd0679a 100644--- a/frontend/src/entity/backups/model/BackupStatus.ts+++ b/frontend/src/entity/backups/model/BackupStatus.ts@@ -3,4 +3,5 @@ export enum BackupStatus {COMPLETED = 'COMPLETED',FAILED = 'FAILED',DELETED = 'DELETED',+ CANCELED = 'CANCELED',}diff --git a/frontend/src/features/backups/ui/BackupsComponent.tsx b/frontend/src/features/backups/ui/BackupsComponent.tsxindex 19b4f416..ff1ce068 100644--- a/frontend/src/features/backups/ui/BackupsComponent.tsx+++ b/frontend/src/features/backups/ui/BackupsComponent.tsx@@ -1,5 +1,6 @@import {CheckCircleOutlined,+ CloseCircleOutlined,CloudUploadOutlined,DeleteOutlined,DownloadOutlined,@@ -57,6 +58,7 @@ export const BackupsComponent = ({ database, isCanManageDBs, scrollContainerRefconst isLazyLoadInProgress = useRef(false);const [downloadingBackupId, setDownloadingBackupId] = useState<string | undefined>();+ const [cancellingBackupId, setCancellingBackupId] = useState<string | undefined>();const downloadBackup = async (backupId: string) => {try {@@ -204,6 +206,19 @@ export const BackupsComponent = ({ database, isCanManageDBs, scrollContainerRefsetDeleteConfimationId(undefined);};+ const cancelBackup = async (backupId: string) => {+ setCancellingBackupId(backupId);++ try {+ await backupsApi.cancelBackup(backupId);+ await reloadInProgressBackups();+ } catch (e) {+ alert((e as Error).message);+ }++ setCancellingBackupId(undefined);+ };+useEffect(() => {setIsBackupConfigLoading(true);setCurrentLimit(BACKUPS_PAGE_SIZE);@@ -324,6 +339,15 @@ export const BackupsComponent = ({ database, isCanManageDBs, scrollContainerRef);}+ if (status === BackupStatus.CANCELED) {+ return (+ <div className="flex items-center text-gray-600">+ <CloseCircleOutlined className="mr-2" style={{ fontSize: 16 }} />+ <div>Canceled</div>+ </div>+ );+ }+return <span className="font-bold">{status}</span>;},filters: [@@ -343,6 +367,10 @@ export const BackupsComponent = ({ database, isCanManageDBs, scrollContainerRefvalue: BackupStatus.DELETED,text: 'Deleted',},+ {+ value: BackupStatus.CANCELED,+ text: 'Canceled',+ },],onFilter: (value, record) => record.status === value,},@@ -393,6 +421,25 @@ export const BackupsComponent = ({ database, isCanManageDBs, scrollContainerRefrender: (_, record: Backup) => {return (<div className="flex gap-2 text-lg">+ {record.status === BackupStatus.IN_PROGRESS && isCanManageDBs && (+ <div className="flex gap-2">+ {cancellingBackupId === record.id ? (+ <SyncOutlined spin />+ ) : (+ <Tooltip title="Cancel backup">+ <CloseCircleOutlined+ className="cursor-pointer"+ onClick={() => {+ if (cancellingBackupId) return;+ cancelBackup(record.id);+ }}+ style={{ color: '#ff0000', opacity: cancellingBackupId ? 0.2 : 1 }}+ />+ </Tooltip>+ )}+ </div>+ )}+{record.status === BackupStatus.COMPLETED && (<div className="flex gap-2">{deletingBackupId === record.id ? (