PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (verification): Scale the per-job disk gap so a 5 GB agent can claim small jobs
Rostislav Duginf9dde57e37ba
diff --git a/backend/internal/features/verification/runs/agent_controller_test.go b/backend/internal/features/verification/runs/agent_controller_test.goindex 96bbed14..12abd3c5 100644--- a/backend/internal/features/verification/runs/agent_controller_test.go+++ b/backend/internal/features/verification/runs/agent_controller_test.go@@ -74,6 +74,37 @@ func Test_ClaimVerification_WhenPendingFitsBudget_ReturnsJobAssignment(t *testinassert.Equal(t, agent.Agent.ID, *updated.AgentID)}+func Test_ClaimVerification_WhenAgentHasFiveGbAndBackupIsTiny_ReturnsJobAssignment(t *testing.T) {+ router := createTestRouter()+ owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleAdmin)+ workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "ws "+uuid.New().String(), owner, router)+ defer workspaces_testing.RemoveTestWorkspace(t.Context(), workspace, router)++ testStorage := storages.CreateTestStorage(workspace.ID)+ defer storages.RemoveTestStorage(t.Context(), testStorage.ID)++ notifier := notifiers.CreateTestNotifier(workspace.ID)+ defer notifiers.RemoveTestNotifier(notifier)++ database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier)+ defer databases.RemoveTestDatabase(t.Context(), database)++ tinyBackup := backuping_logical.SeedTestBackup(t, database.ID, testStorage.ID, 0.01)++ enqueuedVerification := EnqueueManualVerificationViaAPI(t, router, owner.Token, tinyBackup.ID)++ agent := verification_agents.CreateTestVerificationAgent(t, router, owner.Token, "small-disk-"+uuid.New().String())+ defer verification_agents.RemoveTestVerificationAgent(t, router, owner.Token, agent.Agent.ID)++ assignment := ClaimVerificationViaAPI(+ t, router, agent.Agent.ID, agent.Token,+ AgentCapacity{MaxCPU: 2, MaxRAMMb: 1024, MaxDiskGb: 5, MaxConcurrentJobs: 1},+ )++ assert.Equal(t, enqueuedVerification.ID, assignment.VerificationID)+ assert.Equal(t, tinyBackup.ID, assignment.BackupID)+}+func Test_ClaimVerification_WhenBackupUsesTimescaleDB_ForwardsTimescaledbVersion(t *testing.T) {router := createTestRouter()owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleAdmin)diff --git a/backend/internal/features/verification/runs/capacity.go b/backend/internal/features/verification/runs/capacity.goindex 27986cf0..2490fa52 100644--- a/backend/internal/features/verification/runs/capacity.go+++ b/backend/internal/features/verification/runs/capacity.go@@ -6,51 +6,58 @@ import (backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical")-// diskCostPerJobGapMb is the per-job safe gap added on top of the downloaded-// backup file and the restored database. The restore's peak on-disk footprint-// exceeds backup + final DB size: pg_restore writes WAL into pg_wal (inside-// PGDATA, counted by the agent's disk watcher), parallel index builds spill-// sort/temp space, and the whole-cluster baseline plus FS slack add fixed cost.-// 5 GB of absolute headroom covers this across the realistic DB-size range.-const diskCostPerJobGapMb = 5120--// IsVerificationFitWithinRemainedDiskCapacity reports whether-// running candidateBackup on this agent alongside runningBackups-// stays within the agent's declared disk capacity.-func IsVerificationFitWithinRemainedDiskCapacity(+// A restore peaks above archive + final DB size: WAL, index sort spill and the+// cluster baseline all land inside the disk the agent watches. That overhead+// roughly tracks the data loaded, hence the ratio; the floor covers a near-empty+// cluster, and the ceiling keeps a huge database from demanding unbounded+// headroom. BackupRawDbSizeMb is 0 when the size probe failed, so 0 is+// "unknown", not "empty".+const (+ minDiskGapMb = 1536+ maxDiskGapMb = 5120+ diskGapToRestoredSizeRatio = 0.5+)++type DiskAdmission struct {+ IsFit bool+ CandidateRequiredMb int64+ RunningUsedMb int64+ TotalBudgetMb int64+}++func EvaluateDiskAdmission(capacity AgentCapacity,runningBackups []*backups_core_logical.LogicalBackup,candidateBackup *backups_core_logical.LogicalBackup,-) bool {+) DiskAdmission {+ admission := DiskAdmission{+ RunningUsedMb: sumEstimatedRequiredDiskMb(runningBackups),+ TotalBudgetMb: int64(capacity.MaxDiskGb) * 1024,+ }++ if candidateBackup != nil {+ admission.CandidateRequiredMb = EstimateRequiredForRestoreDiskMb(candidateBackup)+ }+if capacity.MaxDiskGb <= 0 || candidateBackup == nil {- return false+ return admission}- totalBudgetMb := int64(capacity.MaxDiskGb) * 1024- usedMb := sumEstimatedRequiredDiskMb(runningBackups)- candidateCostMb := EstimateRequiredForRestoreDiskMb(candidateBackup)+ admission.IsFit = admission.RunningUsedMb+admission.CandidateRequiredMb <= admission.TotalBudgetMb- return usedMb+candidateCostMb <= totalBudgetMb+ return admission}-// EstimateRequiredForRestoreDiskMb estimates expected job's on-disk cost.-//-// Includes:-// - Space needed for backup file (if archived - decompressed on the fly while streaming)-// - Space needed for restored database-// - Safe gap (WAL, indexes, sort/temp spills, FS slack)func EstimateRequiredForRestoreDiskMb(backup *backups_core_logical.LogicalBackup) int64 {- archiveSizeMb := backup.BackupSizeMb- if archiveSizeMb < 0 {- archiveSizeMb = 0- }+ archiveSizeMb := max(backup.BackupSizeMb, 0)+ restoredSizeMb := max(backup.BackupRawDbSizeMb, 0)- restoredSizeMb := backup.BackupRawDbSizeMb- if restoredSizeMb < 0 {- restoredSizeMb = 0+ gapMb := float64(maxDiskGapMb)+ if restoredSizeMb > 0 {+ gapMb = min(maxDiskGapMb, max(minDiskGapMb, restoredSizeMb*diskGapToRestoredSizeRatio))}- return int64(math.Ceil(archiveSizeMb+restoredSizeMb)) + diskCostPerJobGapMb+ return int64(math.Ceil(archiveSizeMb + restoredSizeMb + gapMb))}func sumEstimatedRequiredDiskMb(backups []*backups_core_logical.LogicalBackup) int64 {diff --git a/backend/internal/features/verification/runs/capacity_test.go b/backend/internal/features/verification/runs/capacity_test.goindex fe96527e..56ff8dae 100644--- a/backend/internal/features/verification/runs/capacity_test.go+++ b/backend/internal/features/verification/runs/capacity_test.go@@ -15,35 +15,35 @@ func backupWithSize(archiveMb, restoredMb float64) *backups_core_logical.Logical}}-func Test_DoesVerificationFit_AcrossScenarios_RespectsAgentBudget(t *testing.T) {+func Test_EvaluateDiskAdmission_AcrossScenarios_RespectsAgentBudget(t *testing.T) {cases := []struct {- name string- capacity AgentCapacity- runningBackups []*backups_core_logical.LogicalBackup- candidate *backups_core_logical.LogicalBackup- wantFits bool+ name string+ capacity AgentCapacity+ runningBackups []*backups_core_logical.LogicalBackup+ candidate *backups_core_logical.LogicalBackup+ isFitWithinBudget bool}{{- name: "empty agent fits small candidate",- capacity: AgentCapacity{MaxDiskGb: 10},- runningBackups: nil,- candidate: backupWithSize(50, 100),- wantFits: true,+ name: "empty agent fits small candidate",+ capacity: AgentCapacity{MaxDiskGb: 10},+ runningBackups: nil,+ candidate: backupWithSize(50, 100),+ isFitWithinBudget: true,},{name: "empty agent fits candidate at exact budget boundary",capacity: AgentCapacity{MaxDiskGb: 10},runningBackups: nil,- // archive + restored + per-job gap = 10240 (full 10 GB budget)- candidate: backupWithSize(4000, 1120),- wantFits: true,+ // archive 4096 + restored 4096 + gap 2048 = 10240 (the full 10 GB budget)+ candidate: backupWithSize(4096, 4096),+ isFitWithinBudget: true,},{- name: "empty agent rejects candidate one MB over budget",- capacity: AgentCapacity{MaxDiskGb: 10},- runningBackups: nil,- candidate: backupWithSize(4000, 1121),- wantFits: false,+ name: "empty agent rejects candidate one MB over budget",+ capacity: AgentCapacity{MaxDiskGb: 10},+ runningBackups: nil,+ candidate: backupWithSize(4097, 4096),+ isFitWithinBudget: false,},{name: "one running, room for another small",@@ -51,8 +51,8 @@ func Test_DoesVerificationFit_AcrossScenarios_RespectsAgentBudget(t *testing.T)runningBackups: []*backups_core_logical.LogicalBackup{backupWithSize(50, 200),},- candidate: backupWithSize(50, 200),- wantFits: true,+ candidate: backupWithSize(50, 200),+ isFitWithinBudget: true,},{name: "one running, restored size of candidate blows budget",@@ -60,8 +60,8 @@ func Test_DoesVerificationFit_AcrossScenarios_RespectsAgentBudget(t *testing.T)runningBackups: []*backups_core_logical.LogicalBackup{backupWithSize(500, 4000),},- candidate: backupWithSize(500, 4000),- wantFits: false,+ candidate: backupWithSize(500, 4000),+ isFitWithinBudget: false,},{name: "three concurrent small jobs fit in 25 GB",@@ -71,8 +71,8 @@ func Test_DoesVerificationFit_AcrossScenarios_RespectsAgentBudget(t *testing.T)backupWithSize(100, 500),backupWithSize(100, 500),},- candidate: backupWithSize(100, 500),- wantFits: true,+ candidate: backupWithSize(100, 500),+ isFitWithinBudget: true,},{name: "per-job gap saturation: three RUNNING already over a 5 GB agent",@@ -82,55 +82,116 @@ func Test_DoesVerificationFit_AcrossScenarios_RespectsAgentBudget(t *testing.T)backupWithSize(100, 400),backupWithSize(100, 400),},- candidate: backupWithSize(10, 10),- wantFits: false,+ candidate: backupWithSize(10, 10),+ isFitWithinBudget: false,},{- name: "zero-size candidate still consumes the per-job gap and fits",- capacity: AgentCapacity{MaxDiskGb: 10},- runningBackups: nil,- candidate: backupWithSize(0, 0),- wantFits: true,+ name: "tiny database fits a 5 GB agent",+ capacity: AgentCapacity{MaxDiskGb: 5},+ runningBackups: nil,+ candidate: backupWithSize(0, 0.01),+ isFitWithinBudget: true,},{- name: "negative sizes clamp to zero and still fit",- capacity: AgentCapacity{MaxDiskGb: 10},- runningBackups: nil,- candidate: backupWithSize(-50, -100),- wantFits: true,+ name: "unknown raw DB size keeps the full gap and fits a roomy agent",+ capacity: AgentCapacity{MaxDiskGb: 10},+ runningBackups: nil,+ candidate: backupWithSize(200, 0),+ isFitWithinBudget: true,},{- name: "zero-capacity agent rejects everything",- capacity: AgentCapacity{MaxDiskGb: 0},- runningBackups: nil,- candidate: backupWithSize(1, 1),- wantFits: false,+ name: "unknown raw DB size keeps the full gap and does not fit a 5 GB agent",+ capacity: AgentCapacity{MaxDiskGb: 5},+ runningBackups: nil,+ candidate: backupWithSize(200, 0),+ isFitWithinBudget: false,},{- name: "nil candidate is rejected",- capacity: AgentCapacity{MaxDiskGb: 10},- runningBackups: nil,- candidate: nil,- wantFits: false,+ name: "zero-size candidate still consumes the per-job gap and fits",+ capacity: AgentCapacity{MaxDiskGb: 10},+ runningBackups: nil,+ candidate: backupWithSize(0, 0),+ isFitWithinBudget: true,},{- name: "raw DB size dominates archive size in the cost",- capacity: AgentCapacity{MaxDiskGb: 2},- runningBackups: nil,- candidate: backupWithSize(10, 2000),- wantFits: false,+ name: "negative sizes clamp to zero and still fit",+ capacity: AgentCapacity{MaxDiskGb: 10},+ runningBackups: nil,+ candidate: backupWithSize(-50, -100),+ isFitWithinBudget: true,+ },+ {+ name: "zero-capacity agent rejects everything",+ capacity: AgentCapacity{MaxDiskGb: 0},+ runningBackups: nil,+ candidate: backupWithSize(1, 1),+ isFitWithinBudget: false,+ },+ {+ name: "nil candidate is rejected",+ capacity: AgentCapacity{MaxDiskGb: 10},+ runningBackups: nil,+ candidate: nil,+ isFitWithinBudget: false,+ },+ {+ name: "raw DB size dominates archive size in the cost",+ capacity: AgentCapacity{MaxDiskGb: 2},+ runningBackups: nil,+ candidate: backupWithSize(10, 2000),+ isFitWithinBudget: false,},}for _, testCase := range cases {t.Run(testCase.name, func(t *testing.T) {- fits := IsVerificationFitWithinRemainedDiskCapacity(+ admission := EvaluateDiskAdmission(testCase.capacity,testCase.runningBackups,testCase.candidate,)- assert.Equal(t, testCase.wantFits, fits)+ assert.Equal(t, testCase.isFitWithinBudget, admission.IsFit)+ })+ }+}++func Test_EstimateRequiredForRestoreDiskMb_AcrossRestoredSizes_ClampsGapBetweenFloorAndCap(t *testing.T) {+ cases := []struct {+ name string+ backup *backups_core_logical.LogicalBackup+ requiredDiskMb int64+ }{+ {+ name: "unknown raw DB size falls back to the gap cap",+ backup: backupWithSize(200, 0),+ requiredDiskMb: 200 + maxDiskGapMb,+ },+ {+ name: "tiny restored size uses the gap floor",+ backup: backupWithSize(0, 0.01),+ requiredDiskMb: 1 + minDiskGapMb,+ },+ {+ name: "restored size below the floor crossover uses the gap floor",+ backup: backupWithSize(100, 2000),+ requiredDiskMb: 2100 + minDiskGapMb,+ },+ {+ name: "restored size between floor and cap scales the gap",+ backup: backupWithSize(500, 6000),+ requiredDiskMb: 6500 + 3000,+ },+ {+ name: "restored size above 10 GB pins the gap at the cap",+ backup: backupWithSize(4000, 40000),+ requiredDiskMb: 44000 + maxDiskGapMb,+ },+ }++ for _, testCase := range cases {+ t.Run(testCase.name, func(t *testing.T) {+ assert.Equal(t, testCase.requiredDiskMb, EstimateRequiredForRestoreDiskMb(testCase.backup))})}}diff --git a/backend/internal/features/verification/runs/service.go b/backend/internal/features/verification/runs/service.goindex 90287113..1a117d45 100644--- a/backend/internal/features/verification/runs/service.go+++ b/backend/internal/features/verification/runs/service.go@@ -258,6 +258,8 @@ func (s *VerificationService) ClaimVerification(agent *verification_agents.Agent,req *ClaimRequest,) (*JobAssignment, error) {+ logger := s.logger.With("agent_id", agent.ID)+runningBackups, err := s.verificationRepository.ListRunningBackupsByAgentID(agent.ID)if err != nil {return nil, err@@ -272,7 +274,18 @@ func (s *VerificationService) ClaimVerification(}for _, candidate := range candidates {- if !IsVerificationFitWithinRemainedDiskCapacity(req.Capacity, runningBackups, candidate.Backup) {+ candidateLogger := logger.With("verification_id", candidate.Verification.ID)++ admission := EvaluateDiskAdmission(req.Capacity, runningBackups, candidate.Backup)+ if !admission.IsFit {+ candidateLogger.DebugContext(+ ctx,+ fmt.Sprintf(+ "skipping claim: needs %d MB, %d MB already used of %d MB budget",+ admission.CandidateRequiredMb, admission.RunningUsedMb, admission.TotalBudgetMb,+ ),+ )+continue}@@ -282,10 +295,9 @@ func (s *VerificationService) ClaimVerification(}if validateErr := validateDatabaseIsVerifiable(database); validateErr != nil {- s.logger.WarnContext(+ candidateLogger.WarnContext(ctx,"skipping claim: database not verifiable",- "verification_id", candidate.Verification.ID,"database_id", candidate.Verification.DatabaseID,"error", validateErr,)diff --git a/website/app/restore-verification/page.tsx b/website/app/restore-verification/page.tsxindex 7c116dad..eae4a3f9 100644--- a/website/app/restore-verification/page.tsx+++ b/website/app/restore-verification/page.tsx@@ -177,7 +177,7 @@ export default function RestoreVerificationPage() {Disk capacity for each verification job covering the{" "}<strong>backup file size</strong>, the{" "}<strong>database raw size</strong> and a{" "}- <strong>5 GB safety gap</strong> on top.+ <strong>safety gap</strong> on top.</li><li>At least 1 CPU core and 512 MB of RAM available per concurrent@@ -319,7 +319,7 @@ export default function RestoreVerificationPage() {concurrency. The disk budget is the easiest to get wrong: eachjob needs enough space for the <strong>backup file size</strong>, the <strong>database raw size</strong> and a{" "}- <strong>5 GB safety gap</strong> on top, so set{" "}+ <strong>safety gap of up to 5 GB</strong> on top, so set{" "}<code>--max-disk-gb</code> comfortably above that for yourlargest database.</p>