Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (verification): Scale the per-job disk gap so a 5 GB agent can claim small jobs

Rostislav Duginf9dde57e37ba
diff --git a/backend/internal/features/verification/runs/agent_controller_test.go b/backend/internal/features/verification/runs/agent_controller_test.go
index 96bbed14..12abd3c5 100644
--- a/backend/internal/features/verification/runs/agent_controller_test.go
+++ b/backend/internal/features/verification/runs/agent_controller_test.go
@@ -74,6 +74,37 @@ func Test_ClaimVerification_WhenPendingFitsBudget_ReturnsJobAssignment(t *testin
assert.Equal(t, agent.Agent.ID, *updated.AgentID)
}
+func Test_ClaimVerification_WhenAgentHasFiveGbAndBackupIsTiny_ReturnsJobAssignment(t *testing.T) {
+ router := createTestRouter()
+ owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleAdmin)
+ workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "ws "+uuid.New().String(), owner, router)
+ defer workspaces_testing.RemoveTestWorkspace(t.Context(), workspace, router)
+
+ testStorage := storages.CreateTestStorage(workspace.ID)
+ defer storages.RemoveTestStorage(t.Context(), testStorage.ID)
+
+ notifier := notifiers.CreateTestNotifier(workspace.ID)
+ defer notifiers.RemoveTestNotifier(notifier)
+
+ database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier)
+ defer databases.RemoveTestDatabase(t.Context(), database)
+
+ tinyBackup := backuping_logical.SeedTestBackup(t, database.ID, testStorage.ID, 0.01)
+
+ enqueuedVerification := EnqueueManualVerificationViaAPI(t, router, owner.Token, tinyBackup.ID)
+
+ agent := verification_agents.CreateTestVerificationAgent(t, router, owner.Token, "small-disk-"+uuid.New().String())
+ defer verification_agents.RemoveTestVerificationAgent(t, router, owner.Token, agent.Agent.ID)
+
+ assignment := ClaimVerificationViaAPI(
+ t, router, agent.Agent.ID, agent.Token,
+ AgentCapacity{MaxCPU: 2, MaxRAMMb: 1024, MaxDiskGb: 5, MaxConcurrentJobs: 1},
+ )
+
+ assert.Equal(t, enqueuedVerification.ID, assignment.VerificationID)
+ assert.Equal(t, tinyBackup.ID, assignment.BackupID)
+}
+
func Test_ClaimVerification_WhenBackupUsesTimescaleDB_ForwardsTimescaledbVersion(t *testing.T) {
router := createTestRouter()
owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleAdmin)
diff --git a/backend/internal/features/verification/runs/capacity.go b/backend/internal/features/verification/runs/capacity.go
index 27986cf0..2490fa52 100644
--- a/backend/internal/features/verification/runs/capacity.go
+++ b/backend/internal/features/verification/runs/capacity.go
@@ -6,51 +6,58 @@ import (
backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"
)
-// diskCostPerJobGapMb is the per-job safe gap added on top of the downloaded
-// backup file and the restored database. The restore's peak on-disk footprint
-// exceeds backup + final DB size: pg_restore writes WAL into pg_wal (inside
-// PGDATA, counted by the agent's disk watcher), parallel index builds spill
-// sort/temp space, and the whole-cluster baseline plus FS slack add fixed cost.
-// 5 GB of absolute headroom covers this across the realistic DB-size range.
-const diskCostPerJobGapMb = 5120
-
-// IsVerificationFitWithinRemainedDiskCapacity reports whether
-// running candidateBackup on this agent alongside runningBackups
-// stays within the agent's declared disk capacity.
-func IsVerificationFitWithinRemainedDiskCapacity(
+// A restore peaks above archive + final DB size: WAL, index sort spill and the
+// cluster baseline all land inside the disk the agent watches. That overhead
+// roughly tracks the data loaded, hence the ratio; the floor covers a near-empty
+// cluster, and the ceiling keeps a huge database from demanding unbounded
+// headroom. BackupRawDbSizeMb is 0 when the size probe failed, so 0 is
+// "unknown", not "empty".
+const (
+ minDiskGapMb = 1536
+ maxDiskGapMb = 5120
+ diskGapToRestoredSizeRatio = 0.5
+)
+
+type DiskAdmission struct {
+ IsFit bool
+ CandidateRequiredMb int64
+ RunningUsedMb int64
+ TotalBudgetMb int64
+}
+
+func EvaluateDiskAdmission(
capacity AgentCapacity,
runningBackups []*backups_core_logical.LogicalBackup,
candidateBackup *backups_core_logical.LogicalBackup,
-) bool {
+) DiskAdmission {
+ admission := DiskAdmission{
+ RunningUsedMb: sumEstimatedRequiredDiskMb(runningBackups),
+ TotalBudgetMb: int64(capacity.MaxDiskGb) * 1024,
+ }
+
+ if candidateBackup != nil {
+ admission.CandidateRequiredMb = EstimateRequiredForRestoreDiskMb(candidateBackup)
+ }
+
if capacity.MaxDiskGb <= 0 || candidateBackup == nil {
- return false
+ return admission
}
- totalBudgetMb := int64(capacity.MaxDiskGb) * 1024
- usedMb := sumEstimatedRequiredDiskMb(runningBackups)
- candidateCostMb := EstimateRequiredForRestoreDiskMb(candidateBackup)
+ admission.IsFit = admission.RunningUsedMb+admission.CandidateRequiredMb <= admission.TotalBudgetMb
- return usedMb+candidateCostMb <= totalBudgetMb
+ return admission
}
-// EstimateRequiredForRestoreDiskMb estimates expected job's on-disk cost.
-//
-// Includes:
-// - Space needed for backup file (if archived - decompressed on the fly while streaming)
-// - Space needed for restored database
-// - Safe gap (WAL, indexes, sort/temp spills, FS slack)
func EstimateRequiredForRestoreDiskMb(backup *backups_core_logical.LogicalBackup) int64 {
- archiveSizeMb := backup.BackupSizeMb
- if archiveSizeMb < 0 {
- archiveSizeMb = 0
- }
+ archiveSizeMb := max(backup.BackupSizeMb, 0)
+ restoredSizeMb := max(backup.BackupRawDbSizeMb, 0)
- restoredSizeMb := backup.BackupRawDbSizeMb
- if restoredSizeMb < 0 {
- restoredSizeMb = 0
+ gapMb := float64(maxDiskGapMb)
+ if restoredSizeMb > 0 {
+ gapMb = min(maxDiskGapMb, max(minDiskGapMb, restoredSizeMb*diskGapToRestoredSizeRatio))
}
- return int64(math.Ceil(archiveSizeMb+restoredSizeMb)) + diskCostPerJobGapMb
+ return int64(math.Ceil(archiveSizeMb + restoredSizeMb + gapMb))
}
func sumEstimatedRequiredDiskMb(backups []*backups_core_logical.LogicalBackup) int64 {
diff --git a/backend/internal/features/verification/runs/capacity_test.go b/backend/internal/features/verification/runs/capacity_test.go
index fe96527e..56ff8dae 100644
--- a/backend/internal/features/verification/runs/capacity_test.go
+++ b/backend/internal/features/verification/runs/capacity_test.go
@@ -15,35 +15,35 @@ func backupWithSize(archiveMb, restoredMb float64) *backups_core_logical.Logical
}
}
-func Test_DoesVerificationFit_AcrossScenarios_RespectsAgentBudget(t *testing.T) {
+func Test_EvaluateDiskAdmission_AcrossScenarios_RespectsAgentBudget(t *testing.T) {
cases := []struct {
- name string
- capacity AgentCapacity
- runningBackups []*backups_core_logical.LogicalBackup
- candidate *backups_core_logical.LogicalBackup
- wantFits bool
+ name string
+ capacity AgentCapacity
+ runningBackups []*backups_core_logical.LogicalBackup
+ candidate *backups_core_logical.LogicalBackup
+ isFitWithinBudget bool
}{
{
- name: "empty agent fits small candidate",
- capacity: AgentCapacity{MaxDiskGb: 10},
- runningBackups: nil,
- candidate: backupWithSize(50, 100),
- wantFits: true,
+ name: "empty agent fits small candidate",
+ capacity: AgentCapacity{MaxDiskGb: 10},
+ runningBackups: nil,
+ candidate: backupWithSize(50, 100),
+ isFitWithinBudget: true,
},
{
name: "empty agent fits candidate at exact budget boundary",
capacity: AgentCapacity{MaxDiskGb: 10},
runningBackups: nil,
- // archive + restored + per-job gap = 10240 (full 10 GB budget)
- candidate: backupWithSize(4000, 1120),
- wantFits: true,
+ // archive 4096 + restored 4096 + gap 2048 = 10240 (the full 10 GB budget)
+ candidate: backupWithSize(4096, 4096),
+ isFitWithinBudget: true,
},
{
- name: "empty agent rejects candidate one MB over budget",
- capacity: AgentCapacity{MaxDiskGb: 10},
- runningBackups: nil,
- candidate: backupWithSize(4000, 1121),
- wantFits: false,
+ name: "empty agent rejects candidate one MB over budget",
+ capacity: AgentCapacity{MaxDiskGb: 10},
+ runningBackups: nil,
+ candidate: backupWithSize(4097, 4096),
+ isFitWithinBudget: false,
},
{
name: "one running, room for another small",
@@ -51,8 +51,8 @@ func Test_DoesVerificationFit_AcrossScenarios_RespectsAgentBudget(t *testing.T)
runningBackups: []*backups_core_logical.LogicalBackup{
backupWithSize(50, 200),
},
- candidate: backupWithSize(50, 200),
- wantFits: true,
+ candidate: backupWithSize(50, 200),
+ isFitWithinBudget: true,
},
{
name: "one running, restored size of candidate blows budget",
@@ -60,8 +60,8 @@ func Test_DoesVerificationFit_AcrossScenarios_RespectsAgentBudget(t *testing.T)
runningBackups: []*backups_core_logical.LogicalBackup{
backupWithSize(500, 4000),
},
- candidate: backupWithSize(500, 4000),
- wantFits: false,
+ candidate: backupWithSize(500, 4000),
+ isFitWithinBudget: false,
},
{
name: "three concurrent small jobs fit in 25 GB",
@@ -71,8 +71,8 @@ func Test_DoesVerificationFit_AcrossScenarios_RespectsAgentBudget(t *testing.T)
backupWithSize(100, 500),
backupWithSize(100, 500),
},
- candidate: backupWithSize(100, 500),
- wantFits: true,
+ candidate: backupWithSize(100, 500),
+ isFitWithinBudget: true,
},
{
name: "per-job gap saturation: three RUNNING already over a 5 GB agent",
@@ -82,55 +82,116 @@ func Test_DoesVerificationFit_AcrossScenarios_RespectsAgentBudget(t *testing.T)
backupWithSize(100, 400),
backupWithSize(100, 400),
},
- candidate: backupWithSize(10, 10),
- wantFits: false,
+ candidate: backupWithSize(10, 10),
+ isFitWithinBudget: false,
},
{
- name: "zero-size candidate still consumes the per-job gap and fits",
- capacity: AgentCapacity{MaxDiskGb: 10},
- runningBackups: nil,
- candidate: backupWithSize(0, 0),
- wantFits: true,
+ name: "tiny database fits a 5 GB agent",
+ capacity: AgentCapacity{MaxDiskGb: 5},
+ runningBackups: nil,
+ candidate: backupWithSize(0, 0.01),
+ isFitWithinBudget: true,
},
{
- name: "negative sizes clamp to zero and still fit",
- capacity: AgentCapacity{MaxDiskGb: 10},
- runningBackups: nil,
- candidate: backupWithSize(-50, -100),
- wantFits: true,
+ name: "unknown raw DB size keeps the full gap and fits a roomy agent",
+ capacity: AgentCapacity{MaxDiskGb: 10},
+ runningBackups: nil,
+ candidate: backupWithSize(200, 0),
+ isFitWithinBudget: true,
},
{
- name: "zero-capacity agent rejects everything",
- capacity: AgentCapacity{MaxDiskGb: 0},
- runningBackups: nil,
- candidate: backupWithSize(1, 1),
- wantFits: false,
+ name: "unknown raw DB size keeps the full gap and does not fit a 5 GB agent",
+ capacity: AgentCapacity{MaxDiskGb: 5},
+ runningBackups: nil,
+ candidate: backupWithSize(200, 0),
+ isFitWithinBudget: false,
},
{
- name: "nil candidate is rejected",
- capacity: AgentCapacity{MaxDiskGb: 10},
- runningBackups: nil,
- candidate: nil,
- wantFits: false,
+ name: "zero-size candidate still consumes the per-job gap and fits",
+ capacity: AgentCapacity{MaxDiskGb: 10},
+ runningBackups: nil,
+ candidate: backupWithSize(0, 0),
+ isFitWithinBudget: true,
},
{
- name: "raw DB size dominates archive size in the cost",
- capacity: AgentCapacity{MaxDiskGb: 2},
- runningBackups: nil,
- candidate: backupWithSize(10, 2000),
- wantFits: false,
+ name: "negative sizes clamp to zero and still fit",
+ capacity: AgentCapacity{MaxDiskGb: 10},
+ runningBackups: nil,
+ candidate: backupWithSize(-50, -100),
+ isFitWithinBudget: true,
+ },
+ {
+ name: "zero-capacity agent rejects everything",
+ capacity: AgentCapacity{MaxDiskGb: 0},
+ runningBackups: nil,
+ candidate: backupWithSize(1, 1),
+ isFitWithinBudget: false,
+ },
+ {
+ name: "nil candidate is rejected",
+ capacity: AgentCapacity{MaxDiskGb: 10},
+ runningBackups: nil,
+ candidate: nil,
+ isFitWithinBudget: false,
+ },
+ {
+ name: "raw DB size dominates archive size in the cost",
+ capacity: AgentCapacity{MaxDiskGb: 2},
+ runningBackups: nil,
+ candidate: backupWithSize(10, 2000),
+ isFitWithinBudget: false,
},
}
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
- fits := IsVerificationFitWithinRemainedDiskCapacity(
+ admission := EvaluateDiskAdmission(
testCase.capacity,
testCase.runningBackups,
testCase.candidate,
)
- assert.Equal(t, testCase.wantFits, fits)
+ assert.Equal(t, testCase.isFitWithinBudget, admission.IsFit)
+ })
+ }
+}
+
+func Test_EstimateRequiredForRestoreDiskMb_AcrossRestoredSizes_ClampsGapBetweenFloorAndCap(t *testing.T) {
+ cases := []struct {
+ name string
+ backup *backups_core_logical.LogicalBackup
+ requiredDiskMb int64
+ }{
+ {
+ name: "unknown raw DB size falls back to the gap cap",
+ backup: backupWithSize(200, 0),
+ requiredDiskMb: 200 + maxDiskGapMb,
+ },
+ {
+ name: "tiny restored size uses the gap floor",
+ backup: backupWithSize(0, 0.01),
+ requiredDiskMb: 1 + minDiskGapMb,
+ },
+ {
+ name: "restored size below the floor crossover uses the gap floor",
+ backup: backupWithSize(100, 2000),
+ requiredDiskMb: 2100 + minDiskGapMb,
+ },
+ {
+ name: "restored size between floor and cap scales the gap",
+ backup: backupWithSize(500, 6000),
+ requiredDiskMb: 6500 + 3000,
+ },
+ {
+ name: "restored size above 10 GB pins the gap at the cap",
+ backup: backupWithSize(4000, 40000),
+ requiredDiskMb: 44000 + maxDiskGapMb,
+ },
+ }
+
+ for _, testCase := range cases {
+ t.Run(testCase.name, func(t *testing.T) {
+ assert.Equal(t, testCase.requiredDiskMb, EstimateRequiredForRestoreDiskMb(testCase.backup))
})
}
}
diff --git a/backend/internal/features/verification/runs/service.go b/backend/internal/features/verification/runs/service.go
index 90287113..1a117d45 100644
--- a/backend/internal/features/verification/runs/service.go
+++ b/backend/internal/features/verification/runs/service.go
@@ -258,6 +258,8 @@ func (s *VerificationService) ClaimVerification(
agent *verification_agents.Agent,
req *ClaimRequest,
) (*JobAssignment, error) {
+ logger := s.logger.With("agent_id", agent.ID)
+
runningBackups, err := s.verificationRepository.ListRunningBackupsByAgentID(agent.ID)
if err != nil {
return nil, err
@@ -272,7 +274,18 @@ func (s *VerificationService) ClaimVerification(
}
for _, candidate := range candidates {
- if !IsVerificationFitWithinRemainedDiskCapacity(req.Capacity, runningBackups, candidate.Backup) {
+ candidateLogger := logger.With("verification_id", candidate.Verification.ID)
+
+ admission := EvaluateDiskAdmission(req.Capacity, runningBackups, candidate.Backup)
+ if !admission.IsFit {
+ candidateLogger.DebugContext(
+ ctx,
+ fmt.Sprintf(
+ "skipping claim: needs %d MB, %d MB already used of %d MB budget",
+ admission.CandidateRequiredMb, admission.RunningUsedMb, admission.TotalBudgetMb,
+ ),
+ )
+
continue
}
@@ -282,10 +295,9 @@ func (s *VerificationService) ClaimVerification(
}
if validateErr := validateDatabaseIsVerifiable(database); validateErr != nil {
- s.logger.WarnContext(
+ candidateLogger.WarnContext(
ctx,
"skipping claim: database not verifiable",
- "verification_id", candidate.Verification.ID,
"database_id", candidate.Verification.DatabaseID,
"error", validateErr,
)
diff --git a/website/app/restore-verification/page.tsx b/website/app/restore-verification/page.tsx
index 7c116dad..eae4a3f9 100644
--- a/website/app/restore-verification/page.tsx
+++ b/website/app/restore-verification/page.tsx
@@ -177,7 +177,7 @@ export default function RestoreVerificationPage() {
Disk capacity for each verification job covering the{" "}
<strong>backup file size</strong>, the{" "}
<strong>database raw size</strong> and a{" "}
- <strong>5 GB safety gap</strong> on top.
+ <strong>safety gap</strong> on top.
</li>
<li>
At least 1 CPU core and 512 MB of RAM available per concurrent
@@ -319,7 +319,7 @@ export default function RestoreVerificationPage() {
concurrency. The disk budget is the easiest to get wrong: each
job needs enough space for the <strong>backup file size</strong>
, the <strong>database raw size</strong> and a{" "}
- <strong>5 GB safety gap</strong> on top, so set{" "}
+ <strong>safety gap of up to 5 GB</strong> on top, so set{" "}
<code>--max-disk-gb</code> comfortably above that for your
largest database.
</p>