Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FEATURE (wal): Show a Monitoring tab for WAL-streaming databases that says in plain words whether archiving keeps up and what to do when it does not, backed by samples taken every 30 seconds and a documented statistics API

Rostislav Dugina6e25f62f69a
diff --git a/backend/internal/features/backups/backups/backuping/physical/cleaner.go b/backend/internal/features/backups/backups/backuping/physical/cleaner.go
index 9e9b076e..fed216d1 100644
--- a/backend/internal/features/backups/backups/backuping/physical/cleaner.go
+++ b/backend/internal/features/backups/backups/backuping/physical/cleaner.go
@@ -16,6 +16,7 @@ import (
physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
physical_service "databasus-backend/internal/features/backups/backups/core/physical/service"
+ "databasus-backend/internal/features/backups/backups/core/physical/wal_health"
usecases_physical_postgresql "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"
backups_config_physical "databasus-backend/internal/features/backups/config/physical"
"databasus-backend/internal/features/intervals"
@@ -32,6 +33,7 @@ type PhysicalBackupCleaner struct {
backupConfigService *backups_config_physical.BackupConfigService
fullRepo *physical_repositories.PhysicalFullBackupRepository
walSegmentRepo *physical_repositories.PhysicalWalSegmentRepository
+ walStreamSampleRepo *physical_repositories.PhysicalWalStreamSampleRepository
logger *slog.Logger
hasRun atomic.Bool
@@ -65,10 +67,27 @@ func (c *PhysicalBackupCleaner) Run(ctx context.Context) {
if err := c.cleanOrphans(ctx, logger); err != nil {
logger.ErrorContext(ctx, "failed to clean orphans", "error", err)
}
+
+ c.deleteExpiredWalStreamSamples(ctx, logger, time.Now().UTC())
}
}
}
+// One statement for every database, so samples of a database that no longer
+// streams expire as well; a removed database's samples go with it by cascade.
+func (c *PhysicalBackupCleaner) deleteExpiredWalStreamSamples(ctx context.Context, logger *slog.Logger, now time.Time) {
+ deletedCount, err := c.walStreamSampleRepo.DeleteOlderThan(now.Add(-wal_health.SampleRetention))
+ if err != nil {
+ logger.ErrorContext(ctx, "failed to delete expired wal stream samples", "error", err)
+
+ return
+ }
+
+ if deletedCount > 0 {
+ logger.DebugContext(ctx, fmt.Sprintf("deleted %d expired wal stream samples", deletedCount))
+ }
+}
+
// runStartupSlotCleanup drops orphan per-backup replication slots once at boot.
// In single-instance mode any backup that was running died with the previous
// process, so no slot is protected — the scheduler's restart recovery fails and
diff --git a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go
index a8339763..d747ad01 100644
--- a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go
+++ b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go
@@ -13,6 +13,7 @@ import (
physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
physical_testing "databasus-backend/internal/features/backups/backups/core/physical/testing"
+ "databasus-backend/internal/features/backups/backups/core/physical/wal_health"
postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"
backups_config_physical "databasus-backend/internal/features/backups/config/physical"
"databasus-backend/internal/features/intervals"
@@ -466,3 +467,40 @@ func Test_CleanByChains_WhenInProgressFullExists_NeverDeletesIt(t *testing.T) {
assert.True(t, fullExists(t, keptByCount.ID))
assert.False(t, fullExists(t, evicted.ID), "completed chains beyond the keep-count are still evicted")
}
+
+func Test_DeleteExpiredWalStreamSamples_WhenSamplesOlderThanRetention_DeletesOnlyThose(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ now := time.Now().UTC()
+
+ expiredSample := physical_testing.CreateTestWalStreamSample(t,
+ physical_testing.NewTestWalStreamSample(prereqs.DB.ID, now.Add(-wal_health.SampleRetention-time.Hour)))
+ recentSample := physical_testing.CreateTestWalStreamSample(t,
+ physical_testing.NewTestWalStreamSample(prereqs.DB.ID, now.Add(-24*time.Hour)))
+
+ CreateTestPhysicalCleaner().deleteExpiredWalStreamSamples(t.Context(), logger.GetLogger(), now)
+
+ remainingSamples, err := physical_repositories.GetWalStreamSampleRepository().
+ FindSinceByDatabase(prereqs.DB.ID, time.Time{})
+ require.NoError(t, err)
+ require.Len(t, remainingSamples, 1)
+ require.Equal(t, recentSample.ID, remainingSamples[0].ID)
+ require.NotEqual(t, expiredSample.ID, remainingSamples[0].ID)
+}
+
+func Test_DeleteExpiredWalStreamSamples_WhenDatabaseStoppedStreamingEightDaysAgo_DeletesAllItsSamples(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ now := time.Now().UTC()
+ stoppedStreamingAt := now.Add(-8 * 24 * time.Hour)
+
+ for sampleIndex := range 3 {
+ physical_testing.CreateTestWalStreamSample(t, physical_testing.NewTestWalStreamSample(
+ prereqs.DB.ID, stoppedStreamingAt.Add(-time.Duration(sampleIndex)*wal_health.SampleInterval)))
+ }
+
+ CreateTestPhysicalCleaner().deleteExpiredWalStreamSamples(t.Context(), logger.GetLogger(), now)
+
+ remainingSamples, err := physical_repositories.GetWalStreamSampleRepository().
+ FindSinceByDatabase(prereqs.DB.ID, time.Time{})
+ require.NoError(t, err)
+ require.Empty(t, remainingSamples, "samples expire whether or not the database still streams")
+}
diff --git a/backend/internal/features/backups/backups/backuping/physical/constants.go b/backend/internal/features/backups/backups/backuping/physical/constants.go
index 0405b6a9..d8b67963 100644
--- a/backend/internal/features/backups/backups/backuping/physical/constants.go
+++ b/backend/internal/features/backups/backups/backuping/physical/constants.go
@@ -43,16 +43,10 @@ const (
// The WAL-stream supervisor reconciles owned streamers on the same 15 s
// cadence as the scheduler — cadence-driven work is cheap, and the tighter
- // bound comes from streamer reclaim, where 15 s + streamerHeartbeatStaleness
+ // bound comes from streamer reclaim, where 15 s + wal_health.StreamerHeartbeatStaleness
// keeps detect-to-reclaim under ~2 min after a crash.
walStreamSupervisorTickInterval = 15 * time.Second
- // A streamer row whose heartbeat is older than this is assumed dead and
- // reclaimable by another process. Heartbeats fire every tick (15 s), so 90 s =
- // 6 missed beats — tolerant of GC pauses without leaving a dead streamer
- // unclaimed for long.
- streamerHeartbeatStaleness = 90 * time.Second
-
// Bounds how long a reconcile waits for one streamer goroutine to drain on
// stop so a stuck pg_receivewal teardown can't stall the whole supervisor.
streamerStopTimeout = 30 * time.Second
diff --git a/backend/internal/features/backups/backups/backuping/physical/di.go b/backend/internal/features/backups/backups/backuping/physical/di.go
index b07c2f90..834d1b98 100644
--- a/backend/internal/features/backups/backups/backuping/physical/di.go
+++ b/backend/internal/features/backups/backups/backuping/physical/di.go
@@ -66,6 +66,7 @@ var physicalBackupCleaner = &PhysicalBackupCleaner{
backups_config_physical.GetBackupConfigService(),
physical_repositories.GetFullBackupRepository(),
physical_repositories.GetWalSegmentRepository(),
+ physical_repositories.GetWalStreamSampleRepository(),
logger.GetLogger(),
atomic.Bool{},
}
@@ -80,6 +81,7 @@ var physicalWalStreamSupervisor = &PhysicalWalStreamSupervisor{
physical_repositories.GetWalSegmentRepository(),
physical_repositories.GetWalHistoryRepository(),
physical_repositories.GetWalStreamerRepository(),
+ physical_repositories.GetWalStreamSampleRepository(),
notifiers.GetNotifierService(),
tasks_cancellation.GetRegistry(),
encryption_secrets.GetSecretKeyService(),
diff --git a/backend/internal/features/backups/backups/backuping/physical/mocks_test.go b/backend/internal/features/backups/backups/backuping/physical/mocks_test.go
index a51c006a..4414c5f6 100644
--- a/backend/internal/features/backups/backups/backuping/physical/mocks_test.go
+++ b/backend/internal/features/backups/backups/backuping/physical/mocks_test.go
@@ -97,6 +97,20 @@ type backupPrereqs struct {
func seedBackupPrereqs(t *testing.T) *backupPrereqs {
t.Helper()
+ return seedBackupPrereqsWithDatabase(
+ t,
+ func(workspaceID uuid.UUID, notifier *notifiers.Notifier) *databases.Database {
+ return databases.CreateTestPhysicalPostgresDatabase(workspaceID, notifier, "17")
+ },
+ )
+}
+
+func seedBackupPrereqsWithDatabase(
+ t *testing.T,
+ createDatabase func(workspaceID uuid.UUID, notifier *notifiers.Notifier) *databases.Database,
+) *backupPrereqs {
+ t.Helper()
+
router := backuping_logical.CreateTestRouter()
owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleAdmin)
@@ -110,7 +124,7 @@ func seedBackupPrereqs(t *testing.T) *backupPrereqs {
notifier := notifiers.CreateTestNotifier(workspace.ID)
t.Cleanup(func() { notifiers.RemoveTestNotifier(notifier) })
- db := databases.CreateTestPhysicalPostgresDatabase(workspace.ID, notifier, "17")
+ db := createDatabase(workspace.ID, notifier)
t.Cleanup(func() { databases.RemoveTestDatabase(t.Context(), db) })
cfgService := backups_config_physical.GetBackupConfigService()
diff --git a/backend/internal/features/backups/backups/backuping/physical/testing.go b/backend/internal/features/backups/backups/backuping/physical/testing.go
index 4edec319..39febbf4 100644
--- a/backend/internal/features/backups/backups/backuping/physical/testing.go
+++ b/backend/internal/features/backups/backups/backuping/physical/testing.go
@@ -99,6 +99,7 @@ func CreateTestWalStreamSupervisor(spec WalStreamSupervisorTestSpec) *PhysicalWa
physical_repositories.GetWalSegmentRepository(),
physical_repositories.GetWalHistoryRepository(),
physical_repositories.GetWalStreamerRepository(),
+ physical_repositories.GetWalStreamSampleRepository(),
spec.NotificationSender,
tasks_cancellation.GetRegistry(),
encryption_secrets.GetSecretKeyService(),
@@ -123,6 +124,7 @@ func CreateTestPhysicalCleaner() *PhysicalBackupCleaner {
backups_config_physical.GetBackupConfigService(),
physical_repositories.GetFullBackupRepository(),
physical_repositories.GetWalSegmentRepository(),
+ physical_repositories.GetWalStreamSampleRepository(),
logger.GetLogger(),
atomic.Bool{},
}
diff --git a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go
index 7d7b6835..9de19ab1 100644
--- a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go
+++ b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go
@@ -17,6 +17,7 @@ import (
"databasus-backend/internal/config"
backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
+ "databasus-backend/internal/features/backups/backups/core/physical/wal_health"
postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"
backups_config_physical "databasus-backend/internal/features/backups/config/physical"
"databasus-backend/internal/features/databases"
@@ -41,6 +42,7 @@ type PhysicalWalStreamSupervisor struct {
walSegmentRepo *physical_repositories.PhysicalWalSegmentRepository
historyRepo *physical_repositories.PhysicalWalHistoryRepository
walStreamerRepo *physical_repositories.PhysicalWalStreamerRepository
+ walStreamSampleRepo *physical_repositories.PhysicalWalStreamSampleRepository
notificationSender NotificationSender
taskCancellationRegistry *tasks_cancellation.Registry
secretKeyService *encryption_secrets.SecretKeyService
@@ -124,7 +126,7 @@ func (s *PhysicalWalStreamSupervisor) Run(ctx context.Context) {
}
func (s *PhysicalWalStreamSupervisor) recoverStreamersOnStartup() error {
- failed, err := s.walStreamerRepo.MarkStaleRunningFailed(int(streamerHeartbeatStaleness.Seconds()))
+ failed, err := s.walStreamerRepo.MarkStaleRunningFailed(int(wal_health.StreamerHeartbeatStaleness.Seconds()))
if err != nil {
return err
}
@@ -154,7 +156,7 @@ func (s *PhysicalWalStreamSupervisor) reconcile(ctx context.Context) {
candidates := make(map[uuid.UUID]bool)
for _, backupConfig := range configs {
- if !isWalStreamCandidate(backupConfig) {
+ if !backupConfig.IsWalStreamingExpected() {
continue
}
@@ -172,15 +174,6 @@ func (s *PhysicalWalStreamSupervisor) reconcile(ctx context.Context) {
s.heartbeatOwnedStreamers()
}
-func isWalStreamCandidate(backupConfig *backups_config_physical.PhysicalBackupConfig) bool {
- if backupConfig.PostgresqlPhysical == nil ||
- !backupConfig.PostgresqlPhysical.BackupType.IsWalStreaming() {
- return false
- }
-
- return backupConfig.StorageID != nil
-}
-
func (s *PhysicalWalStreamSupervisor) ensureStreamerRunning(
ctx context.Context,
logger *slog.Logger,
@@ -195,7 +188,7 @@ func (s *PhysicalWalStreamSupervisor) ensureStreamerRunning(
}
claimed, err := s.walStreamerRepo.ClaimIfClaimable(
- backupConfig.DatabaseID, int(streamerHeartbeatStaleness.Seconds()),
+ backupConfig.DatabaseID, int(wal_health.StreamerHeartbeatStaleness.Seconds()),
)
if err != nil {
logger.ErrorContext(ctx, "claim failed", "error", err)
@@ -282,10 +275,12 @@ func (s *PhysicalWalStreamSupervisor) startStreamer(
FieldEncryptor: s.fieldEncryptor,
WalSegmentRepo: s.walSegmentRepo,
HistoryRepo: s.historyRepo,
+ WalStreamSampleRepo: s.walStreamSampleRepo,
+ WalStreamerRepo: s.walStreamerRepo,
WatchDirRoot: config.GetEnv().DataFolder,
WalLagThresholdBytes: backupConfig.WalLagThresholdBytes,
ForcedRotationInterval: postgresql_executor.DefaultForcedRotationInterval,
- ArchiveStalenessThreshold: postgresql_executor.DefaultArchiveStalenessThreshold,
+ ArchiveStalenessThreshold: wal_health.DefaultArchiveStalenessThreshold,
OnGapDetected: s.gapNotifier(ctx, db, backupConfig),
OnSlotRebuilt: s.slotRebuildFullRequester(ctx, logger, db, backupConfig),
OnChainAtRisk: s.chainRiskNotifier(ctx, db, backupConfig),
@@ -534,9 +529,9 @@ func (s *PhysicalWalStreamSupervisor) stopStreamer(databaseID uuid.UUID, shouldR
s.mu.Unlock()
- if err := s.walStreamerRepo.MarkFailed(databaseID); err != nil {
+ if err := s.walStreamerRepo.MarkStopped(databaseID); err != nil {
s.logger.Error(
- "failed to mark stopped streamer failed",
+ "failed to mark stopped streamer released",
"database_id",
databaseID,
"error",
diff --git a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor_test.go b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor_test.go
index 4e4d4112..eb8fee39 100644
--- a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor_test.go
+++ b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor_test.go
@@ -1,18 +1,25 @@
package backuping_physical
import (
+ "context"
"os"
"path/filepath"
+ "strconv"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
+ "databasus-backend/internal/config"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
+ "databasus-backend/internal/features/backups/backups/core/physical/wal_health"
postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"
backups_config_physical "databasus-backend/internal/features/backups/config/physical"
+ "databasus-backend/internal/features/databases"
+ postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
"databasus-backend/internal/features/notifiers"
"databasus-backend/internal/storage"
)
@@ -178,6 +185,7 @@ func Test_StopStreamer_WhenOwnedStreamerStops_MarksRowFailed(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, streamer)
require.Equal(t, physical_enums.PhysicalWalStreamerStatusFailed, streamer.Status)
+ require.Nil(t, streamer.FailingSince, "a deliberate stop is not a failure, so a deploy does not read as one")
}
func Test_RecoverStreamersOnStartup_WhenRunningRowStale_MarksFailed(t *testing.T) {
@@ -185,7 +193,7 @@ func Test_RecoverStreamersOnStartup_WhenRunningRowStale_MarksFailed(t *testing.T
seedStreamerRow(
t,
prereqs.DB.ID,
- time.Now().UTC().Add(-2*streamerHeartbeatStaleness),
+ time.Now().UTC().Add(-2*wal_health.StreamerHeartbeatStaleness),
physical_enums.PhysicalWalStreamerStatusRunning,
)
@@ -380,3 +388,148 @@ func Test_ChainRiskNotifier_WhenReasonsDiffer_UsesSeparateThrottleKinds(t *testi
require.Contains(t, sender.sentNotifications[1].Notification.Message, lastArchivedWalAt.Format(time.RFC3339))
require.Contains(t, sender.sentNotifications[2].Notification.Message, "pg_switch_wal()")
}
+
+func seedBackupPrereqsOnSharedSource(t *testing.T, backupType postgresql_physical.BackupType) *backupPrereqs {
+ t.Helper()
+
+ port, err := strconv.Atoi(config.GetEnv().TestPhysicalPostgres17Port)
+ require.NoError(t, err)
+
+ return seedBackupPrereqsOnEndpoint(t, config.GetEnv().TestLocalhost, port, backupType)
+}
+
+func seedBackupPrereqsOnEndpoint(
+ t *testing.T,
+ host string,
+ port int,
+ backupType postgresql_physical.BackupType,
+) *backupPrereqs {
+ t.Helper()
+
+ return seedBackupPrereqsWithDatabase(
+ t,
+ func(workspaceID uuid.UUID, notifier *notifiers.Notifier) *databases.Database {
+ return databases.CreateTestPhysicalPostgresDatabaseWithType(
+ host,
+ port,
+ workspaceID,
+ notifier,
+ "17",
+ backupType,
+ )
+ },
+ )
+}
+
+// A supervisor that stops its streamers before the test drops their slots, so
+// no receiver re-attaches in between.
+func startReconcilingSupervisor(t *testing.T) (*PhysicalWalStreamSupervisor, context.Context) {
+ t.Helper()
+
+ supervisor := CreateTestWalStreamSupervisor(WalStreamSupervisorTestSpec{
+ NotificationSender: notifiers.GetNotifierService(),
+ })
+
+ ctx, cancel := context.WithCancel(t.Context())
+ t.Cleanup(func() {
+ cancel()
+ supervisor.stopAllStreamers()
+ })
+
+ return supervisor, ctx
+}
+
+func dropStreamerSlotOnCleanup(t *testing.T, prereqs *backupPrereqs) {
+ t.Helper()
+
+ adminConn := postgresql_executor.OpenAdminConn(t, &postgresql_executor.PhysicalDBFixture{DB: prereqs.DB})
+
+ t.Cleanup(func() {
+ postgresql_executor.DropReplicationSlotExternally(
+ t,
+ adminConn,
+ prereqs.DB.PostgresqlPhysical.ReplicationSlotName,
+ )
+ })
+}
+
+func countSamples(t *testing.T, databaseID uuid.UUID) int {
+ t.Helper()
+
+ samples, err := physical_repositories.GetWalStreamSampleRepository().FindSinceByDatabase(databaseID, time.Time{})
+ require.NoError(t, err)
+
+ return len(samples)
+}
+
+func Test_Reconcile_WhenDatabaseIsNotWalStreaming_RecordsNoSamplesForIt(t *testing.T) {
+ if testing.Short() {
+ t.Skip("runs a real streamer until its first sample; skipped in -short")
+ }
+
+ streamingPrereqs := seedBackupPrereqsOnSharedSource(t, postgresql_physical.BackupTypeFullIncrementalAndWalStream)
+ dropStreamerSlotOnCleanup(t, streamingPrereqs)
+
+ incrementalPrereqs := seedBackupPrereqsOnSharedSource(t, postgresql_physical.BackupTypeFullAndIncremental)
+
+ t.Cleanup(func() {
+ _ = physical_repositories.GetWalStreamerRepository().DeleteByDatabaseID(streamingPrereqs.DB.ID)
+ })
+
+ supervisor, ctx := startReconcilingSupervisor(t)
+ supervisor.reconcile(ctx)
+
+ require.Eventually(t, func() bool {
+ return countSamples(t, streamingPrereqs.DB.ID) > 0
+ }, 2*wal_health.SampleInterval+30*time.Second, time.Second,
+ "the WAL streaming database records its first sample within one lag-monitor tick")
+
+ require.Zero(t, countSamples(t, incrementalPrereqs.DB.ID),
+ "a database outside WAL streaming mode is never sampled")
+}
+
+func Test_Reconcile_WhenEverySessionFailsBeforeFirstSample_KeepsFirstFailingTime(t *testing.T) {
+ unreachablePrereqs := seedBackupPrereqsOnEndpoint(t, "127.0.0.1", 1,
+ postgresql_physical.BackupTypeFullIncrementalAndWalStream)
+
+ streamerRepository := physical_repositories.GetWalStreamerRepository()
+ t.Cleanup(func() {
+ _ = streamerRepository.DeleteByDatabaseID(unreachablePrereqs.DB.ID)
+ })
+
+ supervisor, ctx := startReconcilingSupervisor(t)
+
+ waitForFailedSession := func(previousStartedAt time.Time) *physical_models.PhysicalWalStreamer {
+ var failedStreamer *physical_models.PhysicalWalStreamer
+
+ require.Eventually(t, func() bool {
+ supervisor.mu.Lock()
+ _, isRunning := supervisor.running[unreachablePrereqs.DB.ID]
+ supervisor.mu.Unlock()
+
+ streamer, err := streamerRepository.FindByDatabaseID(unreachablePrereqs.DB.ID)
+ if err != nil || streamer == nil || isRunning {
+ return false
+ }
+
+ failedStreamer = streamer
+
+ return streamer.Status == physical_enums.PhysicalWalStreamerStatusFailed &&
+ streamer.StartedAt.After(previousStartedAt)
+ }, 60*time.Second, 200*time.Millisecond, "a session against an unreachable source fails at start")
+
+ return failedStreamer
+ }
+
+ supervisor.reconcile(ctx)
+ firstFailure := waitForFailedSession(time.Time{})
+ require.NotNil(t, firstFailure.FailingSince)
+
+ supervisor.reconcile(ctx)
+ secondFailure := waitForFailedSession(firstFailure.StartedAt)
+
+ require.NotNil(t, secondFailure.FailingSince)
+ require.True(t, firstFailure.FailingSince.Equal(*secondFailure.FailingSince),
+ "a crash loop keeps the time of its first failure")
+ require.Zero(t, countSamples(t, unreachablePrereqs.DB.ID))
+}
diff --git a/backend/internal/features/backups/backups/controllers/physical/controller.go b/backend/internal/features/backups/backups/controllers/physical/controller.go
index 613837cf..bfd85e46 100644
--- a/backend/internal/features/backups/backups/controllers/physical/controller.go
+++ b/backend/internal/features/backups/backups/controllers/physical/controller.go
@@ -27,6 +27,7 @@ type PhysicalBackupController struct {
func (c *PhysicalBackupController) RegisterRoutes(router *gin.RouterGroup) {
router.GET("/backups/physical/database/:id/backups", c.GetBackups)
+ router.GET("/backups/physical/database/:id/wal-stream/statistics", c.GetWalStreamStatistics)
router.POST("/backups/physical/database/:id/trigger", c.TriggerBackup)
router.POST("/backups/physical/database/:id/restore-token", c.GenerateRestoreToken)
@@ -97,6 +98,46 @@ func (c *PhysicalBackupController) GetBackups(ctx *gin.Context) {
ctx.JSON(http.StatusOK, response)
}
+// GetWalStreamStatistics
+// @Summary Get WAL stream statistics for a database
+// @Description Returns how healthy WAL archiving of a physical PostgreSQL database is: one verdict with its reasons and notices, the streamer's state, the values of the newest sample, a keeping-up summary of the last 10 minutes and a chart series for the requested period. Amounts carry their unit in their name, times are UTC, and the verdict, states, reasons and notices are codes from the documented enums, never display text. Samples are taken every 30 seconds and kept for 7 days. A physical database outside WAL streaming mode gets the NOT_STREAMING verdict with the samples it still has; any other database type, an unknown period or a database the user cannot view returns 400.
+// @Tags backups-physical
+// @Produce json
+// @Security BearerAuth
+// @Param id path string true "Database ID"
+// @Param period query string true "Chart period: the last hour in 1-minute points, the last day in 4-minute points or the last week in 28-minute points" Enums(HOUR, DAY, WEEK)
+// @Success 200 {object} backups_dto_physical.GetWalStreamStatisticsResponse
+// @Failure 400 {object} map[string]string
+// @Failure 401 {object} map[string]string
+// @Router /backups/physical/database/{id}/wal-stream/statistics [get]
+func (c *PhysicalBackupController) GetWalStreamStatistics(ctx *gin.Context) {
+ user, ok := users_middleware.GetUserFromContext(ctx)
+ if !ok {
+ ctx.JSON(http.StatusUnauthorized, gin.H{"error": "User not authenticated"})
+ return
+ }
+
+ databaseID, err := uuid.Parse(ctx.Param("id"))
+ if err != nil {
+ ctx.JSON(http.StatusBadRequest, gin.H{"error": "invalid database ID"})
+ return
+ }
+
+ var request backups_dto_physical.GetWalStreamStatisticsRequest
+ if err := ctx.ShouldBindQuery(&request); err != nil {
+ ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+
+ response, err := c.physicalBackupService.GetWalStreamStatistics(ctx.Request.Context(), user, databaseID, &request)
+ if err != nil {
+ ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+
+ ctx.JSON(http.StatusOK, response)
+}
+
// TriggerBackup
// @Summary Trigger a physical backup
// @Description Requests an out-of-cadence backup, honored on the scheduler's next tick. type=full forces a new full; type=incremental extends the current chain (409 if none); type=auto lets the scheduler pick.
diff --git a/backend/internal/features/backups/backups/controllers/physical/controller_test.go b/backend/internal/features/backups/backups/controllers/physical/controller_test.go
index 0a297143..cce3ee98 100644
--- a/backend/internal/features/backups/backups/controllers/physical/controller_test.go
+++ b/backend/internal/features/backups/backups/controllers/physical/controller_test.go
@@ -7,6 +7,7 @@ import (
"encoding/json"
"io"
"net/http"
+ "strconv"
"strings"
"testing"
"time"
@@ -17,18 +18,22 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "databasus-backend/internal/config"
audit_logs "databasus-backend/internal/features/audit_logs"
backuping_physical "databasus-backend/internal/features/backups/backups/backuping/physical"
backups_controllers_physical "databasus-backend/internal/features/backups/backups/controllers/physical"
backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"
+ physical_dto "databasus-backend/internal/features/backups/backups/core/physical/dto"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
physical_testing "databasus-backend/internal/features/backups/backups/core/physical/testing"
+ "databasus-backend/internal/features/backups/backups/core/physical/wal_health"
backups_download "databasus-backend/internal/features/backups/backups/download"
backups_dto_physical "databasus-backend/internal/features/backups/backups/dto/physical"
backups_config_physical "databasus-backend/internal/features/backups/config/physical"
"databasus-backend/internal/features/databases"
+ postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
"databasus-backend/internal/features/notifiers"
"databasus-backend/internal/features/storages"
users_dto "databasus-backend/internal/features/users/dto"
@@ -60,12 +65,24 @@ type physicalControllerPrereqs struct {
func createPhysicalControllerPrereqs(t *testing.T) *physicalControllerPrereqs {
t.Helper()
+ return createPhysicalControllerPrereqsWithDatabase(t,
+ func(workspaceID uuid.UUID, notifier *notifiers.Notifier) *databases.Database {
+ return databases.CreateTestPhysicalPostgresDatabase(workspaceID, notifier, "17")
+ })
+}
+
+func createPhysicalControllerPrereqsWithDatabase(
+ t *testing.T,
+ createDatabase func(workspaceID uuid.UUID, notifier *notifiers.Notifier) *databases.Database,
+) *physicalControllerPrereqs {
+ t.Helper()
+
router := newPhysicalControllerRouter()
user := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Physical Ctrl "+uuid.NewString(), user, router)
storage := storages.CreateTestStorage(workspace.ID)
notifier := notifiers.CreateTestNotifier(workspace.ID)
- database := databases.CreateTestPhysicalPostgresDatabase(workspace.ID, notifier, "17")
+ database := createDatabase(workspace.ID, notifier)
t.Cleanup(func() {
physical_testing.DeleteAllPhysicalCatalogForDatabase(t, database.ID)
@@ -1590,3 +1607,604 @@ func readTarEntryNames(t *testing.T, body []byte) []string {
return names
}
+
+func createWalStreamingControllerPrereqs(t *testing.T) *physicalControllerPrereqs {
+ t.Helper()
+
+ port, err := strconv.Atoi(config.GetEnv().TestPhysicalPostgres17Port)
+ require.NoError(t, err)
+
+ prereqs := createPhysicalControllerPrereqsWithDatabase(t,
+ func(workspaceID uuid.UUID, notifier *notifiers.Notifier) *databases.Database {
+ return databases.CreateTestPhysicalPostgresDatabaseWithType(
+ config.GetEnv().TestLocalhost, port, workspaceID, notifier, "17",
+ postgresql_physical.BackupTypeFullIncrementalAndWalStream,
+ )
+ })
+
+ backupConfig, err := backups_config_physical.GetBackupConfigService().GetBackupConfigByDbId(prereqs.database.ID)
+ require.NoError(t, err)
+
+ backupConfig.IsBackupsEnabled = true
+ backupConfig.StorageID = &prereqs.storage.ID
+ backupConfig.Storage = prereqs.storage
+
+ _, err = backups_config_physical.GetBackupConfigService().SaveBackupConfig(t.Context(), backupConfig)
+ require.NoError(t, err)
+
+ return prereqs
+}
+
+func createHealthyTestWalStreamer(t *testing.T, databaseID uuid.UUID) *physical_models.PhysicalWalStreamer {
+ t.Helper()
+
+ return physical_testing.CreateTestWalStreamer(t, &physical_models.PhysicalWalStreamer{
+ DatabaseID: databaseID,
+ StartedAt: time.Now().UTC().Add(-time.Hour),
+ LastHeartbeatAt: time.Now().UTC(),
+ Status: physical_enums.PhysicalWalStreamerStatusRunning,
+ })
+}
+
+// Seeds one sample every 30 seconds, the newest taken at newestAt.
+func seedTestWalStreamSamples(
+ t *testing.T,
+ databaseID uuid.UUID,
+ count int,
+ newestAt time.Time,
+ modify func(sample *physical_models.PhysicalWalStreamSample),
+) []*physical_models.PhysicalWalStreamSample {
+ t.Helper()
+
+ samples := make([]*physical_models.PhysicalWalStreamSample, 0, count)
+
+ for sampleIndex := range count {
+ sample := physical_testing.NewTestWalStreamSample(
+ databaseID, newestAt.Add(-time.Duration(count-1-sampleIndex)*wal_health.SampleInterval))
+
+ if modify != nil {
+ modify(sample)
+ }
+
+ samples = append(samples, physical_testing.CreateTestWalStreamSample(t, sample))
+ }
+
+ return samples
+}
+
+func getWalStreamStatistics(
+ t *testing.T,
+ prereqs *physicalControllerPrereqs,
+ period string,
+) *backups_dto_physical.GetWalStreamStatisticsResponse {
+ t.Helper()
+
+ var response backups_dto_physical.GetWalStreamStatisticsResponse
+
+ test_utils.MakeGetRequestAndUnmarshal(t, prereqs.router,
+ getWalStreamStatisticsURL(prereqs.database.ID, period),
+ "Bearer "+prereqs.user.Token, http.StatusOK, &response)
+
+ return &response
+}
+
+func getWalStreamStatisticsURL(databaseID uuid.UUID, period string) string {
+ return "/api/v1/backups/physical/database/" + databaseID.String() + "/wal-stream/statistics?period=" + period
+}
+
+func getReasonCodes(reasons []wal_health.Reason) []wal_health.ReasonCode {
+ codes := make([]wal_health.ReasonCode, 0, len(reasons))
+
+ for _, reason := range reasons {
+ codes = append(codes, reason.Code)
+ }
+
+ return codes
+}
+
+func getNoticeCodes(notices []wal_health.Notice) []wal_health.NoticeCode {
+ codes := make([]wal_health.NoticeCode, 0, len(notices))
+
+ for _, notice := range notices {
+ codes = append(codes, notice.Code)
+ }
+
+ return codes
+}
+
+func Test_GetWalStreamStatistics_WhenWorkspaceMember_ReturnsCurrentValuesAndSeries(t *testing.T) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+ createHealthyTestWalStreamer(t, prereqs.database.ID)
+
+ newestSample := seedTestWalStreamSamples(t, prereqs.database.ID, 10, time.Now().UTC(), nil)[9]
+
+ response := getWalStreamStatistics(t, prereqs, "HOUR")
+
+ assert.Equal(t, wal_health.HealthLevelHealthy, response.Health.Level)
+ assert.Empty(t, response.Health.Reasons)
+ assert.Equal(t, wal_health.StreamerStateRunning, response.Streamer.State)
+
+ require.NotNil(t, response.Current)
+ assert.WithinDuration(t, newestSample.SampledAt, response.Current.SampledAt, time.Millisecond)
+ assert.Equal(t, physical_enums.PhysicalWalStreamSourceStateReachable, response.Current.SourceState)
+ assert.Equal(t, *newestSample.SlotLagBytes, *response.Current.SlotLagBytes)
+ assert.Equal(t, *newestSample.UnarchivedWalBytes, *response.Current.UnarchivedWalBytes)
+ assert.Equal(t, newestSample.LocalQueuePauseBytes, response.Current.LocalQueuePauseBytes)
+
+ assert.Equal(t, wal_health.KeepingUpStateKeepingUp, response.KeepingUp.State)
+ require.NotNil(t, response.KeepingUp.SourceWriteBytesPerSecond)
+
+ assert.Equal(t, backups_dto_physical.WalStreamStatisticsPeriodHour, response.Series.Period)
+ assert.Len(t, response.Series.Points, 60)
+}
+
+func Test_GetWalStreamStatistics_WhenRequestNotAllowed_RefusesWithoutStatistics(t *testing.T) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+ seedTestWalStreamSamples(t, prereqs.database.ID, 2, time.Now().UTC(), nil)
+
+ outsider := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)
+
+ logicalDatabase := databases.CreateTestDatabase(prereqs.workspace.ID, prereqs.storage, prereqs.notifier)
+ t.Cleanup(func() { databases.RemoveTestDatabase(context.Background(), logicalDatabase) })
+
+ testCases := []struct {
+ name string
+ url string
+ authToken string
+ expectedStatus int
+ }{
+ {
+ name: "User from another workspace",
+ url: getWalStreamStatisticsURL(prereqs.database.ID, "HOUR"),
+ authToken: "Bearer " + outsider.Token,
+ expectedStatus: http.StatusBadRequest,
+ },
+ {
+ name: "Unauthenticated request",
+ url: getWalStreamStatisticsURL(prereqs.database.ID, "HOUR"),
+ authToken: "",
+ expectedStatus: http.StatusUnauthorized,
+ },
+ {
+ name: "Nonexistent database",
+ url: getWalStreamStatisticsURL(uuid.New(), "HOUR"),
+ authToken: "Bearer " + prereqs.user.Token,
+ expectedStatus: http.StatusBadRequest,
+ },
+ {
+ name: "Logical database",
+ url: getWalStreamStatisticsURL(logicalDatabase.ID, "HOUR"),
+ authToken: "Bearer " + prereqs.user.Token,
+ expectedStatus: http.StatusBadRequest,
+ },
+ {
+ name: "Unknown period",
+ url: getWalStreamStatisticsURL(prereqs.database.ID, "MONTH"),
+ authToken: "Bearer " + prereqs.user.Token,
+ expectedStatus: http.StatusBadRequest,
+ },
+ {
+ name: "missing period",
+ url: "/api/v1/backups/physical/database/" + prereqs.database.ID.String() + "/wal-stream/statistics",
+ authToken: "Bearer " + prereqs.user.Token,
+ expectedStatus: http.StatusBadRequest,
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ response := test_utils.MakeGetRequest(t, prereqs.router, testCase.url, testCase.authToken,
+ testCase.expectedStatus)
+
+ assert.NotContains(t, string(response.Body), `"health"`)
+ assert.NotContains(t, string(response.Body), `"series"`)
+ })
+ }
+}
+
+func Test_GetWalStreamStatistics_ForWeek_ReturnsAtMost360PointsWithArchiveRatePerCoveredSecond(t *testing.T) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+ createHealthyTestWalStreamer(t, prereqs.database.ID)
+
+ firstResponse := getWalStreamStatistics(t, prereqs, "WEEK")
+ bucketDuration := time.Duration(firstResponse.Series.BucketSeconds) * time.Second
+ require.Equal(t, 28*time.Minute, bucketDuration)
+
+ busyBucketStartedAt := firstResponse.Series.From.Add(100 * bucketDuration)
+ quietBucketStartedAt := firstResponse.Series.From.Add(200 * bucketDuration)
+
+ busyArchivedBytes := []int64{30 * segmentBytes, 60 * segmentBytes, 90 * segmentBytes}
+ for sampleIndex, archivedBytes := range busyArchivedBytes {
+ sample := physical_testing.NewTestWalStreamSample(
+ prereqs.database.ID, busyBucketStartedAt.Add(time.Duration(sampleIndex+1)*time.Minute))
+ sample.ArchivedWalBytes = archivedBytes
+ physical_testing.CreateTestWalStreamSample(t, sample)
+ }
+
+ quietSample := physical_testing.NewTestWalStreamSample(prereqs.database.ID, quietBucketStartedAt.Add(time.Minute))
+ quietSample.ArchivedWalBytes = 3 * segmentBytes
+ physical_testing.CreateTestWalStreamSample(t, quietSample)
+
+ response := getWalStreamStatistics(t, prereqs, "WEEK")
+ require.LessOrEqual(t, len(response.Series.Points), 360)
+
+ pointsByStart := make(map[time.Time]*physical_dto.PhysicalWalStreamSeriesPoint)
+ for _, point := range response.Series.Points {
+ pointsByStart[point.StartedAt.UTC()] = point
+ }
+
+ busyPoint := pointsByStart[busyBucketStartedAt]
+ require.NotNil(t, busyPoint)
+ require.NotNil(t, busyPoint.ArchiveBytesPerSecond)
+ assert.InDelta(t, float64(180*segmentBytes)/(3*wal_health.SampleInterval.Seconds()),
+ *busyPoint.ArchiveBytesPerSecond, 1)
+
+ quietPoint := pointsByStart[quietBucketStartedAt]
+ require.NotNil(t, quietPoint)
+ require.NotNil(t, quietPoint.ArchiveBytesPerSecond)
+ assert.InDelta(t, float64(3*segmentBytes)/wal_health.SampleInterval.Seconds(),
+ *quietPoint.ArchiveBytesPerSecond, 1)
+}
+
+func Test_GetWalStreamStatistics_ForHour_Returns60MinutePointsEndingWithCurrentMoment(t *testing.T) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+
+ requestedAfter := time.Now().UTC()
+ response := getWalStreamStatistics(t, prereqs, "HOUR")
+ requestedBefore := time.Now().UTC()
+
+ require.Len(t, response.Series.Points, 60)
+ assert.Equal(t, int64(60), response.Series.BucketSeconds)
+
+ lastPoint := response.Series.Points[59]
+ assert.False(t, lastPoint.StartedAt.After(requestedBefore), "the last point starts no later than the request")
+ assert.True(t, lastPoint.StartedAt.Add(time.Minute).After(requestedAfter), "the last point contains the request")
+ assert.Equal(t, response.Series.To, lastPoint.StartedAt.Add(time.Minute))
+
+ for pointIndex := 1; pointIndex < len(response.Series.Points); pointIndex++ {
+ assert.Equal(t, time.Minute,
+ response.Series.Points[pointIndex].StartedAt.Sub(response.Series.Points[pointIndex-1].StartedAt))
+ }
+}
+
+func Test_GetWalStreamStatistics_WhenStreamerWasDownForAnHour_PointsCoveringHoleHaveNoValues(t *testing.T) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+ createHealthyTestWalStreamer(t, prereqs.database.ID)
+
+ now := time.Now().UTC()
+ holeStartedAt := now.Add(-2 * time.Hour)
+ holeEndedAt := now.Add(-time.Hour)
+
+ seedTestWalStreamSamples(t, prereqs.database.ID, 120, holeStartedAt, nil)
+ seedTestWalStreamSamples(t, prereqs.database.ID, 120, now, nil)
+
+ response := getWalStreamStatistics(t, prereqs, "DAY")
+ bucketDuration := time.Duration(response.Series.BucketSeconds) * time.Second
+
+ holePointCount := 0
+
+ for _, point := range response.Series.Points {
+ isInsideHole := point.StartedAt.After(holeStartedAt) && point.StartedAt.Add(bucketDuration).Before(holeEndedAt)
+ if !isInsideHole {
+ continue
+ }
+
+ holePointCount++
+
+ assert.Nil(t, point.ArchiveBytesPerSecond)
+ assert.Nil(t, point.SourceWriteBytesPerSecond)
+ assert.Nil(t, point.MaxUnarchivedWalBytes)
+ assert.Nil(t, point.Restarts)
+ }
+
+ assert.Greater(t, holePointCount, 10)
+}
+
+func Test_GetWalStreamStatistics_WhenDatabaseRemoved_LeavesNoSamples(t *testing.T) {
+ prereqs := createPhysicalControllerPrereqs(t)
+ seedTestWalStreamSamples(t, prereqs.database.ID, 3, time.Now().UTC(), nil)
+
+ test_utils.MakeDeleteRequest(t, prereqs.router,
+ "/api/v1/databases/"+prereqs.database.ID.String(), "Bearer "+prereqs.user.Token, http.StatusNoContent)
+
+ remainingSamples, err := physical_repositories.GetWalStreamSampleRepository().
+ FindSinceByDatabase(prereqs.database.ID, time.Time{})
+ require.NoError(t, err)
+ assert.Empty(t, remainingSamples)
+}
+
+func Test_GetWalStreamStatistics_WithCommittedSegments_ReportsNewestSegmentReceiveTime(t *testing.T) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+ createHealthyTestWalStreamer(t, prereqs.database.ID)
+ seedTestWalStreamSamples(t, prereqs.database.ID, 2, time.Now().UTC(), nil)
+
+ olderSegment := physical_testing.NewTestWalSegment(prereqs.database.ID, prereqs.storage.ID, 1,
+ "000000010000000000000010", walmath.LSN(0x10*segmentBytes), walmath.LSN(0x11*segmentBytes))
+ olderSegment.ReceivedAt = time.Now().UTC().Add(-10 * time.Minute)
+ physical_testing.CreateTestWalSegment(t, olderSegment)
+
+ newerSegment := physical_testing.NewTestWalSegment(prereqs.database.ID, prereqs.storage.ID, 1,
+ "000000010000000000000011", walmath.LSN(0x11*segmentBytes), walmath.LSN(0x12*segmentBytes))
+ newerSegment.ReceivedAt = time.Now().UTC().Add(-2 * time.Minute)
+ physical_testing.CreateTestWalSegment(t, newerSegment)
+
+ response := getWalStreamStatistics(t, prereqs, "HOUR")
+
+ require.NotNil(t, response.Current)
+ require.NotNil(t, response.Current.LastArchivedWalAt)
+ assert.WithinDuration(t, newerSegment.ReceivedAt, *response.Current.LastArchivedWalAt, time.Millisecond)
+}
+
+func Test_GetWalStreamStatistics_WhenNeedsAttention_ReturnsOnlyDocumentedCodes(t *testing.T) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+ createHealthyTestWalStreamer(t, prereqs.database.ID)
+
+ seedTestWalStreamSamples(t, prereqs.database.ID, 10, time.Now().UTC(),
+ func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.LocalQueueBytes = sample.LocalQueuePauseBytes
+ sample.SlotSafeWalSizeBytes = nil
+ })
+
+ response := test_utils.MakeGetRequest(t, prereqs.router, getWalStreamStatisticsURL(prereqs.database.ID, "DAY"),
+ "Bearer "+prereqs.user.Token, http.StatusOK)
+
+ var statistics backups_dto_physical.GetWalStreamStatisticsResponse
+ require.NoError(t, json.Unmarshal(response.Body, &statistics))
+
+ assert.Equal(t, wal_health.HealthLevelNeedsAttention, statistics.Health.Level)
+ assert.Equal(t, []wal_health.ReasonCode{wal_health.ReasonCodeLocalQueueFilling},
+ getReasonCodes(statistics.Health.Reasons))
+ assert.Contains(t, getNoticeCodes(statistics.Health.Notices), wal_health.NoticeCodeNoSlotWalLimit)
+
+ var body any
+ require.NoError(t, json.Unmarshal(response.Body, &body))
+ assertOnlyCodesTimesAndPositionsAreStrings(t, "", body)
+}
+
+// Every string in the statistics is a code from a documented list, a time or a
+// WAL position, so no field carries a sentence meant for display.
+func assertOnlyCodesTimesAndPositionsAreStrings(t *testing.T, fieldName string, value any) {
+ t.Helper()
+
+ documentedCodes := map[string][]string{
+ "level": {
+ string(wal_health.HealthLevelNotStreaming), string(wal_health.HealthLevelCollectingData),
+ string(wal_health.HealthLevelBroken), string(wal_health.HealthLevelNeedsAttention),
+ string(wal_health.HealthLevelHealthy), string(wal_health.ReasonLevelInfo),
+ },
+ "code": {
+ string(wal_health.ReasonCodeStreamingStopped), string(wal_health.ReasonCodeSourceUnreachable),
+ string(wal_health.ReasonCodeWalChainBroken), string(wal_health.ReasonCodeSlotMissing),
+ string(wal_health.ReasonCodeSlotLost), string(wal_health.ReasonCodeArchiveNotAdvancing),
+ string(wal_health.ReasonCodeArchiveSlowerThanSource), string(wal_health.ReasonCodeSlotKeepsExtraWal),
+ string(wal_health.ReasonCodeSlotLimitApproaching), string(wal_health.ReasonCodeNearSlotRebuild),
+ string(wal_health.ReasonCodeLocalQueueFilling), string(wal_health.ReasonCodeFrequentRestarts),
+ string(wal_health.ReasonCodeForcedRotationRefused), string(wal_health.NoticeCodeNoSlotWalLimit),
+ string(wal_health.NoticeCodeSlotKeepsExtraWalSteadily), string(wal_health.NoticeCodeNoFullBackupYet),
+ },
+ "state": {
+ string(wal_health.StreamerStateRunning), string(wal_health.StreamerStateNotRunning),
+ string(wal_health.StreamerStateNotStarted), string(wal_health.KeepingUpStateUnknown),
+ string(wal_health.KeepingUpStateFallingBehind), string(wal_health.KeepingUpStateCatchingUp),
+ string(wal_health.KeepingUpStateIdle), string(wal_health.KeepingUpStateKeepingUp),
+ },
+ "sourceState": {
+ string(physical_enums.PhysicalWalStreamSourceStateReachable),
+ string(physical_enums.PhysicalWalStreamSourceStateUnreachable),
+ string(physical_enums.PhysicalWalStreamSourceStateSlotMissing),
+ },
+ "slotWalStatus": {"reserved", "extended", "unreserved", "lost"},
+ "period": {"HOUR", "DAY", "WEEK"},
+ }
+
+ switch typedValue := value.(type) {
+ case map[string]any:
+ for nestedFieldName, nestedValue := range typedValue {
+ assertOnlyCodesTimesAndPositionsAreStrings(t, nestedFieldName, nestedValue)
+ }
+
+ case []any:
+ for _, nestedValue := range typedValue {
+ assertOnlyCodesTimesAndPositionsAreStrings(t, fieldName, nestedValue)
+ }
+
+ case string:
+ if allowedCodes, isCodeField := documentedCodes[fieldName]; isCodeField {
+ assert.Contains(t, allowedCodes, typedValue, "field %s holds an undocumented code", fieldName)
+
+ return
+ }
+
+ if strings.HasSuffix(fieldName, "Lsn") {
+ _, err := walmath.ParseLSN(typedValue)
+ assert.NoError(t, err, "field %s holds a WAL position", fieldName)
+
+ return
+ }
+
+ _, err := time.Parse(time.RFC3339Nano, typedValue)
+ assert.NoError(t, err, "field %s holds text that is neither a code, a time nor a position", fieldName)
+ }
+}
+
+func Test_GetWalStreamStatistics_WhenStreamerFailedTenMinutesAgo_ReportsStreamingStoppedSinceNewestSample(
+ t *testing.T,
+) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+
+ physical_testing.CreateTestWalStreamer(t, &physical_models.PhysicalWalStreamer{
+ DatabaseID: prereqs.database.ID,
+ StartedAt: time.Now().UTC().Add(-time.Hour),
+ LastHeartbeatAt: time.Now().UTC().Add(-10 * time.Minute),
+ Status: physical_enums.PhysicalWalStreamerStatusFailed,
+ FailingSince: new(time.Now().UTC().Add(-10 * time.Minute)),
+ })
+
+ newestSample := seedTestWalStreamSamples(t, prereqs.database.ID, 5, time.Now().UTC().Add(-10*time.Minute),
+ func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.LocalQueueBytes = sample.LocalQueuePauseBytes
+ })[4]
+
+ response := getWalStreamStatistics(t, prereqs, "HOUR")
+
+ assert.Equal(t, wal_health.HealthLevelBroken, response.Health.Level)
+ require.Equal(t, []wal_health.ReasonCode{wal_health.ReasonCodeStreamingStopped},
+ getReasonCodes(response.Health.Reasons), "no other reason is read from a sample that old")
+ require.NotNil(t, response.Health.Reasons[0].Since)
+ assert.WithinDuration(t, newestSample.SampledAt, *response.Health.Reasons[0].Since, time.Millisecond)
+ assert.Equal(t, wal_health.StreamerStateNotRunning, response.Streamer.State)
+}
+
+// Two segments on each side of a missing one leave a gap that ends well below
+// the archive progress point of the seeded samples.
+func seedCompletedFullWithWalGap(t *testing.T, prereqs *physicalControllerPrereqs) *physical_models.PhysicalFullBackup {
+ t.Helper()
+
+ fullStartLSN := walmath.LSN(0x10*segmentBytes + physical_testing.FirstRecordOffset)
+ full := physical_testing.CreateTestFullBackup(t, physical_testing.NewTestCompletedFullBackup(
+ prereqs.database.ID, prereqs.storage.ID, 1, fullStartLSN, fullStartLSN+physical_testing.FullLSNSpan))
+
+ for _, segmentNo := range []uint64{0x10, 0x11, 0x13, 0x14} {
+ physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(
+ prereqs.database.ID, prereqs.storage.ID, 1, walmath.WalSegmentNo(segmentNo).GetFilename(1),
+ walmath.LSN(segmentNo*segmentBytes), walmath.LSN((segmentNo+1)*segmentBytes)))
+ }
+
+ return full
+}
+
+func Test_GetWalStreamStatistics_WhenChainHasGapBelowProgressPoint_ReportsWalChainBrokenUntilNewerFull(t *testing.T) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+ createHealthyTestWalStreamer(t, prereqs.database.ID)
+ seedTestWalStreamSamples(t, prereqs.database.ID, 4, time.Now().UTC(), nil)
+
+ seedCompletedFullWithWalGap(t, prereqs)
+
+ brokenResponse := getWalStreamStatistics(t, prereqs, "HOUR")
+ assert.Equal(t, wal_health.HealthLevelBroken, brokenResponse.Health.Level)
+ assert.Equal(t, []wal_health.ReasonCode{wal_health.ReasonCodeWalChainBroken},
+ getReasonCodes(brokenResponse.Health.Reasons), "a healthy slot does not hide a broken chain")
+
+ newerFullStartLSN := walmath.LSN(0x20*segmentBytes + physical_testing.FirstRecordOffset)
+ newerFull := physical_testing.NewTestCompletedFullBackup(prereqs.database.ID, prereqs.storage.ID, 1,
+ newerFullStartLSN, newerFullStartLSN+physical_testing.FullLSNSpan)
+ newerFull.CreatedAt = time.Now().UTC().Add(time.Second)
+ physical_testing.CreateTestFullBackup(t, newerFull)
+
+ repairedResponse := getWalStreamStatistics(t, prereqs, "HOUR")
+ assert.Equal(t, wal_health.HealthLevelHealthy, repairedResponse.Health.Level)
+ assert.Empty(t, repairedResponse.Health.Reasons)
+}
+
+func Test_GetWalStreamStatistics_WhenIncrementalBrokeChainWithGap_StillReportsWalChainBroken(t *testing.T) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+ createHealthyTestWalStreamer(t, prereqs.database.ID)
+ seedTestWalStreamSamples(t, prereqs.database.ID, 4, time.Now().UTC(), nil)
+
+ full := seedCompletedFullWithWalGap(t, prereqs)
+
+ incrementalStartLSN := walmath.LSN(0x14*segmentBytes + physical_testing.FirstRecordOffset)
+ brokenIncremental := physical_testing.NewTestCompletedIncrementalBackup(prereqs.database.ID, prereqs.storage.ID,
+ full.ID, nil, 1, incrementalStartLSN, incrementalStartLSN+physical_testing.FullLSNSpan)
+ brokenIncremental.Status = physical_enums.PhysicalBackupStatusChainBroken
+ physical_testing.CreateTestIncrementalBackup(t, brokenIncremental)
+
+ response := getWalStreamStatistics(t, prereqs, "HOUR")
+
+ assert.Contains(t, getReasonCodes(response.Health.Reasons), wal_health.ReasonCodeWalChainBroken)
+ assert.NotContains(t, getNoticeCodes(response.Health.Notices), wal_health.NoticeCodeNoFullBackupYet,
+ "a chain broken by an incremental still has a completed full backup")
+}
+
+func Test_GetWalStreamStatistics_ForFullBackupRequestOrRun_ReportsFullBackupPending(t *testing.T) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+
+ assert.False(t, getWalStreamStatistics(t, prereqs, "HOUR").Health.IsFullBackupPending)
+
+ inFlightFull := physical_testing.CreateTestFullBackup(t,
+ physical_testing.NewTestInProgressFullBackup(prereqs.database.ID, prereqs.storage.ID, 1))
+ physical_testing.CreateTestInFlightClaim(t, prereqs.database.ID, inFlightFull.ID,
+ physical_enums.PhysicalBackupTypeFull)
+
+ assert.True(t, getWalStreamStatistics(t, prereqs, "HOUR").Health.IsFullBackupPending,
+ "a running full backup is pending")
+
+ require.NoError(t, physical_repositories.GetInFlightBackupRepository().Release(prereqs.database.ID))
+ assert.False(t, getWalStreamStatistics(t, prereqs, "HOUR").Health.IsFullBackupPending)
+
+ test_utils.MakePostRequest(t, prereqs.router,
+ "/api/v1/backups/physical/database/"+prereqs.database.ID.String()+"/trigger",
+ "Bearer "+prereqs.user.Token,
+ backups_dto_physical.TriggerBackupRequest{Type: backups_dto_physical.TriggerBackupTypeFull},
+ http.StatusAccepted)
+
+ assert.True(t, getWalStreamStatistics(t, prereqs, "HOUR").Health.IsFullBackupPending,
+ "a requested full backup is pending")
+}
+
+func Test_GetWalStreamStatistics_WhenStreamingNotExpected_ReportsNotStreamingWithRecordedSeries(t *testing.T) {
+ testCases := []struct {
+ name string
+ stopStreaming func(t *testing.T, prereqs *physicalControllerPrereqs)
+ }{
+ {
+ name: "Backups disabled",
+ stopStreaming: func(t *testing.T, prereqs *physicalControllerPrereqs) {
+ backupConfig, err := backups_config_physical.GetBackupConfigService().
+ GetBackupConfigByDbId(prereqs.database.ID)
+ require.NoError(t, err)
+
+ backupConfig.IsBackupsEnabled = false
+ _, err = backups_config_physical.GetBackupConfigService().SaveBackupConfig(t.Context(), backupConfig)
+ require.NoError(t, err)
+ },
+ },
+ {
+ name: "Switched out of WAL streaming",
+ stopStreaming: func(t *testing.T, prereqs *physicalControllerPrereqs) {
+ require.NoError(t, storage.GetDb().
+ Model(&postgresql_physical.PostgresqlPhysicalDatabase{}).
+ Where("database_id = ?", prereqs.database.ID).
+ Update("backup_type", postgresql_physical.BackupTypeFullAndIncremental).Error)
+ },
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+ createHealthyTestWalStreamer(t, prereqs.database.ID)
+ seedTestWalStreamSamples(t, prereqs.database.ID, 20, time.Now().UTC().Add(-3*time.Hour), nil)
+
+ testCase.stopStreaming(t, prereqs)
+
+ response := getWalStreamStatistics(t, prereqs, "DAY")
+
+ assert.Equal(t, wal_health.HealthLevelNotStreaming, response.Health.Level)
+
+ pointsWithValues := 0
+ for _, point := range response.Series.Points {
+ if point.ArchiveBytesPerSecond != nil {
+ pointsWithValues++
+ }
+ }
+
+ assert.Positive(t, pointsWithValues, "the samples recorded before streaming stopped stay in the series")
+ })
+ }
+}
+
+func Test_GetWalStreamStatistics_WhenRunningStreamerHeartbeatIsStale_ReportsNotRunning(t *testing.T) {
+ prereqs := createWalStreamingControllerPrereqs(t)
+
+ physical_testing.CreateTestWalStreamer(t, &physical_models.PhysicalWalStreamer{
+ DatabaseID: prereqs.database.ID,
+ StartedAt: time.Now().UTC().Add(-time.Hour),
+ LastHeartbeatAt: time.Now().UTC().Add(-wal_health.StreamerHeartbeatStaleness - time.Minute),
+ Status: physical_enums.PhysicalWalStreamerStatusRunning,
+ })
+
+ response := getWalStreamStatistics(t, prereqs, "HOUR")
+
+ assert.Equal(t, wal_health.StreamerStateNotRunning, response.Streamer.State)
+}
diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/service.go b/backend/internal/features/backups/backups/core/physical/chain_view/service.go
index 0e7e9f25..03510b56 100644
--- a/backend/internal/features/backups/backups/core/physical/chain_view/service.go
+++ b/backend/internal/features/backups/backups/core/physical/chain_view/service.go
@@ -44,6 +44,29 @@ func (s *ChainViewService) FindLastExtendableChainByDatabase(databaseID uuid.UUI
return newestView, nil
}
+// GetNewestCompletedFullWithGaps returns the newest completed full backup,
+// whether or not its chain is still extendable, with the WAL gaps of its chain;
+// nil when no full backup has completed. A chain broken by an incremental still
+// has a full backup and can still have a gap, which FindLastExtendableChainByDatabase
+// would hide.
+func (s *ChainViewService) GetNewestCompletedFullWithGaps(databaseID uuid.UUID) (*FullBackupWithGaps, error) {
+ fulls, err := s.fullBackupRepository.FindCompletedNewestFirstByDatabase(databaseID)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(fulls) == 0 {
+ return nil, nil
+ }
+
+ walGaps, err := s.FindWalGapsInChain(fulls[0].ID)
+ if err != nil {
+ return nil, err
+ }
+
+ return &FullBackupWithGaps{FullBackup: fulls[0], WalGaps: walGaps}, nil
+}
+
func (s *ChainViewService) FindNonExtendableChainsByDatabase(databaseID uuid.UUID) ([]*ChainView, error) {
fulls, err := s.fullBackupRepository.FindCompletedNewestFirstByDatabase(databaseID)
if err != nil {
diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/view.go b/backend/internal/features/backups/backups/core/physical/chain_view/view.go
index a7162079..414a5085 100644
--- a/backend/internal/features/backups/backups/core/physical/chain_view/view.go
+++ b/backend/internal/features/backups/backups/core/physical/chain_view/view.go
@@ -17,6 +17,11 @@ type ChainView struct {
Span LSNRange
}
+type FullBackupWithGaps struct {
+ FullBackup *physical_models.PhysicalFullBackup
+ WalGaps []LSNRange
+}
+
// LSNMax is the unbounded upper sentinel for the latest chain on a timeline
// (no successor FULL exists). Encoded as the maximum uint64 LSN so range
// predicates against pg_lsn columns still work without a special case.
diff --git a/backend/internal/features/backups/backups/core/physical/dto/dto.go b/backend/internal/features/backups/backups/core/physical/dto/dto.go
index 4a43c021..85889290 100644
--- a/backend/internal/features/backups/backups/core/physical/dto/dto.go
+++ b/backend/internal/features/backups/backups/core/physical/dto/dto.go
@@ -68,3 +68,26 @@ type PhysicalWalHistoryMetadata struct {
CreatedAt time.Time `json:"createdAt"`
}
+
+// One interval of the WAL stream statistics series. Every value is nil when no
+// sample falls into the interval, so the hole stays a hole instead of a zero.
+type PhysicalWalStreamSeriesPoint struct {
+ StartedAt time.Time `json:"startedAt" gorm:"column:started_at"`
+
+ SourceWriteBytesPerSecond *float64 `json:"sourceWriteBytesPerSecond" gorm:"column:source_write_bytes_per_second"`
+ ArchiveBytesPerSecond *float64 `json:"archiveBytesPerSecond" gorm:"column:archive_bytes_per_second"`
+ ArchivedStoredBytesPerSecond *float64 `json:"archivedStoredBytesPerSecond" gorm:"column:archived_stored_bytes_per_second"`
+
+ MaxSlotLagBytes *int64 `json:"maxSlotLagBytes" gorm:"column:max_slot_lag_bytes"`
+ MaxUnarchivedWalBytes *int64 `json:"maxUnarchivedWalBytes" gorm:"column:max_unarchived_wal_bytes"`
+ MaxLocalQueueBytes *int64 `json:"maxLocalQueueBytes" gorm:"column:max_local_queue_bytes"`
+ LocalQueuePauseBytes *int64 `json:"localQueuePauseBytes" gorm:"column:local_queue_pause_bytes"`
+
+ Restarts *int64 `json:"restarts" gorm:"column:restarts"`
+ Pauses *int64 `json:"pauses" gorm:"column:pauses"`
+ SessionStarts *int64 `json:"sessionStarts" gorm:"column:session_starts"`
+
+ HighestConcurrentUploads *int64 `json:"highestConcurrentUploads" gorm:"column:highest_concurrent_uploads"`
+ SegmentWriteAvgMillis *int64 `json:"segmentWriteAvgMillis" gorm:"column:segment_write_avg_millis"`
+ SegmentWriteMaxMillis *int64 `json:"segmentWriteMaxMillis" gorm:"column:segment_write_max_millis"`
+}
diff --git a/backend/internal/features/backups/backups/core/physical/enums/enums.go b/backend/internal/features/backups/backups/core/physical/enums/enums.go
index 7ddc61ea..1f08d739 100644
--- a/backend/internal/features/backups/backups/core/physical/enums/enums.go
+++ b/backend/internal/features/backups/backups/core/physical/enums/enums.go
@@ -93,3 +93,14 @@ const (
// the scheduler creates.
PhysicalBackupTypeWal PhysicalBackupType = "WAL"
)
+
+type PhysicalWalStreamSourceState string
+
+const (
+ PhysicalWalStreamSourceStateReachable PhysicalWalStreamSourceState = "REACHABLE"
+ PhysicalWalStreamSourceStateUnreachable PhysicalWalStreamSourceState = "UNREACHABLE"
+
+ // The source answered, but Databasus's replication slot no longer exists there.
+ // Unlike an unreachable source, this means the WAL chain is already broken.
+ PhysicalWalStreamSourceStateSlotMissing PhysicalWalStreamSourceState = "SLOT_MISSING"
+)
diff --git a/backend/internal/features/backups/backups/core/physical/models/wal_stream_sample.go b/backend/internal/features/backups/backups/core/physical/models/wal_stream_sample.go
new file mode 100644
index 00000000..6c161aa4
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/models/wal_stream_sample.go
@@ -0,0 +1,61 @@
+package physical_models
+
+import (
+ "time"
+
+ "github.com/google/uuid"
+
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ "databasus-backend/internal/util/walmath"
+)
+
+// One sample of a WAL-streaming database's archiving health, taken by the
+// streamer's lag monitor. Source-side values are nil whenever SourceState is not
+// REACHABLE; on a reachable sample a nil SlotSafeWalSizeBytes means the source
+// sets no limit on the WAL it keeps for the slot.
+type PhysicalWalStreamSample struct {
+ ID uuid.UUID `gorm:"column:id;type:uuid;primaryKey;default:gen_random_uuid()"`
+ DatabaseID uuid.UUID `gorm:"column:database_id;type:uuid;not null"`
+
+ SampledAt time.Time `gorm:"column:sampled_at;not null"`
+ SessionStartedAt time.Time `gorm:"column:session_started_at;not null"`
+
+ SourceState physical_enums.PhysicalWalStreamSourceState `gorm:"column:source_state;type:text;not null"`
+ SourceUnreachableSince *time.Time `gorm:"column:source_unreachable_since"`
+
+ SourceCurrentLSN *walmath.LSN `gorm:"column:source_current_lsn;type:pg_lsn"`
+ ArchiveProgressLSN *walmath.LSN `gorm:"column:archive_progress_lsn;type:pg_lsn"`
+ SourceWrittenBytes *int64 `gorm:"column:source_written_bytes"`
+
+ ArchivedSegments int `gorm:"column:archived_segments;not null"`
+ ArchivedWalBytes int64 `gorm:"column:archived_wal_bytes;not null"`
+ ArchivedStoredBytes int64 `gorm:"column:archived_stored_bytes;not null"`
+ WalSegmentSizeBytes int64 `gorm:"column:wal_segment_size_bytes;not null"`
+ HighestConcurrentUploads int `gorm:"column:highest_concurrent_uploads;not null"`
+ SegmentWriteAvgMillis *int64 `gorm:"column:segment_write_avg_millis"`
+ SegmentWriteMaxMillis *int64 `gorm:"column:segment_write_max_millis"`
+
+ SlotWalStatus *string `gorm:"column:slot_wal_status;type:text"`
+ SlotExtraWalSince *time.Time `gorm:"column:slot_extra_wal_since"`
+ SlotLagBytes *int64 `gorm:"column:slot_lag_bytes"`
+ SlotSafeWalSizeBytes *int64 `gorm:"column:slot_safe_wal_size_bytes"`
+
+ UnarchivedWalBytes *int64 `gorm:"column:unarchived_wal_bytes"`
+ ArchiveStalledSince *time.Time `gorm:"column:archive_stalled_since"`
+
+ LocalQueueBytes int64 `gorm:"column:local_queue_bytes;not null"`
+ LocalQueuePauseBytes int64 `gorm:"column:local_queue_pause_bytes;not null"`
+
+ ReceiverRestarts int `gorm:"column:receiver_restarts;not null"`
+ ReceiverPauses int `gorm:"column:receiver_pauses;not null"`
+ IsReceiverRunning bool `gorm:"column:is_receiver_running;not null"`
+ IsForcedRotationDenied bool `gorm:"column:is_forced_rotation_denied;not null"`
+}
+
+func (PhysicalWalStreamSample) TableName() string {
+ return "physical_wal_stream_samples"
+}
+
+func (s *PhysicalWalStreamSample) IsSourceReachable() bool {
+ return s.SourceState == physical_enums.PhysicalWalStreamSourceStateReachable
+}
diff --git a/backend/internal/features/backups/backups/core/physical/models/wal_streamer.go b/backend/internal/features/backups/backups/core/physical/models/wal_streamer.go
index 82832deb..7114bc21 100644
--- a/backend/internal/features/backups/backups/core/physical/models/wal_streamer.go
+++ b/backend/internal/features/backups/backups/core/physical/models/wal_streamer.go
@@ -18,6 +18,11 @@ type PhysicalWalStreamer struct {
StartedAt time.Time `json:"startedAt" gorm:"column:started_at"`
LastHeartbeatAt time.Time `json:"lastHeartbeatAt" gorm:"column:last_heartbeat_at"`
Status physical_enums.PhysicalWalStreamerStatus `json:"status" gorm:"column:status;type:text;not null"`
+
+ // The first failure since a session last recorded a sample. A crash loop keeps
+ // it, so a streamer that never gets as far as its first sample still shows how
+ // long it has been failing.
+ FailingSince *time.Time `json:"failingSince" gorm:"column:failing_since"`
}
func (PhysicalWalStreamer) TableName() string {
diff --git a/backend/internal/features/backups/backups/core/physical/repositories/di.go b/backend/internal/features/backups/backups/core/physical/repositories/di.go
index 6d6d25fe..d9c4ae14 100644
--- a/backend/internal/features/backups/backups/core/physical/repositories/di.go
+++ b/backend/internal/features/backups/backups/core/physical/repositories/di.go
@@ -7,6 +7,7 @@ var (
walSegmentRepository = &PhysicalWalSegmentRepository{}
walHistoryRepository = &PhysicalWalHistoryRepository{}
walStreamerRepository = &PhysicalWalStreamerRepository{}
+ walStreamSampleRepository = &PhysicalWalStreamSampleRepository{}
)
func GetFullBackupRepository() *PhysicalFullBackupRepository {
@@ -32,3 +33,7 @@ func GetWalHistoryRepository() *PhysicalWalHistoryRepository {
func GetWalStreamerRepository() *PhysicalWalStreamerRepository {
return walStreamerRepository
}
+
+func GetWalStreamSampleRepository() *PhysicalWalStreamSampleRepository {
+ return walStreamSampleRepository
+}
diff --git a/backend/internal/features/backups/backups/core/physical/repositories/wal_stream_sample_repository.go b/backend/internal/features/backups/backups/core/physical/repositories/wal_stream_sample_repository.go
new file mode 100644
index 00000000..e58e767d
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/repositories/wal_stream_sample_repository.go
@@ -0,0 +1,213 @@
+package physical_repositories
+
+import (
+ "errors"
+ "time"
+
+ "github.com/google/uuid"
+ "gorm.io/gorm"
+
+ physical_dto "databasus-backend/internal/features/backups/backups/core/physical/dto"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+ "databasus-backend/internal/storage"
+)
+
+// Buckets align to date_bin with this origin, the same one Go uses to compute
+// the first bucket, so a sample always lands in the bucket the series lists.
+var seriesBucketOrigin = time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC)
+
+type PhysicalWalStreamSampleRepository struct{}
+
+type WalStreamSeriesQuery struct {
+ DatabaseID uuid.UUID
+ FirstBucketAt time.Time
+ BucketDuration time.Duration
+ BucketCount int
+
+ // Every sample stands for this much time, so a rate divides by the time the
+ // samples of a bucket cover, not by the bucket's length, and a partly filled
+ // bucket is not understated.
+ SampleInterval time.Duration
+}
+
+func (r *PhysicalWalStreamSampleRepository) Insert(sample *physical_models.PhysicalWalStreamSample) error {
+ if sample.DatabaseID == uuid.Nil {
+ return errors.New("database ID is required")
+ }
+
+ if sample.ID == uuid.Nil {
+ sample.ID = uuid.New()
+ }
+
+ return storage.GetDb().Create(sample).Error
+}
+
+func (r *PhysicalWalStreamSampleRepository) DeleteOlderThan(cutoff time.Time) (int64, error) {
+ result := storage.
+ GetDb().
+ Where("sampled_at < ?", cutoff).
+ Delete(&physical_models.PhysicalWalStreamSample{})
+
+ return result.RowsAffected, result.Error
+}
+
+func (r *PhysicalWalStreamSampleRepository) FindNewestByDatabase(
+ databaseID uuid.UUID,
+) (*physical_models.PhysicalWalStreamSample, error) {
+ var sample physical_models.PhysicalWalStreamSample
+
+ err := storage.
+ GetDb().
+ Where("database_id = ?", databaseID).
+ Order("sampled_at DESC").
+ First(&sample).Error
+ if err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil, nil
+ }
+
+ return nil, err
+ }
+
+ return &sample, nil
+}
+
+// FindSinceByDatabase returns the samples taken at or after since, oldest first.
+func (r *PhysicalWalStreamSampleRepository) FindSinceByDatabase(
+ databaseID uuid.UUID,
+ since time.Time,
+) ([]*physical_models.PhysicalWalStreamSample, error) {
+ var samples []*physical_models.PhysicalWalStreamSample
+
+ err := storage.
+ GetDb().
+ Where("database_id = ? AND sampled_at >= ?", databaseID, since).
+ Order("sampled_at ASC").
+ Find(&samples).Error
+
+ return samples, err
+}
+
+// A session that crash-looped stays counted by when it started, even when its
+// samples have already left the window.
+func (r *PhysicalWalStreamSampleRepository) CountSessionStartsSince(
+ databaseID uuid.UUID,
+ since time.Time,
+) (int, error) {
+ var count int64
+
+ err := storage.
+ GetDb().
+ Model(&physical_models.PhysicalWalStreamSample{}).
+ Where("database_id = ? AND sampled_at >= ? AND session_started_at >= ?", databaseID, since, since).
+ Distinct("session_started_at").
+ Count(&count).Error
+
+ return int(count), err
+}
+
+// GetSeries returns exactly BucketCount points, oldest first. Buckets without
+// samples come back with every value nil.
+func (r *PhysicalWalStreamSampleRepository) GetSeries(
+ query WalStreamSeriesQuery,
+) ([]*physical_dto.PhysicalWalStreamSeriesPoint, error) {
+ if query.BucketCount <= 0 || query.BucketDuration <= 0 || query.SampleInterval <= 0 {
+ return nil, errors.New("bucket count, bucket duration and sample interval must be positive")
+ }
+
+ var points []*physical_dto.PhysicalWalStreamSeriesPoint
+
+ seriesEndAt := query.FirstBucketAt.Add(query.BucketDuration * time.Duration(query.BucketCount))
+
+ err := storage.
+ GetDb().
+ Raw(`
+ WITH buckets AS (
+ SELECT generate_series(
+ CAST(@first_bucket_at AS TIMESTAMPTZ),
+ CAST(@last_bucket_at AS TIMESTAMPTZ),
+ make_interval(secs => @bucket_seconds)
+ ) AS started_at
+ ),
+ binned_samples AS (
+ SELECT
+ date_bin(make_interval(secs => @bucket_seconds), sampled_at, CAST(@origin AS TIMESTAMPTZ)) AS started_at,
+ SUM(source_written_bytes)::float8
+ / NULLIF(COUNT(source_written_bytes) * @sample_seconds, 0) AS source_write_bytes_per_second,
+ SUM(archived_wal_bytes)::float8 / (COUNT(*) * @sample_seconds) AS archive_bytes_per_second,
+ SUM(archived_stored_bytes)::float8 / (COUNT(*) * @sample_seconds) AS archived_stored_bytes_per_second,
+ MAX(slot_lag_bytes) AS max_slot_lag_bytes,
+ MAX(unarchived_wal_bytes) AS max_unarchived_wal_bytes,
+ MAX(local_queue_bytes) AS max_local_queue_bytes,
+ MAX(local_queue_pause_bytes) AS local_queue_pause_bytes,
+ SUM(receiver_restarts)::bigint AS restarts,
+ SUM(receiver_pauses)::bigint AS pauses,
+ MAX(highest_concurrent_uploads)::bigint AS highest_concurrent_uploads,
+ (
+ SUM(segment_write_avg_millis::bigint * archived_segments)
+ FILTER (WHERE segment_write_avg_millis IS NOT NULL)
+ / NULLIF(SUM(archived_segments) FILTER (WHERE segment_write_avg_millis IS NOT NULL), 0)
+ )::bigint AS segment_write_avg_millis,
+ MAX(segment_write_max_millis)::bigint AS segment_write_max_millis
+ FROM physical_wal_stream_samples
+ WHERE database_id = @database_id
+ AND sampled_at >= @first_bucket_at
+ AND sampled_at < @series_end_at
+ GROUP BY 1
+ ),
+ binned_session_starts AS (
+ SELECT
+ date_bin(make_interval(secs => @bucket_seconds), session_started_at, CAST(@origin AS TIMESTAMPTZ)) AS started_at,
+ COUNT(DISTINCT session_started_at) AS session_starts
+ FROM physical_wal_stream_samples
+ WHERE database_id = @database_id
+ AND sampled_at >= @first_bucket_at
+ AND session_started_at >= @first_bucket_at
+ AND session_started_at < @series_end_at
+ GROUP BY 1
+ )
+ SELECT
+ buckets.started_at,
+ binned_samples.source_write_bytes_per_second,
+ binned_samples.archive_bytes_per_second,
+ binned_samples.archived_stored_bytes_per_second,
+ binned_samples.max_slot_lag_bytes,
+ binned_samples.max_unarchived_wal_bytes,
+ binned_samples.max_local_queue_bytes,
+ binned_samples.local_queue_pause_bytes,
+ binned_samples.restarts,
+ binned_samples.pauses,
+ CASE
+ WHEN binned_samples.started_at IS NULL AND binned_session_starts.started_at IS NULL THEN NULL
+ ELSE COALESCE(binned_session_starts.session_starts, 0)
+ END AS session_starts,
+ binned_samples.highest_concurrent_uploads,
+ binned_samples.segment_write_avg_millis,
+ binned_samples.segment_write_max_millis
+ FROM buckets
+ LEFT JOIN binned_samples ON binned_samples.started_at = buckets.started_at
+ LEFT JOIN binned_session_starts ON binned_session_starts.started_at = buckets.started_at
+ ORDER BY buckets.started_at
+ `, map[string]any{
+ "database_id": query.DatabaseID,
+ "first_bucket_at": query.FirstBucketAt,
+ "last_bucket_at": seriesEndAt.Add(-query.BucketDuration),
+ "series_end_at": seriesEndAt,
+ "bucket_seconds": query.BucketDuration.Seconds(),
+ "sample_seconds": query.SampleInterval.Seconds(),
+ "origin": seriesBucketOrigin,
+ }).
+ Scan(&points).Error
+
+ return points, err
+}
+
+// GetFirstSeriesBucketAt returns the start of the oldest bucket of a series that
+// covers period and ends with the bucket holding now: one bucket after the
+// bucket that holds now minus period, so the series has period ÷ bucket buckets.
+func GetFirstSeriesBucketAt(now time.Time, period, bucketDuration time.Duration) time.Time {
+ periodStart := now.Add(-period)
+ bucketsSinceOrigin := periodStart.Sub(seriesBucketOrigin) / bucketDuration
+
+ return seriesBucketOrigin.Add(bucketsSinceOrigin*bucketDuration + bucketDuration)
+}
diff --git a/backend/internal/features/backups/backups/core/physical/repositories/wal_streamer_repository.go b/backend/internal/features/backups/backups/core/physical/repositories/wal_streamer_repository.go
index 7d189e2e..831f67de 100644
--- a/backend/internal/features/backups/backups/core/physical/repositories/wal_streamer_repository.go
+++ b/backend/internal/features/backups/backups/core/physical/repositories/wal_streamer_repository.go
@@ -88,7 +88,23 @@ func (r *PhysicalWalStreamerRepository) Heartbeat(databaseID uuid.UUID) error {
Update("last_heartbeat_at", gorm.Expr("NOW()")).Error
}
+// A crash loop keeps the failing_since of its first failure, so the time a
+// streamer has been failing survives every reclaim that fails again.
func (r *PhysicalWalStreamerRepository) MarkFailed(databaseID uuid.UUID) error {
+ return storage.
+ GetDb().
+ Model(&physical_models.PhysicalWalStreamer{}).
+ Where("database_id = ?", databaseID).
+ Updates(map[string]any{
+ "status": physical_enums.PhysicalWalStreamerStatusFailed,
+ "failing_since": gorm.Expr("COALESCE(failing_since, NOW())"),
+ }).Error
+}
+
+// A deliberate stop (shutdown, streaming switched off) releases the row for
+// reclaim like a failure does, but it is not one, so it leaves failing_since
+// alone and a deploy does not read as a streamer that has been failing.
+func (r *PhysicalWalStreamerRepository) MarkStopped(databaseID uuid.UUID) error {
return storage.
GetDb().
Model(&physical_models.PhysicalWalStreamer{}).
@@ -96,13 +112,25 @@ func (r *PhysicalWalStreamerRepository) MarkFailed(databaseID uuid.UUID) error {
Update("status", physical_enums.PhysicalWalStreamerStatusFailed).Error
}
+func (r *PhysicalWalStreamerRepository) ClearFailingSince(databaseID uuid.UUID) error {
+ return storage.
+ GetDb().
+ Model(&physical_models.PhysicalWalStreamer{}).
+ Where("database_id = ?", databaseID).
+ Where("failing_since IS NOT NULL").
+ Update("failing_since", nil).Error
+}
+
func (r *PhysicalWalStreamerRepository) MarkStaleRunningFailed(stalenessSeconds int) (int64, error) {
result := storage.
GetDb().
Model(&physical_models.PhysicalWalStreamer{}).
Where("status = ?", physical_enums.PhysicalWalStreamerStatusRunning).
Where("last_heartbeat_at < NOW() - make_interval(secs => ?)", float64(stalenessSeconds)).
- Update("status", physical_enums.PhysicalWalStreamerStatusFailed)
+ Updates(map[string]any{
+ "status": physical_enums.PhysicalWalStreamerStatusFailed,
+ "failing_since": gorm.Expr("COALESCE(failing_since, NOW())"),
+ })
return result.RowsAffected, result.Error
}
diff --git a/backend/internal/features/backups/backups/core/physical/service/di.go b/backend/internal/features/backups/backups/core/physical/service/di.go
index c70fa0d7..85ad15f3 100644
--- a/backend/internal/features/backups/backups/core/physical/service/di.go
+++ b/backend/internal/features/backups/backups/core/physical/service/di.go
@@ -10,6 +10,8 @@ import (
var physicalBackupService = &PhysicalBackupService{
physical_repositories.GetFullBackupRepository(),
physical_repositories.GetWalSegmentRepository(),
+ physical_repositories.GetWalStreamSampleRepository(),
+ physical_repositories.GetWalStreamerRepository(),
storages.GetStorageService(),
storages.GetStorageFileStore(),
encryption.GetFieldEncryptor(),
diff --git a/backend/internal/features/backups/backups/core/physical/service/dto.go b/backend/internal/features/backups/backups/core/physical/service/dto.go
index a3472060..af80e82c 100644
--- a/backend/internal/features/backups/backups/core/physical/service/dto.go
+++ b/backend/internal/features/backups/backups/core/physical/service/dto.go
@@ -7,6 +7,9 @@ import (
"github.com/google/uuid"
"databasus-backend/internal/features/backups/backups/core/physical/chain_view"
+ physical_dto "databasus-backend/internal/features/backups/backups/core/physical/dto"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+ "databasus-backend/internal/features/backups/backups/core/physical/wal_health"
)
type DeleteOrphanWalSegmentsSpec struct {
@@ -114,3 +117,34 @@ type DependentsSummary struct {
HistoryFiles int
TotalSizeMB float64
}
+
+type WalStreamStatisticsQuery struct {
+ DatabaseID uuid.UUID
+ Period time.Duration
+ BucketDuration time.Duration
+
+ SlotRebuildLagBytes int64
+ IsStreamingExpected bool
+ HasCompletedFullBackup bool
+ NewestChainGaps []chain_view.LSNRange
+
+ Now time.Time
+}
+
+// Everything the WAL stream statistics need from the catalog, decided at one
+// moment. NewestSample, Streamer and LastArchivedWalAt are nil when they do not
+// exist yet.
+type WalStreamStatistics struct {
+ Verdict wal_health.Verdict
+ KeepingUp wal_health.KeepingUp
+ StreamerState wal_health.StreamerState
+ RestartsLastHour int
+
+ Streamer *physical_models.PhysicalWalStreamer
+ NewestSample *physical_models.PhysicalWalStreamSample
+ LastArchivedWalAt *time.Time
+
+ SeriesFrom time.Time
+ SeriesTo time.Time
+ SeriesPoints []*physical_dto.PhysicalWalStreamSeriesPoint
+}
diff --git a/backend/internal/features/backups/backups/core/physical/service/service.go b/backend/internal/features/backups/backups/core/physical/service/service.go
index f72889fa..f752beb6 100644
--- a/backend/internal/features/backups/backups/core/physical/service/service.go
+++ b/backend/internal/features/backups/backups/core/physical/service/service.go
@@ -43,12 +43,14 @@ const (
// the single seam through which the cleaner mutates catalog rows — the cleaner
// never issues raw DELETEs.
type PhysicalBackupService struct {
- fullBackupRepository *physical_repositories.PhysicalFullBackupRepository
- walSegmentRepository *physical_repositories.PhysicalWalSegmentRepository
- storageService *storages.StorageService
- fileStore *storage_files.Store
- fieldEncryptor util_encryption.FieldEncryptor
- logger *slog.Logger
+ fullBackupRepository *physical_repositories.PhysicalFullBackupRepository
+ walSegmentRepository *physical_repositories.PhysicalWalSegmentRepository
+ walStreamSampleRepository *physical_repositories.PhysicalWalStreamSampleRepository
+ walStreamerRepository *physical_repositories.PhysicalWalStreamerRepository
+ storageService *storages.StorageService
+ fileStore *storage_files.Store
+ fieldEncryptor util_encryption.FieldEncryptor
+ logger *slog.Logger
}
// GetLatestCompletedFullBackup returns the most recent completed FULL backup for
diff --git a/backend/internal/features/backups/backups/core/physical/service/wal_stream_statistics.go b/backend/internal/features/backups/backups/core/physical/service/wal_stream_statistics.go
new file mode 100644
index 00000000..a3e4d704
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/service/wal_stream_statistics.go
@@ -0,0 +1,83 @@
+package physical_service
+
+import (
+ "time"
+
+ physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
+ "databasus-backend/internal/features/backups/backups/core/physical/wal_health"
+)
+
+// GetWalStreamStatistics reads the samples, the streamer row and the newest
+// archived segment of one database and decides its verdict, keeping-up summary
+// and chart series at query.Now. The verdict reads the last hour of samples,
+// the series reads the requested period.
+func (s *PhysicalBackupService) GetWalStreamStatistics(query WalStreamStatisticsQuery) (*WalStreamStatistics, error) {
+ newestSample, err := s.walStreamSampleRepository.FindNewestByDatabase(query.DatabaseID)
+ if err != nil {
+ return nil, err
+ }
+
+ hourAgo := query.Now.Add(-time.Hour)
+
+ recentSamples, err := s.walStreamSampleRepository.FindSinceByDatabase(query.DatabaseID, hourAgo)
+ if err != nil {
+ return nil, err
+ }
+
+ sessionStartsLastHour, err := s.walStreamSampleRepository.CountSessionStartsSince(query.DatabaseID, hourAgo)
+ if err != nil {
+ return nil, err
+ }
+
+ streamer, err := s.walStreamerRepository.FindByDatabaseID(query.DatabaseID)
+ if err != nil {
+ return nil, err
+ }
+
+ newestCommittedSegment, err := s.walSegmentRepository.FindLatestCommittedSegment(query.DatabaseID)
+ if err != nil {
+ return nil, err
+ }
+
+ seriesFrom := physical_repositories.GetFirstSeriesBucketAt(query.Now, query.Period, query.BucketDuration)
+ bucketCount := int(query.Period / query.BucketDuration)
+
+ seriesPoints, err := s.walStreamSampleRepository.GetSeries(physical_repositories.WalStreamSeriesQuery{
+ DatabaseID: query.DatabaseID,
+ FirstBucketAt: seriesFrom,
+ BucketDuration: query.BucketDuration,
+ BucketCount: bucketCount,
+ SampleInterval: wal_health.SampleInterval,
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ statistics := &WalStreamStatistics{
+ Verdict: wal_health.GetVerdict(wal_health.VerdictInput{
+ NewestSample: newestSample,
+ RecentSamples: recentSamples,
+ Streamer: streamer,
+ SessionStartsLastHour: sessionStartsLastHour,
+ SlotRebuildLagBytes: query.SlotRebuildLagBytes,
+ IsStreamingExpected: query.IsStreamingExpected,
+ HasCompletedFullBackup: query.HasCompletedFullBackup,
+ NewestChainGaps: query.NewestChainGaps,
+ Now: query.Now,
+ }),
+ KeepingUp: wal_health.GetKeepingUp(recentSamples, query.SlotRebuildLagBytes, query.Now),
+ StreamerState: wal_health.GetStreamerState(streamer, query.Now),
+ RestartsLastHour: wal_health.GetRestartsLastHour(recentSamples, sessionStartsLastHour, query.Now),
+ Streamer: streamer,
+ NewestSample: newestSample,
+ SeriesFrom: seriesFrom,
+ SeriesTo: seriesFrom.Add(time.Duration(bucketCount) * query.BucketDuration),
+ SeriesPoints: seriesPoints,
+ }
+
+ if newestCommittedSegment != nil {
+ statistics.LastArchivedWalAt = new(newestCommittedSegment.ReceivedAt)
+ }
+
+ return statistics, nil
+}
diff --git a/backend/internal/features/backups/backups/core/physical/testing/testing.go b/backend/internal/features/backups/backups/core/physical/testing/testing.go
index 1c504ccf..27518378 100644
--- a/backend/internal/features/backups/backups/core/physical/testing/testing.go
+++ b/backend/internal/features/backups/backups/core/physical/testing/testing.go
@@ -226,3 +226,65 @@ func DeleteAllPhysicalCatalogForDatabase(t *testing.T, databaseID uuid.UUID) {
t.Fatalf("cleanup full backups: %v", err)
}
}
+
+// NewTestWalStreamSample builds a healthy reachable sample of a streaming
+// database; tests change the fields their scenario is about.
+func NewTestWalStreamSample(databaseID uuid.UUID, sampledAt time.Time) *physical_models.PhysicalWalStreamSample {
+ const segmentSizeBytes = 16 * 1024 * 1024
+
+ return &physical_models.PhysicalWalStreamSample{
+ DatabaseID: databaseID,
+ SampledAt: sampledAt,
+ SessionStartedAt: sampledAt.Add(-time.Hour),
+ SourceState: physical_enums.PhysicalWalStreamSourceStateReachable,
+ SourceCurrentLSN: new(walmath.LSN(0x50000000)),
+ ArchiveProgressLSN: new(walmath.LSN(0x4F000000)),
+ SourceWrittenBytes: new(int64(segmentSizeBytes)),
+ ArchivedSegments: 1,
+ ArchivedWalBytes: segmentSizeBytes,
+ ArchivedStoredBytes: 4 * 1024 * 1024,
+ WalSegmentSizeBytes: segmentSizeBytes,
+ HighestConcurrentUploads: 1,
+ SegmentWriteAvgMillis: new(int64(100)),
+ SegmentWriteMaxMillis: new(int64(150)),
+ SlotWalStatus: new("reserved"),
+ SlotLagBytes: new(int64(1024 * 1024)),
+ SlotSafeWalSizeBytes: new(int64(10 * 1024 * 1024 * 1024)),
+ UnarchivedWalBytes: new(int64(segmentSizeBytes)),
+ LocalQueueBytes: segmentSizeBytes,
+ LocalQueuePauseBytes: 512 * 1024 * 1024,
+ IsReceiverRunning: true,
+ }
+}
+
+func CreateTestWalStreamSample(
+ t *testing.T,
+ sample *physical_models.PhysicalWalStreamSample,
+) *physical_models.PhysicalWalStreamSample {
+ t.Helper()
+
+ if err := physical_repositories.GetWalStreamSampleRepository().Insert(sample); err != nil {
+ t.Fatalf("save test wal stream sample: %v", err)
+ }
+
+ return sample
+}
+
+// CreateTestWalStreamer writes a streamer row with the exact times a scenario
+// needs, which Claim cannot do because it stamps every time with NOW().
+func CreateTestWalStreamer(
+ t *testing.T,
+ walStreamer *physical_models.PhysicalWalStreamer,
+) *physical_models.PhysicalWalStreamer {
+ t.Helper()
+
+ if err := storage.GetDb().Create(walStreamer).Error; err != nil {
+ t.Fatalf("save test wal streamer: %v", err)
+ }
+
+ t.Cleanup(func() {
+ _ = physical_repositories.GetWalStreamerRepository().DeleteByDatabaseID(walStreamer.DatabaseID)
+ })
+
+ return walStreamer
+}
diff --git a/backend/internal/features/backups/backups/core/physical/wal_health/constants.go b/backend/internal/features/backups/backups/core/physical/wal_health/constants.go
new file mode 100644
index 00000000..7aa44dc3
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/wal_health/constants.go
@@ -0,0 +1,70 @@
+package wal_health
+
+import "time"
+
+const (
+ // The WAL streamer's lag monitor probes the source and records one sample on
+ // this cadence. It balances detection latency against query load on the
+ // source, and every rate divides by it: one sample stands for this much time.
+ SampleInterval = 30 * time.Second
+
+ SampleRetention = 7 * 24 * time.Hour
+
+ // A warning wal_status must persist this long before the WAL streamer alerts
+ // on it, so a transient write burst stays quiet.
+ WarningSlotStatusHoldPeriod = 5 * time.Minute
+
+ // A rarely-written database can legitimately hold its newest WAL locally for a
+ // while, but past this the recovery point has silently stopped tracking the
+ // source and only the operator can decide what to do about it.
+ DefaultArchiveStalenessThreshold = 30 * time.Minute
+
+ // A streamer row whose heartbeat is older than this is assumed dead and
+ // reclaimable by another process. Heartbeats fire every supervisor tick
+ // (15 s), so 90 s = 6 missed beats — tolerant of GC pauses without leaving a
+ // dead streamer unclaimed for long.
+ StreamerHeartbeatStaleness = 90 * time.Second
+)
+
+// Verdict thresholds. Each sits just ahead of something that already acts, so
+// the screen warns before Databasus pauses, rebuilds or notifies.
+const (
+ // Four missed samples: past this the newest sample no longer describes now.
+ staleSampleAge = 2 * time.Minute
+
+ // A session younger than this without a sample is still starting up.
+ collectingDataPeriod = 2 * time.Minute
+
+ streamingStoppedAfter = 1 * time.Minute
+
+ sourceUnreachableAttentionAfter = 1 * time.Minute
+ sourceUnreachableBrokenAfter = 5 * time.Minute
+
+ archiveNotAdvancingAttentionAfter = 10 * time.Minute
+
+ // A 30-second rate swings with every upload batch, while an hour hides a
+ // backlog that started five minutes ago.
+ keepingUpWindow = 10 * time.Minute
+
+ // The lowest value at each end of the window strips the sawtooth of in-flight
+ // uploads, which reaches four segments plus the open one in normal operation.
+ keepingUpWindowEdge = 150 * time.Second
+
+ minBacklogThresholdBytes int64 = 64 * 1024 * 1024
+ backlogThresholdSegments int64 = 4
+
+ slotLimitApproachingHorizon = 1 * time.Hour
+
+ // Percent of the configured rebuild lag: warns before Databasus sacrifices the
+ // chain by rebuilding the slot.
+ nearSlotRebuildPercent int64 = 80
+
+ // Percent of the pause size: warns before back pressure stops the receiver.
+ localQueueFillingPercent int64 = 50
+
+ // Tolerates a deploy and one network blip an hour.
+ frequentRestartsCount = 3
+ frequentRestartsWindow = 1 * time.Hour
+
+ idleSourceWrittenBytes int64 = 1024 * 1024
+)
diff --git a/backend/internal/features/backups/backups/core/physical/wal_health/doc.go b/backend/internal/features/backups/backups/core/physical/wal_health/doc.go
new file mode 100644
index 00000000..333b4f2c
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/wal_health/doc.go
@@ -0,0 +1,6 @@
+// Package wal_health decides how healthy WAL archiving of a streaming database
+// is: the verdict with its reasons and notices, the keeping-up summary and the
+// streamer state. It also owns every threshold those decisions use, including
+// the ones the WAL streamer's notifications and the stream supervisor act on,
+// so the screen and the notifications agree about when a condition began.
+package wal_health
diff --git a/backend/internal/features/backups/backups/core/physical/wal_health/dto.go b/backend/internal/features/backups/backups/core/physical/wal_health/dto.go
new file mode 100644
index 00000000..553ac10d
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/wal_health/dto.go
@@ -0,0 +1,73 @@
+package wal_health
+
+import (
+ "time"
+
+ "databasus-backend/internal/features/backups/backups/core/physical/chain_view"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+)
+
+type VerdictInput struct {
+ // The newest sample of the database, whatever its age; nil when none exists.
+ NewestSample *physical_models.PhysicalWalStreamSample
+
+ // The samples of the last hour, oldest first. The newest sample is the last
+ // one whenever it is younger than an hour.
+ RecentSamples []*physical_models.PhysicalWalStreamSample
+
+ // Nil when no streamer has ever run for the database.
+ Streamer *physical_models.PhysicalWalStreamer
+
+ SessionStartsLastHour int
+
+ SlotRebuildLagBytes int64
+
+ IsStreamingExpected bool
+ HasCompletedFullBackup bool
+
+ // The WAL gaps of the newest completed full backup's chain.
+ NewestChainGaps []chain_view.LSNRange
+
+ Now time.Time
+}
+
+type Reason struct {
+ Code ReasonCode `json:"code"`
+ Level ReasonLevel `json:"level"`
+ Since *time.Time `json:"since,omitempty"`
+ ValueBytes *int64 `json:"valueBytes,omitempty"`
+ ThresholdBytes *int64 `json:"thresholdBytes,omitempty"`
+ Count *int `json:"count,omitempty"`
+ EstimatedSeconds *int64 `json:"estimatedSeconds,omitempty"`
+}
+
+type Notice struct {
+ Code NoticeCode `json:"code"`
+ Level ReasonLevel `json:"level"`
+ Since *time.Time `json:"since,omitempty"`
+}
+
+type Verdict struct {
+ Level HealthLevel `json:"level"`
+ Reasons []Reason `json:"reasons"`
+ Notices []Notice `json:"notices"`
+}
+
+type KeepingUp struct {
+ State KeepingUpState `json:"state"`
+
+ SourceWriteBytesPerSecond *float64 `json:"sourceWriteBytesPerSecond"`
+ ArchiveBytesPerSecond *float64 `json:"archiveBytesPerSecond"`
+
+ UnarchivedWalBytes *int64 `json:"unarchivedWalBytes"`
+ UnarchivedWalChangeBytesPerSecond *float64 `json:"unarchivedWalChangeBytesPerSecond"`
+
+ SlotLagBytes *int64 `json:"slotLagBytes"`
+ SlotLagChangeBytesPerSecond *float64 `json:"slotLagChangeBytesPerSecond"`
+
+ SlotRebuildLagBytes int64 `json:"slotRebuildLagBytes"`
+
+ EstimatedCatchUpSeconds *int64 `json:"estimatedCatchUpSeconds"`
+ EstimatedSlotRebuildSeconds *int64 `json:"estimatedSlotRebuildSeconds"`
+ EstimatedSlotLimitSeconds *int64 `json:"estimatedSlotLimitSeconds"`
+}
diff --git a/backend/internal/features/backups/backups/core/physical/wal_health/enums.go b/backend/internal/features/backups/backups/core/physical/wal_health/enums.go
new file mode 100644
index 00000000..977529a1
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/wal_health/enums.go
@@ -0,0 +1,70 @@
+package wal_health
+
+type HealthLevel string
+
+const (
+ HealthLevelNotStreaming HealthLevel = "NOT_STREAMING"
+ HealthLevelCollectingData HealthLevel = "COLLECTING_DATA"
+ HealthLevelBroken HealthLevel = "BROKEN"
+ HealthLevelNeedsAttention HealthLevel = "NEEDS_ATTENTION"
+ HealthLevelHealthy HealthLevel = "HEALTHY"
+)
+
+type ReasonLevel string
+
+const (
+ ReasonLevelBroken ReasonLevel = "BROKEN"
+ ReasonLevelNeedsAttention ReasonLevel = "NEEDS_ATTENTION"
+ ReasonLevelInfo ReasonLevel = "INFO"
+)
+
+type ReasonCode string
+
+const (
+ ReasonCodeStreamingStopped ReasonCode = "STREAMING_STOPPED"
+ ReasonCodeSourceUnreachable ReasonCode = "SOURCE_UNREACHABLE"
+ ReasonCodeWalChainBroken ReasonCode = "WAL_CHAIN_BROKEN"
+ ReasonCodeSlotMissing ReasonCode = "SLOT_MISSING"
+ ReasonCodeSlotLost ReasonCode = "SLOT_LOST"
+ ReasonCodeArchiveNotAdvancing ReasonCode = "ARCHIVE_NOT_ADVANCING"
+ ReasonCodeArchiveSlowerThanSource ReasonCode = "ARCHIVE_SLOWER_THAN_SOURCE"
+ ReasonCodeSlotKeepsExtraWal ReasonCode = "SLOT_KEEPS_EXTRA_WAL"
+ ReasonCodeSlotLimitApproaching ReasonCode = "SLOT_LIMIT_APPROACHING"
+ ReasonCodeNearSlotRebuild ReasonCode = "NEAR_SLOT_REBUILD"
+ ReasonCodeLocalQueueFilling ReasonCode = "LOCAL_QUEUE_FILLING"
+ ReasonCodeFrequentRestarts ReasonCode = "FREQUENT_RESTARTS"
+ ReasonCodeForcedRotationRefused ReasonCode = "FORCED_ROTATION_REFUSED"
+)
+
+type NoticeCode string
+
+const (
+ NoticeCodeNoSlotWalLimit NoticeCode = "NO_SLOT_WAL_LIMIT"
+ NoticeCodeSlotKeepsExtraWalSteadily NoticeCode = "SLOT_KEEPS_EXTRA_WAL_STEADILY"
+ NoticeCodeNoFullBackupYet NoticeCode = "NO_FULL_BACKUP_YET"
+)
+
+type StreamerState string
+
+const (
+ StreamerStateRunning StreamerState = "RUNNING"
+ StreamerStateNotRunning StreamerState = "NOT_RUNNING"
+ StreamerStateNotStarted StreamerState = "NOT_STARTED"
+)
+
+type KeepingUpState string
+
+const (
+ KeepingUpStateUnknown KeepingUpState = "UNKNOWN"
+ KeepingUpStateFallingBehind KeepingUpState = "FALLING_BEHIND"
+ KeepingUpStateCatchingUp KeepingUpState = "CATCHING_UP"
+ KeepingUpStateIdle KeepingUpState = "IDLE"
+ KeepingUpStateKeepingUp KeepingUpState = "KEEPING_UP"
+)
+
+// Values of the source's pg_replication_slots.wal_status that the verdict reads.
+const (
+ slotWalStatusExtended = "extended"
+ slotWalStatusUnreserved = "unreserved"
+ slotWalStatusLost = "lost"
+)
diff --git a/backend/internal/features/backups/backups/core/physical/wal_health/keeping_up.go b/backend/internal/features/backups/backups/core/physical/wal_health/keeping_up.go
new file mode 100644
index 00000000..3fe4dbe7
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/wal_health/keeping_up.go
@@ -0,0 +1,143 @@
+package wal_health
+
+import (
+ "time"
+
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+)
+
+// GetKeepingUp summarizes the keeping-up window that ends at now: how fast the
+// source writes against how fast Databasus archives, how the backlog moves, and
+// when it clears or when the source or Databasus acts on it.
+func GetKeepingUp(
+ recentSamples []*physical_models.PhysicalWalStreamSample,
+ slotRebuildLagBytes int64,
+ now time.Time,
+) KeepingUp {
+ keepingUp := KeepingUp{State: KeepingUpStateUnknown, SlotRebuildLagBytes: slotRebuildLagBytes}
+
+ windowSamples := getSamplesInWindow(recentSamples, now)
+ if len(windowSamples) == 0 {
+ return keepingUp
+ }
+
+ newest := windowSamples[len(windowSamples)-1]
+ keepingUp.UnarchivedWalBytes = newest.UnarchivedWalBytes
+ keepingUp.SlotLagBytes = newest.SlotLagBytes
+
+ writtenBytes, hasWriteValues := fillRates(&keepingUp, windowSamples)
+
+ if changeBytesPerSecond, isKnown := getChangeBytesPerSecond(recentSamples, now, readUnarchivedWalBytes); isKnown {
+ keepingUp.UnarchivedWalChangeBytesPerSecond = new(changeBytesPerSecond)
+ }
+
+ if changeBytesPerSecond, isKnown := getChangeBytesPerSecond(recentSamples, now, readSlotLagBytes); isKnown {
+ keepingUp.SlotLagChangeBytesPerSecond = new(changeBytesPerSecond)
+ }
+
+ fillEstimates(&keepingUp, newest)
+
+ if hasWriteValues {
+ keepingUp.State = getKeepingUpState(keepingUp, newest, writtenBytes)
+ }
+
+ return keepingUp
+}
+
+// Both rates divide by the seconds covered by samples that hold a write value,
+// so a hole in the window lowers neither rate alone.
+func fillRates(
+ keepingUp *KeepingUp,
+ windowSamples []*physical_models.PhysicalWalStreamSample,
+) (writtenBytes int64, hasWriteValues bool) {
+ var archivedBytes int64
+
+ writeSampleCount := 0
+
+ for _, sample := range windowSamples {
+ if sample.SourceWrittenBytes == nil {
+ continue
+ }
+
+ writtenBytes += *sample.SourceWrittenBytes
+ archivedBytes += sample.ArchivedWalBytes
+ writeSampleCount++
+ }
+
+ if writeSampleCount == 0 {
+ return 0, false
+ }
+
+ coveredSeconds := float64(writeSampleCount) * SampleInterval.Seconds()
+
+ keepingUp.SourceWriteBytesPerSecond = new(float64(writtenBytes) / coveredSeconds)
+ keepingUp.ArchiveBytesPerSecond = new(float64(archivedBytes) / coveredSeconds)
+
+ return writtenBytes, true
+}
+
+// A slot held in extended or unreserved is never rebuilt for its lag, so it
+// gets no rebuild estimate.
+func fillEstimates(keepingUp *KeepingUp, newest *physical_models.PhysicalWalStreamSample) {
+ if keepingUp.UnarchivedWalBytes != nil &&
+ keepingUp.UnarchivedWalChangeBytesPerSecond != nil &&
+ *keepingUp.UnarchivedWalChangeBytesPerSecond < 0 {
+ keepingUp.EstimatedCatchUpSeconds = new(
+ int64(float64(*keepingUp.UnarchivedWalBytes) / -*keepingUp.UnarchivedWalChangeBytesPerSecond),
+ )
+ }
+
+ if keepingUp.SlotLagBytes == nil ||
+ keepingUp.SlotLagChangeBytesPerSecond == nil ||
+ *keepingUp.SlotLagChangeBytesPerSecond <= 0 {
+ return
+ }
+
+ slotLagChangeBytesPerSecond := *keepingUp.SlotLagChangeBytesPerSecond
+
+ if keepingUp.SlotRebuildLagBytes > *keepingUp.SlotLagBytes &&
+ !hasSlotWalStatus(newest, slotWalStatusExtended) &&
+ !hasSlotWalStatus(newest, slotWalStatusUnreserved) {
+ keepingUp.EstimatedSlotRebuildSeconds = new(
+ int64(float64(keepingUp.SlotRebuildLagBytes-*keepingUp.SlotLagBytes) / slotLagChangeBytesPerSecond),
+ )
+ }
+
+ if newest.SlotSafeWalSizeBytes != nil {
+ keepingUp.EstimatedSlotLimitSeconds = new(
+ int64(float64(*newest.SlotSafeWalSizeBytes) / slotLagChangeBytesPerSecond),
+ )
+ }
+}
+
+// First match wins, so a shrinking backlog on a quiet source reads as catching
+// up rather than idle.
+func getKeepingUpState(
+ keepingUp KeepingUp,
+ newest *physical_models.PhysicalWalStreamSample,
+ writtenBytes int64,
+) KeepingUpState {
+ thresholdBytes := getBacklogThresholdBytes(newest)
+ unarchivedChange := keepingUp.UnarchivedWalChangeBytesPerSecond
+
+ if newest.IsSourceReachable() &&
+ unarchivedChange != nil &&
+ getGrowthOverWindowBytes(*unarchivedChange) >= thresholdBytes {
+ return KeepingUpStateFallingBehind
+ }
+
+ if keepingUp.UnarchivedWalBytes != nil &&
+ *keepingUp.UnarchivedWalBytes >= thresholdBytes &&
+ unarchivedChange != nil &&
+ *unarchivedChange < 0 {
+ return KeepingUpStateCatchingUp
+ }
+
+ if writtenBytes < idleSourceWrittenBytes &&
+ keepingUp.UnarchivedWalBytes != nil &&
+ *keepingUp.UnarchivedWalBytes <= newest.WalSegmentSizeBytes {
+ return KeepingUpStateIdle
+ }
+
+ return KeepingUpStateKeepingUp
+}
diff --git a/backend/internal/features/backups/backups/core/physical/wal_health/keeping_up_test.go b/backend/internal/features/backups/backups/core/physical/wal_health/keeping_up_test.go
new file mode 100644
index 00000000..896e3ae5
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/wal_health/keeping_up_test.go
@@ -0,0 +1,175 @@
+package wal_health
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+)
+
+const testMB int64 = 1000 * 1000
+
+func perSample(bytesPerSecond int64) int64 {
+ return bytesPerSecond * int64(SampleInterval.Seconds())
+}
+
+func writingAndArchiving(
+ writeBytesPerSecond, archiveBytesPerSecond int64,
+) func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ return func(_ int, sample *physical_models.PhysicalWalStreamSample) {
+ sample.SourceWrittenBytes = new(perSample(writeBytesPerSecond))
+ sample.ArchivedWalBytes = perSample(archiveBytesPerSecond)
+ }
+}
+
+func combineModifiers(
+ modifiers ...func(index int, sample *physical_models.PhysicalWalStreamSample),
+) func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ return func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ for _, modify := range modifiers {
+ modify(index, sample)
+ }
+ }
+}
+
+func unarchivedWalOf(
+ getBytes func(index int) int64,
+) func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ return func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ sample.UnarchivedWalBytes = new(getBytes(index))
+ }
+}
+
+// 200 KiB over the 20 samples of the window.
+func quietSource() func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ return func(_ int, sample *physical_models.PhysicalWalStreamSample) {
+ sample.SourceWrittenBytes = new(int64(10 * 1024))
+ sample.ArchivedWalBytes = 0
+ }
+}
+
+func getTestKeepingUp(modify func(index int, sample *physical_models.PhysicalWalStreamSample)) KeepingUp {
+ return GetKeepingUp(buildTestSamples(0, modify), testRebuildLagBytes, testNow)
+}
+
+func Test_GetKeepingUp_WithSpecScenarios_ReturnsStateRatesAndEstimates(t *testing.T) {
+ samplesFromNewest := func(index int) int64 { return int64(testSamplesPerHour - 1 - index) }
+
+ t.Run("Archiving keeps up", func(t *testing.T) {
+ keepingUp := getTestKeepingUp(writingAndArchiving(10*testMB, 10*testMB))
+
+ assert.Equal(t, KeepingUpStateKeepingUp, keepingUp.State)
+ assert.InDelta(t, 10*testMB, *keepingUp.SourceWriteBytesPerSecond, 1)
+ assert.InDelta(t, 10*testMB, *keepingUp.ArchiveBytesPerSecond, 1)
+ })
+
+ t.Run("Archiving falls behind", func(t *testing.T) {
+ keepingUp := getTestKeepingUp(combineModifiers(
+ writingAndArchiving(21*testMB, 12*testMB),
+ unarchivedWalOf(func(index int) int64 { return int64(index) * perSample(9*testMB) }),
+ ))
+
+ assert.Equal(t, KeepingUpStateFallingBehind, keepingUp.State)
+ require.NotNil(t, keepingUp.UnarchivedWalChangeBytesPerSecond)
+ assert.InDelta(t, 9*testMB, *keepingUp.UnarchivedWalChangeBytesPerSecond, 1)
+ })
+
+ t.Run("Catching up after a burst", func(t *testing.T) {
+ shrinkPerSample := 600 * testMiB / 20
+
+ keepingUp := getTestKeepingUp(combineModifiers(
+ writingAndArchiving(testMB, 2*testMB),
+ unarchivedWalOf(func(index int) int64 { return 3*testGiB + samplesFromNewest(index)*shrinkPerSample }),
+ ))
+
+ assert.Equal(t, KeepingUpStateCatchingUp, keepingUp.State)
+ require.NotNil(t, keepingUp.EstimatedCatchUpSeconds)
+ assert.InDelta(t, 50*60, *keepingUp.EstimatedCatchUpSeconds, 3*60)
+ })
+
+ t.Run("Running out of allowance", func(t *testing.T) {
+ keepingUp := getTestKeepingUp(func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ sample.SlotLagBytes = new(int64(index) * testGiB / 20)
+ sample.SlotSafeWalSizeBytes = new(5 * testGiB)
+ })
+
+ require.NotNil(t, keepingUp.EstimatedSlotLimitSeconds)
+ assert.InDelta(t, 50*60, *keepingUp.EstimatedSlotLimitSeconds, 60)
+ })
+
+ t.Run("Quiet database", func(t *testing.T) {
+ keepingUp := getTestKeepingUp(combineModifiers(
+ quietSource(),
+ unarchivedWalOf(func(int) int64 { return 100 * 1024 }),
+ ))
+
+ assert.Equal(t, KeepingUpStateIdle, keepingUp.State)
+ })
+
+ t.Run("Quiet database with a stuck backlog", func(t *testing.T) {
+ keepingUp := getTestKeepingUp(combineModifiers(
+ quietSource(),
+ unarchivedWalOf(func(int) int64 { return 2 * testGiB }),
+ ))
+
+ assert.Equal(t, KeepingUpStateKeepingUp, keepingUp.State)
+ })
+}
+
+func Test_GetKeepingUp_WhenWindowHasNoWriteValues_ReportsUnknown(t *testing.T) {
+ keepingUp := getTestKeepingUp(func(_ int, sample *physical_models.PhysicalWalStreamSample) {
+ sample.SourceWrittenBytes = nil
+ })
+
+ assert.Equal(t, KeepingUpStateUnknown, keepingUp.State)
+ assert.Nil(t, keepingUp.SourceWriteBytesPerSecond)
+ assert.Nil(t, keepingUp.ArchiveBytesPerSecond)
+}
+
+func Test_GetKeepingUp_WhenQuietSourceShrinksBacklog_ReportsCatchingUpBeforeIdle(t *testing.T) {
+ keepingUp := getTestKeepingUp(combineModifiers(
+ quietSource(),
+ unarchivedWalOf(func(index int) int64 { return 2*testGiB - int64(index)*testMiB }),
+ ))
+
+ assert.Equal(t, KeepingUpStateCatchingUp, keepingUp.State)
+}
+
+func Test_GetKeepingUp_WhenWindowHasHole_BothRatesShareOneDenominator(t *testing.T) {
+ keepingUp := getTestKeepingUp(func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ writingAndArchiving(10*testMB, 10*testMB)(index, sample)
+
+ if index%2 == 0 {
+ sample.SourceWrittenBytes = nil
+ }
+ })
+
+ assert.InDelta(t, 10*testMB, *keepingUp.SourceWriteBytesPerSecond, 1)
+ assert.InDelta(t, 10*testMB, *keepingUp.ArchiveBytesPerSecond, 1)
+}
+
+func Test_GetKeepingUp_WhenSlotLagGrows_EstimatesRebuildOnlyOutsideExtendedStatus(t *testing.T) {
+ growingLag := func(walStatus string) func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ return func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ growingSlotLag(testMiB)(index, sample)
+ sample.SlotWalStatus = new(walStatus)
+ }
+ }
+
+ reservedKeepingUp := getTestKeepingUp(growingLag("reserved"))
+ require.NotNil(t, reservedKeepingUp.EstimatedSlotRebuildSeconds)
+ assert.Positive(t, *reservedKeepingUp.EstimatedSlotRebuildSeconds)
+
+ extendedKeepingUp := getTestKeepingUp(growingLag("extended"))
+ assert.Nil(t, extendedKeepingUp.EstimatedSlotRebuildSeconds)
+}
+
+func Test_GetKeepingUp_WhenNewestSampleIsStale_ReportsUnknown(t *testing.T) {
+ keepingUp := GetKeepingUp(buildTestSamples(15*time.Minute, nil), testRebuildLagBytes, testNow)
+
+ assert.Equal(t, KeepingUpStateUnknown, keepingUp.State)
+ assert.Equal(t, testRebuildLagBytes, keepingUp.SlotRebuildLagBytes)
+}
diff --git a/backend/internal/features/backups/backups/core/physical/wal_health/streamer_state.go b/backend/internal/features/backups/backups/core/physical/wal_health/streamer_state.go
new file mode 100644
index 00000000..3039ebbb
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/wal_health/streamer_state.go
@@ -0,0 +1,42 @@
+package wal_health
+
+import (
+ "time"
+
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+)
+
+// A RUNNING row whose heartbeat is past the takeover age may belong to a
+// process that is gone, so only a fresh heartbeat counts as running.
+func GetStreamerState(streamer *physical_models.PhysicalWalStreamer, now time.Time) StreamerState {
+ if streamer == nil {
+ return StreamerStateNotStarted
+ }
+
+ if streamer.Status == physical_enums.PhysicalWalStreamerStatusRunning &&
+ now.Sub(streamer.LastHeartbeatAt) < StreamerHeartbeatStaleness {
+ return StreamerStateRunning
+ }
+
+ return StreamerStateNotRunning
+}
+
+// Receiver restarts and whole-session starts both mean streaming was
+// interrupted, so they add up.
+func GetRestartsLastHour(
+ recentSamples []*physical_models.PhysicalWalStreamSample,
+ sessionStartsLastHour int,
+ now time.Time,
+) int {
+ restarts := sessionStartsLastHour
+ windowStart := now.Add(-frequentRestartsWindow)
+
+ for _, sample := range recentSamples {
+ if sample.SampledAt.After(windowStart) {
+ restarts += sample.ReceiverRestarts
+ }
+ }
+
+ return restarts
+}
diff --git a/backend/internal/features/backups/backups/core/physical/wal_health/testing_test.go b/backend/internal/features/backups/backups/core/physical/wal_health/testing_test.go
new file mode 100644
index 00000000..53dc6e0f
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/wal_health/testing_test.go
@@ -0,0 +1,165 @@
+package wal_health
+
+import (
+ "time"
+
+ "github.com/google/uuid"
+
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+ "databasus-backend/internal/util/walmath"
+)
+
+const (
+ testMiB int64 = 1024 * 1024
+ testGiB int64 = 1024 * testMiB
+
+ testSegmentSizeBytes = 16 * testMiB
+ testRebuildLagBytes = 256 * testMiB
+ testPauseBytes = 512 * testMiB
+ testSamplesPerHour = 120
+ testArchiveProgressAt = walmath.LSN(0x40000000)
+)
+
+var testNow = time.Date(2026, time.October, 18, 12, 0, 0, 0, time.UTC)
+
+type reasonExpectation struct {
+ Code ReasonCode
+ Level ReasonLevel
+}
+
+func newHealthyTestSample(sampledAt time.Time) *physical_models.PhysicalWalStreamSample {
+ return &physical_models.PhysicalWalStreamSample{
+ ID: uuid.New(),
+ DatabaseID: uuid.Nil,
+ SampledAt: sampledAt,
+ SessionStartedAt: testNow.Add(-3 * time.Hour),
+ SourceState: physical_enums.PhysicalWalStreamSourceStateReachable,
+ SourceCurrentLSN: new(testArchiveProgressAt + walmath.LSN(8*testMiB)),
+ ArchiveProgressLSN: new(testArchiveProgressAt),
+ SourceWrittenBytes: new(10 * testMiB),
+ ArchivedSegments: 1,
+ ArchivedWalBytes: 10 * testMiB,
+ ArchivedStoredBytes: 4 * testMiB,
+ WalSegmentSizeBytes: testSegmentSizeBytes,
+ HighestConcurrentUploads: 1,
+ SlotWalStatus: new("reserved"),
+ SlotLagBytes: new(2 * testMiB),
+ SlotSafeWalSizeBytes: new(10 * testGiB),
+ UnarchivedWalBytes: new(8 * testMiB),
+ LocalQueueBytes: testSegmentSizeBytes,
+ LocalQueuePauseBytes: testPauseBytes,
+ IsReceiverRunning: true,
+ }
+}
+
+// buildTestSamples returns an hour of healthy samples, one every 30 seconds,
+// the newest taken newestAge before testNow. modify sees each sample with its
+// index, oldest first, so a test can shape a trend across the hour.
+func buildTestSamples(
+ newestAge time.Duration,
+ modify func(index int, sample *physical_models.PhysicalWalStreamSample),
+) []*physical_models.PhysicalWalStreamSample {
+ samples := make([]*physical_models.PhysicalWalStreamSample, 0, testSamplesPerHour)
+ newestAt := testNow.Add(-newestAge)
+
+ for index := range testSamplesPerHour {
+ sampledAt := newestAt.Add(-time.Duration(testSamplesPerHour-1-index) * SampleInterval)
+ sample := newHealthyTestSample(sampledAt)
+
+ if modify != nil {
+ modify(index, sample)
+ }
+
+ samples = append(samples, sample)
+ }
+
+ return samples
+}
+
+func modifyNewestTwo(
+ modify func(sample *physical_models.PhysicalWalStreamSample),
+) func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ return func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ if index >= testSamplesPerHour-2 {
+ modify(sample)
+ }
+ }
+}
+
+func modifyNewestOnly(
+ modify func(sample *physical_models.PhysicalWalStreamSample),
+) func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ return func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ if index == testSamplesPerHour-1 {
+ modify(sample)
+ }
+ }
+}
+
+func newRunningTestStreamer() *physical_models.PhysicalWalStreamer {
+ return &physical_models.PhysicalWalStreamer{
+ DatabaseID: uuid.Nil,
+ StartedAt: testNow.Add(-3 * time.Hour),
+ LastHeartbeatAt: testNow.Add(-10 * time.Second),
+ Status: physical_enums.PhysicalWalStreamerStatusRunning,
+ }
+}
+
+func newFailedTestStreamer(startedAgo, failingFor time.Duration) *physical_models.PhysicalWalStreamer {
+ return &physical_models.PhysicalWalStreamer{
+ DatabaseID: uuid.Nil,
+ StartedAt: testNow.Add(-startedAgo),
+ LastHeartbeatAt: testNow.Add(-startedAgo),
+ Status: physical_enums.PhysicalWalStreamerStatusFailed,
+ FailingSince: new(testNow.Add(-failingFor)),
+ }
+}
+
+func newTestVerdictInput(samples []*physical_models.PhysicalWalStreamSample) VerdictInput {
+ input := VerdictInput{
+ RecentSamples: samples,
+ Streamer: newRunningTestStreamer(),
+ SessionStartsLastHour: 0,
+ SlotRebuildLagBytes: testRebuildLagBytes,
+ IsStreamingExpected: true,
+ HasCompletedFullBackup: true,
+ Now: testNow,
+ }
+
+ if len(samples) > 0 {
+ input.NewestSample = samples[len(samples)-1]
+ }
+
+ return input
+}
+
+func getReasonExpectations(reasons []Reason) []reasonExpectation {
+ expectations := make([]reasonExpectation, 0, len(reasons))
+
+ for _, reason := range reasons {
+ expectations = append(expectations, reasonExpectation{Code: reason.Code, Level: reason.Level})
+ }
+
+ return expectations
+}
+
+func getNoticeCodes(notices []Notice) []NoticeCode {
+ codes := make([]NoticeCode, 0, len(notices))
+
+ for _, notice := range notices {
+ codes = append(codes, notice.Code)
+ }
+
+ return codes
+}
+
+func findReason(reasons []Reason, code ReasonCode) *Reason {
+ for _, reason := range reasons {
+ if reason.Code == code {
+ return &reason
+ }
+ }
+
+ return nil
+}
diff --git a/backend/internal/features/backups/backups/core/physical/wal_health/verdict.go b/backend/internal/features/backups/backups/core/physical/wal_health/verdict.go
new file mode 100644
index 00000000..8480609e
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/wal_health/verdict.go
@@ -0,0 +1,493 @@
+package wal_health
+
+import (
+ "slices"
+ "time"
+
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+)
+
+// GetVerdict decides the one health level of a WAL-streaming database and lists
+// the reasons behind it, broken ones first, and the notices that never change
+// it. The first matching level wins: not streaming, collecting data, broken,
+// needs attention, healthy.
+func GetVerdict(input VerdictInput) Verdict {
+ if !input.IsStreamingExpected {
+ return Verdict{Level: HealthLevelNotStreaming, Reasons: []Reason{}, Notices: []Notice{}}
+ }
+
+ if isCollectingData(input) {
+ return Verdict{Level: HealthLevelCollectingData, Reasons: []Reason{}, Notices: []Notice{}}
+ }
+
+ verdictSamples := newVerdictSamples(input)
+
+ reasons := collectReasons(input, verdictSamples)
+
+ slices.SortStableFunc(reasons, func(first, second Reason) int {
+ return getReasonLevelRank(first.Level) - getReasonLevelRank(second.Level)
+ })
+
+ return Verdict{
+ Level: getHealthLevel(reasons),
+ Reasons: reasons,
+ Notices: collectNotices(input, verdictSamples),
+ }
+}
+
+// The newest sample and the one before it, as far as the verdict trusts them.
+// A sample older than staleSampleAge no longer describes now, and a sample
+// without source-side values says nothing about the slot or the archive.
+type verdictSamples struct {
+ newest *physical_models.PhysicalWalStreamSample
+ previous *physical_models.PhysicalWalStreamSample
+
+ isNewestFresh bool
+ isNewestWithSourceValues bool
+}
+
+func newVerdictSamples(input VerdictInput) verdictSamples {
+ newest := input.NewestSample
+ if newest == nil {
+ return verdictSamples{}
+ }
+
+ isNewestFresh := input.Now.Sub(newest.SampledAt) <= staleSampleAge
+
+ var previous *physical_models.PhysicalWalStreamSample
+
+ sampleCount := len(input.RecentSamples)
+ if sampleCount >= 2 && input.RecentSamples[sampleCount-1].ID == newest.ID {
+ previous = input.RecentSamples[sampleCount-2]
+ }
+
+ return verdictSamples{
+ newest: newest,
+ previous: previous,
+ isNewestFresh: isNewestFresh,
+ isNewestWithSourceValues: isNewestFresh && newest.IsSourceReachable(),
+ }
+}
+
+// A single unusual sample must not change the verdict, so a condition read
+// from samples counts only when it also held in the sample before.
+func (s verdictSamples) isTrueForNewestTwo(
+ condition func(sample *physical_models.PhysicalWalStreamSample) bool,
+) bool {
+ return s.previous != nil && condition(s.newest) && condition(s.previous)
+}
+
+// Right after a start or a restart no sample of the session exists yet, which
+// is not a failure. A streamer that has already been failing for a while is
+// not collecting anything, however young its current session is.
+func isCollectingData(input VerdictInput) bool {
+ streamer := input.Streamer
+ if streamer == nil {
+ return true
+ }
+
+ if isStreamerFailingLongEnough(streamer, input.Now) {
+ return false
+ }
+
+ hasSampleInSession := input.NewestSample != nil && input.NewestSample.SampledAt.After(streamer.StartedAt)
+
+ return !hasSampleInSession && input.Now.Sub(streamer.StartedAt) < collectingDataPeriod
+}
+
+func isStreamerFailingLongEnough(streamer *physical_models.PhysicalWalStreamer, now time.Time) bool {
+ return streamer.FailingSince != nil && now.Sub(*streamer.FailingSince) >= streamingStoppedAfter
+}
+
+func getHealthLevel(reasons []Reason) HealthLevel {
+ if len(reasons) == 0 {
+ return HealthLevelHealthy
+ }
+
+ if reasons[0].Level == ReasonLevelBroken {
+ return HealthLevelBroken
+ }
+
+ return HealthLevelNeedsAttention
+}
+
+func getReasonLevelRank(level ReasonLevel) int {
+ switch level {
+ case ReasonLevelBroken:
+ return 0
+ case ReasonLevelNeedsAttention:
+ return 1
+ default:
+ return 2
+ }
+}
+
+func collectReasons(input VerdictInput, samples verdictSamples) []Reason {
+ reasonGetters := []func(VerdictInput, verdictSamples) *Reason{
+ getStreamingStoppedReason,
+ getSourceUnreachableReason,
+ getWalChainBrokenReason,
+ getSlotMissingReason,
+ getSlotLostReason,
+ getArchiveNotAdvancingReason,
+ getArchiveSlowerThanSourceReason,
+ getSlotKeepsExtraWalReason,
+ getSlotLimitApproachingReason,
+ getNearSlotRebuildReason,
+ getLocalQueueFillingReason,
+ getFrequentRestartsReason,
+ getForcedRotationRefusedReason,
+ }
+
+ reasons := make([]Reason, 0)
+
+ for _, getReason := range reasonGetters {
+ if reason := getReason(input, samples); reason != nil {
+ reasons = append(reasons, *reason)
+ }
+ }
+
+ return reasons
+}
+
+// An unreachable source is its own reason, and it explains why streaming
+// stopped, so it is not reported twice.
+func getStreamingStoppedReason(input VerdictInput, samples verdictSamples) *Reason {
+ newest := samples.newest
+
+ if newest != nil &&
+ newest.SourceState == physical_enums.PhysicalWalStreamSourceStateUnreachable {
+ return nil
+ }
+
+ if newest != nil && !samples.isNewestFresh {
+ return &Reason{Code: ReasonCodeStreamingStopped, Level: ReasonLevelBroken, Since: new(newest.SampledAt)}
+ }
+
+ streamer := input.Streamer
+ if streamer == nil {
+ return nil
+ }
+
+ if isStreamerFailingLongEnough(streamer, input.Now) {
+ return &Reason{Code: ReasonCodeStreamingStopped, Level: ReasonLevelBroken, Since: new(*streamer.FailingSince)}
+ }
+
+ if newest == nil {
+ return &Reason{Code: ReasonCodeStreamingStopped, Level: ReasonLevelBroken, Since: new(streamer.StartedAt)}
+ }
+
+ return nil
+}
+
+// Read from the newest sample even when streaming stopped after it: a source
+// that went down is what stopped streaming.
+func getSourceUnreachableReason(input VerdictInput, samples verdictSamples) *Reason {
+ newest := samples.newest
+
+ if newest == nil ||
+ newest.SourceState != physical_enums.PhysicalWalStreamSourceStateUnreachable ||
+ newest.SourceUnreachableSince == nil {
+ return nil
+ }
+
+ unreachableFor := input.Now.Sub(*newest.SourceUnreachableSince)
+
+ switch {
+ case unreachableFor >= sourceUnreachableBrokenAfter:
+ return &Reason{
+ Code: ReasonCodeSourceUnreachable,
+ Level: ReasonLevelBroken,
+ Since: new(*newest.SourceUnreachableSince),
+ }
+
+ case unreachableFor >= sourceUnreachableAttentionAfter:
+ return &Reason{
+ Code: ReasonCodeSourceUnreachable,
+ Level: ReasonLevelNeedsAttention,
+ Since: new(*newest.SourceUnreachableSince),
+ }
+
+ default:
+ return nil
+ }
+}
+
+// Reads the chain, not the slot: a rebuilt slot looks healthy within one tick
+// while the chain stays broken until a newer full backup starts a new one. A
+// gap counts only once the archive progress point has passed it, because that
+// point never passes a pending segment, so a hole left by out-of-order parallel
+// uploads is still above it.
+func getWalChainBrokenReason(input VerdictInput, samples verdictSamples) *Reason {
+ newest := samples.newest
+ if newest == nil || newest.ArchiveProgressLSN == nil {
+ return nil
+ }
+
+ for _, gap := range input.NewestChainGaps {
+ if gap.End <= *newest.ArchiveProgressLSN {
+ return &Reason{Code: ReasonCodeWalChainBroken, Level: ReasonLevelBroken}
+ }
+ }
+
+ return nil
+}
+
+func getSlotMissingReason(_ VerdictInput, samples verdictSamples) *Reason {
+ if !samples.isNewestFresh ||
+ samples.newest.SourceState != physical_enums.PhysicalWalStreamSourceStateSlotMissing {
+ return nil
+ }
+
+ return &Reason{Code: ReasonCodeSlotMissing, Level: ReasonLevelBroken}
+}
+
+func getSlotLostReason(_ VerdictInput, samples verdictSamples) *Reason {
+ if !samples.isNewestWithSourceValues || !hasSlotWalStatus(samples.newest, slotWalStatusLost) {
+ return nil
+ }
+
+ return &Reason{Code: ReasonCodeSlotLost, Level: ReasonLevelBroken}
+}
+
+// The broken level fires together with the "archiving fell behind"
+// notification, from the same tracker and the same threshold.
+func getArchiveNotAdvancingReason(input VerdictInput, samples verdictSamples) *Reason {
+ if !samples.isNewestWithSourceValues || samples.newest.ArchiveStalledSince == nil {
+ return nil
+ }
+
+ stalledSince := *samples.newest.ArchiveStalledSince
+ stalledFor := input.Now.Sub(stalledSince)
+
+ switch {
+ case stalledFor >= DefaultArchiveStalenessThreshold:
+ return &Reason{Code: ReasonCodeArchiveNotAdvancing, Level: ReasonLevelBroken, Since: new(stalledSince)}
+
+ case stalledFor >= archiveNotAdvancingAttentionAfter:
+ return &Reason{Code: ReasonCodeArchiveNotAdvancing, Level: ReasonLevelNeedsAttention, Since: new(stalledSince)}
+
+ default:
+ return nil
+ }
+}
+
+func getArchiveSlowerThanSourceReason(input VerdictInput, samples verdictSamples) *Reason {
+ if !samples.isNewestWithSourceValues {
+ return nil
+ }
+
+ thresholdBytes := getBacklogThresholdBytes(samples.newest)
+
+ isFallingBehind := samples.isTrueForNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) bool {
+ if !sample.IsSourceReachable() {
+ return false
+ }
+
+ growthBytes, isKnown := getUnarchivedWalGrowthBytes(input.RecentSamples, sample.SampledAt)
+
+ return isKnown && growthBytes >= thresholdBytes
+ })
+ if !isFallingBehind {
+ return nil
+ }
+
+ growthBytes, _ := getUnarchivedWalGrowthBytes(input.RecentSamples, samples.newest.SampledAt)
+
+ return &Reason{
+ Code: ReasonCodeArchiveSlowerThanSource,
+ Level: ReasonLevelNeedsAttention,
+ ValueBytes: new(growthBytes),
+ ThresholdBytes: new(thresholdBytes),
+ }
+}
+
+// An unreserved slot is about to lose WAL at the next checkpoint, and an
+// extended one is only a problem while it keeps growing: on a busy cluster a
+// steadily extended slot is routine and shows as a notice instead.
+func getSlotKeepsExtraWalReason(input VerdictInput, samples verdictSamples) *Reason {
+ if !isSlotKeepingExtraWalLongEnough(input, samples) {
+ return nil
+ }
+
+ newest := samples.newest
+
+ if !hasSlotWalStatus(newest, slotWalStatusUnreserved) && !isSlotLagGrowing(input, newest) {
+ return nil
+ }
+
+ return &Reason{
+ Code: ReasonCodeSlotKeepsExtraWal,
+ Level: ReasonLevelNeedsAttention,
+ Since: new(*newest.SlotExtraWalSince),
+ }
+}
+
+func getSlotLimitApproachingReason(input VerdictInput, samples verdictSamples) *Reason {
+ if !samples.isNewestWithSourceValues {
+ return nil
+ }
+
+ isApproaching := samples.isTrueForNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) bool {
+ estimatedSeconds, isKnown := getEstimatedSlotLimitSeconds(input.RecentSamples, sample)
+
+ return isKnown && estimatedSeconds < int64(slotLimitApproachingHorizon.Seconds())
+ })
+ if !isApproaching {
+ return nil
+ }
+
+ estimatedSeconds, _ := getEstimatedSlotLimitSeconds(input.RecentSamples, samples.newest)
+
+ return &Reason{
+ Code: ReasonCodeSlotLimitApproaching,
+ Level: ReasonLevelNeedsAttention,
+ ValueBytes: new(*samples.newest.SlotSafeWalSizeBytes),
+ EstimatedSeconds: new(estimatedSeconds),
+ }
+}
+
+// Stays listed while the receiver is paused: the slot is rebuilt on the first
+// tick the receiver runs again above the rebuild lag, so a paused receiver
+// above it is exactly the risky case.
+func getNearSlotRebuildReason(input VerdictInput, samples verdictSamples) *Reason {
+ if !samples.isNewestWithSourceValues || input.SlotRebuildLagBytes <= 0 {
+ return nil
+ }
+
+ isNearRebuild := samples.isTrueForNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) bool {
+ return sample.SlotLagBytes != nil &&
+ *sample.SlotLagBytes*100 >= input.SlotRebuildLagBytes*nearSlotRebuildPercent &&
+ !hasSlotWalStatus(sample, slotWalStatusExtended) &&
+ !hasSlotWalStatus(sample, slotWalStatusUnreserved)
+ })
+ if !isNearRebuild {
+ return nil
+ }
+
+ return &Reason{
+ Code: ReasonCodeNearSlotRebuild,
+ Level: ReasonLevelNeedsAttention,
+ ValueBytes: new(*samples.newest.SlotLagBytes),
+ ThresholdBytes: new(input.SlotRebuildLagBytes),
+ }
+}
+
+func getLocalQueueFillingReason(_ VerdictInput, samples verdictSamples) *Reason {
+ if !samples.isNewestFresh {
+ return nil
+ }
+
+ isFilling := samples.isTrueForNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) bool {
+ return sample.LocalQueuePauseBytes > 0 &&
+ sample.LocalQueueBytes*100 >= sample.LocalQueuePauseBytes*localQueueFillingPercent
+ })
+ if !isFilling {
+ return nil
+ }
+
+ return &Reason{
+ Code: ReasonCodeLocalQueueFilling,
+ Level: ReasonLevelNeedsAttention,
+ ValueBytes: new(samples.newest.LocalQueueBytes),
+ ThresholdBytes: new(samples.newest.LocalQueuePauseBytes),
+ }
+}
+
+func getFrequentRestartsReason(input VerdictInput, samples verdictSamples) *Reason {
+ if !samples.isNewestFresh {
+ return nil
+ }
+
+ restarts := GetRestartsLastHour(input.RecentSamples, input.SessionStartsLastHour, input.Now)
+ if restarts < frequentRestartsCount {
+ return nil
+ }
+
+ return &Reason{Code: ReasonCodeFrequentRestarts, Level: ReasonLevelNeedsAttention, Count: new(restarts)}
+}
+
+func getForcedRotationRefusedReason(_ VerdictInput, samples verdictSamples) *Reason {
+ if !samples.isNewestFresh || !samples.newest.IsForcedRotationDenied {
+ return nil
+ }
+
+ return &Reason{Code: ReasonCodeForcedRotationRefused, Level: ReasonLevelNeedsAttention}
+}
+
+// PostgreSQL keeps WAL for a slot without a limit by default, and a steadily
+// extended slot is routine on a busy cluster, so neither changes the level:
+// shown as needs attention, they would stay orange for good and teach the
+// reader to ignore orange.
+func collectNotices(input VerdictInput, samples verdictSamples) []Notice {
+ notices := make([]Notice, 0)
+
+ if samples.isNewestWithSourceValues &&
+ samples.newest.SlotSafeWalSizeBytes == nil &&
+ !hasSlotWalStatus(samples.newest, slotWalStatusLost) {
+ notices = append(notices, Notice{Code: NoticeCodeNoSlotWalLimit, Level: ReasonLevelInfo})
+ }
+
+ if isSlotKeepingExtraWalLongEnough(input, samples) &&
+ hasSlotWalStatus(samples.newest, slotWalStatusExtended) &&
+ !isSlotLagGrowing(input, samples.newest) {
+ notices = append(notices, Notice{
+ Code: NoticeCodeSlotKeepsExtraWalSteadily,
+ Level: ReasonLevelInfo,
+ Since: new(*samples.newest.SlotExtraWalSince),
+ })
+ }
+
+ if !input.HasCompletedFullBackup {
+ notices = append(notices, Notice{Code: NoticeCodeNoFullBackupYet, Level: ReasonLevelInfo})
+ }
+
+ return notices
+}
+
+func isSlotKeepingExtraWalLongEnough(input VerdictInput, samples verdictSamples) bool {
+ return samples.isNewestWithSourceValues &&
+ samples.newest.SlotExtraWalSince != nil &&
+ input.Now.Sub(*samples.newest.SlotExtraWalSince) > WarningSlotStatusHoldPeriod
+}
+
+// Growth below the backlog threshold is the same upload noise the unarchived
+// WAL shows, so it does not count as growing.
+func isSlotLagGrowing(input VerdictInput, newest *physical_models.PhysicalWalStreamSample) bool {
+ changeBytesPerSecond, isKnown := getChangeBytesPerSecond(input.RecentSamples, newest.SampledAt, readSlotLagBytes)
+
+ return isKnown && getGrowthOverWindowBytes(changeBytesPerSecond) >= getBacklogThresholdBytes(newest)
+}
+
+func getUnarchivedWalGrowthBytes(
+ samples []*physical_models.PhysicalWalStreamSample,
+ windowEnd time.Time,
+) (growthBytes int64, isKnown bool) {
+ changeBytesPerSecond, isKnown := getChangeBytesPerSecond(samples, windowEnd, readUnarchivedWalBytes)
+ if !isKnown {
+ return 0, false
+ }
+
+ return getGrowthOverWindowBytes(changeBytesPerSecond), true
+}
+
+func getEstimatedSlotLimitSeconds(
+ samples []*physical_models.PhysicalWalStreamSample,
+ sample *physical_models.PhysicalWalStreamSample,
+) (estimatedSeconds int64, isKnown bool) {
+ if !sample.IsSourceReachable() || sample.SlotSafeWalSizeBytes == nil {
+ return 0, false
+ }
+
+ changeBytesPerSecond, isKnown := getChangeBytesPerSecond(samples, sample.SampledAt, readSlotLagBytes)
+ if !isKnown || changeBytesPerSecond <= 0 {
+ return 0, false
+ }
+
+ return int64(float64(*sample.SlotSafeWalSizeBytes) / changeBytesPerSecond), true
+}
+
+func hasSlotWalStatus(sample *physical_models.PhysicalWalStreamSample, walStatus string) bool {
+ return sample.SlotWalStatus != nil && *sample.SlotWalStatus == walStatus
+}
diff --git a/backend/internal/features/backups/backups/core/physical/wal_health/verdict_test.go b/backend/internal/features/backups/backups/core/physical/wal_health/verdict_test.go
new file mode 100644
index 00000000..d955c8e3
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/wal_health/verdict_test.go
@@ -0,0 +1,602 @@
+package wal_health
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "databasus-backend/internal/features/backups/backups/core/physical/chain_view"
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+ "databasus-backend/internal/util/walmath"
+)
+
+type verdictTestCase struct {
+ name string
+ buildInput func() VerdictInput
+ expectedLevel HealthLevel
+ expectedReasons []reasonExpectation
+ expectedNotices []NoticeCode
+}
+
+func runVerdictTestCases(t *testing.T, testCases []verdictTestCase) {
+ t.Helper()
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ verdict := GetVerdict(testCase.buildInput())
+
+ assert.Equal(t, testCase.expectedLevel, verdict.Level)
+ assert.Equal(t, append([]reasonExpectation{}, testCase.expectedReasons...),
+ getReasonExpectations(verdict.Reasons))
+ assert.Equal(t, append([]NoticeCode{}, testCase.expectedNotices...), getNoticeCodes(verdict.Notices))
+ })
+ }
+}
+
+func inputWithNewestTwo(modify func(sample *physical_models.PhysicalWalStreamSample)) func() VerdictInput {
+ return func() VerdictInput {
+ return newTestVerdictInput(buildTestSamples(0, modifyNewestTwo(modify)))
+ }
+}
+
+func inputWithEverySample(
+ modify func(index int, sample *physical_models.PhysicalWalStreamSample),
+) func() VerdictInput {
+ return func() VerdictInput {
+ return newTestVerdictInput(buildTestSamples(0, modify))
+ }
+}
+
+func unreachableSince(unreachableFor time.Duration) func(sample *physical_models.PhysicalWalStreamSample) {
+ return func(sample *physical_models.PhysicalWalStreamSample) {
+ makeUnreachable(sample, testNow.Add(-unreachableFor))
+ }
+}
+
+func makeUnreachable(sample *physical_models.PhysicalWalStreamSample, since time.Time) {
+ sample.SourceState = physical_enums.PhysicalWalStreamSourceStateUnreachable
+ sample.SourceUnreachableSince = new(since)
+ sample.SourceCurrentLSN = nil
+ sample.SourceWrittenBytes = nil
+ sample.SlotWalStatus = nil
+ sample.SlotLagBytes = nil
+ sample.SlotSafeWalSizeBytes = nil
+ sample.UnarchivedWalBytes = nil
+}
+
+func archiveStalledFor(stalledFor time.Duration) func(sample *physical_models.PhysicalWalStreamSample) {
+ return func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.ArchiveStalledSince = new(testNow.Add(-stalledFor))
+ }
+}
+
+func slotStatusSince(walStatus string, heldFor time.Duration) func(sample *physical_models.PhysicalWalStreamSample) {
+ return func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.SlotWalStatus = new(walStatus)
+ sample.SlotExtraWalSince = new(testNow.Add(-heldFor))
+ }
+}
+
+// Each sample adds stepBytes to the value, so the value grows by twenty steps
+// over the keeping-up window.
+func growingUnarchivedWal(stepBytes int64) func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ return func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ sample.UnarchivedWalBytes = new(8*testMiB + int64(index)*stepBytes)
+ }
+}
+
+func growingSlotLag(stepBytes int64) func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ return func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ sample.SlotLagBytes = new(2*testMiB + int64(index)*stepBytes)
+ }
+}
+
+func Test_GetVerdict_WithSpecScenarios_ReturnsLevelReasonsAndNotices(t *testing.T) {
+ runVerdictTestCases(t, []verdictTestCase{
+ {
+ name: "Everything in order",
+ buildInput: inputWithNewestTwo(func(*physical_models.PhysicalWalStreamSample) {}),
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "Single unusual sample",
+ buildInput: func() VerdictInput {
+ return newTestVerdictInput(buildTestSamples(0, modifyNewestOnly(fillLocalQueue(testPauseBytes/2))))
+ },
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "Short network blip",
+ buildInput: inputWithNewestTwo(unreachableSince(30 * time.Second)),
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "Source unreachable for a few minutes",
+ buildInput: inputWithNewestTwo(unreachableSince(3 * time.Minute)),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeSourceUnreachable, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "Source down for a while",
+ buildInput: func() VerdictInput {
+ samples := buildTestSamples(4*time.Minute, modifyNewestTwo(unreachableSince(6*time.Minute)))
+ input := newTestVerdictInput(samples)
+ input.Streamer = newFailedTestStreamer(time.Minute, 3*time.Minute)
+
+ return input
+ },
+ expectedLevel: HealthLevelBroken,
+ expectedReasons: []reasonExpectation{{ReasonCodeSourceUnreachable, ReasonLevelBroken}},
+ },
+ {
+ name: "Archive stuck for 12 minutes",
+ buildInput: inputWithNewestTwo(archiveStalledFor(12 * time.Minute)),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeArchiveNotAdvancing, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "Archive stuck for 30 minutes",
+ buildInput: inputWithNewestTwo(archiveStalledFor(30 * time.Minute)),
+ expectedLevel: HealthLevelBroken,
+ expectedReasons: []reasonExpectation{{ReasonCodeArchiveNotAdvancing, ReasonLevelBroken}},
+ },
+ {
+ name: "Streamer stopped",
+ buildInput: func() VerdictInput {
+ samples := buildTestSamples(
+ 10*time.Minute,
+ modifyNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.LocalQueueBytes = testPauseBytes
+ sample.ArchiveStalledSince = new(testNow.Add(-time.Hour))
+ }),
+ )
+ input := newTestVerdictInput(samples)
+ input.Streamer = newFailedTestStreamer(10*time.Minute, 10*time.Minute)
+
+ return input
+ },
+ expectedLevel: HealthLevelBroken,
+ expectedReasons: []reasonExpectation{{ReasonCodeStreamingStopped, ReasonLevelBroken}},
+ },
+ {
+ name: "Restart after a deploy",
+ buildInput: func() VerdictInput {
+ input := newTestVerdictInput(buildTestSamples(10*time.Minute, nil))
+ input.Streamer = newRunningTestStreamer()
+ input.Streamer.StartedAt = testNow.Add(-20 * time.Second)
+ input.Streamer.FailingSince = new(testNow.Add(-20 * time.Second))
+
+ return input
+ },
+ expectedLevel: HealthLevelCollectingData,
+ },
+ {
+ name: "Crash loop on a new database",
+ buildInput: func() VerdictInput {
+ input := newTestVerdictInput(nil)
+ input.Streamer = newFailedTestStreamer(10*time.Second, 5*time.Minute)
+
+ return input
+ },
+ expectedLevel: HealthLevelBroken,
+ expectedReasons: []reasonExpectation{{ReasonCodeStreamingStopped, ReasonLevelBroken}},
+ },
+ {
+ name: "Slot removed on the source",
+ buildInput: inputWithNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) {
+ makeUnreachable(sample, testNow)
+ sample.SourceState = physical_enums.PhysicalWalStreamSourceStateSlotMissing
+ sample.SourceUnreachableSince = nil
+ }),
+ expectedLevel: HealthLevelBroken,
+ expectedReasons: []reasonExpectation{{ReasonCodeSlotMissing, ReasonLevelBroken}},
+ },
+ {
+ name: "Just enabled",
+ buildInput: func() VerdictInput {
+ input := newTestVerdictInput(nil)
+ input.Streamer = nil
+
+ return input
+ },
+ expectedLevel: HealthLevelCollectingData,
+ },
+ {
+ name: "Backups disabled",
+ buildInput: func() VerdictInput {
+ input := newTestVerdictInput(buildTestSamples(0, nil))
+ input.IsStreamingExpected = false
+
+ return input
+ },
+ expectedLevel: HealthLevelNotStreaming,
+ },
+ {
+ name: "Several problems at once",
+ buildInput: inputWithNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) {
+ makeUnreachable(sample, testNow)
+ sample.SourceState = physical_enums.PhysicalWalStreamSourceStateSlotMissing
+ sample.SourceUnreachableSince = nil
+ sample.LocalQueueBytes = testPauseBytes / 2
+ }),
+ expectedLevel: HealthLevelBroken,
+ expectedReasons: []reasonExpectation{
+ {ReasonCodeSlotMissing, ReasonLevelBroken},
+ {ReasonCodeLocalQueueFilling, ReasonLevelNeedsAttention},
+ },
+ },
+ {
+ name: "Busy cluster with steady extra WAL",
+ buildInput: inputWithEverySample(func(_ int, sample *physical_models.PhysicalWalStreamSample) {
+ slotStatusSince("extended", time.Hour)(sample)
+ }),
+ expectedLevel: HealthLevelHealthy,
+ expectedNotices: []NoticeCode{NoticeCodeSlotKeepsExtraWalSteadily},
+ },
+ {
+ name: "Paused receiver above the rebuild lag",
+ buildInput: inputWithNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.IsReceiverRunning = false
+ sample.SlotLagBytes = new(testRebuildLagBytes * 9 / 10)
+ }),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeNearSlotRebuild, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "No limit on the source",
+ buildInput: inputWithEverySample(func(_ int, sample *physical_models.PhysicalWalStreamSample) {
+ sample.SlotSafeWalSizeBytes = nil
+ }),
+ expectedLevel: HealthLevelHealthy,
+ expectedNotices: []NoticeCode{NoticeCodeNoSlotWalLimit},
+ },
+ {
+ name: "Slot already lost",
+ buildInput: inputWithNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.SlotWalStatus = new("lost")
+ sample.SlotSafeWalSizeBytes = nil
+ }),
+ expectedLevel: HealthLevelBroken,
+ expectedReasons: []reasonExpectation{{ReasonCodeSlotLost, ReasonLevelBroken}},
+ },
+ {
+ name: "No full backup yet",
+ buildInput: func() VerdictInput {
+ input := newTestVerdictInput(buildTestSamples(0, nil))
+ input.HasCompletedFullBackup = false
+
+ return input
+ },
+ expectedLevel: HealthLevelHealthy,
+ expectedNotices: []NoticeCode{NoticeCodeNoFullBackupYet},
+ },
+ })
+}
+
+func fillLocalQueue(queueBytes int64) func(sample *physical_models.PhysicalWalStreamSample) {
+ return func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.LocalQueueBytes = queueBytes
+ }
+}
+
+func Test_GetVerdict_AtReasonThresholds_ListsReasonOnlyPastThreshold(t *testing.T) {
+ backlogStepAboveThreshold := (minBacklogThresholdBytes + 2*testMiB) / 20
+ backlogStepBelowThreshold := (minBacklogThresholdBytes - 2*testMiB) / 20
+
+ // The slot lag grows by stepBytes every 30 seconds, so the remaining allowance
+ // runs out after allowance ÷ (stepBytes ÷ 30) seconds.
+ slotLagStep := testMiB
+ allowanceForSeconds := func(seconds int64) int64 { return seconds * slotLagStep / 30 }
+
+ runVerdictTestCases(t, []verdictTestCase{
+ {
+ name: "source unreachable just under a minute",
+ buildInput: inputWithNewestTwo(unreachableSince(59 * time.Second)),
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "source unreachable just over a minute",
+ buildInput: inputWithNewestTwo(unreachableSince(61 * time.Second)),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeSourceUnreachable, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "source unreachable just under five minutes",
+ buildInput: inputWithNewestTwo(unreachableSince(5*time.Minute - time.Second)),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeSourceUnreachable, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "source unreachable just over five minutes",
+ buildInput: inputWithNewestTwo(unreachableSince(5*time.Minute + time.Second)),
+ expectedLevel: HealthLevelBroken,
+ expectedReasons: []reasonExpectation{{ReasonCodeSourceUnreachable, ReasonLevelBroken}},
+ },
+ {
+ name: "newest sample just under two minutes old",
+ buildInput: func() VerdictInput {
+ return newTestVerdictInput(buildTestSamples(2*time.Minute-time.Second, nil))
+ },
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "newest sample just over two minutes old",
+ buildInput: func() VerdictInput {
+ return newTestVerdictInput(buildTestSamples(2*time.Minute+time.Second, nil))
+ },
+ expectedLevel: HealthLevelBroken,
+ expectedReasons: []reasonExpectation{{ReasonCodeStreamingStopped, ReasonLevelBroken}},
+ },
+ {
+ name: "streamer failing just under a minute",
+ buildInput: func() VerdictInput {
+ input := newTestVerdictInput(buildTestSamples(0, nil))
+ input.Streamer = newFailedTestStreamer(time.Hour, time.Minute-time.Second)
+
+ return input
+ },
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "streamer failing just over a minute",
+ buildInput: func() VerdictInput {
+ input := newTestVerdictInput(buildTestSamples(0, nil))
+ input.Streamer = newFailedTestStreamer(time.Hour, time.Minute+time.Second)
+
+ return input
+ },
+ expectedLevel: HealthLevelBroken,
+ expectedReasons: []reasonExpectation{{ReasonCodeStreamingStopped, ReasonLevelBroken}},
+ },
+ {
+ name: "archive stalled just under ten minutes",
+ buildInput: inputWithNewestTwo(archiveStalledFor(10*time.Minute - time.Second)),
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "archive stalled just over ten minutes",
+ buildInput: inputWithNewestTwo(archiveStalledFor(10*time.Minute + time.Second)),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeArchiveNotAdvancing, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "archive stalled just under thirty minutes",
+ buildInput: inputWithNewestTwo(archiveStalledFor(30*time.Minute - time.Second)),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeArchiveNotAdvancing, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "archive stalled just over thirty minutes",
+ buildInput: inputWithNewestTwo(archiveStalledFor(30*time.Minute + time.Second)),
+ expectedLevel: HealthLevelBroken,
+ expectedReasons: []reasonExpectation{{ReasonCodeArchiveNotAdvancing, ReasonLevelBroken}},
+ },
+ {
+ name: "unarchived wal growth just under the backlog threshold",
+ buildInput: inputWithEverySample(growingUnarchivedWal(backlogStepBelowThreshold)),
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "unarchived wal growth just over the backlog threshold",
+ buildInput: inputWithEverySample(growingUnarchivedWal(backlogStepAboveThreshold)),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeArchiveSlowerThanSource, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "unarchived wal sawtooth of four segments plus one",
+ buildInput: inputWithEverySample(func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ sample.UnarchivedWalBytes = new(8*testMiB + int64(index%6)*testSegmentSizeBytes)
+ }),
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "slot unreserved just under five minutes",
+ buildInput: inputWithNewestTwo(slotStatusSince("unreserved", 5*time.Minute-time.Second)),
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "slot unreserved just over five minutes",
+ buildInput: inputWithNewestTwo(slotStatusSince("unreserved", 5*time.Minute+time.Second)),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeSlotKeepsExtraWal, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "slot extended over five minutes while its lag grows",
+ buildInput: inputWithEverySample(func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ growingSlotLag(backlogStepAboveThreshold)(index, sample)
+ slotStatusSince("extended", 20*time.Minute)(sample)
+ }),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeSlotKeepsExtraWal, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "slot allowance runs out just over an hour",
+ buildInput: inputWithEverySample(func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ growingSlotLag(slotLagStep)(index, sample)
+ sample.SlotSafeWalSizeBytes = new(allowanceForSeconds(3700))
+ }),
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "slot allowance runs out just under an hour",
+ buildInput: inputWithEverySample(func(index int, sample *physical_models.PhysicalWalStreamSample) {
+ growingSlotLag(slotLagStep)(index, sample)
+ sample.SlotSafeWalSizeBytes = new(allowanceForSeconds(3500))
+ }),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeSlotLimitApproaching, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "slot lag just under 80% of the rebuild lag",
+ buildInput: inputWithNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.SlotLagBytes = new(testRebuildLagBytes*8/10 - 1)
+ }),
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "slot lag just over 80% of the rebuild lag",
+ buildInput: inputWithNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.SlotLagBytes = new(testRebuildLagBytes*8/10 + 1)
+ }),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeNearSlotRebuild, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "slot lag past the rebuild lag while extended",
+ buildInput: inputWithNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.SlotLagBytes = new(testRebuildLagBytes * 2)
+ sample.SlotWalStatus = new("extended")
+ }),
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "local queue just under half the pause size",
+ buildInput: inputWithNewestTwo(fillLocalQueue(testPauseBytes/2 - 1)),
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "local queue at half the pause size",
+ buildInput: inputWithNewestTwo(fillLocalQueue(testPauseBytes / 2)),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeLocalQueueFilling, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "two restarts in the last hour",
+ buildInput: func() VerdictInput {
+ input := newTestVerdictInput(
+ buildTestSamples(0, modifyNewestOnly(func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.ReceiverRestarts = 1
+ })),
+ )
+ input.SessionStartsLastHour = 1
+
+ return input
+ },
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "three restarts in the last hour",
+ buildInput: func() VerdictInput {
+ input := newTestVerdictInput(
+ buildTestSamples(0, modifyNewestOnly(func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.ReceiverRestarts = 2
+ })),
+ )
+ input.SessionStartsLastHour = 1
+
+ return input
+ },
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeFrequentRestarts, ReasonLevelNeedsAttention}},
+ },
+ {
+ name: "forced rotation refused",
+ buildInput: inputWithNewestTwo(func(sample *physical_models.PhysicalWalStreamSample) {
+ sample.IsForcedRotationDenied = true
+ }),
+ expectedLevel: HealthLevelNeedsAttention,
+ expectedReasons: []reasonExpectation{{ReasonCodeForcedRotationRefused, ReasonLevelNeedsAttention}},
+ },
+ })
+}
+
+func Test_GetVerdict_WhenChainHasGap_ListsWalChainBrokenOnlyForGapBelowProgressPoint(t *testing.T) {
+ gapBelowProgressPoint := chain_view.LSNRange{
+ Start: testArchiveProgressAt - walmath.LSN(2*testSegmentSizeBytes),
+ End: testArchiveProgressAt - walmath.LSN(testSegmentSizeBytes),
+ }
+ gapAboveProgressPoint := chain_view.LSNRange{
+ Start: testArchiveProgressAt,
+ End: testArchiveProgressAt + walmath.LSN(testSegmentSizeBytes),
+ }
+
+ runVerdictTestCases(t, []verdictTestCase{
+ {
+ name: "Slot rebuilt",
+ buildInput: func() VerdictInput {
+ input := newTestVerdictInput(buildTestSamples(0, nil))
+ input.NewestChainGaps = []chain_view.LSNRange{gapBelowProgressPoint}
+
+ return input
+ },
+ expectedLevel: HealthLevelBroken,
+ expectedReasons: []reasonExpectation{{ReasonCodeWalChainBroken, ReasonLevelBroken}},
+ },
+ {
+ name: "same chain without a gap",
+ buildInput: func() VerdictInput {
+ return newTestVerdictInput(buildTestSamples(0, nil))
+ },
+ expectedLevel: HealthLevelHealthy,
+ },
+ {
+ name: "gap above the archive progress point",
+ buildInput: func() VerdictInput {
+ input := newTestVerdictInput(buildTestSamples(0, nil))
+ input.NewestChainGaps = []chain_view.LSNRange{gapAboveProgressPoint}
+
+ return input
+ },
+ expectedLevel: HealthLevelHealthy,
+ },
+ })
+}
+
+func Test_GetVerdict_WhenReasonsCarryValues_ReportsSinceAmountsAndThresholds(t *testing.T) {
+ unreachableAt := testNow.Add(-3 * time.Minute)
+ verdict := GetVerdict(inputWithNewestTwo(unreachableSince(3 * time.Minute))())
+
+ sourceUnreachable := findReason(verdict.Reasons, ReasonCodeSourceUnreachable)
+ require.NotNil(t, sourceUnreachable)
+ require.NotNil(t, sourceUnreachable.Since)
+ assert.Equal(t, unreachableAt, *sourceUnreachable.Since)
+
+ newestSampleAt := testNow.Add(-10 * time.Minute)
+ stoppedInput := newTestVerdictInput(buildTestSamples(10*time.Minute, nil))
+ stoppedInput.Streamer = newFailedTestStreamer(10*time.Minute, 10*time.Minute)
+
+ streamingStopped := findReason(GetVerdict(stoppedInput).Reasons, ReasonCodeStreamingStopped)
+ require.NotNil(t, streamingStopped)
+ require.NotNil(t, streamingStopped.Since)
+ assert.Equal(t, newestSampleAt, *streamingStopped.Since)
+
+ queueVerdict := GetVerdict(inputWithNewestTwo(fillLocalQueue(300 * testMiB))())
+
+ localQueueFilling := findReason(queueVerdict.Reasons, ReasonCodeLocalQueueFilling)
+ require.NotNil(t, localQueueFilling)
+ assert.Equal(t, 300*testMiB, *localQueueFilling.ValueBytes)
+ assert.Equal(t, testPauseBytes, *localQueueFilling.ThresholdBytes)
+}
+
+func Test_GetStreamerState_ForEachStreamerRow_ReportsRunningOnlyWithFreshHeartbeat(t *testing.T) {
+ staleStreamer := newRunningTestStreamer()
+ staleStreamer.LastHeartbeatAt = testNow.Add(-StreamerHeartbeatStaleness - time.Second)
+
+ testCases := []struct {
+ name string
+ streamer *physical_models.PhysicalWalStreamer
+ expectedState StreamerState
+ }{
+ {
+ name: "running with a fresh heartbeat",
+ streamer: newRunningTestStreamer(),
+ expectedState: StreamerStateRunning,
+ },
+ {name: "Stale heartbeat", streamer: staleStreamer, expectedState: StreamerStateNotRunning},
+ {
+ name: "failed",
+ streamer: newFailedTestStreamer(time.Minute, time.Minute),
+ expectedState: StreamerStateNotRunning,
+ },
+ {name: "Streamer never started", streamer: nil, expectedState: StreamerStateNotStarted},
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ assert.Equal(t, testCase.expectedState, GetStreamerState(testCase.streamer, testNow))
+ })
+ }
+}
diff --git a/backend/internal/features/backups/backups/core/physical/wal_health/window.go b/backend/internal/features/backups/backups/core/physical/wal_health/window.go
new file mode 100644
index 00000000..0162cb0a
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/wal_health/window.go
@@ -0,0 +1,92 @@
+package wal_health
+
+import (
+ "time"
+
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+)
+
+type sampleValueReader func(sample *physical_models.PhysicalWalStreamSample) *int64
+
+func readUnarchivedWalBytes(sample *physical_models.PhysicalWalStreamSample) *int64 {
+ return sample.UnarchivedWalBytes
+}
+
+func readSlotLagBytes(sample *physical_models.PhysicalWalStreamSample) *int64 {
+ return sample.SlotLagBytes
+}
+
+// getSamplesInWindow returns the samples of the keeping-up window that ends at
+// windowEnd. Each sample stands for the interval before it, so the window holds
+// the samples after its start, not at it.
+func getSamplesInWindow(
+ samples []*physical_models.PhysicalWalStreamSample,
+ windowEnd time.Time,
+) []*physical_models.PhysicalWalStreamSample {
+ windowStart := windowEnd.Add(-keepingUpWindow)
+
+ windowSamples := make([]*physical_models.PhysicalWalStreamSample, 0, len(samples))
+
+ for _, sample := range samples {
+ if sample.SampledAt.After(windowStart) && !sample.SampledAt.After(windowEnd) {
+ windowSamples = append(windowSamples, sample)
+ }
+ }
+
+ return windowSamples
+}
+
+// getChangeBytesPerSecond measures how a value moved over the keeping-up window
+// ending at windowEnd, from the lowest value in the window's first edge to the
+// lowest in its last edge. The edges are at least five minutes apart, so the
+// rate never divides by a short span.
+func getChangeBytesPerSecond(
+ samples []*physical_models.PhysicalWalStreamSample,
+ windowEnd time.Time,
+ readValue sampleValueReader,
+) (changeBytesPerSecond float64, isKnown bool) {
+ windowStart := windowEnd.Add(-keepingUpWindow)
+ firstEdgeEnd := windowStart.Add(keepingUpWindowEdge)
+ lastEdgeStart := windowEnd.Add(-keepingUpWindowEdge)
+
+ var firstLowest, lastLowest *physical_models.PhysicalWalStreamSample
+
+ for _, sample := range getSamplesInWindow(samples, windowEnd) {
+ value := readValue(sample)
+ if value == nil {
+ continue
+ }
+
+ if !sample.SampledAt.After(firstEdgeEnd) &&
+ (firstLowest == nil || *value < *readValue(firstLowest)) {
+ firstLowest = sample
+ }
+
+ if !sample.SampledAt.Before(lastEdgeStart) &&
+ (lastLowest == nil || *value <= *readValue(lastLowest)) {
+ lastLowest = sample
+ }
+ }
+
+ if firstLowest == nil || lastLowest == nil {
+ return 0, false
+ }
+
+ spanSeconds := lastLowest.SampledAt.Sub(firstLowest.SampledAt).Seconds()
+ if spanSeconds <= 0 {
+ return 0, false
+ }
+
+ return float64(*readValue(lastLowest)-*readValue(firstLowest)) / spanSeconds, true
+}
+
+func getGrowthOverWindowBytes(changeBytesPerSecond float64) int64 {
+ return int64(changeBytesPerSecond * keepingUpWindow.Seconds())
+}
+
+// The unarchived WAL normally holds the uploads in flight plus the open
+// segment, so growth below four segments is noise on a cluster with large
+// segments.
+func getBacklogThresholdBytes(sample *physical_models.PhysicalWalStreamSample) int64 {
+ return max(minBacklogThresholdBytes, backlogThresholdSegments*sample.WalSegmentSizeBytes)
+}
diff --git a/backend/internal/features/backups/backups/dto/physical/wal_stream_statistics.go b/backend/internal/features/backups/backups/dto/physical/wal_stream_statistics.go
new file mode 100644
index 00000000..5136a9fa
--- /dev/null
+++ b/backend/internal/features/backups/backups/dto/physical/wal_stream_statistics.go
@@ -0,0 +1,116 @@
+package backups_dto_physical
+
+import (
+ "time"
+
+ physical_dto "databasus-backend/internal/features/backups/backups/core/physical/dto"
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ "databasus-backend/internal/features/backups/backups/core/physical/wal_health"
+)
+
+type WalStreamStatisticsPeriod string
+
+const (
+ WalStreamStatisticsPeriodHour WalStreamStatisticsPeriod = "HOUR"
+ WalStreamStatisticsPeriodDay WalStreamStatisticsPeriod = "DAY"
+ WalStreamStatisticsPeriodWeek WalStreamStatisticsPeriod = "WEEK"
+)
+
+// One-minute buckets for the hour: 30-second ones would hold zero or two samples
+// depending on the sampling phase and draw false gaps and doubled rates. The day
+// and the week get 360 buckets each, more than a chart can draw distinctly.
+func (p WalStreamStatisticsPeriod) GetDurations() (period, bucket time.Duration, isKnown bool) {
+ switch p {
+ case WalStreamStatisticsPeriodHour:
+ return time.Hour, time.Minute, true
+ case WalStreamStatisticsPeriodDay:
+ return 24 * time.Hour, 4 * time.Minute, true
+ case WalStreamStatisticsPeriodWeek:
+ return 7 * 24 * time.Hour, 28 * time.Minute, true
+ default:
+ return 0, 0, false
+ }
+}
+
+type GetWalStreamStatisticsRequest struct {
+ Period WalStreamStatisticsPeriod `form:"period" binding:"required"`
+}
+
+// The WAL stream statistics of one database. Every amount carries its unit in
+// its name, every time is UTC, and the verdict, states, reasons and notices are
+// codes, never display text.
+type GetWalStreamStatisticsResponse struct {
+ Health WalStreamHealth `json:"health"`
+ Streamer WalStreamStreamer `json:"streamer"`
+ Current *WalStreamCurrent `json:"current"`
+ KeepingUp wal_health.KeepingUp `json:"keepingUp"`
+ Series WalStreamSeries `json:"series"`
+}
+
+type WalStreamHealth struct {
+ Level wal_health.HealthLevel `json:"level"`
+ Reasons []wal_health.Reason `json:"reasons"`
+ Notices []wal_health.Notice `json:"notices"`
+
+ // A full backup is requested or running, so a broken chain is already being
+ // repaired.
+ IsFullBackupPending bool `json:"isFullBackupPending"`
+}
+
+type WalStreamStreamer struct {
+ State wal_health.StreamerState `json:"state"`
+ SessionStartedAt *time.Time `json:"sessionStartedAt"`
+ FailingSince *time.Time `json:"failingSince"`
+ RestartsLastHour int `json:"restartsLastHour"`
+}
+
+// The newest sample. Source-side values are null while the source is
+// unreachable or without the slot; on a reachable sample a null
+// slotSafeWalSizeBytes means the source sets no limit.
+type WalStreamCurrent struct {
+ SampledAt time.Time `json:"sampledAt"`
+ LastArchivedWalAt *time.Time `json:"lastArchivedWalAt"`
+ SessionStartedAt time.Time `json:"sessionStartedAt"`
+
+ SourceState physical_enums.PhysicalWalStreamSourceState `json:"sourceState"`
+ SourceUnreachableSince *time.Time `json:"sourceUnreachableSince"`
+
+ SourceCurrentLsn *string `json:"sourceCurrentLsn"`
+ ArchiveProgressLsn *string `json:"archiveProgressLsn"`
+ SourceWrittenBytes *int64 `json:"sourceWrittenBytes"`
+
+ ArchivedSegments int `json:"archivedSegments"`
+ ArchivedWalBytes int64 `json:"archivedWalBytes"`
+ ArchivedStoredBytes int64 `json:"archivedStoredBytes"`
+ WalSegmentSizeBytes int64 `json:"walSegmentSizeBytes"`
+ HighestConcurrentUploads int `json:"highestConcurrentUploads"`
+ SegmentWriteAvgMillis *int64 `json:"segmentWriteAvgMillis"`
+ SegmentWriteMaxMillis *int64 `json:"segmentWriteMaxMillis"`
+
+ // pg_replication_slots.wal_status as the source reports it.
+ SlotWalStatus *string `json:"slotWalStatus"`
+ SlotExtraWalSince *time.Time `json:"slotExtraWalSince"`
+ SlotLagBytes *int64 `json:"slotLagBytes"`
+ SlotSafeWalSizeBytes *int64 `json:"slotSafeWalSizeBytes"`
+
+ UnarchivedWalBytes *int64 `json:"unarchivedWalBytes"`
+ ArchiveStalledSince *time.Time `json:"archiveStalledSince"`
+
+ LocalQueueBytes int64 `json:"localQueueBytes"`
+ LocalQueuePauseBytes int64 `json:"localQueuePauseBytes"`
+
+ ReceiverRestarts int `json:"receiverRestarts"`
+ ReceiverPauses int `json:"receiverPauses"`
+ IsReceiverRunning bool `json:"isReceiverRunning"`
+ IsForcedRotationDenied bool `json:"isForcedRotationDenied"`
+}
+
+// Exactly period ÷ bucketSeconds points, oldest first; the last one holds the
+// moment of the request. A point without samples has every value null.
+type WalStreamSeries struct {
+ Period WalStreamStatisticsPeriod `json:"period"`
+ BucketSeconds int64 `json:"bucketSeconds"`
+ From time.Time `json:"from"`
+ To time.Time `json:"to"`
+ Points []*physical_dto.PhysicalWalStreamSeriesPoint `json:"points"`
+}
diff --git a/backend/internal/features/backups/backups/services/physical_wal_stream_statistics.go b/backend/internal/features/backups/backups/services/physical_wal_stream_statistics.go
new file mode 100644
index 00000000..1d3e6335
--- /dev/null
+++ b/backend/internal/features/backups/backups/services/physical_wal_stream_statistics.go
@@ -0,0 +1,218 @@
+package backups_services
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/google/uuid"
+
+ physical_dto "databasus-backend/internal/features/backups/backups/core/physical/dto"
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_core_service "databasus-backend/internal/features/backups/backups/core/physical/service"
+ "databasus-backend/internal/features/backups/backups/core/physical/wal_health"
+ backups_dto_physical "databasus-backend/internal/features/backups/backups/dto/physical"
+ backups_config_physical "databasus-backend/internal/features/backups/config/physical"
+ "databasus-backend/internal/features/databases"
+ users_models "databasus-backend/internal/features/users/models"
+)
+
+var ErrWalStreamStatisticsNotPhysical = errors.New(
+ "WAL stream statistics are only available for physical PostgreSQL databases",
+)
+
+var ErrUnknownWalStreamStatisticsPeriod = errors.New("period must be one of HOUR, DAY or WEEK")
+
+// GetWalStreamStatistics returns the WAL stream statistics of a physical
+// PostgreSQL database to a user who can view it. A physical database outside
+// WAL streaming mode still gets an answer: the not streaming verdict and the
+// samples it recorded before.
+func (s *PhysicalBackupService) GetWalStreamStatistics(
+ ctx context.Context,
+ user *users_models.User,
+ databaseID uuid.UUID,
+ request *backups_dto_physical.GetWalStreamStatisticsRequest,
+) (*backups_dto_physical.GetWalStreamStatisticsResponse, error) {
+ database, err := s.databaseService.GetDatabase(ctx, user, databaseID)
+ if err != nil {
+ return nil, err
+ }
+
+ if database.Type != databases.DatabaseTypePostgresPhysical {
+ return nil, ErrWalStreamStatisticsNotPhysical
+ }
+
+ period, bucketDuration, isKnownPeriod := request.Period.GetDurations()
+ if !isKnownPeriod {
+ return nil, ErrUnknownWalStreamStatisticsPeriod
+ }
+
+ backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(databaseID)
+ if err != nil {
+ return nil, err
+ }
+
+ newestFull, err := s.chainViewService.GetNewestCompletedFullWithGaps(databaseID)
+ if err != nil {
+ return nil, err
+ }
+
+ isFullBackupPending, err := s.isFullBackupPending(databaseID, backupConfig)
+ if err != nil {
+ return nil, err
+ }
+
+ query := physical_core_service.WalStreamStatisticsQuery{
+ DatabaseID: databaseID,
+ Period: period,
+ BucketDuration: bucketDuration,
+ SlotRebuildLagBytes: backupConfig.WalLagThresholdBytes,
+ IsStreamingExpected: backupConfig.IsWalStreamingExpected(),
+ Now: time.Now().UTC(),
+ }
+
+ if newestFull != nil {
+ query.HasCompletedFullBackup = true
+ query.NewestChainGaps = newestFull.WalGaps
+ }
+
+ statistics, err := s.physicalCoreService.GetWalStreamStatistics(query)
+ if err != nil {
+ return nil, err
+ }
+
+ return toWalStreamStatisticsResponse(request.Period, bucketDuration, isFullBackupPending, statistics), nil
+}
+
+// Databasus requests a full backup by itself only after a slot rebuild, so the
+// screen tells the user to start one unless one is already requested or running.
+func (s *PhysicalBackupService) isFullBackupPending(
+ databaseID uuid.UUID,
+ backupConfig *backups_config_physical.PhysicalBackupConfig,
+) (bool, error) {
+ if backupConfig.ForceFullRequestedAt != nil {
+ return true, nil
+ }
+
+ inFlightBackup, err := s.inFlightBackupRepository.FindByDatabaseID(databaseID)
+ if err != nil {
+ return false, err
+ }
+
+ return inFlightBackup != nil && inFlightBackup.BackupType == physical_enums.PhysicalBackupTypeFull, nil
+}
+
+func toWalStreamStatisticsResponse(
+ period backups_dto_physical.WalStreamStatisticsPeriod,
+ bucketDuration time.Duration,
+ isFullBackupPending bool,
+ statistics *physical_core_service.WalStreamStatistics,
+) *backups_dto_physical.GetWalStreamStatisticsResponse {
+ response := &backups_dto_physical.GetWalStreamStatisticsResponse{
+ Health: backups_dto_physical.WalStreamHealth{
+ Level: statistics.Verdict.Level,
+ Reasons: getReasonsInUTC(statistics.Verdict.Reasons),
+ Notices: getNoticesInUTC(statistics.Verdict.Notices),
+ IsFullBackupPending: isFullBackupPending,
+ },
+ Streamer: backups_dto_physical.WalStreamStreamer{
+ State: statistics.StreamerState,
+ RestartsLastHour: statistics.RestartsLastHour,
+ },
+ KeepingUp: statistics.KeepingUp,
+ Series: backups_dto_physical.WalStreamSeries{
+ Period: period,
+ BucketSeconds: int64(bucketDuration.Seconds()),
+ From: statistics.SeriesFrom,
+ To: statistics.SeriesTo,
+ Points: getSeriesPointsInUTC(statistics.SeriesPoints),
+ },
+ }
+
+ if statistics.Streamer != nil {
+ response.Streamer.SessionStartedAt = new(statistics.Streamer.StartedAt.UTC())
+ response.Streamer.FailingSince = getTimeInUTC(statistics.Streamer.FailingSince)
+ }
+
+ if sample := statistics.NewestSample; sample != nil {
+ response.Current = &backups_dto_physical.WalStreamCurrent{
+ SampledAt: sample.SampledAt.UTC(),
+ LastArchivedWalAt: getTimeInUTC(statistics.LastArchivedWalAt),
+ SessionStartedAt: sample.SessionStartedAt.UTC(),
+ SourceState: sample.SourceState,
+ SourceUnreachableSince: getTimeInUTC(sample.SourceUnreachableSince),
+ SourceWrittenBytes: sample.SourceWrittenBytes,
+ ArchivedSegments: sample.ArchivedSegments,
+ ArchivedWalBytes: sample.ArchivedWalBytes,
+ ArchivedStoredBytes: sample.ArchivedStoredBytes,
+ WalSegmentSizeBytes: sample.WalSegmentSizeBytes,
+ HighestConcurrentUploads: sample.HighestConcurrentUploads,
+ SegmentWriteAvgMillis: sample.SegmentWriteAvgMillis,
+ SegmentWriteMaxMillis: sample.SegmentWriteMaxMillis,
+ SlotWalStatus: sample.SlotWalStatus,
+ SlotExtraWalSince: getTimeInUTC(sample.SlotExtraWalSince),
+ SlotLagBytes: sample.SlotLagBytes,
+ SlotSafeWalSizeBytes: sample.SlotSafeWalSizeBytes,
+ UnarchivedWalBytes: sample.UnarchivedWalBytes,
+ ArchiveStalledSince: getTimeInUTC(sample.ArchiveStalledSince),
+ LocalQueueBytes: sample.LocalQueueBytes,
+ LocalQueuePauseBytes: sample.LocalQueuePauseBytes,
+ ReceiverRestarts: sample.ReceiverRestarts,
+ ReceiverPauses: sample.ReceiverPauses,
+ IsReceiverRunning: sample.IsReceiverRunning,
+ IsForcedRotationDenied: sample.IsForcedRotationDenied,
+ }
+
+ if sample.SourceCurrentLSN != nil {
+ response.Current.SourceCurrentLsn = new(sample.SourceCurrentLSN.String())
+ }
+
+ if sample.ArchiveProgressLSN != nil {
+ response.Current.ArchiveProgressLsn = new(sample.ArchiveProgressLSN.String())
+ }
+ }
+
+ return response
+}
+
+// The database driver hands times back in the server's local zone, and the
+// statistics promise UTC.
+func getTimeInUTC(value *time.Time) *time.Time {
+ if value == nil {
+ return nil
+ }
+
+ return new(value.UTC())
+}
+
+func getReasonsInUTC(reasons []wal_health.Reason) []wal_health.Reason {
+ utcReasons := make([]wal_health.Reason, 0, len(reasons))
+
+ for _, reason := range reasons {
+ reason.Since = getTimeInUTC(reason.Since)
+ utcReasons = append(utcReasons, reason)
+ }
+
+ return utcReasons
+}
+
+func getNoticesInUTC(notices []wal_health.Notice) []wal_health.Notice {
+ utcNotices := make([]wal_health.Notice, 0, len(notices))
+
+ for _, notice := range notices {
+ notice.Since = getTimeInUTC(notice.Since)
+ utcNotices = append(utcNotices, notice)
+ }
+
+ return utcNotices
+}
+
+func getSeriesPointsInUTC(
+ points []*physical_dto.PhysicalWalStreamSeriesPoint,
+) []*physical_dto.PhysicalWalStreamSeriesPoint {
+ for _, point := range points {
+ point.StartedAt = point.StartedAt.UTC()
+ }
+
+ return points
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go
index 57e40ba0..ada9fe36 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go
@@ -6,18 +6,12 @@ import (
"log/slog"
"time"
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ "databasus-backend/internal/features/backups/backups/core/physical/wal_health"
"databasus-backend/internal/util/walmath"
)
const (
- // Balances detection latency against query load on the source (one cheap
- // indexed row).
- lagMonitorPollInterval = 30 * time.Second
-
- // A warning wal_status must persist this long before we alert on it (vs a
- // transient write burst).
- warningSlotStatusHoldPeriod = 5 * time.Minute
-
// Beyond this many rebuild ATTEMPTS in a sliding hour (counted regardless of
// outcome), mechanical retry won't help (creds rotated, pg_hba changed,
// source dead); stop and surface the condition instead of dropping+recreating
@@ -61,6 +55,41 @@ type slotBreakClassifier struct {
warningStatusSince time.Time
}
+// What the lag monitor keeps from one tick to the next within a session. The
+// trackers decide the notifications, and their start times go into every
+// sample, so the samples and the notifications agree about when a condition
+// began.
+type lagMonitorState struct {
+ classifier slotBreakClassifier
+ stalenessTracker walArchiveStalenessTracker
+
+ previousSourceLSN *walmath.LSN
+ sourceUnreachableSince *time.Time
+ hasRecordedSample bool
+}
+
+func (m *lagMonitorState) recordProbeOutcome(probe slotProbe, now time.Time) {
+ if probe.Outcome != slotProbeOutcomeUnreachable {
+ m.sourceUnreachableSince = nil
+
+ return
+ }
+
+ if m.sourceUnreachableSince == nil {
+ m.sourceUnreachableSince = new(now)
+ }
+}
+
+// A tick without source-side values forgets the position, so the next sample
+// has no write volume instead of one that spans the gap.
+func (m *lagMonitorState) rememberSourcePosition(probe slotProbe) {
+ m.previousSourceLSN = nil
+
+ if probe.IsReachableWithSlot() {
+ m.previousSourceLSN = new(probe.State.CurrentLSN)
+ }
+}
+
// Only 'lost' and a stolen slot are unrecoverable enough to justify dropping the
// slot, which always costs a WAL gap and a fresh FULL. 'extended' merely means
// the slot retains WAL beyond max_wal_size — routine on a busy cluster, and
@@ -98,7 +127,7 @@ func (c *slotBreakClassifier) recordSampleAndClassifyBreak(
c.warningStatusSince = sample.ObservedAt
}
- if sample.ObservedAt.Sub(c.warningStatusSince) > warningSlotStatusHoldPeriod {
+ if sample.ObservedAt.Sub(c.warningStatusSince) > wal_health.WarningSlotStatusHoldPeriod {
return breakReasonSlotRetention, slotBreakActionAlert
}
@@ -119,12 +148,10 @@ func (c *slotBreakClassifier) recordSampleAndClassifyBreak(
// Source-side slot state only; consumer-side liveness is the slot-LSN watcher's
// job (slot_lsn_watcher.go).
func (s *WalStreamSupervisor) runLagMonitor(ctx context.Context, logger *slog.Logger) {
- ticker := time.NewTicker(lagMonitorPollInterval)
+ ticker := time.NewTicker(wal_health.SampleInterval)
defer ticker.Stop()
- var classifier slotBreakClassifier
-
- var stalenessTracker walArchiveStalenessTracker
+ monitorState := s.newLagMonitorState(logger)
for {
select {
@@ -132,41 +159,150 @@ func (s *WalStreamSupervisor) runLagMonitor(ctx context.Context, logger *slog.Lo
return
case <-ticker.C:
- state, isSourceReachable := s.GetSlotStateIfReachable(ctx, logger)
- if !isSourceReachable {
- continue
- }
+ s.runLagMonitorTick(ctx, logger, monitorState, time.Now().UTC())
+ }
+ }
+}
+
+// A down source makes every session fail within a minute, so the time it has
+// been unreachable is seeded from the newest stored sample instead of starting
+// over with each session.
+func (s *WalStreamSupervisor) newLagMonitorState(logger *slog.Logger) *lagMonitorState {
+ monitorState := &lagMonitorState{}
- s.reportArchivingProgress(ctx, logger, state)
+ if s.spec.WalStreamSampleRepo == nil {
+ return monitorState
+ }
- s.reportArchiveStalenessIfDue(logger, &stalenessTracker, state)
+ newestSample, err := s.spec.WalStreamSampleRepo.FindNewestByDatabase(s.spec.DatabaseID)
+ if err != nil {
+ logger.Error("failed to read the newest wal stream sample", "error", err)
+
+ return monitorState
+ }
+
+ if newestSample != nil &&
+ newestSample.SourceState == physical_enums.PhysicalWalStreamSourceStateUnreachable {
+ monitorState.sourceUnreachableSince = newestSample.SourceUnreachableSince
+ }
+
+ return monitorState
+}
- reason, action := classifier.recordSampleAndClassifyBreak(slotBreakSample{
- SlotState: state,
- WalLagThresholdBytes: s.spec.WalLagThresholdBytes,
- IsReceiverRunning: s.isReceiverRunning.Load(),
- ObservedAt: time.Now().UTC(),
- })
+// The sample is recorded after classifying and before acting, so a rebuild in
+// this tick leaves the slot as it was probed in the sample rather than the fresh
+// slot it is replaced with. The upload snapshot resets on read, so it is taken
+// once and shared by the log line and the sample.
+func (s *WalStreamSupervisor) runLagMonitorTick(
+ ctx context.Context,
+ logger *slog.Logger,
+ monitorState *lagMonitorState,
+ now time.Time,
+) {
+ probe := s.probeSlot(ctx, logger)
+ snapshot := s.uploader.TakeArchivingSnapshot()
+
+ newestCommitted, lastCommittedWalAt := s.getLastCommittedWal(logger)
+ archiveProgressLSN := getArchiveProgressPoint(s.getLowestQueuedSegment(), newestCommitted)
+
+ var (
+ reason walBreakReason
+ action slotBreakAction
+ )
- switch action {
- case slotBreakActionNone:
- continue
+ if probe.IsReachableWithSlot() {
+ state := probe.State
- case slotBreakActionAlert:
- s.reportChainAtRisk(logger, reason, state)
+ s.reportArchivingProgress(ctx, logger, state, snapshot)
- case slotBreakActionRebuild:
- logger.WarnContext(ctx, "wal stream break observed", "reason", string(reason),
- "lag_bytes", state.LagBytes, "wal_status", state.WalStatus)
+ s.reportArchiveStalenessIfDue(logger, &monitorState.stalenessTracker, state, walArchiveSample{
+ SourceCurrentLSN: state.CurrentLSN,
+ ArchiveProgressPoint: archiveProgressLSN,
+ ObservedAt: now,
+ StalenessThreshold: s.spec.ArchiveStalenessThreshold,
+ }, lastCommittedWalAt)
- if err := s.rebuildSlot(ctx, logger, reason); err != nil {
- logger.ErrorContext(ctx, "slot rebuild failed", "reason", string(reason), "error", err)
- }
- }
+ reason, action = monitorState.classifier.recordSampleAndClassifyBreak(slotBreakSample{
+ SlotState: state,
+ WalLagThresholdBytes: s.spec.WalLagThresholdBytes,
+ IsReceiverRunning: s.isReceiverRunning.Load(),
+ ObservedAt: now,
+ })
+ }
+
+ monitorState.recordProbeOutcome(probe, now)
+
+ s.recordWalStreamSample(logger, monitorState, walStreamSampleInput{
+ DatabaseID: s.spec.DatabaseID,
+ SampledAt: now,
+ SessionStartedAt: s.sessionStartedAt,
+ Probe: probe,
+ SourceUnreachableSince: monitorState.sourceUnreachableSince,
+ PreviousSourceLSN: monitorState.previousSourceLSN,
+ ArchiveProgressLSN: archiveProgressLSN,
+ Snapshot: snapshot,
+ WalSegmentSizeBytes: walSegmentSizeBytes(s.spec.SourceDB),
+ SlotExtraWalSince: monitorState.classifier.warningStatusSince,
+ ArchiveStalledSince: monitorState.stalenessTracker.fallingBehindSince,
+ LocalQueueBytes: s.backlogBytes(),
+ LocalQueuePauseBytes: s.highWatermarkBytes.Load(),
+ IsReceiverRunning: s.isReceiverRunning.Load(),
+ IsForcedRotationDenied: s.isForcedRotationDenied.Load(),
+ })
+
+ monitorState.rememberSourcePosition(probe)
+
+ switch action {
+ case slotBreakActionNone:
+ return
+
+ case slotBreakActionAlert:
+ s.reportChainAtRisk(logger, reason, probe.State)
+
+ case slotBreakActionRebuild:
+ logger.WarnContext(ctx, "wal stream break observed", "reason", string(reason),
+ "lag_bytes", probe.State.LagBytes, "wal_status", probe.State.WalStatus)
+
+ if err := s.rebuildSlot(ctx, logger, reason); err != nil {
+ logger.ErrorContext(ctx, "slot rebuild failed", "reason", string(reason), "error", err)
}
}
}
+// Without a sample repository, as in tests that drive the streamer alone,
+// nothing is recorded and the receiver counters stay for the test to read.
+func (s *WalStreamSupervisor) recordWalStreamSample(
+ logger *slog.Logger,
+ monitorState *lagMonitorState,
+ input walStreamSampleInput,
+) {
+ if s.spec.WalStreamSampleRepo == nil {
+ return
+ }
+
+ input.ReceiverRestarts, input.ReceiverPauses = s.receiverRespawns.takeCounts()
+
+ if err := s.spec.WalStreamSampleRepo.Insert(buildWalStreamSample(input)); err != nil {
+ logger.Error("failed to record a wal stream sample", "error", err)
+
+ return
+ }
+
+ if monitorState.hasRecordedSample {
+ return
+ }
+
+ monitorState.hasRecordedSample = true
+
+ if s.spec.WalStreamerRepo == nil {
+ return
+ }
+
+ if err := s.spec.WalStreamerRepo.ClearFailingSince(s.spec.DatabaseID); err != nil {
+ logger.Error("failed to clear the wal streamer's failing time", "error", err)
+ }
+}
+
// Archiving throughput and lag are only visible today once something has already broken, which
// leaves no history to read after an incident. This rides the existing poll rather than adding a
// ticker, and stays at INFO only when segments actually moved. The step timings tell latency-bound
@@ -176,9 +312,8 @@ func (s *WalStreamSupervisor) reportArchivingProgress(
ctx context.Context,
logger *slog.Logger,
state *SlotState,
+ snapshot ArchivingSnapshot,
) {
- snapshot := s.uploader.TakeArchivingSnapshot()
-
if snapshot.ArchivedSegments == 0 {
logger.DebugContext(ctx, fmt.Sprintf("no wal archived since the last report, %d bytes behind",
state.LagBytes), "wal_status", state.WalStatus)
@@ -206,21 +341,14 @@ func formatUploadStepTiming(timing uploadStepTiming, archivedSegments int64) str
timing.Longest.Round(time.Millisecond))
}
-// The slot's restart_lsn plus its lag is where the source currently is, so the
-// staleness check needs no extra query on the source.
func (s *WalStreamSupervisor) reportArchiveStalenessIfDue(
logger *slog.Logger,
stalenessTracker *walArchiveStalenessTracker,
state *SlotState,
+ archiveSample walArchiveSample,
+ lastCommittedWalAt *time.Time,
) {
- newestCommitted, lastCommittedWalAt := s.getLastCommittedWal(logger)
-
- isArchiveStale := stalenessTracker.recordSampleAndDetectStaleness(walArchiveSample{
- SourceCurrentLSN: state.RestartLSN + walmath.LSN(state.LagBytes),
- ArchiveProgressPoint: getArchiveProgressPoint(s.getLowestQueuedSegment(), newestCommitted),
- ObservedAt: time.Now().UTC(),
- StalenessThreshold: s.spec.ArchiveStalenessThreshold,
- })
+ isArchiveStale := stalenessTracker.recordSampleAndDetectStaleness(archiveSample)
if !isArchiveStale {
return
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go
index 7d06b0ba..2aac7cd1 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go
@@ -1,11 +1,24 @@
package usecases_physical_postgresql
import (
+ "context"
"log/slog"
"testing"
"time"
+ "github.com/jackc/pgx/v5"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+ physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
+ physical_testing "databasus-backend/internal/features/backups/backups/core/physical/testing"
+ "databasus-backend/internal/features/backups/backups/core/physical/wal_health"
+ postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
+ "databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/logger"
+ "databasus-backend/internal/util/walmath"
)
func runningSlotBreakSample(state *SlotState, walLagThresholdBytes int64, observedAt time.Time) slotBreakSample {
@@ -39,7 +52,7 @@ func Test_RecordSampleAndClassifyBreak_WhenWalStatusUnreserved_AlertsWithoutRebu
"unreserved is a warning; it must hold before it is worth telling anyone about")
reason, heldAction := classifier.recordSampleAndClassifyBreak(
- runningSlotBreakSample(state, 0, firstSeenAt.Add(warningSlotStatusHoldPeriod+time.Second)),
+ runningSlotBreakSample(state, 0, firstSeenAt.Add(wal_health.WarningSlotStatusHoldPeriod+time.Second)),
)
require.Equal(t, slotBreakActionAlert, heldAction,
@@ -56,7 +69,7 @@ func Test_RecordSampleAndClassifyBreak_WhenWalStatusExtendedPastHold_AlertsWitho
classifier.recordSampleAndClassifyBreak(runningSlotBreakSample(state, 0, firstSeenAt))
reason, action := classifier.recordSampleAndClassifyBreak(
- runningSlotBreakSample(state, 0, firstSeenAt.Add(warningSlotStatusHoldPeriod+time.Second)),
+ runningSlotBreakSample(state, 0, firstSeenAt.Add(wal_health.WarningSlotStatusHoldPeriod+time.Second)),
)
require.Equal(t, slotBreakActionAlert, action,
@@ -137,7 +150,7 @@ func Test_RecordSampleAndClassifyBreak_WhenSlotHealthy_DoesNotRebuildAndClearsWa
_, actionAfterRecovery := classifier.recordSampleAndClassifyBreak(
runningSlotBreakSample(
- &SlotState{WalStatus: "extended"}, 100, firstSeenAt.Add(warningSlotStatusHoldPeriod+time.Second),
+ &SlotState{WalStatus: "extended"}, 100, firstSeenAt.Add(wal_health.WarningSlotStatusHoldPeriod+time.Second),
),
)
@@ -168,7 +181,12 @@ func Test_ReportArchivingProgress_AfterBusyInterval_LogsCountsStepTimingsAndConc
})
supervisor.uploader.RecordConcurrentUploads(4)
- supervisor.reportArchivingProgress(t.Context(), supervisorLogger, &SlotState{LagBytes: 2048, WalStatus: "reserved"})
+ supervisor.reportArchivingProgress(
+ t.Context(),
+ supervisorLogger,
+ &SlotState{LagBytes: 2048, WalStatus: "reserved"},
+ supervisor.uploader.TakeArchivingSnapshot(),
+ )
logLine := log.String()
require.Contains(t, logLine, "level=INFO")
@@ -185,10 +203,304 @@ func Test_ReportArchivingProgress_AfterQuietInterval_LogsOnlyBytesBehindAtDebug(
log := &capturedLog{}
supervisor, supervisorLogger := newSupervisorReportingTo(log)
- supervisor.reportArchivingProgress(t.Context(), supervisorLogger, &SlotState{LagBytes: 4096, WalStatus: "reserved"})
+ supervisor.reportArchivingProgress(
+ t.Context(),
+ supervisorLogger,
+ &SlotState{LagBytes: 4096, WalStatus: "reserved"},
+ supervisor.uploader.TakeArchivingSnapshot(),
+ )
logLine := log.String()
require.Contains(t, logLine, "level=DEBUG")
require.Contains(t, logLine, "no wal archived since the last report, 4096 bytes behind")
require.NotContains(t, logLine, "avg")
}
+
+var lagMonitorTickStartedAt = time.Date(2026, time.October, 18, 12, 0, 0, 0, time.UTC)
+
+func newLagMonitorTickSupervisor(
+ t *testing.T,
+ fixture *PhysicalDBFixture,
+ sourceDB *postgresql_physical.PostgresqlPhysicalDatabase,
+ walLagThresholdBytes int64,
+) *WalStreamSupervisor {
+ t.Helper()
+
+ return NewWalStreamSupervisor(WalStreamSpec{
+ DatabaseID: fixture.DB.ID,
+ SourceDB: sourceDB,
+ StorageID: fixture.Storage.ID,
+ FieldEncryptor: encryption.GetFieldEncryptor(),
+ WalSegmentRepo: physical_repositories.GetWalSegmentRepository(),
+ HistoryRepo: physical_repositories.GetWalHistoryRepository(),
+ WalStreamSampleRepo: physical_repositories.GetWalStreamSampleRepository(),
+ WalStreamerRepo: physical_repositories.GetWalStreamerRepository(),
+ WatchDirRoot: t.TempDir(),
+ WalLagThresholdBytes: walLagThresholdBytes,
+ Logger: logger.GetLogger(),
+ })
+}
+
+func createStreamerSlot(t *testing.T, fixture *PhysicalDBFixture) {
+ t.Helper()
+
+ adminConn := OpenAdminConn(t, fixture)
+
+ require.NoError(t, fixture.DB.PostgresqlPhysical.VerifyWalSlot(
+ t.Context(), logger.GetLogger(), encryption.GetFieldEncryptor(),
+ ))
+
+ t.Cleanup(func() {
+ DropReplicationSlotExternally(t, adminConn, fixture.DB.PostgresqlPhysical.ReplicationSlotName)
+ })
+}
+
+// A committed segment below the source's position gives the tick an archive
+// progress point, which the unarchived WAL is measured from.
+func seedCommittedSegmentBelowCurrentPosition(t *testing.T, fixture *PhysicalDBFixture) {
+ t.Helper()
+
+ currentLSN, err := readCurrentWalLSN(t.Context(), OpenAdminConn(t, fixture))
+ require.NoError(t, err)
+
+ segmentSizeBytes := walSegmentSizeBytes(fixture.DB.PostgresqlPhysical)
+ segmentEndLSN := walmath.LSN(uint64(currentLSN) / uint64(segmentSizeBytes) * uint64(segmentSizeBytes))
+ segmentStartLSN := segmentEndLSN - walmath.LSN(segmentSizeBytes)
+
+ physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(
+ fixture.DB.ID, fixture.Storage.ID, 1, "seeded-segment", segmentStartLSN, segmentEndLSN,
+ ))
+}
+
+func runLagMonitorTicks(
+ t *testing.T,
+ supervisor *WalStreamSupervisor,
+ monitorState *lagMonitorState,
+ firstTickAt time.Time,
+ tickCount int,
+ beforeEachTick func(),
+) {
+ t.Helper()
+
+ for tickIndex := range tickCount {
+ if beforeEachTick != nil {
+ beforeEachTick()
+ }
+
+ supervisor.runLagMonitorTick(
+ t.Context(),
+ logger.GetLogger(),
+ monitorState,
+ firstTickAt.Add(time.Duration(tickIndex)*wal_health.SampleInterval),
+ )
+ }
+}
+
+func getRecordedSamples(t *testing.T, fixture *PhysicalDBFixture) []*physical_models.PhysicalWalStreamSample {
+ t.Helper()
+
+ samples, err := physical_repositories.GetWalStreamSampleRepository().
+ FindSinceByDatabase(fixture.DB.ID, time.Time{})
+ require.NoError(t, err)
+
+ return samples
+}
+
+func Test_RunLagMonitorTick_WhenSourceHealthy_RecordsReachableSamplesWithSlotAndQueueValues(t *testing.T) {
+ fixture := SetupPhysicalDBForBackup(t)
+ adminConn := OpenAdminConn(t, fixture)
+
+ createStreamerSlot(t, fixture)
+ seedCommittedSegmentBelowCurrentPosition(t, fixture)
+
+ supervisor := newLagMonitorTickSupervisor(t, fixture, fixture.DB.PostgresqlPhysical, 0)
+
+ runLagMonitorTicks(t, supervisor, supervisor.newLagMonitorState(logger.GetLogger()), lagMonitorTickStartedAt, 4,
+ func() {
+ _, err := GenerateWalActivity(t.Context(), adminConn, 4096)
+ require.NoError(t, err)
+ })
+
+ samples := getRecordedSamples(t, fixture)
+ require.Len(t, samples, 4)
+
+ for index, sample := range samples {
+ assert.Equal(t, physical_enums.PhysicalWalStreamSourceStateReachable, sample.SourceState)
+ assert.Equal(
+ t,
+ lagMonitorTickStartedAt.Add(time.Duration(index)*wal_health.SampleInterval),
+ sample.SampledAt.UTC(),
+ )
+ assert.NotNil(t, sample.SourceCurrentLSN)
+ assert.NotNil(t, sample.SlotWalStatus)
+ assert.NotNil(t, sample.SlotLagBytes)
+ assert.NotNil(t, sample.UnarchivedWalBytes)
+ assert.Positive(t, sample.LocalQueuePauseBytes)
+
+ if index > 0 {
+ require.NotNil(t, sample.SourceWrittenBytes)
+ assert.Positive(t, *sample.SourceWrittenBytes)
+ }
+ }
+
+ assert.Nil(t, samples[0].SourceWrittenBytes)
+}
+
+func Test_RunLagMonitorTick_WhenSourceUnreachableAcrossSessions_KeepsOneUnreachableSinceTime(t *testing.T) {
+ fixture := SetupPhysicalDBForBackup(t)
+
+ unreachableSourceDB := *fixture.DB.PostgresqlPhysical
+ unreachableSourceDB.Host = "127.0.0.1"
+ unreachableSourceDB.Port = 1
+
+ firstSession := newLagMonitorTickSupervisor(t, fixture, &unreachableSourceDB, 0)
+ runLagMonitorTicks(t, firstSession, firstSession.newLagMonitorState(logger.GetLogger()),
+ lagMonitorTickStartedAt, 3, nil)
+
+ secondSessionStartedAt := lagMonitorTickStartedAt.Add(5 * time.Minute)
+ secondSession := newLagMonitorTickSupervisor(t, fixture, &unreachableSourceDB, 0)
+ runLagMonitorTicks(t, secondSession, secondSession.newLagMonitorState(logger.GetLogger()),
+ secondSessionStartedAt, 1, nil)
+
+ samples := getRecordedSamples(t, fixture)
+ require.Len(t, samples, 4)
+
+ for _, sample := range samples {
+ assert.Equal(t, physical_enums.PhysicalWalStreamSourceStateUnreachable, sample.SourceState)
+ require.NotNil(t, sample.SourceUnreachableSince)
+ assert.Equal(t, lagMonitorTickStartedAt, sample.SourceUnreachableSince.UTC(),
+ "the time a source has been unreachable survives the session restart its outage causes")
+ assert.Nil(t, sample.SlotWalStatus)
+ assert.Nil(t, sample.SlotLagBytes)
+ assert.Nil(t, sample.SlotSafeWalSizeBytes)
+ assert.Nil(t, sample.UnarchivedWalBytes)
+ assert.Nil(t, sample.SourceWrittenBytes)
+ }
+}
+
+func Test_RunLagMonitorTick_WhenSlotHeldExtended_RecordsTimeClassifierStartedItsHold(t *testing.T) {
+ fixture := SetupPhysicalDBForBackup(t)
+ adminConn := OpenAdminConn(t, fixture)
+
+ createStreamerSlot(t, fixture)
+ setMaxWalSizeForTest(t, adminConn, "32MB")
+
+ slotName := fixture.DB.PostgresqlPhysical.ReplicationSlotName
+
+ require.Eventually(t, func() bool {
+ ForceReplicationLag(t, adminConn, 16*1024*1024)
+
+ slotState, err := InspectSlot(t.Context(), adminConn, slotName)
+
+ return err == nil && slotState != nil && slotState.WalStatus == slotWalStatusExtendedForTest
+ }, 90*time.Second, time.Second, "a slot lagging past max_wal_size is reported as extended")
+
+ supervisor := newLagMonitorTickSupervisor(t, fixture, fixture.DB.PostgresqlPhysical, 0)
+ runLagMonitorTicks(
+ t,
+ supervisor,
+ supervisor.newLagMonitorState(logger.GetLogger()),
+ lagMonitorTickStartedAt,
+ 2,
+ nil,
+ )
+
+ samples := getRecordedSamples(t, fixture)
+ require.Len(t, samples, 2)
+
+ for _, sample := range samples {
+ require.NotNil(t, sample.SlotExtraWalSince)
+ assert.Equal(t, lagMonitorTickStartedAt, sample.SlotExtraWalSince.UTC())
+ }
+}
+
+const slotWalStatusExtendedForTest = "extended"
+
+// max_wal_size is reloadable, so the test restores it instead of leaving the
+// shared source checkpointing every two segments.
+func setMaxWalSizeForTest(t *testing.T, adminConn *pgx.Conn, size string) {
+ t.Helper()
+
+ _, err := adminConn.Exec(t.Context(), "ALTER SYSTEM SET max_wal_size = '"+size+"'")
+ require.NoError(t, err)
+
+ _, err = adminConn.Exec(t.Context(), "SELECT pg_reload_conf()")
+ require.NoError(t, err)
+
+ t.Cleanup(func() {
+ _, _ = adminConn.Exec(context.Background(), "ALTER SYSTEM RESET max_wal_size")
+ _, _ = adminConn.Exec(context.Background(), "SELECT pg_reload_conf()")
+ })
+}
+
+func Test_RunLagMonitorTick_WhenTickRebuildsSlot_RecordsSlotAsProbedBeforeRebuild(t *testing.T) {
+ fixture := SetupPhysicalDBForBackup(t)
+ adminConn := OpenAdminConn(t, fixture)
+
+ createStreamerSlot(t, fixture)
+
+ const walLagThresholdBytes = 64 * 1024
+
+ slotName := fixture.DB.PostgresqlPhysical.ReplicationSlotName
+ ForceReplicationLag(t, adminConn, 4*walLagThresholdBytes)
+ WaitUntilSlotLag(t, adminConn, slotName, 2*walLagThresholdBytes, 30*time.Second)
+
+ supervisor := newLagMonitorTickSupervisor(t, fixture, fixture.DB.PostgresqlPhysical, walLagThresholdBytes)
+ supervisor.isReceiverRunning.Store(true)
+
+ sampleCountAtRebuild := -1
+ supervisor.spec.OnSlotRebuilt = func(context.Context, string) error {
+ sampleCountAtRebuild = len(getRecordedSamples(t, fixture))
+
+ return nil
+ }
+
+ runLagMonitorTicks(
+ t,
+ supervisor,
+ supervisor.newLagMonitorState(logger.GetLogger()),
+ lagMonitorTickStartedAt,
+ 1,
+ nil,
+ )
+
+ require.Equal(t, 1, sampleCountAtRebuild, "the tick records its sample before it rebuilds the slot")
+
+ samples := getRecordedSamples(t, fixture)
+ require.Len(t, samples, 1)
+ require.NotNil(t, samples[0].SlotLagBytes)
+ assert.Greater(t, *samples[0].SlotLagBytes, int64(walLagThresholdBytes),
+ "the sample shows the slot that caused the rebuild")
+}
+
+func Test_RunLagMonitorTick_OnFirstSampleOfSession_ClearsStreamerFailingTime(t *testing.T) {
+ fixture := SetupPhysicalDBForBackup(t)
+
+ streamerRepository := physical_repositories.GetWalStreamerRepository()
+ t.Cleanup(func() {
+ _ = streamerRepository.DeleteByDatabaseID(fixture.DB.ID)
+ })
+
+ require.NoError(t, streamerRepository.Claim(fixture.DB.ID))
+ require.NoError(t, streamerRepository.MarkFailed(fixture.DB.ID))
+
+ failedStreamer, err := streamerRepository.FindByDatabaseID(fixture.DB.ID)
+ require.NoError(t, err)
+ require.NotNil(t, failedStreamer.FailingSince)
+
+ createStreamerSlot(t, fixture)
+
+ supervisor := newLagMonitorTickSupervisor(t, fixture, fixture.DB.PostgresqlPhysical, 0)
+ runLagMonitorTicks(
+ t,
+ supervisor,
+ supervisor.newLagMonitorState(logger.GetLogger()),
+ lagMonitorTickStartedAt,
+ 1,
+ nil,
+ )
+
+ recoveredStreamer, err := streamerRepository.FindByDatabaseID(fixture.DB.ID)
+ require.NoError(t, err)
+ assert.Nil(t, recoveredStreamer.FailingSince)
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receiver_respawn_counter.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receiver_respawn_counter.go
new file mode 100644
index 00000000..6f3c82b0
--- /dev/null
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receiver_respawn_counter.go
@@ -0,0 +1,24 @@
+package usecases_physical_postgresql
+
+import "sync/atomic"
+
+// A pause for a full local queue is Databasus working as designed, so it is
+// counted apart from a restart and never looks like a crash.
+type receiverRespawnCounter struct {
+ restarts atomic.Int64
+ pauses atomic.Int64
+}
+
+func (c *receiverRespawnCounter) recordRespawn(isAfterBackpressurePause bool) {
+ if isAfterBackpressurePause {
+ c.pauses.Add(1)
+
+ return
+ }
+
+ c.restarts.Add(1)
+}
+
+func (c *receiverRespawnCounter) takeCounts() (restarts, pauses int) {
+ return int(c.restarts.Swap(0)), int(c.pauses.Swap(0))
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot.go
index c57d1059..74254ed3 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot.go
@@ -20,6 +20,15 @@ type SlotState struct {
RestartLSN walmath.LSN
FlushLSN walmath.LSN
LagBytes int64
+
+ // The source's own position, read in the same query. Unlike restart_lsn plus
+ // the lag, it stays right for a lost slot, whose restart_lsn is NULL.
+ CurrentLSN walmath.LSN
+
+ // How much WAL the source can still write before the slot loses WAL. Nil when
+ // max_slot_wal_keep_size is -1 (no limit) and when the slot is already lost;
+ // WalStatus tells the two apart.
+ SafeWalSizeBytes *int64
}
// InspectSlot reads the row for slotName from pg_replication_slots and
@@ -39,7 +48,9 @@ func InspectSlot(ctx context.Context, conn *pgx.Conn, slotName string) (*SlotSta
COALESCE(s.wal_status, ''),
COALESCE(s.restart_lsn, '0/0'::pg_lsn),
COALESCE(s.confirmed_flush_lsn, '0/0'::pg_lsn),
- COALESCE(pg_wal_lsn_diff(pg_current_wal_lsn(), s.restart_lsn), 0)::bigint
+ COALESCE(pg_wal_lsn_diff(pg_current_wal_lsn(), s.restart_lsn), 0)::bigint,
+ pg_current_wal_lsn(),
+ s.safe_wal_size
FROM pg_replication_slots s
LEFT JOIN pg_stat_replication r ON r.pid = s.active_pid
WHERE s.slot_name = $1
@@ -51,6 +62,8 @@ func InspectSlot(ctx context.Context, conn *pgx.Conn, slotName string) (*SlotSta
&restartLSN,
&flushLSN,
&state.LagBytes,
+ &state.CurrentLSN,
+ &state.SafeWalSizeBytes,
)
switch {
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go
index af4268c9..bc47e77e 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go
@@ -17,6 +17,8 @@ const (
// restart_lsn unchanged this long while WAL is waiting means pg_receivewal is
// stuck (alive but flushing nothing); restart it locally.
slotLsnStallTimeout = 60 * time.Second
+
+ slotProbeTimeout = 10 * time.Second
)
type slotLivenessSample struct {
@@ -70,42 +72,63 @@ func (t *stallTracker) reset() {
t.lastAdvanceAt = time.Time{}
}
-// isSourceReachable=false means defer to the lag monitor.
-func (s *WalStreamSupervisor) GetSlotStateIfReachable(
- ctx context.Context,
- logger *slog.Logger,
-) (state *SlotState, isSourceReachable bool) {
- conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)
+type slotProbeOutcome int
+
+const (
+ slotProbeOutcomeReachable slotProbeOutcome = iota
+
+ // The source answered, but our slot is gone from it, so WAL is no longer
+ // retained for us and the chain is already broken.
+ slotProbeOutcomeSlotMissing
+
+ slotProbeOutcomeUnreachable
+)
+
+// Only a reachable probe carries a slot state.
+type slotProbe struct {
+ Outcome slotProbeOutcome
+ State *SlotState
+}
+
+func (p slotProbe) IsReachableWithSlot() bool {
+ return p.Outcome == slotProbeOutcomeReachable
+}
+
+// A source that accepts connections but never answers would otherwise hold
+// every caller's tick for as long as the TCP connection lives.
+func (s *WalStreamSupervisor) probeSlot(ctx context.Context, logger *slog.Logger) slotProbe {
+ probeCtx, cancel := context.WithTimeout(ctx, slotProbeTimeout)
+ defer cancel()
+
+ conn, err := s.spec.SourceDB.OpenInspectionConn(probeCtx, s.spec.FieldEncryptor)
if err != nil {
- logger.DebugContext(ctx, "source unreachable, deferring to the lag monitor", "error", err)
+ logger.DebugContext(ctx, "source unreachable during the slot probe", "error", err)
- return nil, false
+ return slotProbe{Outcome: slotProbeOutcomeUnreachable}
}
defer func() { _ = conn.Close(context.Background()) }()
- state, err = InspectSlot(ctx, conn, s.slotName)
+ state, err := InspectSlot(probeCtx, conn, s.slotName)
if err != nil {
logger.DebugContext(ctx, "failed to inspect the replication slot", "error", err)
- return nil, false
+ return slotProbe{Outcome: slotProbeOutcomeUnreachable}
}
if state == nil {
- // The slot is gone from the source while we still believe we are streaming from it, so WAL
- // is no longer retained for us and the chain is already at risk.
logger.WarnContext(ctx, "the replication slot no longer exists on the source")
- return nil, false
+ return slotProbe{Outcome: slotProbeOutcomeSlotMissing}
}
var alive int
- if err := conn.QueryRow(ctx, "SELECT 1").Scan(&alive); err != nil {
+ if err := conn.QueryRow(probeCtx, "SELECT 1").Scan(&alive); err != nil {
logger.DebugContext(ctx, "source stopped responding during the slot probe", "error", err)
- return nil, false
+ return slotProbe{Outcome: slotProbeOutcomeUnreachable}
}
- return state, true
+ return slotProbe{Outcome: slotProbeOutcomeReachable, State: state}
}
// A stall with an unreachable server is left to the lag monitor (slot loss /
@@ -122,11 +145,13 @@ func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slo
return
case <-ticker.C:
- state, isSourceReachable := s.GetSlotStateIfReachable(ctx, logger)
- if !isSourceReachable {
+ probe := s.probeSlot(ctx, logger)
+ if !probe.IsReachableWithSlot() {
continue
}
+ state := probe.State
+
sample := slotLivenessSample{
RestartLSN: state.RestartLSN,
LagBytes: state.LagBytes,
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go
index dcaeef34..2731a87a 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go
@@ -1,11 +1,19 @@
package usecases_physical_postgresql
import (
+ "context"
+ "net"
+ "strings"
+ "sync"
"testing"
"time"
+ "github.com/google/uuid"
"github.com/stretchr/testify/require"
+ postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
+ "databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/logger"
"databasus-backend/internal/util/walmath"
)
@@ -115,3 +123,108 @@ func Test_RecordSampleAndDetectStall_WhenReceiverNotRunning_DoesNotDetectStall(t
require.False(t, tracker.recordSampleAndDetectStall(stoppedSample(base.Add(10*time.Minute)), time.Minute),
"a receiver we are deliberately holding down is not a stalled one")
}
+
+func newProbeOnlySupervisor(
+ t *testing.T,
+ sourceDB *postgresql_physical.PostgresqlPhysicalDatabase,
+) *WalStreamSupervisor {
+ t.Helper()
+
+ return NewWalStreamSupervisor(WalStreamSpec{
+ DatabaseID: uuid.New(),
+ SourceDB: sourceDB,
+ FieldEncryptor: encryption.GetFieldEncryptor(),
+ WatchDirRoot: t.TempDir(),
+ Logger: logger.GetLogger(),
+ })
+}
+
+func Test_ProbeSlot_WhenSlotRemovedOnReachableSource_ReportsSlotMissing(t *testing.T) {
+ fixture := SetupPhysicalDBForBackup(t)
+ adminConn := OpenAdminConn(t, fixture)
+
+ sourceDB := *fixture.DB.PostgresqlPhysical
+ sourceDB.ReplicationSlotName = "databasus_probe_" + strings.ReplaceAll(uuid.New().String(), "-", "")[:12]
+
+ _, err := adminConn.Exec(
+ t.Context(),
+ "SELECT pg_create_physical_replication_slot($1, true)",
+ sourceDB.ReplicationSlotName,
+ )
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _, _ = adminConn.Exec(context.Background(), "SELECT pg_drop_replication_slot($1)", sourceDB.ReplicationSlotName)
+ })
+
+ supervisor := newProbeOnlySupervisor(t, &sourceDB)
+
+ probe := supervisor.probeSlot(t.Context(), logger.GetLogger())
+ require.Equal(t, slotProbeOutcomeReachable, probe.Outcome)
+ require.NotNil(t, probe.State)
+
+ DropReplicationSlotExternally(t, adminConn, sourceDB.ReplicationSlotName)
+
+ probe = supervisor.probeSlot(t.Context(), logger.GetLogger())
+ require.Equal(t, slotProbeOutcomeSlotMissing, probe.Outcome,
+ "a source that answers without the slot is not unreachable: the chain is already broken")
+ require.Nil(t, probe.State)
+}
+
+func Test_ProbeSlot_WhenSourceAcceptsButNeverAnswers_ReportsUnreachableWithinTimeout(t *testing.T) {
+ silentListener := startSilentListener(t)
+
+ supervisor := newProbeOnlySupervisor(t, &postgresql_physical.PostgresqlPhysicalDatabase{
+ Host: "127.0.0.1",
+ Port: silentListener.Addr().(*net.TCPAddr).Port,
+ Username: "streamer",
+ Password: "streamer",
+ ReplicationSlotName: "databasus_silent_source",
+ })
+
+ probeStartedAt := time.Now().UTC()
+ probe := supervisor.probeSlot(t.Context(), logger.GetLogger())
+
+ require.Equal(t, slotProbeOutcomeUnreachable, probe.Outcome)
+ require.Less(t, time.Since(probeStartedAt), 11*time.Second,
+ "a silent source must not hold the lag monitor's tick past the probe timeout")
+}
+
+// Accepts every connection and holds it open without ever writing a byte, the
+// way a hung server or a black-holing middlebox behaves.
+func startSilentListener(t *testing.T) net.Listener {
+ t.Helper()
+
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+
+ var (
+ acceptedConnsMutex sync.Mutex
+ acceptedConns []net.Conn
+ )
+
+ go func() {
+ for {
+ conn, err := listener.Accept()
+ if err != nil {
+ return
+ }
+
+ acceptedConnsMutex.Lock()
+ acceptedConns = append(acceptedConns, conn)
+ acceptedConnsMutex.Unlock()
+ }
+ }()
+
+ t.Cleanup(func() {
+ _ = listener.Close()
+
+ acceptedConnsMutex.Lock()
+ defer acceptedConnsMutex.Unlock()
+
+ for _, conn := range acceptedConns {
+ _ = conn.Close()
+ }
+ })
+
+ return listener
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_test.go
new file mode 100644
index 00000000..fc5b47d9
--- /dev/null
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_test.go
@@ -0,0 +1,83 @@
+package usecases_physical_postgresql_test
+
+import (
+ "context"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"
+ "databasus-backend/internal/util/walmath"
+)
+
+func Test_InspectSlot_WithAndWithoutSlotWalLimit_ReportsAllowanceAndCurrentPosition(t *testing.T) {
+ fixture := postgresql_executor.SetupPhysicalDBForBackup(t)
+ adminConn := postgresql_executor.OpenAdminConn(t, fixture)
+
+ slotName := createReservedTestSlot(t, adminConn)
+
+ var currentLSNBeforeInspect string
+ require.NoError(
+ t,
+ adminConn.QueryRow(t.Context(), "SELECT pg_current_wal_lsn()::text").Scan(&currentLSNBeforeInspect),
+ )
+
+ unlimitedSlotState, err := postgresql_executor.InspectSlot(t.Context(), adminConn, slotName)
+ require.NoError(t, err)
+ require.NotNil(t, unlimitedSlotState)
+
+ assert.Nil(t, unlimitedSlotState.SafeWalSizeBytes, "max_slot_wal_keep_size = -1 means no limit")
+
+ positionBeforeInspect, err := walmath.ParseLSN(currentLSNBeforeInspect)
+ require.NoError(t, err)
+ assert.GreaterOrEqual(t, unlimitedSlotState.CurrentLSN, positionBeforeInspect,
+ "the position InspectSlot reads must not be below one read before it")
+
+ setSlotWalLimit(t, adminConn, "100GB")
+
+ require.Eventually(t, func() bool {
+ limitedSlotState, err := postgresql_executor.InspectSlot(t.Context(), adminConn, slotName)
+ if err != nil || limitedSlotState == nil {
+ return false
+ }
+
+ return limitedSlotState.SafeWalSizeBytes != nil && *limitedSlotState.SafeWalSizeBytes > 0
+ }, 15*time.Second, 200*time.Millisecond, "a source with a slot WAL limit reports how much WAL is left")
+}
+
+func createReservedTestSlot(t *testing.T, adminConn *pgx.Conn) string {
+ t.Helper()
+
+ slotName := "databasus_inspect_" + strings.ReplaceAll(uuid.New().String(), "-", "")[:12]
+
+ _, err := adminConn.Exec(t.Context(), "SELECT pg_create_physical_replication_slot($1, true)", slotName)
+ require.NoError(t, err)
+
+ t.Cleanup(func() {
+ _, _ = adminConn.Exec(context.Background(), "SELECT pg_drop_replication_slot($1)", slotName)
+ })
+
+ return slotName
+}
+
+// The setting is reloadable, so the test restores the default instead of
+// leaving a limit behind for the tests that share this source.
+func setSlotWalLimit(t *testing.T, adminConn *pgx.Conn, limit string) {
+ t.Helper()
+
+ _, err := adminConn.Exec(t.Context(), "ALTER SYSTEM SET max_slot_wal_keep_size = '"+limit+"'")
+ require.NoError(t, err)
+
+ _, err = adminConn.Exec(t.Context(), "SELECT pg_reload_conf()")
+ require.NoError(t, err)
+
+ t.Cleanup(func() {
+ _, _ = adminConn.Exec(context.Background(), "ALTER SYSTEM RESET max_slot_wal_keep_size")
+ _, _ = adminConn.Exec(context.Background(), "SELECT pg_reload_conf()")
+ })
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go
index 477f5bfe..79d8067c 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go
@@ -818,6 +818,11 @@ type WalStreamerTestSpec struct {
ArchiveStalenessThreshold time.Duration
OnSlotRebuilt func(ctx context.Context, reason string) error
OnChainAtRisk func(report ChainRiskReport)
+
+ // Overrides the local queue size at which the receiver is paused, so a test
+ // can reach back pressure with a few segments instead of half a gigabyte.
+ // Zero keeps the production size.
+ LocalQueuePauseBytes int64
}
// A replication slot is meant to outlive a streamer restart (so WAL is never
@@ -857,6 +862,10 @@ func StartWalStreamerForTest(t *testing.T, spec WalStreamerTestSpec) *WalStreame
Logger: logger.GetLogger(),
})
+ if spec.LocalQueuePauseBytes > 0 {
+ supervisor.setLocalQueuePauseBytes(spec.LocalQueuePauseBytes)
+ }
+
ctx, cancel := context.WithCancel(t.Context())
done := make(chan struct{})
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go
index bdb51405..d56aa579 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go
@@ -7,6 +7,7 @@ import (
"path/filepath"
"time"
+ postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
"databasus-backend/internal/util/walmath"
)
@@ -31,16 +32,29 @@ func (s *WalStreamSupervisor) runBackpressureMonitor(ctx context.Context, _ *slo
return
case <-ticker.C:
- if s.backlogBytes() >= s.highWatermarkBytes {
+ if s.backlogBytes() >= s.highWatermarkBytes.Load() {
+ s.isBackpressureStopPending.Store(true)
s.signalRestart()
}
}
}
}
-func (s *WalStreamSupervisor) waitForBacklogBelowLow(ctx context.Context, logger *slog.Logger) bool {
- if s.backlogBytes() < s.highWatermarkBytes {
- return true
+func getDefaultLocalQueuePauseBytes(sourceDB *postgresql_physical.PostgresqlPhysicalDatabase) int64 {
+ return max(walLocalMinHighWatermarkBytes, 8*walSegmentSizeBytes(sourceDB))
+}
+
+func (s *WalStreamSupervisor) setLocalQueuePauseBytes(pauseBytes int64) {
+ s.highWatermarkBytes.Store(pauseBytes)
+ s.lowWatermarkBytes.Store(pauseBytes / 5)
+}
+
+func (s *WalStreamSupervisor) waitForBacklogBelowLow(
+ ctx context.Context,
+ logger *slog.Logger,
+) (hasWaited, isRunning bool) {
+ if s.backlogBytes() < s.highWatermarkBytes.Load() {
+ return false, true
}
logger.WarnContext(ctx, "wal backlog over high watermark; pausing pg_receivewal until it drains")
@@ -51,13 +65,13 @@ func (s *WalStreamSupervisor) waitForBacklogBelowLow(ctx context.Context, logger
for {
select {
case <-ctx.Done():
- return false
+ return true, false
case <-ticker.C:
- if s.backlogBytes() < s.lowWatermarkBytes {
+ if s.backlogBytes() < s.lowWatermarkBytes.Load() {
logger.InfoContext(ctx, "wal backlog drained below low watermark; resuming pg_receivewal")
- return true
+ return true, true
}
}
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go
index 19a71189..3e9263b8 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go
@@ -22,8 +22,8 @@ func Test_WalStream_BackpressureWatermarks_ScaleWithWalSegmentSize(t *testing.T)
Logger: logger.GetLogger(),
})
- require.Equal(t, 8*customSegSize, supervisor.highWatermarkBytes)
- require.Equal(t, 8*customSegSize/5, supervisor.lowWatermarkBytes)
+ require.Equal(t, 8*customSegSize, supervisor.highWatermarkBytes.Load())
+ require.Equal(t, 8*customSegSize/5, supervisor.lowWatermarkBytes.Load())
}
func Test_BacklogBytes_WhenSegmentsStagedOutOfResumePath_CountsThemTowardTheWatermark(t *testing.T) {
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume.go
index b54c85f8..803116c1 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume.go
@@ -106,11 +106,13 @@ func segmentNoAtLSN(lsn walmath.LSN, segmentSizeBytes int64) walmath.WalSegmentN
}
func (s *WalStreamSupervisor) realignResumePath(ctx context.Context, logger *slog.Logger) {
- state, isSourceReachable := s.GetSlotStateIfReachable(ctx, logger)
- if !isSourceReachable {
+ probe := s.probeSlot(ctx, logger)
+ if !probe.IsReachableWithSlot() {
return
}
+ state := probe.State
+
segmentSizeBytes := walSegmentSizeBytes(s.spec.SourceDB)
resumeSegmentNo, isResumeSegmentFound := GetResumeSegmentNo(s.watchDir, segmentSizeBytes)
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_rotation.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_rotation.go
index d28f77f2..e5003c71 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_rotation.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_rotation.go
@@ -108,6 +108,8 @@ func (s *WalStreamSupervisor) runForcedWalRotation(ctx context.Context, logger *
logger.WarnContext(ctx, "source refuses pg_switch_wal; forced wal rotation disabled for this streamer",
"error", err)
+ s.isForcedRotationDenied.Store(true)
+
s.reportChainAtRisk(logger, breakReasonRotationDenied, nil)
return
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_rotation_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_rotation_test.go
index 5845c799..a608a8fe 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_rotation_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_rotation_test.go
@@ -6,9 +6,12 @@ import (
"testing"
"time"
+ "github.com/google/uuid"
"github.com/jackc/pgx/v5/pgconn"
"github.com/stretchr/testify/require"
+ "databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/logger"
"databasus-backend/internal/util/walmath"
)
@@ -102,3 +105,39 @@ func Test_RecordRotation_WhenSwitchMovedTheInsertPoint_DoesNotCountItAsFreshWal(
ObservedAt: base.Add(10 * testRotationInterval),
}, testRotationInterval), "the switch itself moved the insert point; that is not the source writing")
}
+
+func Test_RunForcedWalRotation_WhenSourceRefusesSwitchWal_SetsRefusalFlag(t *testing.T) {
+ fixture := SetupPhysicalDBForBackup(t)
+ adminConn := OpenAdminConn(t, fixture)
+
+ supervisor := NewWalStreamSupervisor(WalStreamSpec{
+ DatabaseID: uuid.New(),
+ SourceDB: createReplicationOnlyRole(t, adminConn, fixture),
+ FieldEncryptor: encryption.GetFieldEncryptor(),
+ WatchDirRoot: t.TempDir(),
+ ForcedRotationInterval: time.Second,
+ Logger: logger.GetLogger(),
+ })
+
+ rotationDone := make(chan struct{})
+
+ go func() {
+ defer close(rotationDone)
+
+ supervisor.runForcedWalRotation(t.Context(), logger.GetLogger())
+ }()
+
+ require.False(t, supervisor.isForcedRotationDenied.Load())
+
+ require.Eventually(t, func() bool {
+ _, err := GenerateWalActivity(t.Context(), adminConn, 4096)
+
+ return err == nil && supervisor.isForcedRotationDenied.Load()
+ }, 60*time.Second, time.Second, "a refused pg_switch_wal must be visible to the samples")
+
+ select {
+ case <-rotationDone:
+ case <-time.After(10 * time.Second):
+ t.Fatal("the rotation loop must stop after a permanent refusal")
+ }
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_staleness.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_staleness.go
index 6dda11fe..c9f75965 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_staleness.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_staleness.go
@@ -7,11 +7,6 @@ import (
"databasus-backend/internal/util/walmath"
)
-// A rarely-written database can legitimately hold its newest WAL locally for a
-// while, but past this the recovery point has silently stopped tracking the
-// source and only the operator can decide what to do about it.
-const DefaultArchiveStalenessThreshold = 30 * time.Minute
-
type walArchiveSample struct {
SourceCurrentLSN walmath.LSN
ArchiveProgressPoint walmath.LSN
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
index 23a80abc..7c36a10b 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
@@ -143,6 +143,13 @@ type WalStreamSpec struct {
WalSegmentRepo *physical_repositories.PhysicalWalSegmentRepository
HistoryRepo *physical_repositories.PhysicalWalHistoryRepository
+ // Where the lag monitor records one sample per tick; nil records none.
+ WalStreamSampleRepo *physical_repositories.PhysicalWalStreamSampleRepository
+
+ // The lag monitor clears the streamer's failing time with a session's first
+ // sample; nil leaves it alone.
+ WalStreamerRepo *physical_repositories.PhysicalWalStreamerRepository
+
// config.DataFolder; the per-DB queue lives under
// <root>/wal-queue/<database_id>/. It must survive a process restart so crash
// recovery can re-process finalized-but-not-uploaded segments.
@@ -189,10 +196,16 @@ type WalStreamSupervisor struct {
watchDir string
slotName string
+ // Every sample of this session carries it, so the read path can count session
+ // starts within a window.
+ sessionStartedAt time.Time
+
// Derived once from the source's (immutable) wal_segment_size; recomputing
- // them on every poll tick would be wasted work.
- highWatermarkBytes int64
- lowWatermarkBytes int64
+ // them on every poll tick would be wasted work. Atomic because the
+ // back-pressure monitor, the supervision loop and the lag monitor read them
+ // from their own goroutines while a test may lower them.
+ highWatermarkBytes atomic.Int64
+ lowWatermarkBytes atomic.Int64
// Asks the supervision loop to SIGTERM the current pg_receivewal and respawn
// (sent by the back-pressure monitor and the slot-LSN watcher). Buffered size
@@ -207,6 +220,16 @@ type WalStreamSupervisor struct {
// we are causing ourselves by holding the receiver down.
isReceiverRunning atomic.Bool
+ // Set by the back-pressure monitor when it stops the receiver, so the respawn
+ // that follows counts as a pause rather than a restart.
+ isBackpressureStopPending atomic.Bool
+
+ // Taken and reset by the lag monitor with each sample.
+ receiverRespawns receiverRespawnCounter
+
+ // The source refused pg_switch_wal in this session; every later sample says so.
+ isForcedRotationDenied atomic.Bool
+
// rebuildMu serializes slot rebuilds in this process; rebuildTimestamps powers
// the per-hour loop-protection cap. One supervisor owns a DB at a time (the
// physical_wal_streamers heartbeat claim), so this is the only guard needed.
@@ -242,18 +265,19 @@ func NewWalStreamSupervisor(spec WalStreamSpec) *WalStreamSupervisor {
OnGapDetected: spec.OnGapDetected,
})
- highWatermarkBytes := max(walLocalMinHighWatermarkBytes, 8*walSegmentSizeBytes(spec.SourceDB))
-
- return &WalStreamSupervisor{
- spec: spec,
- logger: scopedLogger,
- uploader: uploader,
- watchDir: watchDir,
- slotName: spec.SourceDB.ReplicationSlotName,
- highWatermarkBytes: highWatermarkBytes,
- lowWatermarkBytes: highWatermarkBytes / 5,
- restartSignal: make(chan struct{}, 1),
+ supervisor := &WalStreamSupervisor{
+ spec: spec,
+ logger: scopedLogger,
+ uploader: uploader,
+ watchDir: watchDir,
+ slotName: spec.SourceDB.ReplicationSlotName,
+ sessionStartedAt: time.Now().UTC(),
+ restartSignal: make(chan struct{}, 1),
}
+
+ supervisor.setLocalQueuePauseBytes(getDefaultLocalQueuePauseBytes(spec.SourceDB))
+
+ return supervisor
}
func (s *WalStreamSupervisor) Run(ctx context.Context) error {
@@ -328,6 +352,8 @@ func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, logg
failureEscalator rapidFailureEscalator
)
+ hasSpawnedReceiver := false
+
for {
if ctx.Err() != nil {
return nil
@@ -337,7 +363,8 @@ func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, logg
return nil
}
- if !s.waitForBacklogBelowLow(ctx, logger) {
+ hasWaitedForBacklogDrain, isRunning := s.waitForBacklogBelowLow(ctx, logger)
+ if !isRunning {
return nil
}
@@ -347,6 +374,14 @@ func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, logg
s.removePartials(logger)
s.realignResumePath(ctx, logger)
+ isBackpressureStop := s.isBackpressureStopPending.Swap(false)
+
+ if hasSpawnedReceiver {
+ s.receiverRespawns.recordRespawn(hasWaitedForBacklogDrain || isBackpressureStop)
+ }
+
+ hasSpawnedReceiver = true
+
receiverRun := s.spawnAndSupervise(ctx, logger, pgBin)
// One probe per iteration, shared by both dispositions that judge the source: asking twice
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_sample.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_sample.go
new file mode 100644
index 00000000..80bec7eb
--- /dev/null
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_sample.go
@@ -0,0 +1,130 @@
+package usecases_physical_postgresql
+
+import (
+ "time"
+
+ "github.com/google/uuid"
+
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+ "databasus-backend/internal/util/walmath"
+)
+
+// Everything one lag-monitor tick knows when it records its sample. The trackers'
+// times carry over ticks without source-side values, so their zero value, not a
+// missing probe, means "no condition".
+type walStreamSampleInput struct {
+ DatabaseID uuid.UUID
+ SampledAt time.Time
+ SessionStartedAt time.Time
+
+ Probe slotProbe
+ SourceUnreachableSince *time.Time
+
+ // The source position of the previous tick, nil after a tick without
+ // source-side values and on the first tick of a session.
+ PreviousSourceLSN *walmath.LSN
+
+ // Zero while nothing has been archived for the database.
+ ArchiveProgressLSN walmath.LSN
+
+ Snapshot ArchivingSnapshot
+ WalSegmentSizeBytes int64
+
+ SlotExtraWalSince time.Time
+ ArchiveStalledSince time.Time
+
+ LocalQueueBytes int64
+ LocalQueuePauseBytes int64
+
+ ReceiverRestarts int
+ ReceiverPauses int
+ IsReceiverRunning bool
+ IsForcedRotationDenied bool
+}
+
+func buildWalStreamSample(input walStreamSampleInput) *physical_models.PhysicalWalStreamSample {
+ sample := &physical_models.PhysicalWalStreamSample{
+ DatabaseID: input.DatabaseID,
+ SampledAt: input.SampledAt,
+ SessionStartedAt: input.SessionStartedAt,
+ SourceState: getSampleSourceState(input.Probe),
+ ArchivedSegments: int(input.Snapshot.ArchivedSegments),
+ ArchivedWalBytes: input.Snapshot.ArchivedSegments * input.WalSegmentSizeBytes,
+ ArchivedStoredBytes: input.Snapshot.ArchivedBytes,
+ WalSegmentSizeBytes: input.WalSegmentSizeBytes,
+ HighestConcurrentUploads: input.Snapshot.HighestConcurrentUploads,
+ SlotExtraWalSince: getTimeIfSet(input.SlotExtraWalSince),
+ ArchiveStalledSince: getTimeIfSet(input.ArchiveStalledSince),
+ LocalQueueBytes: input.LocalQueueBytes,
+ LocalQueuePauseBytes: input.LocalQueuePauseBytes,
+ ReceiverRestarts: input.ReceiverRestarts,
+ ReceiverPauses: input.ReceiverPauses,
+ IsReceiverRunning: input.IsReceiverRunning,
+ IsForcedRotationDenied: input.IsForcedRotationDenied,
+ }
+
+ if input.Probe.Outcome == slotProbeOutcomeUnreachable {
+ sample.SourceUnreachableSince = input.SourceUnreachableSince
+ }
+
+ if input.ArchiveProgressLSN > 0 {
+ sample.ArchiveProgressLSN = new(input.ArchiveProgressLSN)
+ }
+
+ if input.Snapshot.ArchivedSegments > 0 {
+ sample.SegmentWriteAvgMillis = new(
+ input.Snapshot.SegmentWrite.GetAverage(input.Snapshot.ArchivedSegments).Milliseconds(),
+ )
+ sample.SegmentWriteMaxMillis = new(input.Snapshot.SegmentWrite.Longest.Milliseconds())
+ }
+
+ if input.Probe.IsReachableWithSlot() {
+ fillSourceSideValues(sample, input)
+ }
+
+ return sample
+}
+
+// The current position comes from the source itself rather than from the slot,
+// so a lost slot, whose restart_lsn is gone, still yields a real position. A
+// position behind the previous one means the source switched to a new
+// timeline, and that tick's write volume is unknown rather than negative.
+func fillSourceSideValues(sample *physical_models.PhysicalWalStreamSample, input walStreamSampleInput) {
+ state := input.Probe.State
+ currentLSN := state.CurrentLSN
+
+ sample.SourceCurrentLSN = new(currentLSN)
+ sample.SlotWalStatus = new(state.WalStatus)
+ sample.SlotLagBytes = new(state.LagBytes)
+ sample.SlotSafeWalSizeBytes = state.SafeWalSizeBytes
+
+ if input.PreviousSourceLSN != nil && currentLSN >= *input.PreviousSourceLSN {
+ sample.SourceWrittenBytes = new(int64(currentLSN - *input.PreviousSourceLSN))
+ }
+
+ if input.ArchiveProgressLSN > 0 && currentLSN >= input.ArchiveProgressLSN {
+ sample.UnarchivedWalBytes = new(int64(currentLSN - input.ArchiveProgressLSN))
+ }
+}
+
+func getSampleSourceState(probe slotProbe) physical_enums.PhysicalWalStreamSourceState {
+ switch probe.Outcome {
+ case slotProbeOutcomeReachable:
+ return physical_enums.PhysicalWalStreamSourceStateReachable
+
+ case slotProbeOutcomeSlotMissing:
+ return physical_enums.PhysicalWalStreamSourceStateSlotMissing
+
+ default:
+ return physical_enums.PhysicalWalStreamSourceStateUnreachable
+ }
+}
+
+func getTimeIfSet(value time.Time) *time.Time {
+ if value.IsZero() {
+ return nil
+ }
+
+ return new(value)
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_sample_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_sample_test.go
new file mode 100644
index 00000000..fe664a9a
--- /dev/null
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_sample_test.go
@@ -0,0 +1,190 @@
+package usecases_physical_postgresql
+
+import (
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+ "databasus-backend/internal/util/walmath"
+)
+
+const testSampleSegmentSize = int64(16 * 1024 * 1024)
+
+var testSampleStartedAt = time.Date(2026, time.October, 18, 12, 0, 0, 0, time.UTC)
+
+func reachableProbe(currentLSN walmath.LSN, walStatus string) slotProbe {
+ return slotProbe{
+ Outcome: slotProbeOutcomeReachable,
+ State: &SlotState{
+ WalStatus: walStatus,
+ RestartLSN: currentLSN - 4096,
+ LagBytes: 4096,
+ CurrentLSN: currentLSN,
+ },
+ }
+}
+
+func newTestSampleInput(probe slotProbe, archiveProgressLSN walmath.LSN) walStreamSampleInput {
+ return walStreamSampleInput{
+ DatabaseID: uuid.New(),
+ SampledAt: testSampleStartedAt,
+ SessionStartedAt: testSampleStartedAt.Add(-time.Hour),
+ Probe: probe,
+ ArchiveProgressLSN: archiveProgressLSN,
+ WalSegmentSizeBytes: testSampleSegmentSize,
+ LocalQueueBytes: 3 * testSampleSegmentSize,
+ LocalQueuePauseBytes: 32 * testSampleSegmentSize,
+ IsReceiverRunning: true,
+ Snapshot: ArchivingSnapshot{
+ ArchivedSegments: 2,
+ ArchivedBytes: 5 * 1024 * 1024,
+ SegmentWrite: uploadStepTiming{Total: 400 * time.Millisecond, Longest: 300 * time.Millisecond},
+ HighestConcurrentUploads: 3,
+ },
+ }
+}
+
+// Replays ticks the way the lag monitor does, so the source write volume of each
+// sample depends on the ticks before it.
+func buildSamplesForProbes(probes []slotProbe) []*physical_models.PhysicalWalStreamSample {
+ monitorState := &lagMonitorState{}
+ samples := make([]*physical_models.PhysicalWalStreamSample, 0, len(probes))
+
+ for index, probe := range probes {
+ sampledAt := testSampleStartedAt.Add(time.Duration(index) * 30 * time.Second)
+ monitorState.recordProbeOutcome(probe, sampledAt)
+
+ input := newTestSampleInput(probe, walmath.LSN(0x1000000))
+ input.SampledAt = sampledAt
+ input.PreviousSourceLSN = monitorState.previousSourceLSN
+ input.SourceUnreachableSince = monitorState.sourceUnreachableSince
+
+ samples = append(samples, buildWalStreamSample(input))
+ monitorState.rememberSourcePosition(probe)
+ }
+
+ return samples
+}
+
+func Test_BuildWalStreamSample_WhenSlotLost_KeepsRealCurrentPositionWithoutNegativeUnarchivedWal(t *testing.T) {
+ const currentLSN = walmath.LSN(0x50000000)
+
+ lostProbe := slotProbe{
+ Outcome: slotProbeOutcomeReachable,
+ State: &SlotState{WalStatus: "lost", CurrentLSN: currentLSN},
+ }
+
+ sampleBehindProgress := buildWalStreamSample(newTestSampleInput(lostProbe, currentLSN+0x1000))
+ require.NotNil(t, sampleBehindProgress.SourceCurrentLSN)
+ assert.Equal(t, currentLSN, *sampleBehindProgress.SourceCurrentLSN)
+ assert.Nil(t, sampleBehindProgress.UnarchivedWalBytes)
+
+ sampleAheadOfProgress := buildWalStreamSample(newTestSampleInput(lostProbe, currentLSN-0x1000))
+ require.NotNil(t, sampleAheadOfProgress.UnarchivedWalBytes)
+ assert.Equal(t, int64(0x1000), *sampleAheadOfProgress.UnarchivedWalBytes)
+}
+
+func Test_BuildWalStreamSample_WhenSourcePositionMovesBackwards_RecordsNoWriteVolume(t *testing.T) {
+ samples := buildSamplesForProbes([]slotProbe{
+ reachableProbe(0x60000000, "reserved"),
+ reachableProbe(0x50000000, "reserved"),
+ reachableProbe(0x50001000, "reserved"),
+ })
+
+ assert.Nil(t, samples[1].SourceWrittenBytes, "a timeline switch must not be recorded as a negative write")
+ require.NotNil(t, samples[2].SourceWrittenBytes)
+ assert.Equal(t, int64(0x1000), *samples[2].SourceWrittenBytes)
+}
+
+func Test_BuildWalStreamSample_AfterStreamerStarts_RecordsWriteVolumeFromSecondSample(t *testing.T) {
+ samples := buildSamplesForProbes([]slotProbe{
+ reachableProbe(0x50000000, "reserved"),
+ reachableProbe(0x50002000, "reserved"),
+ })
+
+ assert.Nil(t, samples[0].SourceWrittenBytes)
+ require.NotNil(t, samples[1].SourceWrittenBytes)
+ assert.Equal(t, int64(0x2000), *samples[1].SourceWrittenBytes)
+}
+
+func Test_BuildWalStreamSample_WhenSourceReachableAgain_RecordsWriteVolumeFromSecondReachableSample(t *testing.T) {
+ samples := buildSamplesForProbes([]slotProbe{
+ reachableProbe(0x50000000, "reserved"),
+ {Outcome: slotProbeOutcomeUnreachable},
+ reachableProbe(0x50004000, "reserved"),
+ reachableProbe(0x50005000, "reserved"),
+ })
+
+ assert.Nil(t, samples[1].SourceWrittenBytes)
+ assert.Nil(t, samples[2].SourceWrittenBytes, "the write volume across an unreachable tick is unknown")
+ require.NotNil(t, samples[3].SourceWrittenBytes)
+ assert.Equal(t, int64(0x1000), *samples[3].SourceWrittenBytes)
+}
+
+func Test_BuildWalStreamSample_WhileNothingArchived_RecordsNoUnarchivedWal(t *testing.T) {
+ sample := buildWalStreamSample(newTestSampleInput(reachableProbe(0x50000000, "reserved"), 0))
+
+ assert.Nil(t, sample.ArchiveProgressLSN)
+ assert.Nil(t, sample.UnarchivedWalBytes)
+ require.NotNil(t, sample.SlotLagBytes)
+}
+
+func Test_BuildWalStreamSample_WithoutSourceSideValues_LeavesThemEmptyAndFillsTheRest(t *testing.T) {
+ unreachableSince := testSampleStartedAt.Add(-time.Minute)
+
+ testCases := []struct {
+ name string
+ probe slotProbe
+ expectedState physical_enums.PhysicalWalStreamSourceState
+ expectedSourceUnreachableSince *time.Time
+ }{
+ {
+ name: "unreachable",
+ probe: slotProbe{Outcome: slotProbeOutcomeUnreachable},
+ expectedState: physical_enums.PhysicalWalStreamSourceStateUnreachable,
+ expectedSourceUnreachableSince: &unreachableSince,
+ },
+ {
+ name: "without the slot",
+ probe: slotProbe{Outcome: slotProbeOutcomeSlotMissing},
+ expectedState: physical_enums.PhysicalWalStreamSourceStateSlotMissing,
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ input := newTestSampleInput(testCase.probe, walmath.LSN(0x1000000))
+ input.PreviousSourceLSN = new(walmath.LSN(0x1000))
+ input.SourceUnreachableSince = &unreachableSince
+
+ sample := buildWalStreamSample(input)
+
+ assert.Equal(t, testCase.expectedState, sample.SourceState)
+ assert.Equal(t, testCase.expectedSourceUnreachableSince, sample.SourceUnreachableSince)
+
+ assert.Nil(t, sample.SourceCurrentLSN)
+ assert.Nil(t, sample.SourceWrittenBytes)
+ assert.Nil(t, sample.SlotWalStatus)
+ assert.Nil(t, sample.SlotLagBytes)
+ assert.Nil(t, sample.SlotSafeWalSizeBytes)
+ assert.Nil(t, sample.UnarchivedWalBytes)
+
+ require.NotNil(t, sample.ArchiveProgressLSN)
+ assert.Equal(t, 2, sample.ArchivedSegments)
+ assert.Equal(t, 2*testSampleSegmentSize, sample.ArchivedWalBytes)
+ assert.Equal(t, int64(5*1024*1024), sample.ArchivedStoredBytes)
+ assert.Equal(t, 3, sample.HighestConcurrentUploads)
+ require.NotNil(t, sample.SegmentWriteAvgMillis)
+ assert.Equal(t, int64(200), *sample.SegmentWriteAvgMillis)
+ assert.Equal(t, int64(300), *sample.SegmentWriteMaxMillis)
+ assert.Equal(t, 3*testSampleSegmentSize, sample.LocalQueueBytes)
+ assert.Equal(t, 32*testSampleSegmentSize, sample.LocalQueuePauseBytes)
+ assert.True(t, sample.IsReceiverRunning)
+ })
+ }
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
index 0302e573..e7d77a0e 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
@@ -937,3 +937,98 @@ func Test_WalStream_WhenUploadsKeepFailing_AlertsArchiveStaleOnce(t *testing.T)
require.Len(t, chainRisks.getReasons(), 1)
}
+
+func Test_WalStream_WhenQueueFillsAndReceiverExitsOnItsOwn_CountsOnePauseAndOneRestart(t *testing.T) {
+ if testing.Short() {
+ t.Skip("streamer integration test runs pg_receivewal; skipped in -short")
+ }
+
+ fixture := SetupPhysicalDBForBackup(t)
+ t.Cleanup(func() {
+ _ = physical_repositories.GetWalStreamerRepository().DeleteByDatabaseID(fixture.DB.ID)
+ })
+
+ adminConn := OpenAdminConn(t, fixture)
+ segmentSizeBytes := walSegmentSizeBytes(fixture.DB.PostgresqlPhysical)
+
+ store := newMockWalStorage()
+ store.startFailingSaves()
+
+ // Three segments keep the shared source's background WAL from pausing the
+ // receiver a second time once uploads succeed again.
+ streamer := StartWalStreamerForTest(t, WalStreamerTestSpec{
+ Fixture: fixture,
+ FileStore: newMockWalStoreFor(store),
+ WatchDirRoot: t.TempDir(),
+ LocalQueuePauseBytes: 3 * segmentSizeBytes,
+ })
+ t.Cleanup(streamer.Stop)
+
+ supervisor := streamer.Supervisor
+
+ require.Eventually(t, supervisor.isReceiverRunning.Load, 60*time.Second, 200*time.Millisecond)
+
+ // Written once: WAL written during the pause would refill the queue as soon
+ // as the receiver resumes and pause it a second time.
+ ForceReplicationLag(t, adminConn, 4*segmentSizeBytes)
+
+ require.Eventually(t, func() bool {
+ return !supervisor.isReceiverRunning.Load() && supervisor.backlogBytes() >= supervisor.highWatermarkBytes.Load()
+ }, 90*time.Second, 200*time.Millisecond, "a full local queue must pause the receiver")
+
+ store.stopFailingSaves()
+
+ require.Eventually(t, supervisor.isReceiverRunning.Load, 90*time.Second, 200*time.Millisecond,
+ "the receiver resumes once the queue drains")
+
+ var restarts, pauses int
+
+ takeCounts := func() {
+ takenRestarts, takenPauses := supervisor.receiverRespawns.takeCounts()
+ restarts += takenRestarts
+ pauses += takenPauses
+ }
+
+ require.Eventually(t, func() bool {
+ takeCounts()
+
+ return pauses == 1
+ }, 10*time.Second, 200*time.Millisecond)
+
+ require.Zero(t, restarts, "the respawn after back pressure is a pause, not a restart")
+
+ // Other tests write WAL to the same source, and a burst of it would fill a
+ // three-segment queue again; from here on the production pause size applies,
+ // so the only respawn left to count is the one this test causes.
+ supervisor.setLocalQueuePauseBytes(getDefaultLocalQueuePauseBytes(fixture.DB.PostgresqlPhysical))
+
+ terminateWalReceiverBackend(t, adminConn, fixture.DB.PostgresqlPhysical.ReplicationSlotName)
+
+ require.Eventually(t, func() bool {
+ takeCounts()
+
+ return restarts == 1
+ }, 60*time.Second, 200*time.Millisecond, "a receiver that exits on its own is respawned as a restart")
+
+ require.Equal(t, 1, pauses, "back pressure is a pause, not a restart")
+
+ nextRestarts, nextPauses := supervisor.receiverRespawns.takeCounts()
+ require.Zero(t, nextRestarts)
+ require.Zero(t, nextPauses)
+}
+
+func terminateWalReceiverBackend(t *testing.T, adminConn *pgx.Conn, slotName string) {
+ t.Helper()
+
+ require.Eventually(t, func() bool {
+ var terminatedCount int
+
+ err := adminConn.QueryRow(t.Context(), `
+ SELECT COUNT(pg_terminate_backend(active_pid))
+ FROM pg_replication_slots
+ WHERE slot_name = $1 AND active_pid IS NOT NULL
+ `, slotName).Scan(&terminatedCount)
+
+ return err == nil && terminatedCount == 1
+ }, 30*time.Second, 200*time.Millisecond, "the receiver must be attached to the slot before it can be cut off")
+}
diff --git a/backend/internal/features/backups/config/physical/model.go b/backend/internal/features/backups/config/physical/model.go
index 78600fb7..681fb22e 100644
--- a/backend/internal/features/backups/config/physical/model.go
+++ b/backend/internal/features/backups/config/physical/model.go
@@ -118,6 +118,16 @@ func (b *PhysicalBackupConfig) Validate() error {
}
}
+// The WAL stream supervisor runs a streamer exactly for these configs, and the
+// WAL stream statistics call everything else "not streaming".
+func (b *PhysicalBackupConfig) IsWalStreamingExpected() bool {
+ if !b.IsBackupsEnabled || b.StorageID == nil {
+ return false
+ }
+
+ return b.PostgresqlPhysical != nil && b.PostgresqlPhysical.BackupType.IsWalStreaming()
+}
+
func (b *PhysicalBackupConfig) Copy(newDatabaseID uuid.UUID) *PhysicalBackupConfig {
return &PhysicalBackupConfig{
DatabaseID: newDatabaseID,
diff --git a/backend/migrations/20261018195439_add_physical_wal_stream_samples.sql b/backend/migrations/20261018195439_add_physical_wal_stream_samples.sql
new file mode 100644
index 00000000..ccbf3058
--- /dev/null
+++ b/backend/migrations/20261018195439_add_physical_wal_stream_samples.sql
@@ -0,0 +1,74 @@
+-- +goose Up
+-- +goose StatementBegin
+
+-- One row per WAL-streaming database every 30 seconds, written by the WAL
+-- streamer's lag monitor. Source-side columns are NULL whenever source_state is
+-- not REACHABLE, so a NULL slot_safe_wal_size_bytes on a reachable row means the
+-- source sets no limit.
+CREATE TABLE physical_wal_stream_samples (
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+ database_id UUID NOT NULL,
+ sampled_at TIMESTAMPTZ NOT NULL,
+ session_started_at TIMESTAMPTZ NOT NULL,
+ source_state TEXT NOT NULL,
+ source_unreachable_since TIMESTAMPTZ,
+ source_current_lsn PG_LSN,
+ archive_progress_lsn PG_LSN,
+ source_written_bytes BIGINT,
+ archived_segments INTEGER NOT NULL,
+ archived_wal_bytes BIGINT NOT NULL,
+ archived_stored_bytes BIGINT NOT NULL,
+ wal_segment_size_bytes BIGINT NOT NULL,
+ highest_concurrent_uploads INTEGER NOT NULL,
+ segment_write_avg_millis INTEGER,
+ segment_write_max_millis INTEGER,
+ slot_wal_status TEXT,
+ slot_extra_wal_since TIMESTAMPTZ,
+ slot_lag_bytes BIGINT,
+ slot_safe_wal_size_bytes BIGINT,
+ unarchived_wal_bytes BIGINT,
+ archive_stalled_since TIMESTAMPTZ,
+ local_queue_bytes BIGINT NOT NULL,
+ local_queue_pause_bytes BIGINT NOT NULL,
+ receiver_restarts INTEGER NOT NULL,
+ receiver_pauses INTEGER NOT NULL,
+ is_receiver_running BOOLEAN NOT NULL,
+ is_forced_rotation_denied BOOLEAN NOT NULL
+);
+
+ALTER TABLE physical_wal_stream_samples
+ ADD CONSTRAINT fk_physical_wal_stream_samples_database_id
+ FOREIGN KEY (database_id)
+ REFERENCES databases (id)
+ ON DELETE CASCADE;
+
+CREATE INDEX idx_physical_wal_stream_samples_database_id_sampled_at
+ ON physical_wal_stream_samples (database_id, sampled_at);
+
+-- The retention cleaner deletes expired samples of every database in one
+-- statement, including databases that no longer stream.
+CREATE INDEX idx_physical_wal_stream_samples_sampled_at
+ ON physical_wal_stream_samples (sampled_at);
+
+-- A session that fails before its first sample leaves no sample behind, so the
+-- first failure of a crash loop is kept here until a session samples again.
+ALTER TABLE physical_wal_streamers
+ ADD COLUMN failing_since TIMESTAMPTZ;
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+
+ALTER TABLE physical_wal_streamers
+ DROP COLUMN IF EXISTS failing_since;
+
+DROP INDEX IF EXISTS idx_physical_wal_stream_samples_sampled_at;
+DROP INDEX IF EXISTS idx_physical_wal_stream_samples_database_id_sampled_at;
+
+ALTER TABLE physical_wal_stream_samples
+ DROP CONSTRAINT IF EXISTS fk_physical_wal_stream_samples_database_id;
+
+DROP TABLE IF EXISTS physical_wal_stream_samples;
+
+-- +goose StatementEnd
diff --git a/frontend/src/entity/backups/physical/api/physicalWalStreamStatisticsApi.ts b/frontend/src/entity/backups/physical/api/physicalWalStreamStatisticsApi.ts
new file mode 100644
index 00000000..08c3b7d1
--- /dev/null
+++ b/frontend/src/entity/backups/physical/api/physicalWalStreamStatisticsApi.ts
@@ -0,0 +1,17 @@
+import { getApplicationServer } from '../../../../constants';
+import { apiHelper } from '../../../../shared/api/apiHelper';
+import type { WalStreamStatistics } from '../model/WalStreamStatistics';
+import type { WalStreamStatisticsPeriod } from '../model/WalStreamStatisticsPeriod';
+
+export const physicalWalStreamStatisticsApi = {
+ // No retry inside the request: the Monitoring tab polls every 30 seconds, and a request that kept
+ // retrying for a minute and a half would overlap the next poll and hide a failure the tab must
+ // show.
+ async getWalStreamStatistics(databaseId: string, period: WalStreamStatisticsPeriod) {
+ const params = new URLSearchParams({ period });
+
+ return apiHelper.fetchGetJson<WalStreamStatistics>(
+ `${getApplicationServer()}/api/v1/backups/physical/database/${databaseId}/wal-stream/statistics?${params.toString()}`,
+ );
+ },
+};
diff --git a/frontend/src/entity/backups/physical/index.ts b/frontend/src/entity/backups/physical/index.ts
index 79f3968a..2da1bf30 100644
--- a/frontend/src/entity/backups/physical/index.ts
+++ b/frontend/src/entity/backups/physical/index.ts
@@ -1,6 +1,7 @@
export { physicalBackupsApi } from './api/physicalBackupsApi';
export type { TriggerPhysicalBackupType } from './api/physicalBackupsApi';
export { physicalBackupConfigApi } from './api/physicalBackupConfigApi';
+export { physicalWalStreamStatisticsApi } from './api/physicalWalStreamStatisticsApi';
export { PhysicalBackupStatus } from './model/PhysicalBackupStatus';
export { PhysicalBackupType } from './model/PhysicalBackupType';
export type { PhysicalBackupListItem } from './model/PhysicalBackupListItem';
@@ -18,3 +19,28 @@ export { PHYSICAL_RETENTION_LABEL_KEYS } from './model/PhysicalRetentionLabelKey
export { PHYSICAL_BACKUP_TYPE_LABEL_KEYS } from './model/PhysicalBackupTypeLabelKeys';
export { PHYSICAL_FULL_BACKUPS_POLICY_LABEL_KEYS } from './model/PhysicalFullBackupsPolicyLabelKeys';
export { PHYSICAL_BACKUP_NOTIFICATION_TYPE_LABEL_KEYS } from './model/PhysicalBackupNotificationTypeLabelKeys';
+export { WalStreamStatisticsPeriod } from './model/WalStreamStatisticsPeriod';
+export { WalStreamHealthLevel } from './model/WalStreamHealthLevel';
+export { WalStreamReasonLevel } from './model/WalStreamReasonLevel';
+export { WalStreamReasonCode } from './model/WalStreamReasonCode';
+export { WalStreamNoticeCode } from './model/WalStreamNoticeCode';
+export { WalStreamStreamerState } from './model/WalStreamStreamerState';
+export { WalStreamKeepingUpState } from './model/WalStreamKeepingUpState';
+export { WalStreamSourceState } from './model/WalStreamSourceState';
+export type { WalStreamReason } from './model/WalStreamReason';
+export type { WalStreamNotice } from './model/WalStreamNotice';
+export type { WalStreamHealth } from './model/WalStreamHealth';
+export type { WalStreamStreamer } from './model/WalStreamStreamer';
+export type { WalStreamCurrent } from './model/WalStreamCurrent';
+export type { WalStreamKeepingUp } from './model/WalStreamKeepingUp';
+export type { WalStreamSeriesPoint } from './model/WalStreamSeriesPoint';
+export type { WalStreamSeries } from './model/WalStreamSeries';
+export type { WalStreamStatistics } from './model/WalStreamStatistics';
+export { WAL_STREAM_STATISTICS_PERIOD_LABEL_KEYS } from './model/WalStreamStatisticsPeriodLabelKeys';
+export { WAL_STREAM_HEALTH_LEVEL_LABEL_KEYS } from './model/WalStreamHealthLevelLabelKeys';
+export { WAL_STREAM_REASON_LEVEL_LABEL_KEYS } from './model/WalStreamReasonLevelLabelKeys';
+export { WAL_STREAM_REASON_CODE_LABEL_KEYS } from './model/WalStreamReasonCodeLabelKeys';
+export { WAL_STREAM_NOTICE_CODE_LABEL_KEYS } from './model/WalStreamNoticeCodeLabelKeys';
+export { WAL_STREAM_STREAMER_STATE_LABEL_KEYS } from './model/WalStreamStreamerStateLabelKeys';
+export { WAL_STREAM_KEEPING_UP_STATE_LABEL_KEYS } from './model/WalStreamKeepingUpStateLabelKeys';
+export { WAL_STREAM_SOURCE_STATE_LABEL_KEYS } from './model/WalStreamSourceStateLabelKeys';
diff --git a/frontend/src/entity/backups/physical/model/WalStreamCurrent.ts b/frontend/src/entity/backups/physical/model/WalStreamCurrent.ts
new file mode 100644
index 00000000..102ca032
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamCurrent.ts
@@ -0,0 +1,40 @@
+import type { WalStreamSourceState } from './WalStreamSourceState';
+
+// Source-side values are null while the source is unreachable or without the slot. On a reachable
+// sample a null slotSafeWalSizeBytes means the source sets no limit.
+export interface WalStreamCurrent {
+ sampledAt: string;
+ lastArchivedWalAt: string | null;
+ sessionStartedAt: string;
+
+ sourceState: WalStreamSourceState;
+ sourceUnreachableSince: string | null;
+
+ sourceCurrentLsn: string | null;
+ archiveProgressLsn: string | null;
+ sourceWrittenBytes: number | null;
+
+ archivedSegments: number;
+ archivedWalBytes: number;
+ archivedStoredBytes: number;
+ walSegmentSizeBytes: number;
+ highestConcurrentUploads: number;
+ segmentWriteAvgMillis: number | null;
+ segmentWriteMaxMillis: number | null;
+
+ slotWalStatus: string | null;
+ slotExtraWalSince: string | null;
+ slotLagBytes: number | null;
+ slotSafeWalSizeBytes: number | null;
+
+ unarchivedWalBytes: number | null;
+ archiveStalledSince: string | null;
+
+ localQueueBytes: number;
+ localQueuePauseBytes: number;
+
+ receiverRestarts: number;
+ receiverPauses: number;
+ isReceiverRunning: boolean;
+ isForcedRotationDenied: boolean;
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamHealth.ts b/frontend/src/entity/backups/physical/model/WalStreamHealth.ts
new file mode 100644
index 00000000..70d7fd02
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamHealth.ts
@@ -0,0 +1,10 @@
+import type { WalStreamHealthLevel } from './WalStreamHealthLevel';
+import type { WalStreamNotice } from './WalStreamNotice';
+import type { WalStreamReason } from './WalStreamReason';
+
+export interface WalStreamHealth {
+ level: WalStreamHealthLevel;
+ reasons: WalStreamReason[];
+ notices: WalStreamNotice[];
+ isFullBackupPending: boolean;
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamHealthLevel.ts b/frontend/src/entity/backups/physical/model/WalStreamHealthLevel.ts
new file mode 100644
index 00000000..ed3702e8
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamHealthLevel.ts
@@ -0,0 +1,7 @@
+export enum WalStreamHealthLevel {
+ NOT_STREAMING = 'NOT_STREAMING',
+ COLLECTING_DATA = 'COLLECTING_DATA',
+ BROKEN = 'BROKEN',
+ NEEDS_ATTENTION = 'NEEDS_ATTENTION',
+ HEALTHY = 'HEALTHY',
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamHealthLevelLabelKeys.ts b/frontend/src/entity/backups/physical/model/WalStreamHealthLevelLabelKeys.ts
new file mode 100644
index 00000000..7a2dcd5f
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamHealthLevelLabelKeys.ts
@@ -0,0 +1,10 @@
+import type { TranslationKey } from '../../../../shared/i18n';
+import { WalStreamHealthLevel } from './WalStreamHealthLevel';
+
+export const WAL_STREAM_HEALTH_LEVEL_LABEL_KEYS: Record<WalStreamHealthLevel, TranslationKey> = {
+ [WalStreamHealthLevel.NOT_STREAMING]: 'backups.physical.monitoring.levels.notStreaming',
+ [WalStreamHealthLevel.COLLECTING_DATA]: 'backups.physical.monitoring.levels.collectingData',
+ [WalStreamHealthLevel.BROKEN]: 'backups.physical.monitoring.levels.broken',
+ [WalStreamHealthLevel.NEEDS_ATTENTION]: 'backups.physical.monitoring.levels.needsAttention',
+ [WalStreamHealthLevel.HEALTHY]: 'backups.physical.monitoring.levels.healthy',
+};
diff --git a/frontend/src/entity/backups/physical/model/WalStreamKeepingUp.ts b/frontend/src/entity/backups/physical/model/WalStreamKeepingUp.ts
new file mode 100644
index 00000000..ae901b10
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamKeepingUp.ts
@@ -0,0 +1,20 @@
+import type { WalStreamKeepingUpState } from './WalStreamKeepingUpState';
+
+export interface WalStreamKeepingUp {
+ state: WalStreamKeepingUpState;
+
+ sourceWriteBytesPerSecond: number | null;
+ archiveBytesPerSecond: number | null;
+
+ unarchivedWalBytes: number | null;
+ unarchivedWalChangeBytesPerSecond: number | null;
+
+ slotLagBytes: number | null;
+ slotLagChangeBytesPerSecond: number | null;
+
+ slotRebuildLagBytes: number;
+
+ estimatedCatchUpSeconds: number | null;
+ estimatedSlotRebuildSeconds: number | null;
+ estimatedSlotLimitSeconds: number | null;
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamKeepingUpState.ts b/frontend/src/entity/backups/physical/model/WalStreamKeepingUpState.ts
new file mode 100644
index 00000000..7a9428df
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamKeepingUpState.ts
@@ -0,0 +1,7 @@
+export enum WalStreamKeepingUpState {
+ UNKNOWN = 'UNKNOWN',
+ FALLING_BEHIND = 'FALLING_BEHIND',
+ CATCHING_UP = 'CATCHING_UP',
+ IDLE = 'IDLE',
+ KEEPING_UP = 'KEEPING_UP',
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamKeepingUpStateLabelKeys.ts b/frontend/src/entity/backups/physical/model/WalStreamKeepingUpStateLabelKeys.ts
new file mode 100644
index 00000000..3cd3ffb1
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamKeepingUpStateLabelKeys.ts
@@ -0,0 +1,14 @@
+import type { TranslationKey } from '../../../../shared/i18n';
+import { WalStreamKeepingUpState } from './WalStreamKeepingUpState';
+
+export const WAL_STREAM_KEEPING_UP_STATE_LABEL_KEYS: Record<
+ WalStreamKeepingUpState,
+ TranslationKey
+> = {
+ [WalStreamKeepingUpState.UNKNOWN]: 'backups.physical.monitoring.keepingUpStates.unknown',
+ [WalStreamKeepingUpState.FALLING_BEHIND]:
+ 'backups.physical.monitoring.keepingUpStates.fallingBehind',
+ [WalStreamKeepingUpState.CATCHING_UP]: 'backups.physical.monitoring.keepingUpStates.catchingUp',
+ [WalStreamKeepingUpState.IDLE]: 'backups.physical.monitoring.keepingUpStates.idle',
+ [WalStreamKeepingUpState.KEEPING_UP]: 'backups.physical.monitoring.keepingUpStates.keepingUp',
+};
diff --git a/frontend/src/entity/backups/physical/model/WalStreamNotice.ts b/frontend/src/entity/backups/physical/model/WalStreamNotice.ts
new file mode 100644
index 00000000..c01fd967
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamNotice.ts
@@ -0,0 +1,8 @@
+import type { WalStreamNoticeCode } from './WalStreamNoticeCode';
+import type { WalStreamReasonLevel } from './WalStreamReasonLevel';
+
+export interface WalStreamNotice {
+ code: WalStreamNoticeCode;
+ level: WalStreamReasonLevel;
+ since?: string;
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamNoticeCode.ts b/frontend/src/entity/backups/physical/model/WalStreamNoticeCode.ts
new file mode 100644
index 00000000..3dd99c75
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamNoticeCode.ts
@@ -0,0 +1,5 @@
+export enum WalStreamNoticeCode {
+ NO_SLOT_WAL_LIMIT = 'NO_SLOT_WAL_LIMIT',
+ SLOT_KEEPS_EXTRA_WAL_STEADILY = 'SLOT_KEEPS_EXTRA_WAL_STEADILY',
+ NO_FULL_BACKUP_YET = 'NO_FULL_BACKUP_YET',
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamNoticeCodeLabelKeys.ts b/frontend/src/entity/backups/physical/model/WalStreamNoticeCodeLabelKeys.ts
new file mode 100644
index 00000000..20fd25cf
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamNoticeCodeLabelKeys.ts
@@ -0,0 +1,11 @@
+import type { TranslationKey } from '../../../../shared/i18n';
+import { WalStreamNoticeCode } from './WalStreamNoticeCode';
+
+export const WAL_STREAM_NOTICE_CODE_LABEL_KEYS: Record<WalStreamNoticeCode, TranslationKey> = {
+ [WalStreamNoticeCode.NO_SLOT_WAL_LIMIT]:
+ 'backups.physical.monitoring.notices.noSlotWalLimit.title',
+ [WalStreamNoticeCode.SLOT_KEEPS_EXTRA_WAL_STEADILY]:
+ 'backups.physical.monitoring.notices.slotKeepsExtraWalSteadily.title',
+ [WalStreamNoticeCode.NO_FULL_BACKUP_YET]:
+ 'backups.physical.monitoring.notices.noFullBackupYet.title',
+};
diff --git a/frontend/src/entity/backups/physical/model/WalStreamReason.ts b/frontend/src/entity/backups/physical/model/WalStreamReason.ts
new file mode 100644
index 00000000..02bfb88c
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamReason.ts
@@ -0,0 +1,12 @@
+import type { WalStreamReasonCode } from './WalStreamReasonCode';
+import type { WalStreamReasonLevel } from './WalStreamReasonLevel';
+
+export interface WalStreamReason {
+ code: WalStreamReasonCode;
+ level: WalStreamReasonLevel;
+ since?: string;
+ valueBytes?: number;
+ thresholdBytes?: number;
+ count?: number;
+ estimatedSeconds?: number;
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamReasonCode.ts b/frontend/src/entity/backups/physical/model/WalStreamReasonCode.ts
new file mode 100644
index 00000000..8fbd8c1a
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamReasonCode.ts
@@ -0,0 +1,15 @@
+export enum WalStreamReasonCode {
+ STREAMING_STOPPED = 'STREAMING_STOPPED',
+ SOURCE_UNREACHABLE = 'SOURCE_UNREACHABLE',
+ WAL_CHAIN_BROKEN = 'WAL_CHAIN_BROKEN',
+ SLOT_MISSING = 'SLOT_MISSING',
+ SLOT_LOST = 'SLOT_LOST',
+ ARCHIVE_NOT_ADVANCING = 'ARCHIVE_NOT_ADVANCING',
+ ARCHIVE_SLOWER_THAN_SOURCE = 'ARCHIVE_SLOWER_THAN_SOURCE',
+ SLOT_KEEPS_EXTRA_WAL = 'SLOT_KEEPS_EXTRA_WAL',
+ SLOT_LIMIT_APPROACHING = 'SLOT_LIMIT_APPROACHING',
+ NEAR_SLOT_REBUILD = 'NEAR_SLOT_REBUILD',
+ LOCAL_QUEUE_FILLING = 'LOCAL_QUEUE_FILLING',
+ FREQUENT_RESTARTS = 'FREQUENT_RESTARTS',
+ FORCED_ROTATION_REFUSED = 'FORCED_ROTATION_REFUSED',
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamReasonCodeLabelKeys.ts b/frontend/src/entity/backups/physical/model/WalStreamReasonCodeLabelKeys.ts
new file mode 100644
index 00000000..47f9f85e
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamReasonCodeLabelKeys.ts
@@ -0,0 +1,29 @@
+import type { TranslationKey } from '../../../../shared/i18n';
+import { WalStreamReasonCode } from './WalStreamReasonCode';
+
+export const WAL_STREAM_REASON_CODE_LABEL_KEYS: Record<WalStreamReasonCode, TranslationKey> = {
+ [WalStreamReasonCode.STREAMING_STOPPED]:
+ 'backups.physical.monitoring.reasons.streamingStopped.title',
+ [WalStreamReasonCode.SOURCE_UNREACHABLE]:
+ 'backups.physical.monitoring.reasons.sourceUnreachable.title',
+ [WalStreamReasonCode.WAL_CHAIN_BROKEN]:
+ 'backups.physical.monitoring.reasons.walChainBroken.title',
+ [WalStreamReasonCode.SLOT_MISSING]: 'backups.physical.monitoring.reasons.slotMissing.title',
+ [WalStreamReasonCode.SLOT_LOST]: 'backups.physical.monitoring.reasons.slotLost.title',
+ [WalStreamReasonCode.ARCHIVE_NOT_ADVANCING]:
+ 'backups.physical.monitoring.reasons.archiveNotAdvancing.title',
+ [WalStreamReasonCode.ARCHIVE_SLOWER_THAN_SOURCE]:
+ 'backups.physical.monitoring.reasons.archiveSlowerThanSource.title',
+ [WalStreamReasonCode.SLOT_KEEPS_EXTRA_WAL]:
+ 'backups.physical.monitoring.reasons.slotKeepsExtraWal.title',
+ [WalStreamReasonCode.SLOT_LIMIT_APPROACHING]:
+ 'backups.physical.monitoring.reasons.slotLimitApproaching.title',
+ [WalStreamReasonCode.NEAR_SLOT_REBUILD]:
+ 'backups.physical.monitoring.reasons.nearSlotRebuild.title',
+ [WalStreamReasonCode.LOCAL_QUEUE_FILLING]:
+ 'backups.physical.monitoring.reasons.localQueueFilling.title',
+ [WalStreamReasonCode.FREQUENT_RESTARTS]:
+ 'backups.physical.monitoring.reasons.frequentRestarts.title',
+ [WalStreamReasonCode.FORCED_ROTATION_REFUSED]:
+ 'backups.physical.monitoring.reasons.forcedRotationRefused.title',
+};
diff --git a/frontend/src/entity/backups/physical/model/WalStreamReasonLevel.ts b/frontend/src/entity/backups/physical/model/WalStreamReasonLevel.ts
new file mode 100644
index 00000000..2f908862
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamReasonLevel.ts
@@ -0,0 +1,5 @@
+export enum WalStreamReasonLevel {
+ BROKEN = 'BROKEN',
+ NEEDS_ATTENTION = 'NEEDS_ATTENTION',
+ INFO = 'INFO',
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamReasonLevelLabelKeys.ts b/frontend/src/entity/backups/physical/model/WalStreamReasonLevelLabelKeys.ts
new file mode 100644
index 00000000..6c9827a5
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamReasonLevelLabelKeys.ts
@@ -0,0 +1,8 @@
+import type { TranslationKey } from '../../../../shared/i18n';
+import { WalStreamReasonLevel } from './WalStreamReasonLevel';
+
+export const WAL_STREAM_REASON_LEVEL_LABEL_KEYS: Record<WalStreamReasonLevel, TranslationKey> = {
+ [WalStreamReasonLevel.BROKEN]: 'backups.physical.monitoring.reasonLevels.broken',
+ [WalStreamReasonLevel.NEEDS_ATTENTION]: 'backups.physical.monitoring.reasonLevels.needsAttention',
+ [WalStreamReasonLevel.INFO]: 'backups.physical.monitoring.reasonLevels.info',
+};
diff --git a/frontend/src/entity/backups/physical/model/WalStreamSeries.ts b/frontend/src/entity/backups/physical/model/WalStreamSeries.ts
new file mode 100644
index 00000000..5ff5d065
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamSeries.ts
@@ -0,0 +1,10 @@
+import type { WalStreamSeriesPoint } from './WalStreamSeriesPoint';
+import type { WalStreamStatisticsPeriod } from './WalStreamStatisticsPeriod';
+
+export interface WalStreamSeries {
+ period: WalStreamStatisticsPeriod;
+ bucketSeconds: number;
+ from: string;
+ to: string;
+ points: WalStreamSeriesPoint[];
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamSeriesPoint.ts b/frontend/src/entity/backups/physical/model/WalStreamSeriesPoint.ts
new file mode 100644
index 00000000..ce5caacc
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamSeriesPoint.ts
@@ -0,0 +1,21 @@
+// Every value is null when no sample falls into the point's interval.
+export interface WalStreamSeriesPoint {
+ startedAt: string;
+
+ sourceWriteBytesPerSecond: number | null;
+ archiveBytesPerSecond: number | null;
+ archivedStoredBytesPerSecond: number | null;
+
+ maxSlotLagBytes: number | null;
+ maxUnarchivedWalBytes: number | null;
+ maxLocalQueueBytes: number | null;
+ localQueuePauseBytes: number | null;
+
+ restarts: number | null;
+ pauses: number | null;
+ sessionStarts: number | null;
+
+ highestConcurrentUploads: number | null;
+ segmentWriteAvgMillis: number | null;
+ segmentWriteMaxMillis: number | null;
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamSourceState.ts b/frontend/src/entity/backups/physical/model/WalStreamSourceState.ts
new file mode 100644
index 00000000..20104274
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamSourceState.ts
@@ -0,0 +1,5 @@
+export enum WalStreamSourceState {
+ REACHABLE = 'REACHABLE',
+ UNREACHABLE = 'UNREACHABLE',
+ SLOT_MISSING = 'SLOT_MISSING',
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamSourceStateLabelKeys.ts b/frontend/src/entity/backups/physical/model/WalStreamSourceStateLabelKeys.ts
new file mode 100644
index 00000000..447a761c
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamSourceStateLabelKeys.ts
@@ -0,0 +1,10 @@
+import type { TranslationKey } from '../../../../shared/i18n';
+import { WalStreamSourceState } from './WalStreamSourceState';
+
+export const WAL_STREAM_SOURCE_STATE_LABEL_KEYS: Record<WalStreamSourceState, TranslationKey> = {
+ [WalStreamSourceState.REACHABLE]: 'backups.physical.monitoring.details.sourceStates.reachable',
+ [WalStreamSourceState.UNREACHABLE]:
+ 'backups.physical.monitoring.details.sourceStates.unreachable',
+ [WalStreamSourceState.SLOT_MISSING]:
+ 'backups.physical.monitoring.details.sourceStates.slotMissing',
+};
diff --git a/frontend/src/entity/backups/physical/model/WalStreamStatistics.ts b/frontend/src/entity/backups/physical/model/WalStreamStatistics.ts
new file mode 100644
index 00000000..5d8dfecc
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamStatistics.ts
@@ -0,0 +1,13 @@
+import type { WalStreamCurrent } from './WalStreamCurrent';
+import type { WalStreamHealth } from './WalStreamHealth';
+import type { WalStreamKeepingUp } from './WalStreamKeepingUp';
+import type { WalStreamSeries } from './WalStreamSeries';
+import type { WalStreamStreamer } from './WalStreamStreamer';
+
+export interface WalStreamStatistics {
+ health: WalStreamHealth;
+ streamer: WalStreamStreamer;
+ current: WalStreamCurrent | null;
+ keepingUp: WalStreamKeepingUp;
+ series: WalStreamSeries;
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamStatisticsPeriod.ts b/frontend/src/entity/backups/physical/model/WalStreamStatisticsPeriod.ts
new file mode 100644
index 00000000..0e634a2d
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamStatisticsPeriod.ts
@@ -0,0 +1,5 @@
+export enum WalStreamStatisticsPeriod {
+ HOUR = 'HOUR',
+ DAY = 'DAY',
+ WEEK = 'WEEK',
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamStatisticsPeriodLabelKeys.ts b/frontend/src/entity/backups/physical/model/WalStreamStatisticsPeriodLabelKeys.ts
new file mode 100644
index 00000000..f665da22
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamStatisticsPeriodLabelKeys.ts
@@ -0,0 +1,11 @@
+import type { TranslationKey } from '../../../../shared/i18n';
+import { WalStreamStatisticsPeriod } from './WalStreamStatisticsPeriod';
+
+export const WAL_STREAM_STATISTICS_PERIOD_LABEL_KEYS: Record<
+ WalStreamStatisticsPeriod,
+ TranslationKey
+> = {
+ [WalStreamStatisticsPeriod.HOUR]: 'backups.physical.monitoring.periods.hour',
+ [WalStreamStatisticsPeriod.DAY]: 'backups.physical.monitoring.periods.day',
+ [WalStreamStatisticsPeriod.WEEK]: 'backups.physical.monitoring.periods.week',
+};
diff --git a/frontend/src/entity/backups/physical/model/WalStreamStreamer.ts b/frontend/src/entity/backups/physical/model/WalStreamStreamer.ts
new file mode 100644
index 00000000..5d82b94c
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamStreamer.ts
@@ -0,0 +1,8 @@
+import type { WalStreamStreamerState } from './WalStreamStreamerState';
+
+export interface WalStreamStreamer {
+ state: WalStreamStreamerState;
+ sessionStartedAt: string | null;
+ failingSince: string | null;
+ restartsLastHour: number;
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamStreamerState.ts b/frontend/src/entity/backups/physical/model/WalStreamStreamerState.ts
new file mode 100644
index 00000000..175472ad
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamStreamerState.ts
@@ -0,0 +1,5 @@
+export enum WalStreamStreamerState {
+ RUNNING = 'RUNNING',
+ NOT_RUNNING = 'NOT_RUNNING',
+ NOT_STARTED = 'NOT_STARTED',
+}
diff --git a/frontend/src/entity/backups/physical/model/WalStreamStreamerStateLabelKeys.ts b/frontend/src/entity/backups/physical/model/WalStreamStreamerStateLabelKeys.ts
new file mode 100644
index 00000000..e4b17011
--- /dev/null
+++ b/frontend/src/entity/backups/physical/model/WalStreamStreamerStateLabelKeys.ts
@@ -0,0 +1,11 @@
+import type { TranslationKey } from '../../../../shared/i18n';
+import { WalStreamStreamerState } from './WalStreamStreamerState';
+
+export const WAL_STREAM_STREAMER_STATE_LABEL_KEYS: Record<WalStreamStreamerState, TranslationKey> =
+ {
+ [WalStreamStreamerState.RUNNING]: 'backups.physical.monitoring.details.streamerStates.running',
+ [WalStreamStreamerState.NOT_RUNNING]:
+ 'backups.physical.monitoring.details.streamerStates.notRunning',
+ [WalStreamStreamerState.NOT_STARTED]:
+ 'backups.physical.monitoring.details.streamerStates.notStarted',
+ };
diff --git a/frontend/src/features/backups/physical/index.ts b/frontend/src/features/backups/physical/index.ts
index 81b0c919..ddb123a9 100644
--- a/frontend/src/features/backups/physical/index.ts
+++ b/frontend/src/features/backups/physical/index.ts
@@ -1,3 +1,4 @@
export { PhysicalBackupsComponent } from './ui/PhysicalBackupsComponent';
export { EditPhysicalBackupConfigComponent } from './ui/EditPhysicalBackupConfigComponent';
export { ShowPhysicalBackupConfigComponent } from './ui/ShowPhysicalBackupConfigComponent';
+export { WalStreamMonitoringComponent, isMonitoringTabShown } from './monitoring';
diff --git a/frontend/src/features/backups/physical/monitoring/index.ts b/frontend/src/features/backups/physical/monitoring/index.ts
new file mode 100644
index 00000000..c2f7a73f
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/index.ts
@@ -0,0 +1,2 @@
+export { isMonitoringTabShown } from './lib/monitoringTab';
+export { WalStreamMonitoringComponent } from './ui/WalStreamMonitoringComponent';
diff --git a/frontend/src/features/backups/physical/monitoring/lib/chartColors.ts b/frontend/src/features/backups/physical/monitoring/lib/chartColors.ts
new file mode 100644
index 00000000..02542a54
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/chartColors.ts
@@ -0,0 +1,37 @@
+import type { ResolvedTheme } from '../../../../../shared/theme';
+
+export interface WalStreamChartColors {
+ sourceWrites: string;
+ archive: string;
+ notSavedYet: string;
+ notReceivedYet: string;
+ buffer: string;
+ pauseLimit: string;
+ restartMarker: string;
+ pauseMarker: string;
+}
+
+// The lines also differ in dash pattern, so they stay apart in grayscale and for color-blind
+// readers. The dark variants are lighter, so they keep their contrast on a dark background.
+export const WAL_STREAM_CHART_COLORS: Record<ResolvedTheme, WalStreamChartColors> = {
+ light: {
+ sourceWrites: '#1677ff',
+ archive: '#fa8c16',
+ notSavedYet: '#722ed1',
+ notReceivedYet: '#13c2c2',
+ buffer: '#722ed1',
+ pauseLimit: '#8c8c8c',
+ restartMarker: '#f5222d',
+ pauseMarker: '#8c8c8c',
+ },
+ dark: {
+ sourceWrites: '#4096ff',
+ archive: '#ffa940',
+ notSavedYet: '#9254de',
+ notReceivedYet: '#36cfc9',
+ buffer: '#9254de',
+ pauseLimit: '#8c8c8c',
+ restartMarker: '#ff4d4f',
+ pauseMarker: '#8c8c8c',
+ },
+};
diff --git a/frontend/src/features/backups/physical/monitoring/lib/chartRows.test.ts b/frontend/src/features/backups/physical/monitoring/lib/chartRows.test.ts
new file mode 100644
index 00000000..8e55155f
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/chartRows.test.ts
@@ -0,0 +1,118 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import {
+ type WalStreamSeries,
+ type WalStreamSeriesPoint,
+ WalStreamStatisticsPeriod,
+} from '../../../../../entity/backups/physical';
+import { getDataSinceMs, hasAnyChartValues, toChartRows } from './chartRows';
+
+// The entity's public API imports its API client, and the client's config module reads window at
+// import time. The tests run in node, so they supply window.
+vi.hoisted(() => {
+ vi.stubGlobal('window', {});
+});
+
+const MB = 1000 * 1000;
+const seriesStartMs = Date.parse('2026-10-18T11:00:00Z');
+
+const buildEmptyPoint = (minute: number): WalStreamSeriesPoint => ({
+ startedAt: new Date(seriesStartMs + minute * 60_000).toISOString(),
+ sourceWriteBytesPerSecond: null,
+ archiveBytesPerSecond: null,
+ archivedStoredBytesPerSecond: null,
+ maxSlotLagBytes: null,
+ maxUnarchivedWalBytes: null,
+ maxLocalQueueBytes: null,
+ localQueuePauseBytes: null,
+ restarts: null,
+ pauses: null,
+ sessionStarts: null,
+ highestConcurrentUploads: null,
+ segmentWriteAvgMillis: null,
+ segmentWriteMaxMillis: null,
+});
+
+const buildFallingBehindPoint = (minute: number): WalStreamSeriesPoint => ({
+ ...buildEmptyPoint(minute),
+ sourceWriteBytesPerSecond: 21 * MB,
+ archiveBytesPerSecond: 12 * MB,
+ maxUnarchivedWalBytes: minute * 9 * MB * 60,
+ maxLocalQueueBytes: 64 * MB,
+ localQueuePauseBytes: 512 * MB,
+ restarts: 0,
+ pauses: 0,
+});
+
+const buildSeries = (points: WalStreamSeriesPoint[]): WalStreamSeries => ({
+ period: WalStreamStatisticsPeriod.HOUR,
+ bucketSeconds: 60,
+ from: points[0].startedAt,
+ to: new Date(seriesStartMs + points.length * 60_000).toISOString(),
+ points,
+});
+
+describe('toChartRows', () => {
+ it('keeps archiving below the source and the backlog rising while archiving falls behind', () => {
+ const rows = toChartRows(
+ Array.from({ length: 60 }, (_, minute) => buildFallingBehindPoint(minute)),
+ );
+
+ for (const row of rows) {
+ expect(row.archiveBytesPerSecond ?? 0).toBeLessThan(row.sourceWriteBytesPerSecond ?? 0);
+ }
+
+ for (let rowIndex = 1; rowIndex < rows.length; rowIndex++) {
+ expect(rows[rowIndex].maxUnarchivedWalBytes ?? 0).toBeGreaterThan(
+ rows[rowIndex - 1].maxUnarchivedWalBytes ?? 0,
+ );
+ }
+ });
+
+ it('keeps an interval without samples as a gap, not as zero', () => {
+ const [row] = toChartRows([buildEmptyPoint(0)]);
+
+ expect(row.sourceWriteBytesPerSecond).toBeNull();
+ expect(row.archiveBytesPerSecond).toBeNull();
+ expect(row.maxUnarchivedWalBytes).toBeNull();
+ expect(row.maxLocalQueueBytes).toBeNull();
+ expect(row.restartMarkerBytes).toBeNull();
+ });
+
+ it('puts restart and pause markers on the buffer line', () => {
+ const [row] = toChartRows([{ ...buildFallingBehindPoint(0), restarts: 1, pauses: 2 }]);
+
+ expect(row.restartMarkerBytes).toBe(64 * MB);
+ expect(row.pauseMarkerBytes).toBe(64 * MB);
+ });
+});
+
+describe('hasAnyChartValues', () => {
+ it('reports no history when no point has values', () => {
+ expect(hasAnyChartValues(buildSeries([buildEmptyPoint(0), buildEmptyPoint(1)]))).toBe(false);
+ });
+
+ it('reports history once one point has values', () => {
+ expect(hasAnyChartValues(buildSeries([buildEmptyPoint(0), buildFallingBehindPoint(1)]))).toBe(
+ true,
+ );
+ });
+});
+
+describe('getDataSinceMs', () => {
+ it('names the first interval with data when the data starts inside the period', () => {
+ const series = buildSeries([
+ buildEmptyPoint(0),
+ buildEmptyPoint(1),
+ buildFallingBehindPoint(2),
+ ]);
+
+ expect(getDataSinceMs(series)).toBe(seriesStartMs + 2 * 60_000);
+ });
+
+ it('says nothing when the data covers the whole period', () => {
+ expect(
+ getDataSinceMs(buildSeries([buildFallingBehindPoint(0), buildFallingBehindPoint(1)])),
+ ).toBeNull();
+ });
+});
diff --git a/frontend/src/features/backups/physical/monitoring/lib/chartRows.ts b/frontend/src/features/backups/physical/monitoring/lib/chartRows.ts
new file mode 100644
index 00000000..e76bbeaf
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/chartRows.ts
@@ -0,0 +1,39 @@
+import type { WalStreamSeries, WalStreamSeriesPoint } from '../../../../../entity/backups/physical';
+import type { WalStreamChartRow } from '../model/WalStreamChartRow';
+
+export const toChartRows = (points: WalStreamSeriesPoint[]): WalStreamChartRow[] =>
+ points.map((point) => ({
+ timestampMs: new Date(point.startedAt).getTime(),
+ sourceWriteBytesPerSecond: point.sourceWriteBytesPerSecond,
+ archiveBytesPerSecond: point.archiveBytesPerSecond,
+ maxUnarchivedWalBytes: point.maxUnarchivedWalBytes,
+ maxSlotLagBytes: point.maxSlotLagBytes,
+ maxLocalQueueBytes: point.maxLocalQueueBytes,
+ localQueuePauseBytes: point.localQueuePauseBytes,
+ restartMarkerBytes: getMarkerBytes(point.restarts, point),
+ pauseMarkerBytes: getMarkerBytes(point.pauses, point),
+ }));
+
+// A marker sits on the buffer line at its interval, so it needs no axis of its own.
+const getMarkerBytes = (eventCount: number | null, point: WalStreamSeriesPoint): number | null =>
+ eventCount !== null && eventCount > 0 ? (point.maxLocalQueueBytes ?? 0) : null;
+
+const hasPointValues = (point: WalStreamSeriesPoint): boolean =>
+ point.archiveBytesPerSecond !== null ||
+ point.sourceWriteBytesPerSecond !== null ||
+ point.maxLocalQueueBytes !== null;
+
+export const hasAnyChartValues = (series: WalStreamSeries): boolean =>
+ series.points.some(hasPointValues);
+
+// Null when the data covers the whole period, so the caption only appears when the charts start
+// later than their axis.
+export const getDataSinceMs = (series: WalStreamSeries): number | null => {
+ const firstPointWithValues = series.points.find(hasPointValues);
+
+ if (!firstPointWithValues || firstPointWithValues === series.points[0]) {
+ return null;
+ }
+
+ return new Date(firstPointWithValues.startedAt).getTime();
+};
diff --git a/frontend/src/features/backups/physical/monitoring/lib/easedTexts.test.ts b/frontend/src/features/backups/physical/monitoring/lib/easedTexts.test.ts
new file mode 100644
index 00000000..fa94487c
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/easedTexts.test.ts
@@ -0,0 +1,38 @@
+import { describe, expect, it } from 'vitest';
+
+import type { MonitoringText } from '../model/MonitoringText';
+import { interpolateMonitoringTexts } from './easedTexts';
+
+const writeSentence = (bytesPerSecond: number): MonitoringText => ({
+ key: 'backups.physical.monitoring.keepingUp.keepingUp',
+ params: {
+ writeRate: { kind: 'bytesPerSecond', bytesPerSecond },
+ archiveRate: { kind: 'bytesPerSecond', bytesPerSecond },
+ },
+});
+
+describe('interpolateMonitoringTexts', () => {
+ it('moves the amounts of the same sentence between their old and new values', () => {
+ const [halfway] = interpolateMonitoringTexts([writeSentence(10)], [writeSentence(20)], 0.5);
+ const halfwayRate = halfway.params?.writeRate;
+
+ expect(halfwayRate?.kind === 'bytesPerSecond' && halfwayRate.bytesPerSecond).toBeGreaterThan(
+ 10,
+ );
+ expect(halfwayRate?.kind === 'bytesPerSecond' && halfwayRate.bytesPerSecond).toBeLessThan(20);
+ });
+
+ it('ends on the new values', () => {
+ expect(interpolateMonitoringTexts([writeSentence(10)], [writeSentence(20)], 1)).toEqual([
+ writeSentence(20),
+ ]);
+ });
+
+ it('replaces a sentence whose meaning changed at once', () => {
+ const idleSentence: MonitoringText = { key: 'backups.physical.monitoring.keepingUp.idle' };
+
+ expect(interpolateMonitoringTexts([writeSentence(10)], [idleSentence], 0.5)).toEqual([
+ idleSentence,
+ ]);
+ });
+});
diff --git a/frontend/src/features/backups/physical/monitoring/lib/easedTexts.ts b/frontend/src/features/backups/physical/monitoring/lib/easedTexts.ts
new file mode 100644
index 00000000..ba27f528
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/easedTexts.ts
@@ -0,0 +1,51 @@
+import type { MonitoringText } from '../model/MonitoringText';
+import type { MonitoringTextParam } from '../model/MonitoringTextParam';
+
+export const easeOutCubic = (progress: number): number =>
+ 1 - (1 - Math.min(1, Math.max(0, progress))) ** 3;
+
+export const interpolateNumber = (fromValue: number, toValue: number, progress: number): number =>
+ fromValue + (toValue - fromValue) * easeOutCubic(progress);
+
+// Moves the amounts of the same sentence towards their new values, so a refresh reads as a change
+// rather than a jump. A sentence whose key changed is replaced at once: its numbers mean
+// something else now.
+export const interpolateMonitoringTexts = (
+ fromTexts: MonitoringText[],
+ toTexts: MonitoringText[],
+ progress: number,
+): MonitoringText[] =>
+ toTexts.map((toText, textIndex) => {
+ const fromText = fromTexts[textIndex];
+
+ if (progress >= 1 || fromText === undefined || fromText.key !== toText.key || !toText.params) {
+ return toText;
+ }
+
+ const params: Record<string, MonitoringTextParam> = {};
+
+ for (const [name, toParam] of Object.entries(toText.params)) {
+ params[name] = interpolateParam(fromText.params?.[name], toParam, progress);
+ }
+
+ return { key: toText.key, params };
+ });
+
+const interpolateParam = (
+ fromParam: MonitoringTextParam | undefined,
+ toParam: MonitoringTextParam,
+ progress: number,
+): MonitoringTextParam => {
+ if (fromParam?.kind === 'bytes' && toParam.kind === 'bytes') {
+ return { kind: 'bytes', bytes: interpolateNumber(fromParam.bytes, toParam.bytes, progress) };
+ }
+
+ if (fromParam?.kind === 'bytesPerSecond' && toParam.kind === 'bytesPerSecond') {
+ return {
+ kind: 'bytesPerSecond',
+ bytesPerSecond: interpolateNumber(fromParam.bytesPerSecond, toParam.bytesPerSecond, progress),
+ };
+ }
+
+ return toParam;
+};
diff --git a/frontend/src/features/backups/physical/monitoring/lib/keepingUpSentences.test.ts b/frontend/src/features/backups/physical/monitoring/lib/keepingUpSentences.test.ts
new file mode 100644
index 00000000..1bbb2cc2
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/keepingUpSentences.test.ts
@@ -0,0 +1,107 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import {
+ type WalStreamKeepingUp,
+ WalStreamKeepingUpState,
+} from '../../../../../entity/backups/physical';
+import { getKeepingUpSentences } from './keepingUpSentences';
+
+// The entity's public API imports its API client, and the client's config module reads window at
+// import time. The tests run in node, so they supply window.
+vi.hoisted(() => {
+ vi.stubGlobal('window', {});
+});
+
+const MB = 1000 * 1000;
+const GB = 1000 * MB;
+
+const buildKeepingUp = (change: Partial<WalStreamKeepingUp>): WalStreamKeepingUp => ({
+ state: WalStreamKeepingUpState.KEEPING_UP,
+ sourceWriteBytesPerSecond: 10 * MB,
+ archiveBytesPerSecond: 10 * MB,
+ unarchivedWalBytes: 16 * MB,
+ unarchivedWalChangeBytesPerSecond: 0,
+ slotLagBytes: MB,
+ slotLagChangeBytesPerSecond: 0,
+ slotRebuildLagBytes: 256 * 1024 * 1024,
+ estimatedCatchUpSeconds: null,
+ estimatedSlotRebuildSeconds: null,
+ estimatedSlotLimitSeconds: null,
+ ...change,
+});
+
+describe('getKeepingUpSentences', () => {
+ it('explains falling behind with the difference, the backlog and its growth an hour', () => {
+ const sentences = getKeepingUpSentences(
+ buildKeepingUp({
+ state: WalStreamKeepingUpState.FALLING_BEHIND,
+ sourceWriteBytesPerSecond: 21 * MB,
+ archiveBytesPerSecond: 12 * MB,
+ unarchivedWalBytes: 3.4 * GB,
+ unarchivedWalChangeBytesPerSecond: 9 * MB,
+ }),
+ );
+
+ expect(sentences.map((sentence) => sentence.key)).toEqual([
+ 'backups.physical.monitoring.keepingUp.fallingBehind',
+ 'backups.physical.monitoring.keepingUp.fallingBehindBacklog',
+ ]);
+
+ expect(sentences[0].params?.differenceRate).toEqual({
+ kind: 'bytesPerSecond',
+ bytesPerSecond: 9 * MB,
+ });
+ expect(sentences[1].params?.unsavedAmount).toEqual({ kind: 'bytes', bytes: 3.4 * GB });
+
+ const growthPerHour = sentences[1].params?.growthPerHour;
+ expect(growthPerHour?.kind).toBe('bytes');
+ expect(growthPerHour?.kind === 'bytes' && growthPerHour.bytes / GB).toBeCloseTo(32.4, 1);
+ });
+
+ it('adds what happens next only when it is under a day away', () => {
+ const nearSentences = getKeepingUpSentences(
+ buildKeepingUp({ estimatedSlotRebuildSeconds: 40 * 60 }),
+ );
+ const farSentences = getKeepingUpSentences(
+ buildKeepingUp({ estimatedSlotRebuildSeconds: 3 * 24 * 3600 }),
+ );
+
+ expect(nearSentences.map((sentence) => sentence.key)).toContain(
+ 'backups.physical.monitoring.keepingUp.rebuildConsequence',
+ );
+ expect(farSentences.map((sentence) => sentence.key)).not.toContain(
+ 'backups.physical.monitoring.keepingUp.rebuildConsequence',
+ );
+ });
+
+ it('warns about the allowance running out when that is under a day away', () => {
+ const sentences = getKeepingUpSentences(buildKeepingUp({ estimatedSlotLimitSeconds: 50 * 60 }));
+
+ expect(sentences.map((sentence) => sentence.key)).toContain(
+ 'backups.physical.monitoring.keepingUp.limitConsequence',
+ );
+ });
+
+ it.each([
+ [WalStreamKeepingUpState.KEEPING_UP, 'backups.physical.monitoring.keepingUp.keepingUp'],
+ [WalStreamKeepingUpState.IDLE, 'backups.physical.monitoring.keepingUp.idle'],
+ [WalStreamKeepingUpState.UNKNOWN, 'backups.physical.monitoring.keepingUp.unknown'],
+ ])('describes %s in one sentence', (state, expectedKey) => {
+ expect(
+ getKeepingUpSentences(buildKeepingUp({ state })).map((sentence) => sentence.key),
+ ).toEqual([expectedKey]);
+ });
+
+ it('says how much backlog is left and how long it takes while catching up', () => {
+ const [sentence] = getKeepingUpSentences(
+ buildKeepingUp({
+ state: WalStreamKeepingUpState.CATCHING_UP,
+ unarchivedWalBytes: 3 * GB,
+ estimatedCatchUpSeconds: 50 * 60,
+ }),
+ );
+
+ expect(sentence.key).toBe('backups.physical.monitoring.keepingUp.catchingUp');
+ expect(sentence.params?.unsavedAmount).toEqual({ kind: 'bytes', bytes: 3 * GB });
+ });
+});
diff --git a/frontend/src/features/backups/physical/monitoring/lib/keepingUpSentences.ts b/frontend/src/features/backups/physical/monitoring/lib/keepingUpSentences.ts
new file mode 100644
index 00000000..9365f165
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/keepingUpSentences.ts
@@ -0,0 +1,139 @@
+import {
+ type WalStreamKeepingUp,
+ WalStreamKeepingUpState,
+} from '../../../../../entity/backups/physical';
+import type { MonitoringText } from '../model/MonitoringText';
+import { getRoundedDuration } from './roundedDuration';
+
+const SECONDS_PER_HOUR = 3600;
+
+// A consequence further away than this is not news yet, and saying it would only alarm.
+const CONSEQUENCE_HORIZON_SECONDS = 24 * SECONDS_PER_HOUR;
+
+// One or two plain sentences on whether archiving keeps up, then what happens next when the
+// source or Databasus will act within a day.
+export const getKeepingUpSentences = (keepingUp: WalStreamKeepingUp): MonitoringText[] => [
+ ...getStateSentences(keepingUp),
+ ...getConsequenceSentences(keepingUp),
+];
+
+const getStateSentences = (keepingUp: WalStreamKeepingUp): MonitoringText[] => {
+ const writeRate = keepingUp.sourceWriteBytesPerSecond ?? 0;
+ const archiveRate = keepingUp.archiveBytesPerSecond ?? 0;
+
+ switch (keepingUp.state) {
+ case WalStreamKeepingUpState.KEEPING_UP:
+ return [
+ {
+ key: 'backups.physical.monitoring.keepingUp.keepingUp',
+ params: {
+ writeRate: { kind: 'bytesPerSecond', bytesPerSecond: writeRate },
+ archiveRate: { kind: 'bytesPerSecond', bytesPerSecond: archiveRate },
+ },
+ },
+ ];
+
+ case WalStreamKeepingUpState.FALLING_BEHIND:
+ return getFallingBehindSentences(keepingUp, writeRate, archiveRate);
+
+ case WalStreamKeepingUpState.CATCHING_UP:
+ return [getCatchingUpSentence(keepingUp)];
+
+ case WalStreamKeepingUpState.IDLE:
+ return [{ key: 'backups.physical.monitoring.keepingUp.idle' }];
+
+ default:
+ return [{ key: 'backups.physical.monitoring.keepingUp.unknown' }];
+ }
+};
+
+const getFallingBehindSentences = (
+ keepingUp: WalStreamKeepingUp,
+ writeRate: number,
+ archiveRate: number,
+): MonitoringText[] => {
+ const sentences: MonitoringText[] = [
+ {
+ key: 'backups.physical.monitoring.keepingUp.fallingBehind',
+ params: {
+ differenceRate: {
+ kind: 'bytesPerSecond',
+ bytesPerSecond: Math.max(0, writeRate - archiveRate),
+ },
+ writeRate: { kind: 'bytesPerSecond', bytesPerSecond: writeRate },
+ archiveRate: { kind: 'bytesPerSecond', bytesPerSecond: archiveRate },
+ },
+ },
+ ];
+
+ if (
+ keepingUp.unarchivedWalBytes !== null &&
+ keepingUp.unarchivedWalChangeBytesPerSecond !== null &&
+ keepingUp.unarchivedWalChangeBytesPerSecond > 0
+ ) {
+ sentences.push({
+ key: 'backups.physical.monitoring.keepingUp.fallingBehindBacklog',
+ params: {
+ unsavedAmount: { kind: 'bytes', bytes: keepingUp.unarchivedWalBytes },
+ growthPerHour: {
+ kind: 'bytes',
+ bytes: keepingUp.unarchivedWalChangeBytesPerSecond * SECONDS_PER_HOUR,
+ },
+ },
+ });
+ }
+
+ return sentences;
+};
+
+const getCatchingUpSentence = (keepingUp: WalStreamKeepingUp): MonitoringText => {
+ const unsavedAmount = { kind: 'bytes' as const, bytes: keepingUp.unarchivedWalBytes ?? 0 };
+
+ if (keepingUp.estimatedCatchUpSeconds === null) {
+ return {
+ key: 'backups.physical.monitoring.keepingUp.catchingUpWithoutEstimate',
+ params: { unsavedAmount },
+ };
+ }
+
+ return {
+ key: 'backups.physical.monitoring.keepingUp.catchingUp',
+ params: {
+ unsavedAmount,
+ duration: { kind: 'text', text: getRoundedDuration(keepingUp.estimatedCatchUpSeconds) },
+ },
+ };
+};
+
+const getConsequenceSentences = (keepingUp: WalStreamKeepingUp): MonitoringText[] => {
+ const sentences: MonitoringText[] = [];
+
+ if (isWithinConsequenceHorizon(keepingUp.estimatedSlotRebuildSeconds)) {
+ sentences.push({
+ key: 'backups.physical.monitoring.keepingUp.rebuildConsequence',
+ params: {
+ duration: {
+ kind: 'text',
+ text: getRoundedDuration(keepingUp.estimatedSlotRebuildSeconds ?? 0),
+ },
+ },
+ });
+ }
+
+ if (isWithinConsequenceHorizon(keepingUp.estimatedSlotLimitSeconds)) {
+ sentences.push({
+ key: 'backups.physical.monitoring.keepingUp.limitConsequence',
+ params: {
+ duration: {
+ kind: 'text',
+ text: getRoundedDuration(keepingUp.estimatedSlotLimitSeconds ?? 0),
+ },
+ },
+ });
+ }
+
+ return sentences;
+};
+
+const isWithinConsequenceHorizon = (estimatedSeconds: number | null): boolean =>
+ estimatedSeconds !== null && estimatedSeconds < CONSEQUENCE_HORIZON_SECONDS;
diff --git a/frontend/src/features/backups/physical/monitoring/lib/monitoringTab.test.ts b/frontend/src/features/backups/physical/monitoring/lib/monitoringTab.test.ts
new file mode 100644
index 00000000..dad3d0f2
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/monitoringTab.test.ts
@@ -0,0 +1,49 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import {
+ type Database,
+ DatabaseType,
+ PhysicalDatabaseBackupType,
+} from '../../../../../entity/databases';
+import { isMonitoringTabShown } from './monitoringTab';
+
+// The entity's public API imports its API client, and the client's config module reads window at
+// import time. The tests run in node, so they supply window.
+vi.hoisted(() => {
+ vi.stubGlobal('window', {});
+});
+
+const buildDatabase = (type: DatabaseType, backupType?: PhysicalDatabaseBackupType): Database =>
+ ({
+ id: 'database-id',
+ name: 'database',
+ workspaceId: 'workspace-id',
+ type,
+ notifiers: [],
+ postgresqlPhysical: backupType ? { backupType } : undefined,
+ }) as Database;
+
+describe('isMonitoringTabShown', () => {
+ it('shows the tab for a physical database in WAL streaming mode', () => {
+ expect(
+ isMonitoringTabShown(
+ buildDatabase(
+ DatabaseType.POSTGRES_PHYSICAL,
+ PhysicalDatabaseBackupType.FULL_INCREMENTAL_WAL_STREAM,
+ ),
+ ),
+ ).toBe(true);
+ });
+
+ it('hides the tab for a physical database without WAL streaming', () => {
+ expect(
+ isMonitoringTabShown(
+ buildDatabase(DatabaseType.POSTGRES_PHYSICAL, PhysicalDatabaseBackupType.FULL_INCREMENTAL),
+ ),
+ ).toBe(false);
+ });
+
+ it('hides the tab for a logical database', () => {
+ expect(isMonitoringTabShown(buildDatabase(DatabaseType.POSTGRES_LOGICAL))).toBe(false);
+ });
+});
diff --git a/frontend/src/features/backups/physical/monitoring/lib/monitoringTab.ts b/frontend/src/features/backups/physical/monitoring/lib/monitoringTab.ts
new file mode 100644
index 00000000..99ad6436
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/monitoringTab.ts
@@ -0,0 +1,10 @@
+import {
+ type Database,
+ DatabaseType,
+ PhysicalDatabaseBackupType,
+} from '../../../../../entity/databases';
+
+export const isMonitoringTabShown = (database: Database): boolean =>
+ database.type === DatabaseType.POSTGRES_PHYSICAL &&
+ database.postgresqlPhysical?.backupType ===
+ PhysicalDatabaseBackupType.FULL_INCREMENTAL_WAL_STREAM;
diff --git a/frontend/src/features/backups/physical/monitoring/lib/monitoringTextValues.ts b/frontend/src/features/backups/physical/monitoring/lib/monitoringTextValues.ts
new file mode 100644
index 00000000..0beb9f28
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/monitoringTextValues.ts
@@ -0,0 +1,48 @@
+import type { TFunction } from 'i18next';
+
+import { type LocaleContextValue, translateLocalizedText } from '../../../../../shared/i18n';
+import { formatBytes } from '../../../../../shared/lib';
+import type { MonitoringText } from '../model/MonitoringText';
+import type { MonitoringTextParam } from '../model/MonitoringTextParam';
+
+// Runs while a component renders, with that render's language, and turns the numbers and nested
+// keys of a sentence into the values its translation interpolates.
+export const getMonitoringTextValues = (
+ text: MonitoringText,
+ t: TFunction,
+ formatNumber: LocaleContextValue['formatNumber'],
+): Record<string, string> => {
+ const values: Record<string, string> = {};
+
+ for (const [name, param] of Object.entries(text.params ?? {})) {
+ values[name] = formatMonitoringTextParam(param, t, formatNumber);
+ }
+
+ return values;
+};
+
+export const formatBytesPerSecond = (
+ bytesPerSecond: number,
+ t: TFunction,
+ formatNumber: LocaleContextValue['formatNumber'],
+): string =>
+ t('backups.physical.monitoring.charts.perSecond', {
+ amount: formatBytes(bytesPerSecond, formatNumber),
+ });
+
+const formatMonitoringTextParam = (
+ param: MonitoringTextParam,
+ t: TFunction,
+ formatNumber: LocaleContextValue['formatNumber'],
+): string => {
+ switch (param.kind) {
+ case 'bytes':
+ return formatBytes(param.bytes, formatNumber);
+ case 'bytesPerSecond':
+ return formatBytesPerSecond(param.bytesPerSecond, t, formatNumber);
+ case 'count':
+ return formatNumber(param.count);
+ case 'text':
+ return translateLocalizedText(param.text, t);
+ }
+};
diff --git a/frontend/src/features/backups/physical/monitoring/lib/pathStages.test.ts b/frontend/src/features/backups/physical/monitoring/lib/pathStages.test.ts
new file mode 100644
index 00000000..8fbfe81f
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/pathStages.test.ts
@@ -0,0 +1,64 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import {
+ WalStreamHealthLevel,
+ WalStreamReasonCode,
+ WalStreamReasonLevel,
+} from '../../../../../entity/backups/physical';
+import { MonitoringTone } from '../model/MonitoringTone';
+import { PathStage } from '../model/PathStage';
+import { getAmountProgress, getStageTones } from './pathStages';
+
+// The entity's public API imports its API client, and the client's config module reads window at
+// import time. The tests run in node, so they supply window.
+vi.hoisted(() => {
+ vi.stubGlobal('window', {});
+});
+
+describe('getStageTones', () => {
+ it('colors only the Databasus stage when the local queue is filling', () => {
+ const stageTones = getStageTones(WalStreamHealthLevel.NEEDS_ATTENTION, [
+ {
+ code: WalStreamReasonCode.LOCAL_QUEUE_FILLING,
+ level: WalStreamReasonLevel.NEEDS_ATTENTION,
+ },
+ ]);
+
+ expect(stageTones[PathStage.DATABASUS]).toBe(MonitoringTone.WARNING);
+ expect(stageTones[PathStage.SOURCE]).toBe(MonitoringTone.SUCCESS);
+ expect(stageTones[PathStage.STORAGE]).toBe(MonitoringTone.SUCCESS);
+ });
+
+ it('takes the most severe reason of a stage', () => {
+ const stageTones = getStageTones(WalStreamHealthLevel.BROKEN, [
+ { code: WalStreamReasonCode.SLOT_MISSING, level: WalStreamReasonLevel.BROKEN },
+ { code: WalStreamReasonCode.NEAR_SLOT_REBUILD, level: WalStreamReasonLevel.NEEDS_ATTENTION },
+ ]);
+
+ expect(stageTones[PathStage.SOURCE]).toBe(MonitoringTone.ERROR);
+ });
+
+ it('leaves every stage neutral while streaming is off', () => {
+ const stageTones = getStageTones(WalStreamHealthLevel.NOT_STREAMING, []);
+
+ expect(Object.values(stageTones)).toEqual([
+ MonitoringTone.NEUTRAL,
+ MonitoringTone.NEUTRAL,
+ MonitoringTone.NEUTRAL,
+ ]);
+ });
+});
+
+describe('getAmountProgress', () => {
+ it('shows an amount within its limit as a share of it', () => {
+ expect(getAmountProgress(300, 512)).toEqual({ percent: (300 / 512) * 100, isOverLimit: false });
+ });
+
+ it('marks an amount above its limit as over the limit', () => {
+ expect(getAmountProgress(600, 512)).toEqual({ percent: 100, isOverLimit: true });
+ });
+
+ it('shows no progress without a limit', () => {
+ expect(getAmountProgress(600, 0)).toEqual({ percent: 0, isOverLimit: false });
+ });
+});
diff --git a/frontend/src/features/backups/physical/monitoring/lib/pathStages.ts b/frontend/src/features/backups/physical/monitoring/lib/pathStages.ts
new file mode 100644
index 00000000..86fddd91
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/pathStages.ts
@@ -0,0 +1,59 @@
+import {
+ WalStreamHealthLevel,
+ type WalStreamReason,
+ WalStreamReasonLevel,
+} from '../../../../../entity/backups/physical';
+import type { AmountProgress } from '../model/AmountProgress';
+import { MonitoringTone } from '../model/MonitoringTone';
+import { PathStage } from '../model/PathStage';
+import { REASON_STAGES } from './reasonTexts';
+
+// Each stage takes the color of the most severe reason about it. Notices never color a stage, and
+// without a verdict about streaming there is nothing to color.
+export const getStageTones = (
+ level: WalStreamHealthLevel,
+ reasons: WalStreamReason[],
+): Record<PathStage, MonitoringTone> => {
+ const isVerdictAboutStreaming =
+ level !== WalStreamHealthLevel.NOT_STREAMING && level !== WalStreamHealthLevel.COLLECTING_DATA;
+ const defaultTone = isVerdictAboutStreaming ? MonitoringTone.SUCCESS : MonitoringTone.NEUTRAL;
+
+ const stageTones: Record<PathStage, MonitoringTone> = {
+ [PathStage.SOURCE]: defaultTone,
+ [PathStage.DATABASUS]: defaultTone,
+ [PathStage.STORAGE]: defaultTone,
+ };
+
+ if (!isVerdictAboutStreaming) {
+ return stageTones;
+ }
+
+ for (const reason of reasons) {
+ const stage = REASON_STAGES[reason.code];
+
+ if (reason.level === WalStreamReasonLevel.BROKEN) {
+ stageTones[stage] = MonitoringTone.ERROR;
+ } else if (
+ reason.level === WalStreamReasonLevel.NEEDS_ATTENTION &&
+ stageTones[stage] !== MonitoringTone.ERROR
+ ) {
+ stageTones[stage] = MonitoringTone.WARNING;
+ }
+ }
+
+ return stageTones;
+};
+
+// An amount above its limit is shown as over the limit rather than as a bar that merely looks
+// full, so the reader sees that the limit was crossed.
+export const getAmountProgress = (valueBytes: number, limitBytes: number): AmountProgress => {
+ if (limitBytes <= 0) {
+ return { percent: 0, isOverLimit: false };
+ }
+
+ if (valueBytes > limitBytes) {
+ return { percent: 100, isOverLimit: true };
+ }
+
+ return { percent: Math.max(0, (valueBytes / limitBytes) * 100), isOverLimit: false };
+};
diff --git a/frontend/src/features/backups/physical/monitoring/lib/reasonTexts.test.ts b/frontend/src/features/backups/physical/monitoring/lib/reasonTexts.test.ts
new file mode 100644
index 00000000..7591d250
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/reasonTexts.test.ts
@@ -0,0 +1,170 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import {
+ WalStreamNoticeCode,
+ type WalStreamReason,
+ WalStreamReasonCode,
+ WalStreamReasonLevel,
+} from '../../../../../entity/backups/physical';
+import { DICTIONARIES } from '../../../../../shared/i18n';
+import { getNoticeTexts, getReasonTexts, sortReasonsBrokenFirst } from './reasonTexts';
+
+// The entity's public API imports its API client, and the client's config module reads window at
+// import time. The tests run in node, so they supply window.
+vi.hoisted(() => {
+ vi.stubGlobal('window', {});
+});
+
+const now = new Date('2026-10-18T12:00:00Z');
+
+type Dictionary = { [key: string]: string | Dictionary };
+
+const getEnglishText = (key: string): string | undefined => {
+ let node: string | Dictionary | undefined = DICTIONARIES.en as unknown as Dictionary;
+
+ for (const part of key.split('.')) {
+ node = typeof node === 'object' ? node[part] : undefined;
+ }
+
+ return typeof node === 'string' ? node : undefined;
+};
+
+const flattenEnglish = (dictionary: Dictionary, prefix: string): [string, string][] =>
+ Object.entries(dictionary).flatMap(([key, value]) =>
+ typeof value === 'string'
+ ? [[`${prefix}.${key}`, value] as [string, string]]
+ : flattenEnglish(value, `${prefix}.${key}`),
+ );
+
+const buildReason = (
+ code: WalStreamReasonCode,
+ level = WalStreamReasonLevel.BROKEN,
+): WalStreamReason => ({
+ code,
+ level,
+ since: '2026-10-18T11:50:00Z',
+ valueBytes: 300 * 1024 * 1024,
+ thresholdBytes: 512 * 1024 * 1024,
+ count: 3,
+ estimatedSeconds: 40 * 60,
+});
+
+describe('getReasonTexts', () => {
+ it.each(Object.values(WalStreamReasonCode))(
+ 'gives %s a title, what happened and what to do',
+ (code) => {
+ for (const isFullBackupPending of [false, true]) {
+ const texts = getReasonTexts(buildReason(code), isFullBackupPending, now);
+
+ expect(getEnglishText(texts.titleKey)).toBeDefined();
+ expect(getEnglishText(texts.whatHappened.key)).toBeDefined();
+ expect(getEnglishText(texts.whatToDo.key)).toBeDefined();
+ }
+ },
+ );
+
+ it.each(Object.values(WalStreamNoticeCode))(
+ 'gives notice %s a title, what happened and what to do',
+ (code) => {
+ const texts = getNoticeTexts({ code, level: WalStreamReasonLevel.INFO });
+
+ expect(getEnglishText(texts.titleKey)).toBeDefined();
+ expect(getEnglishText(texts.whatHappened.key)).toBeDefined();
+ expect(getEnglishText(texts.whatToDo.key)).toBeDefined();
+ },
+ );
+
+ it.each([WalStreamReasonCode.WAL_CHAIN_BROKEN, WalStreamReasonCode.SLOT_MISSING])(
+ 'asks for a full backup on %s only while none is requested or running',
+ (code) => {
+ const withoutPendingFull = getReasonTexts(buildReason(code), false, now).whatToDo.key;
+ const withPendingFull = getReasonTexts(buildReason(code), true, now).whatToDo.key;
+
+ expect(getEnglishText(withoutPendingFull)).toContain('Start a full backup');
+ expect(getEnglishText(withPendingFull)).toContain('on its way');
+ expect(getEnglishText(withPendingFull)).not.toContain('Start a full backup');
+ },
+ );
+
+ it('does not claim that Databasus repairs a broken chain by itself', () => {
+ const whatToDo = getReasonTexts(
+ buildReason(WalStreamReasonCode.WAL_CHAIN_BROKEN),
+ false,
+ now,
+ ).whatToDo;
+
+ expect(getEnglishText(whatToDo.key)).not.toMatch(/by itself/);
+ });
+
+ it('passes the age, the amounts and the count of a reason to its sentence', () => {
+ const queueTexts = getReasonTexts(
+ buildReason(WalStreamReasonCode.LOCAL_QUEUE_FILLING, WalStreamReasonLevel.NEEDS_ATTENTION),
+ false,
+ now,
+ );
+
+ expect(queueTexts.whatHappened.params?.amount).toEqual({
+ kind: 'bytes',
+ bytes: 300 * 1024 * 1024,
+ });
+ expect(queueTexts.whatHappened.params?.limit).toEqual({
+ kind: 'bytes',
+ bytes: 512 * 1024 * 1024,
+ });
+
+ const stoppedTexts = getReasonTexts(
+ {
+ code: WalStreamReasonCode.STREAMING_STOPPED,
+ level: WalStreamReasonLevel.BROKEN,
+ since: '2026-10-18T11:50:00Z',
+ },
+ false,
+ now,
+ );
+
+ expect(stoppedTexts.whatHappened.params?.duration).toEqual({
+ kind: 'text',
+ text: { key: 'backups.physical.monitoring.durations.minutes', params: { minutes: 10 } },
+ });
+ });
+});
+
+describe('sortReasonsBrokenFirst', () => {
+ it('lists the missing slot before the filling queue', () => {
+ const sortedReasons = sortReasonsBrokenFirst([
+ {
+ code: WalStreamReasonCode.LOCAL_QUEUE_FILLING,
+ level: WalStreamReasonLevel.NEEDS_ATTENTION,
+ },
+ { code: WalStreamReasonCode.SLOT_MISSING, level: WalStreamReasonLevel.BROKEN },
+ ]);
+
+ expect(sortedReasons.map((reason) => reason.code)).toEqual([
+ WalStreamReasonCode.SLOT_MISSING,
+ WalStreamReasonCode.LOCAL_QUEUE_FILLING,
+ ]);
+ });
+});
+
+describe('the English monitoring copy', () => {
+ // Technical names belong only where the reader asks for them: the collapsed details, the
+ // explanations on hover, and a "what to do" sentence that names the setting to change.
+ const isTechnicalPlace = (key: string, text: string) =>
+ key.includes('.details.') ||
+ key.includes('.tooltips.') ||
+ (key.endsWith('.whatToDo') && text.includes('<code>'));
+
+ it('avoids WAL, slot, LSN and lag in every default label and sentence', () => {
+ const monitoringCopy = flattenEnglish(
+ (DICTIONARIES.en as unknown as { backups: { physical: { monitoring: Dictionary } } }).backups
+ .physical.monitoring,
+ 'backups.physical.monitoring',
+ );
+
+ const technicalTexts = monitoringCopy
+ .filter(([key, text]) => !isTechnicalPlace(key, text))
+ .filter(([, text]) => /\b(WAL|slot|LSN|lag)\b/i.test(text));
+
+ expect(technicalTexts).toEqual([]);
+ });
+});
diff --git a/frontend/src/features/backups/physical/monitoring/lib/reasonTexts.ts b/frontend/src/features/backups/physical/monitoring/lib/reasonTexts.ts
new file mode 100644
index 00000000..ca3a4e3a
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/reasonTexts.ts
@@ -0,0 +1,191 @@
+import {
+ WAL_STREAM_NOTICE_CODE_LABEL_KEYS,
+ WAL_STREAM_REASON_CODE_LABEL_KEYS,
+ type WalStreamNotice,
+ WalStreamNoticeCode,
+ type WalStreamReason,
+ WalStreamReasonCode,
+ WalStreamReasonLevel,
+} from '../../../../../entity/backups/physical';
+import type { TranslationKey } from '../../../../../shared/i18n';
+import type { MonitoringText } from '../model/MonitoringText';
+import type { MonitoringTextParam } from '../model/MonitoringTextParam';
+import { PathStage } from '../model/PathStage';
+import type { ReasonTexts } from '../model/ReasonTexts';
+import { getRoundedDuration, getSecondsBetween } from './roundedDuration';
+
+export const REASON_STAGES: Record<WalStreamReasonCode, PathStage> = {
+ [WalStreamReasonCode.STREAMING_STOPPED]: PathStage.DATABASUS,
+ [WalStreamReasonCode.SOURCE_UNREACHABLE]: PathStage.SOURCE,
+ [WalStreamReasonCode.WAL_CHAIN_BROKEN]: PathStage.STORAGE,
+ [WalStreamReasonCode.SLOT_MISSING]: PathStage.SOURCE,
+ [WalStreamReasonCode.SLOT_LOST]: PathStage.SOURCE,
+ [WalStreamReasonCode.ARCHIVE_NOT_ADVANCING]: PathStage.STORAGE,
+ [WalStreamReasonCode.ARCHIVE_SLOWER_THAN_SOURCE]: PathStage.STORAGE,
+ [WalStreamReasonCode.SLOT_KEEPS_EXTRA_WAL]: PathStage.SOURCE,
+ [WalStreamReasonCode.SLOT_LIMIT_APPROACHING]: PathStage.SOURCE,
+ [WalStreamReasonCode.NEAR_SLOT_REBUILD]: PathStage.SOURCE,
+ [WalStreamReasonCode.LOCAL_QUEUE_FILLING]: PathStage.DATABASUS,
+ [WalStreamReasonCode.FREQUENT_RESTARTS]: PathStage.DATABASUS,
+ [WalStreamReasonCode.FORCED_ROTATION_REFUSED]: PathStage.SOURCE,
+};
+
+interface ReasonSentenceKeys {
+ whatHappened: TranslationKey;
+ whatToDo: TranslationKey;
+
+ // Asked instead of whatToDo while a full backup is requested or running, for the conditions
+ // only a new full backup repairs.
+ whatToDoWhilePending?: TranslationKey;
+}
+
+const REASON_SENTENCE_KEYS: Record<WalStreamReasonCode, ReasonSentenceKeys> = {
+ [WalStreamReasonCode.STREAMING_STOPPED]: {
+ whatHappened: 'backups.physical.monitoring.reasons.streamingStopped.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.streamingStopped.whatToDo',
+ },
+ [WalStreamReasonCode.SOURCE_UNREACHABLE]: {
+ whatHappened: 'backups.physical.monitoring.reasons.sourceUnreachable.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.sourceUnreachable.whatToDo',
+ },
+ [WalStreamReasonCode.WAL_CHAIN_BROKEN]: {
+ whatHappened: 'backups.physical.monitoring.reasons.walChainBroken.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.walChainBroken.whatToDo',
+ whatToDoWhilePending: 'backups.physical.monitoring.reasons.walChainBroken.whatToDoPending',
+ },
+ [WalStreamReasonCode.SLOT_MISSING]: {
+ whatHappened: 'backups.physical.monitoring.reasons.slotMissing.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.slotMissing.whatToDo',
+ whatToDoWhilePending: 'backups.physical.monitoring.reasons.slotMissing.whatToDoPending',
+ },
+ [WalStreamReasonCode.SLOT_LOST]: {
+ whatHappened: 'backups.physical.monitoring.reasons.slotLost.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.slotLost.whatToDo',
+ },
+ [WalStreamReasonCode.ARCHIVE_NOT_ADVANCING]: {
+ whatHappened: 'backups.physical.monitoring.reasons.archiveNotAdvancing.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.archiveNotAdvancing.whatToDo',
+ },
+ [WalStreamReasonCode.ARCHIVE_SLOWER_THAN_SOURCE]: {
+ whatHappened: 'backups.physical.monitoring.reasons.archiveSlowerThanSource.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.archiveSlowerThanSource.whatToDo',
+ },
+ [WalStreamReasonCode.SLOT_KEEPS_EXTRA_WAL]: {
+ whatHappened: 'backups.physical.monitoring.reasons.slotKeepsExtraWal.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.slotKeepsExtraWal.whatToDo',
+ },
+ [WalStreamReasonCode.SLOT_LIMIT_APPROACHING]: {
+ whatHappened: 'backups.physical.monitoring.reasons.slotLimitApproaching.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.slotLimitApproaching.whatToDo',
+ },
+ [WalStreamReasonCode.NEAR_SLOT_REBUILD]: {
+ whatHappened: 'backups.physical.monitoring.reasons.nearSlotRebuild.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.nearSlotRebuild.whatToDo',
+ },
+ [WalStreamReasonCode.LOCAL_QUEUE_FILLING]: {
+ whatHappened: 'backups.physical.monitoring.reasons.localQueueFilling.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.localQueueFilling.whatToDo',
+ },
+ [WalStreamReasonCode.FREQUENT_RESTARTS]: {
+ whatHappened: 'backups.physical.monitoring.reasons.frequentRestarts.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.frequentRestarts.whatToDo',
+ },
+ [WalStreamReasonCode.FORCED_ROTATION_REFUSED]: {
+ whatHappened: 'backups.physical.monitoring.reasons.forcedRotationRefused.whatHappened',
+ whatToDo: 'backups.physical.monitoring.reasons.forcedRotationRefused.whatToDo',
+ },
+};
+
+const NOTICE_SENTENCE_KEYS: Record<WalStreamNoticeCode, ReasonSentenceKeys> = {
+ [WalStreamNoticeCode.NO_SLOT_WAL_LIMIT]: {
+ whatHappened: 'backups.physical.monitoring.notices.noSlotWalLimit.whatHappened',
+ whatToDo: 'backups.physical.monitoring.notices.noSlotWalLimit.whatToDo',
+ },
+ [WalStreamNoticeCode.SLOT_KEEPS_EXTRA_WAL_STEADILY]: {
+ whatHappened: 'backups.physical.monitoring.notices.slotKeepsExtraWalSteadily.whatHappened',
+ whatToDo: 'backups.physical.monitoring.notices.slotKeepsExtraWalSteadily.whatToDo',
+ },
+ [WalStreamNoticeCode.NO_FULL_BACKUP_YET]: {
+ whatHappened: 'backups.physical.monitoring.notices.noFullBackupYet.whatHappened',
+ whatToDo: 'backups.physical.monitoring.notices.noFullBackupYet.whatToDo',
+ },
+};
+
+const NOTICE_STAGES: Record<WalStreamNoticeCode, PathStage> = {
+ [WalStreamNoticeCode.NO_SLOT_WAL_LIMIT]: PathStage.SOURCE,
+ [WalStreamNoticeCode.SLOT_KEEPS_EXTRA_WAL_STEADILY]: PathStage.SOURCE,
+ [WalStreamNoticeCode.NO_FULL_BACKUP_YET]: PathStage.STORAGE,
+};
+
+const REASON_LEVEL_ORDER: Record<WalStreamReasonLevel, number> = {
+ [WalStreamReasonLevel.BROKEN]: 0,
+ [WalStreamReasonLevel.NEEDS_ATTENTION]: 1,
+ [WalStreamReasonLevel.INFO]: 2,
+};
+
+// Broken reasons first; within a level the backend's order, which follows the documented list.
+export const sortReasonsBrokenFirst = (reasons: WalStreamReason[]): WalStreamReason[] =>
+ [...reasons].sort(
+ (first, second) => REASON_LEVEL_ORDER[first.level] - REASON_LEVEL_ORDER[second.level],
+ );
+
+export const getReasonTexts = (
+ reason: WalStreamReason,
+ isFullBackupPending: boolean,
+ now: Date,
+): ReasonTexts => {
+ const sentenceKeys = REASON_SENTENCE_KEYS[reason.code];
+ const whatToDoKey =
+ isFullBackupPending && sentenceKeys.whatToDoWhilePending
+ ? sentenceKeys.whatToDoWhilePending
+ : sentenceKeys.whatToDo;
+
+ return {
+ level: reason.level,
+ stage: REASON_STAGES[reason.code],
+ titleKey: WAL_STREAM_REASON_CODE_LABEL_KEYS[reason.code],
+ whatHappened: { key: sentenceKeys.whatHappened, params: getReasonParams(reason, now) },
+ whatToDo: { key: whatToDoKey },
+ };
+};
+
+export const getNoticeTexts = (notice: WalStreamNotice): ReasonTexts => {
+ const sentenceKeys = NOTICE_SENTENCE_KEYS[notice.code];
+
+ return {
+ level: notice.level,
+ stage: NOTICE_STAGES[notice.code],
+ titleKey: WAL_STREAM_NOTICE_CODE_LABEL_KEYS[notice.code],
+ whatHappened: { key: sentenceKeys.whatHappened },
+ whatToDo: { key: sentenceKeys.whatToDo },
+ };
+};
+
+const getReasonParams = (reason: WalStreamReason, now: Date): MonitoringText['params'] => {
+ const params: Record<string, MonitoringTextParam> = {};
+
+ if (reason.since !== undefined) {
+ params.duration = {
+ kind: 'text',
+ text: getRoundedDuration(getSecondsBetween(reason.since, now)),
+ };
+ }
+
+ if (reason.estimatedSeconds !== undefined) {
+ params.duration = { kind: 'text', text: getRoundedDuration(reason.estimatedSeconds) };
+ }
+
+ if (reason.valueBytes !== undefined) {
+ params.amount = { kind: 'bytes', bytes: reason.valueBytes };
+ }
+
+ if (reason.thresholdBytes !== undefined) {
+ params.limit = { kind: 'bytes', bytes: reason.thresholdBytes };
+ }
+
+ if (reason.count !== undefined) {
+ params.count = { kind: 'count', count: reason.count };
+ }
+
+ return params;
+};
diff --git a/frontend/src/features/backups/physical/monitoring/lib/roundedDuration.test.ts b/frontend/src/features/backups/physical/monitoring/lib/roundedDuration.test.ts
new file mode 100644
index 00000000..3201054a
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/roundedDuration.test.ts
@@ -0,0 +1,42 @@
+import { describe, expect, it } from 'vitest';
+
+import { getMeasurementAge, getRoundedDuration } from './roundedDuration';
+
+describe('getRoundedDuration', () => {
+ it.each([
+ [20, 'backups.physical.monitoring.durations.minutes', { minutes: 1 }],
+ [40 * 60 + 20, 'backups.physical.monitoring.durations.minutes', { minutes: 40 }],
+ [59 * 60 + 10, 'backups.physical.monitoring.durations.minutes', { minutes: 59 }],
+ [59 * 60 + 40, 'backups.physical.monitoring.durations.hoursMinutes', { hours: 1, minutes: 0 }],
+ [
+ 2 * 3600 + 15 * 60 + 10,
+ 'backups.physical.monitoring.durations.hoursMinutes',
+ { hours: 2, minutes: 15 },
+ ],
+ [
+ 23 * 3600 + 59 * 60,
+ 'backups.physical.monitoring.durations.hoursMinutes',
+ { hours: 23, minutes: 59 },
+ ],
+ [
+ 3 * 86400 + 4 * 3600 + 20 * 60,
+ 'backups.physical.monitoring.durations.daysHours',
+ { days: 3, hours: 4 },
+ ],
+ ])('rounds %d seconds for reading', (seconds, expectedKey, expectedParams) => {
+ expect(getRoundedDuration(seconds)).toEqual({ key: expectedKey, params: expectedParams });
+ });
+});
+
+describe('getMeasurementAge', () => {
+ it('counts seconds under a minute', () => {
+ expect(getMeasurementAge(12.7)).toEqual({
+ key: 'backups.physical.monitoring.durations.seconds',
+ params: { seconds: 12 },
+ });
+ });
+
+ it('rounds like any other duration from a minute on', () => {
+ expect(getMeasurementAge(3 * 60)).toEqual(getRoundedDuration(3 * 60));
+ });
+});
diff --git a/frontend/src/features/backups/physical/monitoring/lib/roundedDuration.ts b/frontend/src/features/backups/physical/monitoring/lib/roundedDuration.ts
new file mode 100644
index 00000000..e24b01b6
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/roundedDuration.ts
@@ -0,0 +1,53 @@
+import type { LocalizedText } from '../../../../../shared/i18n';
+
+const SECONDS_PER_MINUTE = 60;
+const SECONDS_PER_HOUR = 3600;
+const SECONDS_PER_DAY = 86400;
+
+// Minutes under an hour, hours and minutes under a day, days and hours beyond: precise enough to
+// act on, short enough to read at a glance. A duration is never shown as zero minutes.
+export const getRoundedDuration = (seconds: number): LocalizedText => {
+ const totalMinutes = Math.max(1, Math.round(seconds / SECONDS_PER_MINUTE));
+
+ if (totalMinutes < 60) {
+ return {
+ key: 'backups.physical.monitoring.durations.minutes',
+ params: { minutes: totalMinutes },
+ };
+ }
+
+ if (seconds < SECONDS_PER_DAY) {
+ const hours = Math.floor(totalMinutes / 60);
+ const minutes = totalMinutes % 60;
+
+ if (hours < 24) {
+ return {
+ key: 'backups.physical.monitoring.durations.hoursMinutes',
+ params: { hours, minutes },
+ };
+ }
+ }
+
+ const totalHours = Math.round(seconds / SECONDS_PER_HOUR);
+
+ return {
+ key: 'backups.physical.monitoring.durations.daysHours',
+ params: { days: Math.floor(totalHours / 24), hours: totalHours % 24 },
+ };
+};
+
+// The live label counts seconds, because "1 min" would hide whether the data is 5 or 55 seconds
+// old.
+export const getMeasurementAge = (seconds: number): LocalizedText => {
+ if (seconds < SECONDS_PER_MINUTE) {
+ return {
+ key: 'backups.physical.monitoring.durations.seconds',
+ params: { seconds: Math.max(0, Math.floor(seconds)) },
+ };
+ }
+
+ return getRoundedDuration(seconds);
+};
+
+export const getSecondsBetween = (earlier: string | Date, later: Date): number =>
+ Math.max(0, (later.getTime() - new Date(earlier).getTime()) / 1000);
diff --git a/frontend/src/features/backups/physical/monitoring/lib/toneColors.ts b/frontend/src/features/backups/physical/monitoring/lib/toneColors.ts
new file mode 100644
index 00000000..3113301d
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/toneColors.ts
@@ -0,0 +1,40 @@
+import type { GlobalToken } from 'antd';
+
+import { MonitoringTone } from '../model/MonitoringTone';
+
+interface ToneColors {
+ background: string;
+ border: string;
+ icon: string;
+}
+
+// AntD tokens follow the dark theme by themselves, so the banner and the stages need no colors of
+// their own.
+export const getToneColors = (token: GlobalToken, tone: MonitoringTone): ToneColors => {
+ switch (tone) {
+ case MonitoringTone.SUCCESS:
+ return {
+ background: token.colorSuccessBg,
+ border: token.colorSuccessBorder,
+ icon: token.colorSuccess,
+ };
+ case MonitoringTone.WARNING:
+ return {
+ background: token.colorWarningBg,
+ border: token.colorWarningBorder,
+ icon: token.colorWarning,
+ };
+ case MonitoringTone.ERROR:
+ return {
+ background: token.colorErrorBg,
+ border: token.colorErrorBorder,
+ icon: token.colorError,
+ };
+ default:
+ return {
+ background: token.colorFillTertiary,
+ border: token.colorBorder,
+ icon: token.colorTextSecondary,
+ };
+ }
+};
diff --git a/frontend/src/features/backups/physical/monitoring/lib/verdictBanner.test.ts b/frontend/src/features/backups/physical/monitoring/lib/verdictBanner.test.ts
new file mode 100644
index 00000000..cf15c1e5
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/verdictBanner.test.ts
@@ -0,0 +1,91 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import { WalStreamHealthLevel } from '../../../../../entity/backups/physical';
+import { MonitoringTone } from '../model/MonitoringTone';
+import { VerdictIcon } from '../model/VerdictIcon';
+import { getVerdictBanner } from './verdictBanner';
+
+// The entity's public API imports its API client, and the client's config module reads window at
+// import time. The tests run in node, so they supply window.
+vi.hoisted(() => {
+ vi.stubGlobal('window', {});
+});
+
+const now = new Date('2026-10-18T12:00:00Z');
+
+describe('getVerdictBanner', () => {
+ it.each([
+ [
+ WalStreamHealthLevel.HEALTHY,
+ MonitoringTone.SUCCESS,
+ VerdictIcon.CHECK,
+ 'backups.physical.monitoring.levels.healthy',
+ ],
+ [
+ WalStreamHealthLevel.NEEDS_ATTENTION,
+ MonitoringTone.WARNING,
+ VerdictIcon.EXCLAMATION,
+ 'backups.physical.monitoring.levels.needsAttention',
+ ],
+ [
+ WalStreamHealthLevel.BROKEN,
+ MonitoringTone.ERROR,
+ VerdictIcon.CLOSE,
+ 'backups.physical.monitoring.levels.broken',
+ ],
+ [
+ WalStreamHealthLevel.COLLECTING_DATA,
+ MonitoringTone.NEUTRAL,
+ VerdictIcon.CLOCK,
+ 'backups.physical.monitoring.levels.collectingData',
+ ],
+ [
+ WalStreamHealthLevel.NOT_STREAMING,
+ MonitoringTone.NEUTRAL,
+ VerdictIcon.PAUSE,
+ 'backups.physical.monitoring.levels.notStreaming',
+ ],
+ ])('shows %s with its own color, icon and headline', (level, tone, icon, headlineKey) => {
+ const banner = getVerdictBanner(level, now, now);
+
+ expect(banner.tone).toBe(tone);
+ expect(banner.icon).toBe(icon);
+ expect(banner.headlineKey).toBe(headlineKey);
+ });
+
+ it('links the not streaming headline to the configuration', () => {
+ expect(getVerdictBanner(WalStreamHealthLevel.NOT_STREAMING, now, now).isConfigLinkShown).toBe(
+ true,
+ );
+ expect(getVerdictBanner(WalStreamHealthLevel.HEALTHY, now, now).isConfigLinkShown).toBe(false);
+ });
+
+ it('cannot confirm the state when refreshes have failed for 3 minutes after a healthy response', () => {
+ const lastSuccessfulRefreshAt = new Date(now.getTime() - 3 * 60 * 1000);
+
+ const banner = getVerdictBanner(WalStreamHealthLevel.HEALTHY, lastSuccessfulRefreshAt, now);
+
+ expect(banner.isStateConfirmed).toBe(false);
+ expect(banner.tone).toBe(MonitoringTone.NEUTRAL);
+ expect(banner.icon).toBe(VerdictIcon.QUESTION);
+ expect(banner.headlineKey).toBe('backups.physical.monitoring.banner.cannotConfirm');
+ expect(banner.description).toEqual({
+ key: 'backups.physical.monitoring.banner.cannotConfirmDescription',
+ params: {
+ level: { kind: 'text', text: { key: 'backups.physical.monitoring.levels.healthy' } },
+ duration: {
+ kind: 'text',
+ text: { key: 'backups.physical.monitoring.durations.minutes', params: { minutes: 3 } },
+ },
+ },
+ });
+ });
+
+ it('keeps the verdict while the last refresh is under 2 minutes old', () => {
+ const lastSuccessfulRefreshAt = new Date(now.getTime() - 90 * 1000);
+
+ expect(getVerdictBanner(WalStreamHealthLevel.HEALTHY, lastSuccessfulRefreshAt, now).tone).toBe(
+ MonitoringTone.SUCCESS,
+ );
+ });
+});
diff --git a/frontend/src/features/backups/physical/monitoring/lib/verdictBanner.ts b/frontend/src/features/backups/physical/monitoring/lib/verdictBanner.ts
new file mode 100644
index 00000000..474b08f9
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/verdictBanner.ts
@@ -0,0 +1,74 @@
+import {
+ WAL_STREAM_HEALTH_LEVEL_LABEL_KEYS,
+ WalStreamHealthLevel,
+} from '../../../../../entity/backups/physical';
+import { MonitoringTone } from '../model/MonitoringTone';
+import type { VerdictBanner } from '../model/VerdictBanner';
+import { VerdictIcon } from '../model/VerdictIcon';
+import { getRoundedDuration, getSecondsBetween } from './roundedDuration';
+
+// Four missed refreshes: past this the screen no longer knows the current state, and showing the
+// last verdict in its own color would claim otherwise.
+export const UNCONFIRMED_STATE_AFTER_MS = 2 * 60 * 1000;
+
+const LEVEL_TONES: Record<WalStreamHealthLevel, MonitoringTone> = {
+ [WalStreamHealthLevel.HEALTHY]: MonitoringTone.SUCCESS,
+ [WalStreamHealthLevel.NEEDS_ATTENTION]: MonitoringTone.WARNING,
+ [WalStreamHealthLevel.BROKEN]: MonitoringTone.ERROR,
+ [WalStreamHealthLevel.COLLECTING_DATA]: MonitoringTone.NEUTRAL,
+ [WalStreamHealthLevel.NOT_STREAMING]: MonitoringTone.NEUTRAL,
+};
+
+const LEVEL_ICONS: Record<WalStreamHealthLevel, VerdictIcon> = {
+ [WalStreamHealthLevel.HEALTHY]: VerdictIcon.CHECK,
+ [WalStreamHealthLevel.NEEDS_ATTENTION]: VerdictIcon.EXCLAMATION,
+ [WalStreamHealthLevel.BROKEN]: VerdictIcon.CLOSE,
+ [WalStreamHealthLevel.COLLECTING_DATA]: VerdictIcon.CLOCK,
+ [WalStreamHealthLevel.NOT_STREAMING]: VerdictIcon.PAUSE,
+};
+
+export const getVerdictBanner = (
+ level: WalStreamHealthLevel,
+ lastSuccessfulRefreshAt: Date,
+ now: Date,
+): VerdictBanner => {
+ if (now.getTime() - lastSuccessfulRefreshAt.getTime() > UNCONFIRMED_STATE_AFTER_MS) {
+ return {
+ tone: MonitoringTone.NEUTRAL,
+ icon: VerdictIcon.QUESTION,
+ headlineKey: 'backups.physical.monitoring.banner.cannotConfirm',
+ description: {
+ key: 'backups.physical.monitoring.banner.cannotConfirmDescription',
+ params: {
+ level: { kind: 'text', text: { key: WAL_STREAM_HEALTH_LEVEL_LABEL_KEYS[level] } },
+ duration: {
+ kind: 'text',
+ text: getRoundedDuration(getSecondsBetween(lastSuccessfulRefreshAt, now)),
+ },
+ },
+ },
+ isConfigLinkShown: false,
+ isStateConfirmed: false,
+ };
+ }
+
+ return {
+ tone: LEVEL_TONES[level],
+ icon: LEVEL_ICONS[level],
+ headlineKey: WAL_STREAM_HEALTH_LEVEL_LABEL_KEYS[level],
+ description: getLevelDescription(level),
+ isConfigLinkShown: level === WalStreamHealthLevel.NOT_STREAMING,
+ isStateConfirmed: true,
+ };
+};
+
+const getLevelDescription = (level: WalStreamHealthLevel): VerdictBanner['description'] => {
+ switch (level) {
+ case WalStreamHealthLevel.COLLECTING_DATA:
+ return { key: 'backups.physical.monitoring.banner.collectingDataDescription' };
+ case WalStreamHealthLevel.NOT_STREAMING:
+ return { key: 'backups.physical.monitoring.banner.notStreamingDescription' };
+ default:
+ return undefined;
+ }
+};
diff --git a/frontend/src/features/backups/physical/monitoring/lib/walStreamRefreshController.test.ts b/frontend/src/features/backups/physical/monitoring/lib/walStreamRefreshController.test.ts
new file mode 100644
index 00000000..dc464ba1
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/walStreamRefreshController.test.ts
@@ -0,0 +1,204 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import {
+ type WalStreamStatistics,
+ WalStreamStatisticsPeriod,
+} from '../../../../../entity/backups/physical';
+import type { WalStreamRefreshState } from '../model/WalStreamRefreshState';
+import {
+ type PageVisibility,
+ WAL_STREAM_REFRESH_INTERVAL_MS,
+ WalStreamRefreshController,
+} from './walStreamRefreshController';
+
+// The entity's public API imports its API client, and the client's config module reads window at
+// import time. The tests run in node, so they supply window.
+vi.hoisted(() => {
+ vi.stubGlobal('window', {});
+});
+
+class FakePageVisibility implements PageVisibility {
+ private isPageVisible = true;
+ private listeners: (() => void)[] = [];
+
+ isVisible = () => this.isPageVisible;
+
+ subscribe = (onVisibilityChange: () => void) => {
+ this.listeners.push(onVisibilityChange);
+
+ return () => {
+ this.listeners = this.listeners.filter((listener) => listener !== onVisibilityChange);
+ };
+ };
+
+ setVisible(isPageVisible: boolean) {
+ this.isPageVisible = isPageVisible;
+ this.listeners.forEach((listener) => listener());
+ }
+}
+
+const buildStatistics = (marker: string): WalStreamStatistics =>
+ ({ series: { from: marker } }) as unknown as WalStreamStatistics;
+
+interface ControllerHarness {
+ controller: WalStreamRefreshController;
+ fetchStatistics: ReturnType<typeof vi.fn>;
+ pageVisibility: FakePageVisibility;
+ getLatestState: () => WalStreamRefreshState | undefined;
+}
+
+const createHarness = (): ControllerHarness => {
+ const fetchStatistics = vi.fn(async (period: WalStreamStatisticsPeriod) =>
+ buildStatistics(period),
+ );
+ const pageVisibility = new FakePageVisibility();
+ let latestState: WalStreamRefreshState | undefined;
+
+ const controller = new WalStreamRefreshController(
+ {
+ fetchStatistics,
+ timer: {
+ setInterval: (callback, intervalMs) => setInterval(callback, intervalMs),
+ clearInterval: (handle) => clearInterval(handle as ReturnType<typeof setInterval>),
+ },
+ pageVisibility,
+ getNow: () => new Date(),
+ },
+ WalStreamStatisticsPeriod.DAY,
+ (state) => {
+ latestState = state;
+ },
+ );
+
+ return { controller, fetchStatistics, pageVisibility, getLatestState: () => latestState };
+};
+
+describe('WalStreamRefreshController', () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it('refreshes on open and every 30 seconds in every period', async () => {
+ const { controller, fetchStatistics } = createHarness();
+
+ controller.start();
+ expect(fetchStatistics).toHaveBeenCalledTimes(1);
+ expect(fetchStatistics).toHaveBeenLastCalledWith(WalStreamStatisticsPeriod.DAY);
+
+ await vi.advanceTimersByTimeAsync(3 * WAL_STREAM_REFRESH_INTERVAL_MS);
+ expect(fetchStatistics).toHaveBeenCalledTimes(4);
+
+ controller.stop();
+ });
+
+ it('does not refresh while the browser tab is hidden and refreshes at once when it is shown', async () => {
+ const { controller, fetchStatistics, pageVisibility } = createHarness();
+
+ controller.start();
+ pageVisibility.setVisible(false);
+ fetchStatistics.mockClear();
+
+ await vi.advanceTimersByTimeAsync(10 * 60 * 1000);
+ expect(fetchStatistics).not.toHaveBeenCalled();
+
+ pageVisibility.setVisible(true);
+ expect(fetchStatistics).toHaveBeenCalledTimes(1);
+
+ controller.stop();
+ });
+
+ it('refreshes at once with the new period when the period changes', async () => {
+ const { controller, fetchStatistics } = createHarness();
+
+ controller.start();
+ await vi.advanceTimersByTimeAsync(10_000);
+ fetchStatistics.mockClear();
+
+ controller.setPeriod(WalStreamStatisticsPeriod.WEEK);
+
+ expect(fetchStatistics).toHaveBeenCalledTimes(1);
+ expect(fetchStatistics).toHaveBeenLastCalledWith(WalStreamStatisticsPeriod.WEEK);
+
+ controller.stop();
+ });
+
+ it('keeps the previous response after a failed refresh and still runs the next one', async () => {
+ const { controller, fetchStatistics, getLatestState } = createHarness();
+
+ controller.start();
+ await vi.advanceTimersByTimeAsync(0);
+
+ const firstStatistics = getLatestState()?.statistics;
+ expect(firstStatistics).not.toBeNull();
+
+ fetchStatistics.mockRejectedValueOnce(new Error('network down'));
+ await vi.advanceTimersByTimeAsync(WAL_STREAM_REFRESH_INTERVAL_MS);
+
+ expect(getLatestState()?.statistics).toBe(firstStatistics);
+ expect(getLatestState()?.isLastRefreshFailed).toBe(true);
+ expect(getLatestState()?.firstLoadError).toBeNull();
+
+ await vi.advanceTimersByTimeAsync(WAL_STREAM_REFRESH_INTERVAL_MS);
+
+ expect(fetchStatistics).toHaveBeenCalledTimes(3);
+ expect(getLatestState()?.isLastRefreshFailed).toBe(false);
+
+ controller.stop();
+ });
+
+ it('reports a failed first load as an error with nothing to show', async () => {
+ const { controller, fetchStatistics, getLatestState } = createHarness();
+ const loadError = new Error('forbidden');
+
+ fetchStatistics.mockRejectedValueOnce(loadError);
+ controller.start();
+ await vi.advanceTimersByTimeAsync(0);
+
+ expect(getLatestState()?.statistics).toBeNull();
+ expect(getLatestState()?.firstLoadError).toBe(loadError);
+
+ controller.retry();
+ await vi.advanceTimersByTimeAsync(0);
+
+ expect(getLatestState()?.statistics).not.toBeNull();
+ expect(getLatestState()?.firstLoadError).toBeNull();
+
+ controller.stop();
+ });
+
+ it('drops a response to the previous period that arrives after the period changed', async () => {
+ const { controller, fetchStatistics, getLatestState } = createHarness();
+ let resolveDayResponse: (statistics: WalStreamStatistics) => void = () => {};
+
+ fetchStatistics.mockImplementationOnce(
+ () => new Promise<WalStreamStatistics>((resolve) => (resolveDayResponse = resolve)),
+ );
+
+ controller.start();
+ controller.setPeriod(WalStreamStatisticsPeriod.HOUR);
+ await vi.advanceTimersByTimeAsync(0);
+
+ resolveDayResponse(buildStatistics(WalStreamStatisticsPeriod.DAY));
+ await vi.advanceTimersByTimeAsync(0);
+
+ expect(getLatestState()?.statistics).toEqual(buildStatistics(WalStreamStatisticsPeriod.HOUR));
+
+ controller.stop();
+ });
+
+ it('stops refreshing once stopped', async () => {
+ const { controller, fetchStatistics } = createHarness();
+
+ controller.start();
+ controller.stop();
+ fetchStatistics.mockClear();
+
+ await vi.advanceTimersByTimeAsync(5 * WAL_STREAM_REFRESH_INTERVAL_MS);
+
+ expect(fetchStatistics).not.toHaveBeenCalled();
+ });
+});
diff --git a/frontend/src/features/backups/physical/monitoring/lib/walStreamRefreshController.ts b/frontend/src/features/backups/physical/monitoring/lib/walStreamRefreshController.ts
new file mode 100644
index 00000000..29ec3f6d
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/lib/walStreamRefreshController.ts
@@ -0,0 +1,129 @@
+import type {
+ WalStreamStatistics,
+ WalStreamStatisticsPeriod,
+} from '../../../../../entity/backups/physical';
+import type { WalStreamRefreshState } from '../model/WalStreamRefreshState';
+
+// Samples arrive every 30 seconds, so refreshing faster shows nothing new and refreshing slower
+// lets the screen fall behind the verdict.
+export const WAL_STREAM_REFRESH_INTERVAL_MS = 30_000;
+
+export interface RefreshTimer {
+ setInterval: (callback: () => void, intervalMs: number) => unknown;
+ clearInterval: (handle: unknown) => void;
+}
+
+export interface PageVisibility {
+ isVisible: () => boolean;
+
+ // Returns the unsubscribe function.
+ subscribe: (onVisibilityChange: () => void) => () => void;
+}
+
+export interface WalStreamRefreshDependencies {
+ fetchStatistics: (period: WalStreamStatisticsPeriod) => Promise<WalStreamStatistics>;
+ timer: RefreshTimer;
+ pageVisibility: PageVisibility;
+ getNow: () => Date;
+}
+
+// Refreshes on start, every 30 seconds in every period, at once on a period change and at once
+// when the browser tab becomes visible again, and never while it is hidden. A response that
+// arrives after a newer request started belongs to an older period or moment and is dropped. The
+// class holds no React state, so the schedule is testable with fake timers.
+export class WalStreamRefreshController {
+ private state: WalStreamRefreshState = {
+ statistics: null,
+ lastSuccessfulRefreshAt: null,
+ firstLoadError: null,
+ isLastRefreshFailed: false,
+ };
+
+ private requestGeneration = 0;
+ private intervalHandle: unknown = null;
+ private unsubscribeFromVisibility: (() => void) | null = null;
+
+ constructor(
+ private readonly dependencies: WalStreamRefreshDependencies,
+ private period: WalStreamStatisticsPeriod,
+ private readonly onStateChange: (state: WalStreamRefreshState) => void,
+ ) {}
+
+ start() {
+ this.unsubscribeFromVisibility = this.dependencies.pageVisibility.subscribe(() => {
+ if (this.dependencies.pageVisibility.isVisible()) {
+ this.refreshNowAndRestartInterval();
+ }
+ });
+
+ this.refreshNowAndRestartInterval();
+ }
+
+ stop() {
+ this.requestGeneration += 1;
+ this.clearRefreshInterval();
+ this.unsubscribeFromVisibility?.();
+ this.unsubscribeFromVisibility = null;
+ }
+
+ setPeriod(period: WalStreamStatisticsPeriod) {
+ this.period = period;
+ this.refreshNowAndRestartInterval();
+ }
+
+ retry() {
+ this.refreshNowAndRestartInterval();
+ }
+
+ getState(): WalStreamRefreshState {
+ return this.state;
+ }
+
+ private refreshNowAndRestartInterval() {
+ this.clearRefreshInterval();
+
+ this.intervalHandle = this.dependencies.timer.setInterval(() => {
+ if (this.dependencies.pageVisibility.isVisible()) {
+ void this.refresh();
+ }
+ }, WAL_STREAM_REFRESH_INTERVAL_MS);
+
+ void this.refresh();
+ }
+
+ private clearRefreshInterval() {
+ if (this.intervalHandle !== null) {
+ this.dependencies.timer.clearInterval(this.intervalHandle);
+ this.intervalHandle = null;
+ }
+ }
+
+ private async refresh() {
+ this.requestGeneration += 1;
+ const generation = this.requestGeneration;
+
+ try {
+ const statistics = await this.dependencies.fetchStatistics(this.period);
+ if (generation !== this.requestGeneration) return;
+
+ this.updateState({
+ statistics,
+ lastSuccessfulRefreshAt: this.dependencies.getNow(),
+ firstLoadError: null,
+ isLastRefreshFailed: false,
+ });
+ } catch (e) {
+ if (generation !== this.requestGeneration) return;
+
+ this.updateState({
+ firstLoadError: this.state.statistics === null ? e : null,
+ isLastRefreshFailed: true,
+ });
+ }
+ }
+
+ private updateState(change: Partial<WalStreamRefreshState>) {
+ this.state = { ...this.state, ...change };
+ this.onStateChange(this.state);
+ }
+}
diff --git a/frontend/src/features/backups/physical/monitoring/model/AmountProgress.ts b/frontend/src/features/backups/physical/monitoring/model/AmountProgress.ts
new file mode 100644
index 00000000..3420ac04
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/model/AmountProgress.ts
@@ -0,0 +1,4 @@
+export interface AmountProgress {
+ percent: number;
+ isOverLimit: boolean;
+}
diff --git a/frontend/src/features/backups/physical/monitoring/model/MonitoringText.ts b/frontend/src/features/backups/physical/monitoring/model/MonitoringText.ts
new file mode 100644
index 00000000..ddcf46b8
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/model/MonitoringText.ts
@@ -0,0 +1,9 @@
+import type { TranslationKey } from '../../../../../shared/i18n';
+import type { MonitoringTextParam } from './MonitoringTextParam';
+
+// A sentence decided outside a component. Its amounts stay numbers and its durations stay keys, so
+// the component formats and translates them in the current language while rendering.
+export interface MonitoringText {
+ key: TranslationKey;
+ params?: Record<string, MonitoringTextParam>;
+}
diff --git a/frontend/src/features/backups/physical/monitoring/model/MonitoringTextParam.ts b/frontend/src/features/backups/physical/monitoring/model/MonitoringTextParam.ts
new file mode 100644
index 00000000..324e22e3
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/model/MonitoringTextParam.ts
@@ -0,0 +1,7 @@
+import type { LocalizedText } from '../../../../../shared/i18n';
+
+export type MonitoringTextParam =
+ | { kind: 'bytes'; bytes: number }
+ | { kind: 'bytesPerSecond'; bytesPerSecond: number }
+ | { kind: 'count'; count: number }
+ | { kind: 'text'; text: LocalizedText };
diff --git a/frontend/src/features/backups/physical/monitoring/model/MonitoringTone.ts b/frontend/src/features/backups/physical/monitoring/model/MonitoringTone.ts
new file mode 100644
index 00000000..268d1ad4
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/model/MonitoringTone.ts
@@ -0,0 +1,6 @@
+export enum MonitoringTone {
+ SUCCESS = 'SUCCESS',
+ WARNING = 'WARNING',
+ ERROR = 'ERROR',
+ NEUTRAL = 'NEUTRAL',
+}
diff --git a/frontend/src/features/backups/physical/monitoring/model/PathStage.ts b/frontend/src/features/backups/physical/monitoring/model/PathStage.ts
new file mode 100644
index 00000000..55f71fa6
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/model/PathStage.ts
@@ -0,0 +1,5 @@
+export enum PathStage {
+ SOURCE = 'SOURCE',
+ DATABASUS = 'DATABASUS',
+ STORAGE = 'STORAGE',
+}
diff --git a/frontend/src/features/backups/physical/monitoring/model/ReasonTexts.ts b/frontend/src/features/backups/physical/monitoring/model/ReasonTexts.ts
new file mode 100644
index 00000000..a6108b41
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/model/ReasonTexts.ts
@@ -0,0 +1,12 @@
+import type { WalStreamReasonLevel } from '../../../../../entity/backups/physical';
+import type { TranslationKey } from '../../../../../shared/i18n';
+import type { MonitoringText } from './MonitoringText';
+import type { PathStage } from './PathStage';
+
+export interface ReasonTexts {
+ level: WalStreamReasonLevel;
+ stage: PathStage;
+ titleKey: TranslationKey;
+ whatHappened: MonitoringText;
+ whatToDo: MonitoringText;
+}
diff --git a/frontend/src/features/backups/physical/monitoring/model/VerdictBanner.ts b/frontend/src/features/backups/physical/monitoring/model/VerdictBanner.ts
new file mode 100644
index 00000000..098d57b3
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/model/VerdictBanner.ts
@@ -0,0 +1,15 @@
+import type { TranslationKey } from '../../../../../shared/i18n';
+import type { MonitoringText } from './MonitoringText';
+import type { MonitoringTone } from './MonitoringTone';
+import type { VerdictIcon } from './VerdictIcon';
+
+export interface VerdictBanner {
+ tone: MonitoringTone;
+ icon: VerdictIcon;
+ headlineKey: TranslationKey;
+ description?: MonitoringText;
+ isConfigLinkShown: boolean;
+
+ // False once refreshes have failed for too long to vouch for the verdict on screen.
+ isStateConfirmed: boolean;
+}
diff --git a/frontend/src/features/backups/physical/monitoring/model/VerdictIcon.ts b/frontend/src/features/backups/physical/monitoring/model/VerdictIcon.ts
new file mode 100644
index 00000000..9c1e0bd0
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/model/VerdictIcon.ts
@@ -0,0 +1,8 @@
+export enum VerdictIcon {
+ CHECK = 'CHECK',
+ EXCLAMATION = 'EXCLAMATION',
+ CLOSE = 'CLOSE',
+ CLOCK = 'CLOCK',
+ PAUSE = 'PAUSE',
+ QUESTION = 'QUESTION',
+}
diff --git a/frontend/src/features/backups/physical/monitoring/model/WalStreamChartRow.ts b/frontend/src/features/backups/physical/monitoring/model/WalStreamChartRow.ts
new file mode 100644
index 00000000..6dca55cd
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/model/WalStreamChartRow.ts
@@ -0,0 +1,17 @@
+// One chart point. A null value is a gap the charts draw as a gap, never as zero.
+export interface WalStreamChartRow {
+ timestampMs: number;
+
+ sourceWriteBytesPerSecond: number | null;
+ archiveBytesPerSecond: number | null;
+
+ maxUnarchivedWalBytes: number | null;
+ maxSlotLagBytes: number | null;
+
+ maxLocalQueueBytes: number | null;
+ localQueuePauseBytes: number | null;
+
+ // Where a restart or pause marker sits on the buffer chart, null without one.
+ restartMarkerBytes: number | null;
+ pauseMarkerBytes: number | null;
+}
diff --git a/frontend/src/features/backups/physical/monitoring/model/WalStreamRefreshState.ts b/frontend/src/features/backups/physical/monitoring/model/WalStreamRefreshState.ts
new file mode 100644
index 00000000..1385d415
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/model/WalStreamRefreshState.ts
@@ -0,0 +1,12 @@
+import type { WalStreamStatistics } from '../../../../../entity/backups/physical';
+
+export interface WalStreamRefreshState {
+ // The newest successful response; a failed refresh never replaces it.
+ statistics: WalStreamStatistics | null;
+ lastSuccessfulRefreshAt: Date | null;
+
+ // The first load failed, so there is nothing to show but the error and a retry.
+ firstLoadError: unknown;
+
+ isLastRefreshFailed: boolean;
+}
diff --git a/frontend/src/features/backups/physical/monitoring/ui/ChangesPathComponent.tsx b/frontend/src/features/backups/physical/monitoring/ui/ChangesPathComponent.tsx
new file mode 100644
index 00000000..81cae23f
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/ui/ChangesPathComponent.tsx
@@ -0,0 +1,238 @@
+import { ArrowDownOutlined, ArrowRightOutlined } from '@ant-design/icons';
+import { Progress, Tooltip, theme } from 'antd';
+import type { ReactNode } from 'react';
+import { useTranslation } from 'react-i18next';
+
+import {
+ type WalStreamStatistics,
+ WalStreamSourceState,
+} from '../../../../../entity/backups/physical';
+import { type TranslationKey, useLocale } from '../../../../../shared/i18n';
+import { formatBytes } from '../../../../../shared/lib';
+import { interpolateNumber } from '../lib/easedTexts';
+import { formatBytesPerSecond } from '../lib/monitoringTextValues';
+import { getAmountProgress } from '../lib/pathStages';
+import { getToneColors } from '../lib/toneColors';
+import type { MonitoringTone } from '../model/MonitoringTone';
+import { PathStage } from '../model/PathStage';
+import { useEasedTransition } from './useEasedTransition';
+
+interface Props {
+ statistics: WalStreamStatistics;
+ stageTones: Record<PathStage, MonitoringTone>;
+ isReducedMotion: boolean;
+}
+
+interface StageCardProps {
+ titleKey: TranslationKey;
+ tone: MonitoringTone;
+ children: ReactNode;
+}
+
+interface AmountProps {
+ labelKey: TranslationKey;
+ tooltipKey: TranslationKey;
+ valueBytes: number | null;
+ limitBytes?: number;
+ isReducedMotion: boolean;
+}
+
+interface ArrowProps {
+ labelKey: TranslationKey;
+ tooltipKey: TranslationKey;
+ bytesPerSecond: number | null;
+}
+
+const StageCardComponent = ({ titleKey, tone, children }: StageCardProps) => {
+ const { t } = useTranslation();
+ const { token } = theme.useToken();
+
+ return (
+ <div
+ className="flex-1 rounded-lg border-2 bg-white p-3 dark:bg-gray-800"
+ style={{ borderColor: getToneColors(token, tone).border }}
+ >
+ <div className="mb-2 font-semibold">{t(titleKey)}</div>
+ <div className="space-y-2 text-sm">{children}</div>
+ </div>
+ );
+};
+
+const easeAmount = (fromBytes: number, toBytes: number, progress: number) =>
+ interpolateNumber(fromBytes, toBytes, progress);
+
+const AmountComponent = ({
+ labelKey,
+ tooltipKey,
+ valueBytes,
+ limitBytes,
+ isReducedMotion,
+}: AmountProps) => {
+ const { t } = useTranslation();
+ const { formatNumber } = useLocale();
+
+ const easedBytes = useEasedTransition(valueBytes ?? 0, easeAmount, isReducedMotion);
+
+ const progress =
+ valueBytes !== null && limitBytes !== undefined
+ ? getAmountProgress(valueBytes, limitBytes)
+ : null;
+
+ return (
+ <div>
+ <Tooltip title={t(tooltipKey)}>
+ <span tabIndex={0} className="cursor-help border-b border-dotted border-gray-400">
+ {t(labelKey)}
+ </span>
+ </Tooltip>
+
+ <div className="font-semibold">
+ {valueBytes === null
+ ? t('backups.physical.monitoring.path.unknownAmount')
+ : formatBytes(easedBytes, formatNumber)}
+ </div>
+
+ {progress && (
+ <>
+ <Progress
+ percent={progress.percent}
+ showInfo={false}
+ size="small"
+ status={progress.isOverLimit ? 'exception' : 'normal'}
+ />
+ {progress.isOverLimit && (
+ <div className="text-xs font-semibold text-red-600 dark:text-red-400">
+ {t('backups.physical.monitoring.path.overLimit')}
+ </div>
+ )}
+ </>
+ )}
+ </div>
+ );
+};
+
+const ArrowComponent = ({ labelKey, tooltipKey, bytesPerSecond }: ArrowProps) => {
+ const { t } = useTranslation();
+ const { formatNumber } = useLocale();
+
+ const rate =
+ bytesPerSecond === null
+ ? t('backups.physical.monitoring.path.unknownAmount')
+ : formatBytesPerSecond(bytesPerSecond, t, formatNumber);
+
+ return (
+ <div className="flex flex-col items-center justify-center px-2 text-xs text-gray-600 dark:text-gray-400">
+ <Tooltip title={t(tooltipKey)}>
+ <span tabIndex={0} className="cursor-help text-center">
+ {t(labelKey, { rate })}
+ </span>
+ </Tooltip>
+ <ArrowRightOutlined className="hidden text-lg md:block" aria-hidden />
+ <ArrowDownOutlined className="text-lg md:hidden" aria-hidden />
+ </div>
+ );
+};
+
+export const ChangesPathComponent = ({ statistics, stageTones, isReducedMotion }: Props) => {
+ const { t } = useTranslation();
+ const { formatNumber, formatRelativeTime } = useLocale();
+
+ const { current, keepingUp } = statistics;
+ const isSourceReachable = current?.sourceState === WalStreamSourceState.REACHABLE;
+ const slotRebuildLagBytes = keepingUp.slotRebuildLagBytes;
+
+ return (
+ <div className="mt-4 flex flex-col items-stretch md:flex-row">
+ <StageCardComponent
+ titleKey="backups.physical.monitoring.path.database"
+ tone={stageTones[PathStage.SOURCE]}
+ >
+ <AmountComponent
+ labelKey="backups.physical.monitoring.path.notReceivedYet"
+ tooltipKey="backups.physical.monitoring.tooltips.notReceivedYet"
+ valueBytes={current?.slotLagBytes ?? null}
+ limitBytes={slotRebuildLagBytes > 0 ? slotRebuildLagBytes : undefined}
+ isReducedMotion={isReducedMotion}
+ />
+
+ {slotRebuildLagBytes > 0 && (
+ <Tooltip title={t('backups.physical.monitoring.tooltips.rebuildLimit')}>
+ <div tabIndex={0} className="cursor-help text-xs text-gray-600 dark:text-gray-400">
+ {t('backups.physical.monitoring.path.rebuildLimit', {
+ amount: formatBytes(slotRebuildLagBytes, formatNumber),
+ })}
+ </div>
+ </Tooltip>
+ )}
+
+ {isSourceReachable && (
+ <Tooltip title={t('backups.physical.monitoring.tooltips.allowanceLeft')}>
+ <div tabIndex={0} className="cursor-help text-xs text-gray-600 dark:text-gray-400">
+ {current?.slotSafeWalSizeBytes != null
+ ? t('backups.physical.monitoring.path.allowanceLeft', {
+ amount: formatBytes(current.slotSafeWalSizeBytes, formatNumber),
+ })
+ : t('backups.physical.monitoring.path.noAllowanceLimit')}
+ </div>
+ </Tooltip>
+ )}
+ </StageCardComponent>
+
+ <ArrowComponent
+ labelKey="backups.physical.monitoring.path.writeRate"
+ tooltipKey="backups.physical.monitoring.tooltips.writeRate"
+ bytesPerSecond={keepingUp.sourceWriteBytesPerSecond}
+ />
+
+ <StageCardComponent
+ titleKey="backups.physical.monitoring.path.databasus"
+ tone={stageTones[PathStage.DATABASUS]}
+ >
+ <AmountComponent
+ labelKey="backups.physical.monitoring.path.waitingInBuffer"
+ tooltipKey="backups.physical.monitoring.tooltips.waitingInBuffer"
+ valueBytes={current?.localQueueBytes ?? null}
+ limitBytes={current?.localQueuePauseBytes}
+ isReducedMotion={isReducedMotion}
+ />
+
+ {current && (
+ <Tooltip title={t('backups.physical.monitoring.tooltips.pauseLimit')}>
+ <div tabIndex={0} className="cursor-help text-xs text-gray-600 dark:text-gray-400">
+ {t('backups.physical.monitoring.path.pauseLimit', {
+ amount: formatBytes(current.localQueuePauseBytes, formatNumber),
+ })}
+ </div>
+ </Tooltip>
+ )}
+
+ {current && !current.isReceiverRunning && (
+ <div className="text-xs font-semibold">
+ {t('backups.physical.monitoring.path.receivingPaused')}
+ </div>
+ )}
+ </StageCardComponent>
+
+ <ArrowComponent
+ labelKey="backups.physical.monitoring.path.saveRate"
+ tooltipKey="backups.physical.monitoring.tooltips.saveRate"
+ bytesPerSecond={keepingUp.archiveBytesPerSecond}
+ />
+
+ <StageCardComponent
+ titleKey="backups.physical.monitoring.path.storage"
+ tone={stageTones[PathStage.STORAGE]}
+ >
+ <Tooltip title={t('backups.physical.monitoring.tooltips.lastSaved')}>
+ <div tabIndex={0} className="cursor-help">
+ {current?.lastArchivedWalAt
+ ? t('backups.physical.monitoring.path.lastSaved', {
+ relativeTime: formatRelativeTime(current.lastArchivedWalAt),
+ })
+ : t('backups.physical.monitoring.path.nothingSavedYet')}
+ </div>
+ </Tooltip>
+ </StageCardComponent>
+ </div>
+ );
+};
diff --git a/frontend/src/features/backups/physical/monitoring/ui/MonitoringReasonsComponent.tsx b/frontend/src/features/backups/physical/monitoring/ui/MonitoringReasonsComponent.tsx
new file mode 100644
index 00000000..ed747b42
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/ui/MonitoringReasonsComponent.tsx
@@ -0,0 +1,97 @@
+import { CloseCircleFilled, ExclamationCircleFilled } from '@ant-design/icons';
+import { Alert, Collapse, theme } from 'antd';
+import { useTranslation } from 'react-i18next';
+
+import { WalStreamNoticeCode, WalStreamReasonLevel } from '../../../../../entity/backups/physical';
+import type { ReasonTexts } from '../model/ReasonTexts';
+import { MonitoringTextComponent } from './MonitoringTextComponent';
+
+interface Props {
+ reasons: ReasonTexts[];
+ notices: { code: WalStreamNoticeCode; texts: ReasonTexts }[];
+}
+
+export const MonitoringReasonsComponent = ({ reasons, notices }: Props) => {
+ const { t } = useTranslation();
+ const { token } = theme.useToken();
+
+ if (reasons.length === 0 && notices.length === 0) {
+ return null;
+ }
+
+ return (
+ <div className="mt-4 space-y-3">
+ {reasons.length > 0 && (
+ <ul className="space-y-3">
+ {reasons.map((reason) => {
+ const isBroken = reason.level === WalStreamReasonLevel.BROKEN;
+
+ return (
+ <li key={reason.titleKey} className="flex items-start gap-3">
+ <span
+ className="mt-0.5 text-lg leading-none"
+ style={{ color: isBroken ? token.colorError : token.colorWarning }}
+ aria-hidden
+ >
+ {isBroken ? <CloseCircleFilled /> : <ExclamationCircleFilled />}
+ </span>
+
+ <div>
+ <div className="font-semibold">{t(reason.titleKey)}</div>
+ <div>
+ <MonitoringTextComponent text={reason.whatHappened} />
+ </div>
+ <div className="text-gray-600 dark:text-gray-400">
+ <MonitoringTextComponent text={reason.whatToDo} />
+ </div>
+ </div>
+ </li>
+ );
+ })}
+ </ul>
+ )}
+
+ {notices.length > 0 && (
+ <Alert
+ type="info"
+ showIcon
+ message={
+ <div className="space-y-2">
+ {notices.map(({ code, texts }) =>
+ code === WalStreamNoticeCode.NO_SLOT_WAL_LIMIT ? (
+ <Collapse
+ key={code}
+ ghost
+ size="small"
+ items={[
+ {
+ key: code,
+ label: (
+ <div>
+ <div className="font-semibold">{t(texts.titleKey)}</div>
+ <MonitoringTextComponent text={texts.whatHappened} />
+ </div>
+ ),
+ children: <MonitoringTextComponent text={texts.whatToDo} />,
+ },
+ ]}
+ />
+ ) : (
+ <div key={code}>
+ <div className="font-semibold">{t(texts.titleKey)}</div>
+ <div>
+ <MonitoringTextComponent text={texts.whatHappened} />
+ </div>
+ <div className="text-gray-600 dark:text-gray-400">
+ <MonitoringTextComponent text={texts.whatToDo} />
+ </div>
+ </div>
+ ),
+ )}
+ </div>
+ }
+ />
+ )}
+ </div>
+ );
+};
diff --git a/frontend/src/features/backups/physical/monitoring/ui/MonitoringTextComponent.tsx b/frontend/src/features/backups/physical/monitoring/ui/MonitoringTextComponent.tsx
new file mode 100644
index 00000000..e9ca167c
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/ui/MonitoringTextComponent.tsx
@@ -0,0 +1,36 @@
+import { useTranslation } from 'react-i18next';
+
+import { TransByKey, useLocale } from '../../../../../shared/i18n';
+import { InlineCodeComponent } from '../../../../../shared/ui';
+import { getMonitoringTextValues } from '../lib/monitoringTextValues';
+import type { MonitoringText } from '../model/MonitoringText';
+
+interface Props {
+ text: MonitoringText;
+ onOpenConfig?: () => void;
+}
+
+export const MonitoringTextComponent = ({ text, onOpenConfig }: Props) => {
+ const { t } = useTranslation();
+ const { formatNumber } = useLocale();
+
+ return (
+ <TransByKey
+ i18nKey={text.key}
+ values={getMonitoringTextValues(text, t, formatNumber)}
+ components={{
+ code: <InlineCodeComponent />,
+ configLink: (
+ <a
+ href="#"
+ className="underline"
+ onClick={(event) => {
+ event.preventDefault();
+ onOpenConfig?.();
+ }}
+ />
+ ),
+ }}
+ />
+ );
+};
diff --git a/frontend/src/features/backups/physical/monitoring/ui/VerdictBannerComponent.tsx b/frontend/src/features/backups/physical/monitoring/ui/VerdictBannerComponent.tsx
new file mode 100644
index 00000000..8c629841
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/ui/VerdictBannerComponent.tsx
@@ -0,0 +1,121 @@
+import {
+ CheckCircleFilled,
+ ClockCircleOutlined,
+ CloseCircleFilled,
+ ExclamationCircleFilled,
+ PauseCircleFilled,
+ QuestionCircleFilled,
+} from '@ant-design/icons';
+import { theme } from 'antd';
+import type { ReactNode } from 'react';
+import { useTranslation } from 'react-i18next';
+
+import { translateLocalizedText } from '../../../../../shared/i18n';
+import { interpolateMonitoringTexts } from '../lib/easedTexts';
+import { getMeasurementAge, getSecondsBetween } from '../lib/roundedDuration';
+import { getToneColors } from '../lib/toneColors';
+import { UNCONFIRMED_STATE_AFTER_MS } from '../lib/verdictBanner';
+import type { MonitoringText } from '../model/MonitoringText';
+import type { VerdictBanner } from '../model/VerdictBanner';
+import { VerdictIcon } from '../model/VerdictIcon';
+import { MonitoringTextComponent } from './MonitoringTextComponent';
+import { useEasedTransition } from './useEasedTransition';
+
+interface Props {
+ banner: VerdictBanner;
+ keepingUpSentences: MonitoringText[];
+ sampledAt: string | null;
+ now: Date;
+ isReducedMotion: boolean;
+ onOpenConfig: () => void;
+}
+
+const VERDICT_ICONS: Record<VerdictIcon, ReactNode> = {
+ [VerdictIcon.CHECK]: <CheckCircleFilled />,
+ [VerdictIcon.EXCLAMATION]: <ExclamationCircleFilled />,
+ [VerdictIcon.CLOSE]: <CloseCircleFilled />,
+ [VerdictIcon.CLOCK]: <ClockCircleOutlined />,
+ [VerdictIcon.PAUSE]: <PauseCircleFilled />,
+ [VerdictIcon.QUESTION]: <QuestionCircleFilled />,
+};
+
+export const VerdictBannerComponent = ({
+ banner,
+ keepingUpSentences,
+ sampledAt,
+ now,
+ isReducedMotion,
+ onOpenConfig,
+}: Props) => {
+ const { t } = useTranslation();
+ const { token } = theme.useToken();
+
+ const easedSentences = useEasedTransition(
+ keepingUpSentences,
+ interpolateMonitoringTexts,
+ isReducedMotion,
+ );
+
+ const toneColors = getToneColors(token, banner.tone);
+ const isLive =
+ sampledAt !== null &&
+ now.getTime() - new Date(sampledAt).getTime() < UNCONFIRMED_STATE_AFTER_MS;
+
+ return (
+ <div
+ className="w-full rounded-lg border p-4"
+ style={{
+ backgroundColor: toneColors.background,
+ borderColor: toneColors.border,
+ transition: isReducedMotion ? undefined : 'background-color 300ms, border-color 300ms',
+ }}
+ >
+ <div className="flex flex-col gap-3 md:flex-row md:items-start md:justify-between">
+ <div className="flex items-start gap-3">
+ <div className="text-3xl leading-none" style={{ color: toneColors.icon }} aria-hidden>
+ {VERDICT_ICONS[banner.icon]}
+ </div>
+
+ <div>
+ <div aria-live="polite" className="text-lg font-semibold">
+ {t(banner.headlineKey)}
+ </div>
+
+ {banner.description && (
+ <div className="mt-1">
+ <MonitoringTextComponent text={banner.description} onOpenConfig={onOpenConfig} />
+ </div>
+ )}
+
+ {easedSentences.length > 0 && (
+ <div className="mt-1 text-gray-700 dark:text-gray-300">
+ {easedSentences.map((sentence) => (
+ <span key={sentence.key} className="mr-1">
+ <MonitoringTextComponent text={sentence} />
+ </span>
+ ))}
+ </div>
+ )}
+ </div>
+ </div>
+
+ <div className="flex shrink-0 items-center gap-2 text-sm text-gray-600 dark:text-gray-400">
+ <span
+ aria-hidden
+ className={`inline-block h-2.5 w-2.5 rounded-full ${isLive && !isReducedMotion ? 'animate-pulse' : ''}`}
+ style={{ backgroundColor: isLive ? token.colorSuccess : token.colorTextQuaternary }}
+ />
+
+ {sampledAt
+ ? t('backups.physical.monitoring.banner.measuredAgo', {
+ duration: translateLocalizedText(
+ getMeasurementAge(getSecondsBetween(sampledAt, now)),
+ t,
+ ),
+ })
+ : t('backups.physical.monitoring.banner.notMeasuredYet')}
+ </div>
+ </div>
+ </div>
+ );
+};
diff --git a/frontend/src/features/backups/physical/monitoring/ui/WalStreamChartsComponent.tsx b/frontend/src/features/backups/physical/monitoring/ui/WalStreamChartsComponent.tsx
new file mode 100644
index 00000000..030ad397
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/ui/WalStreamChartsComponent.tsx
@@ -0,0 +1,297 @@
+import { Segmented, theme } from 'antd';
+import dayjs from 'dayjs';
+import { useTranslation } from 'react-i18next';
+import {
+ CartesianGrid,
+ ComposedChart,
+ Legend,
+ Line,
+ ReferenceLine,
+ ResponsiveContainer,
+ Scatter,
+ Tooltip,
+ XAxis,
+ YAxis,
+} from 'recharts';
+
+import {
+ WAL_STREAM_STATISTICS_PERIOD_LABEL_KEYS,
+ type WalStreamSeries,
+ WalStreamStatisticsPeriod,
+} from '../../../../../entity/backups/physical';
+import { type TranslationKey, useLocale } from '../../../../../shared/i18n';
+import { formatBytes } from '../../../../../shared/lib';
+import { useTheme } from '../../../../../shared/theme';
+import { getUserTimeFormat } from '../../../../../shared/time';
+import { WAL_STREAM_CHART_COLORS } from '../lib/chartColors';
+import { getDataSinceMs, hasAnyChartValues, toChartRows } from '../lib/chartRows';
+import { formatBytesPerSecond } from '../lib/monitoringTextValues';
+import type { WalStreamChartRow } from '../model/WalStreamChartRow';
+
+interface Props {
+ series: WalStreamSeries;
+ period: WalStreamStatisticsPeriod;
+ onPeriodChange: (period: WalStreamStatisticsPeriod) => void;
+ isAnimationActive: boolean;
+}
+
+interface ChartLine {
+ dataKey: keyof WalStreamChartRow;
+ labelKey: TranslationKey;
+ color: string;
+ isDashed: boolean;
+}
+
+interface ChartFrameProps {
+ titleKey: TranslationKey;
+ subtitleKey?: TranslationKey;
+ rows: WalStreamChartRow[];
+ series: WalStreamSeries;
+ lines: ChartLine[];
+ isRate: boolean;
+ isAnimationActive: boolean;
+ pauseLimitBytes?: number;
+}
+
+const CHART_HEIGHT_PX = 220;
+
+// eslint-disable-next-line i18next/no-literal-string -- recharts sync group id
+const CHARTS_SYNC_ID = 'walStreamMonitoring';
+
+const PERIODS = [
+ WalStreamStatisticsPeriod.HOUR,
+ WalStreamStatisticsPeriod.DAY,
+ WalStreamStatisticsPeriod.WEEK,
+];
+
+/* eslint-disable i18next/no-literal-string -- dayjs format tokens, not copy */
+const getTimeTickFormat = (period: WalStreamStatisticsPeriod): string => {
+ const clockFormat = getUserTimeFormat().use12Hours ? 'h:mm A' : 'HH:mm';
+
+ return period === WalStreamStatisticsPeriod.WEEK ? `ddd ${clockFormat}` : clockFormat;
+};
+/* eslint-enable i18next/no-literal-string */
+
+const ChartFrameComponent = ({
+ titleKey,
+ subtitleKey,
+ rows,
+ series,
+ lines,
+ isRate,
+ isAnimationActive,
+ pauseLimitBytes,
+}: ChartFrameProps) => {
+ const { t } = useTranslation();
+ const { formatNumber } = useLocale();
+ const { token } = theme.useToken();
+ const { resolvedTheme } = useTheme();
+
+ const chartColors = WAL_STREAM_CHART_COLORS[resolvedTheme];
+ const timeTickFormat = getTimeTickFormat(series.period);
+
+ const formatAmount = (value: number) =>
+ isRate ? formatBytesPerSecond(value, t, formatNumber) : formatBytes(value, formatNumber);
+
+ return (
+ <div>
+ <div className="font-semibold">{t(titleKey)}</div>
+ {subtitleKey && (
+ <div className="text-xs text-gray-600 dark:text-gray-400">{t(subtitleKey)}</div>
+ )}
+
+ <div style={{ width: '100%', height: CHART_HEIGHT_PX }}>
+ <ResponsiveContainer width="100%" height="100%">
+ <ComposedChart
+ data={rows}
+ syncId={CHARTS_SYNC_ID}
+ margin={{ top: 8, right: 16, left: 8, bottom: 0 }}
+ >
+ <CartesianGrid strokeDasharray="3 3" stroke={token.colorBorderSecondary} />
+ <XAxis
+ dataKey="timestampMs"
+ type="number"
+ scale="time"
+ domain={[new Date(series.from).getTime(), new Date(series.to).getTime()]}
+ tickFormatter={(timestampMs: number) => dayjs(timestampMs).format(timeTickFormat)}
+ tick={{ fontSize: 11, fill: token.colorTextSecondary }}
+ stroke={token.colorBorder}
+ />
+ <YAxis
+ width={80}
+ tickFormatter={formatAmount}
+ tick={{ fontSize: 11, fill: token.colorTextSecondary }}
+ stroke={token.colorBorder}
+ />
+ <Tooltip
+ contentStyle={{
+ backgroundColor: token.colorBgElevated,
+ borderColor: token.colorBorder,
+ color: token.colorText,
+ }}
+ labelFormatter={(timestampMs) => dayjs(Number(timestampMs)).format(timeTickFormat)}
+ formatter={(value) => (typeof value === 'number' ? formatAmount(value) : value)}
+ />
+ <Legend verticalAlign="top" height={28} wrapperStyle={{ fontSize: 12 }} />
+
+ {lines.map((line) => (
+ <Line
+ key={line.dataKey}
+ type="monotone"
+ dataKey={line.dataKey}
+ name={t(line.labelKey)}
+ stroke={line.color}
+ strokeWidth={2}
+ strokeDasharray={line.isDashed ? '6 4' : undefined}
+ dot={false}
+ connectNulls={false}
+ isAnimationActive={isAnimationActive}
+ />
+ ))}
+
+ {pauseLimitBytes !== undefined && (
+ <ReferenceLine
+ y={pauseLimitBytes}
+ stroke={chartColors.pauseLimit}
+ strokeDasharray="4 4"
+ label={{
+ value: t('backups.physical.monitoring.charts.pauseLimit'),
+ position: 'insideTopRight',
+ fill: token.colorTextSecondary,
+ fontSize: 11,
+ }}
+ />
+ )}
+
+ {pauseLimitBytes !== undefined && (
+ <Scatter
+ dataKey="restartMarkerBytes"
+ name={t('backups.physical.monitoring.charts.restarts')}
+ fill={chartColors.restartMarker}
+ isAnimationActive={isAnimationActive}
+ />
+ )}
+
+ {pauseLimitBytes !== undefined && (
+ <Scatter
+ dataKey="pauseMarkerBytes"
+ name={t('backups.physical.monitoring.charts.pauses')}
+ fill={chartColors.pauseMarker}
+ shape="diamond"
+ isAnimationActive={isAnimationActive}
+ />
+ )}
+ </ComposedChart>
+ </ResponsiveContainer>
+ </div>
+ </div>
+ );
+};
+
+export const WalStreamChartsComponent = ({
+ series,
+ period,
+ onPeriodChange,
+ isAnimationActive,
+}: Props) => {
+ const { t } = useTranslation();
+ const { resolvedTheme } = useTheme();
+
+ const chartColors = WAL_STREAM_CHART_COLORS[resolvedTheme];
+ const rows = toChartRows(series.points);
+ const dataSinceMs = getDataSinceMs(series);
+ const latestPauseLimitBytes = [...series.points]
+ .reverse()
+ .find((point) => point.localQueuePauseBytes !== null)?.localQueuePauseBytes;
+
+ return (
+ <div className="mt-6">
+ <Segmented
+ value={period}
+ onChange={(value) => onPeriodChange(value as WalStreamStatisticsPeriod)}
+ options={PERIODS.map((periodOption) => ({
+ value: periodOption,
+ label: t(WAL_STREAM_STATISTICS_PERIOD_LABEL_KEYS[periodOption]),
+ }))}
+ />
+
+ {!hasAnyChartValues(series) ? (
+ <div className="mt-4 rounded-lg border border-dashed border-gray-300 p-6 text-center text-gray-600 dark:border-gray-600 dark:text-gray-400">
+ {t('backups.physical.monitoring.charts.noHistory')}
+ </div>
+ ) : (
+ <div className="mt-4 space-y-6">
+ {dataSinceMs !== null && (
+ <div className="text-xs text-gray-600 dark:text-gray-400">
+ {t('backups.physical.monitoring.charts.dataSince', {
+ time: dayjs(dataSinceMs).format(getTimeTickFormat(series.period)),
+ })}
+ </div>
+ )}
+
+ <ChartFrameComponent
+ titleKey="backups.physical.monitoring.charts.keepingUpTitle"
+ rows={rows}
+ series={series}
+ isRate
+ isAnimationActive={isAnimationActive}
+ lines={[
+ {
+ dataKey: 'sourceWriteBytesPerSecond',
+ labelKey: 'backups.physical.monitoring.charts.databaseWrites',
+ color: chartColors.sourceWrites,
+ isDashed: false,
+ },
+ {
+ dataKey: 'archiveBytesPerSecond',
+ labelKey: 'backups.physical.monitoring.charts.savedToStorage',
+ color: chartColors.archive,
+ isDashed: true,
+ },
+ ]}
+ />
+
+ <ChartFrameComponent
+ titleKey="backups.physical.monitoring.charts.unsavedTitle"
+ subtitleKey="backups.physical.monitoring.charts.unsavedSubtitle"
+ rows={rows}
+ series={series}
+ isRate={false}
+ isAnimationActive={isAnimationActive}
+ lines={[
+ {
+ dataKey: 'maxUnarchivedWalBytes',
+ labelKey: 'backups.physical.monitoring.charts.notSavedYet',
+ color: chartColors.notSavedYet,
+ isDashed: false,
+ },
+ {
+ dataKey: 'maxSlotLagBytes',
+ labelKey: 'backups.physical.monitoring.charts.notReceivedYet',
+ color: chartColors.notReceivedYet,
+ isDashed: true,
+ },
+ ]}
+ />
+
+ <ChartFrameComponent
+ titleKey="backups.physical.monitoring.charts.bufferTitle"
+ rows={rows}
+ series={series}
+ isRate={false}
+ isAnimationActive={isAnimationActive}
+ pauseLimitBytes={latestPauseLimitBytes ?? 0}
+ lines={[
+ {
+ dataKey: 'maxLocalQueueBytes',
+ labelKey: 'backups.physical.monitoring.charts.waitingInBuffer',
+ color: chartColors.buffer,
+ isDashed: false,
+ },
+ ]}
+ />
+ </div>
+ )}
+ </div>
+ );
+};
diff --git a/frontend/src/features/backups/physical/monitoring/ui/WalStreamDetailsComponent.tsx b/frontend/src/features/backups/physical/monitoring/ui/WalStreamDetailsComponent.tsx
new file mode 100644
index 00000000..ea110847
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/ui/WalStreamDetailsComponent.tsx
@@ -0,0 +1,121 @@
+import { Collapse } from 'antd';
+import dayjs from 'dayjs';
+import { useTranslation } from 'react-i18next';
+
+import {
+ WAL_STREAM_SOURCE_STATE_LABEL_KEYS,
+ WAL_STREAM_STREAMER_STATE_LABEL_KEYS,
+ type WalStreamStatistics,
+ WalStreamSourceState,
+} from '../../../../../entity/backups/physical';
+import { type TranslationKey, useLocale } from '../../../../../shared/i18n';
+import { formatBytes } from '../../../../../shared/lib';
+import { getUserTimeFormat } from '../../../../../shared/time';
+
+interface Props {
+ statistics: WalStreamStatistics;
+}
+
+// eslint-disable-next-line i18next/no-literal-string -- Collapse item key
+const DETAILS_PANEL_KEY = 'details';
+
+export const WalStreamDetailsComponent = ({ statistics }: Props) => {
+ const { t } = useTranslation();
+ const { formatNumber } = useLocale();
+
+ const { current, streamer } = statistics;
+ const notAvailable = t('backups.physical.monitoring.details.notAvailable');
+
+ const formatMilliseconds = (milliseconds: number | null | undefined) =>
+ milliseconds == null
+ ? notAvailable
+ : t('backups.physical.monitoring.details.milliseconds', {
+ value: formatNumber(milliseconds),
+ });
+
+ const getAllowance = () => {
+ if (current?.sourceState !== WalStreamSourceState.REACHABLE) {
+ return notAvailable;
+ }
+
+ return current.slotSafeWalSizeBytes === null
+ ? t('backups.physical.monitoring.details.noLimit')
+ : formatBytes(current.slotSafeWalSizeBytes, formatNumber);
+ };
+
+ const getWalToStoredRatio = () =>
+ current && current.archivedStoredBytes > 0
+ ? formatNumber(current.archivedWalBytes / current.archivedStoredBytes, {
+ maximumFractionDigits: 1,
+ })
+ : notAvailable;
+
+ const rows: { labelKey: TranslationKey; value: string }[] = [
+ {
+ labelKey: 'backups.physical.monitoring.details.streamerState',
+ value: t(WAL_STREAM_STREAMER_STATE_LABEL_KEYS[streamer.state]),
+ },
+ {
+ labelKey: 'backups.physical.monitoring.details.sourceState',
+ value: current ? t(WAL_STREAM_SOURCE_STATE_LABEL_KEYS[current.sourceState]) : notAvailable,
+ },
+ {
+ labelKey: 'backups.physical.monitoring.details.slotStatus',
+ value: current?.slotWalStatus ?? notAvailable,
+ },
+ { labelKey: 'backups.physical.monitoring.details.allowance', value: getAllowance() },
+ {
+ labelKey: 'backups.physical.monitoring.details.averageSegmentWrite',
+ value: formatMilliseconds(current?.segmentWriteAvgMillis),
+ },
+ {
+ labelKey: 'backups.physical.monitoring.details.longestSegmentWrite',
+ value: formatMilliseconds(current?.segmentWriteMaxMillis),
+ },
+ {
+ labelKey: 'backups.physical.monitoring.details.uploadConcurrency',
+ value: current ? formatNumber(current.highestConcurrentUploads) : notAvailable,
+ },
+ {
+ labelKey: 'backups.physical.monitoring.details.walToStoredRatio',
+ value: getWalToStoredRatio(),
+ },
+ {
+ labelKey: 'backups.physical.monitoring.details.walSegmentSize',
+ value: current ? formatBytes(current.walSegmentSizeBytes, formatNumber) : notAvailable,
+ },
+ {
+ labelKey: 'backups.physical.monitoring.details.sessionStart',
+ value: streamer.sessionStartedAt
+ ? dayjs(streamer.sessionStartedAt).format(getUserTimeFormat().format)
+ : notAvailable,
+ },
+ {
+ labelKey: 'backups.physical.monitoring.details.restartsLastHour',
+ value: formatNumber(streamer.restartsLastHour),
+ },
+ ];
+
+ return (
+ <Collapse
+ className="mt-6"
+ size="small"
+ items={[
+ {
+ key: DETAILS_PANEL_KEY,
+ label: t('backups.physical.monitoring.details.title'),
+ children: (
+ <dl className="grid grid-cols-1 gap-x-6 gap-y-2 text-sm md:grid-cols-2">
+ {rows.map((row) => (
+ <div key={row.labelKey} className="flex justify-between gap-4">
+ <dt className="text-gray-600 dark:text-gray-400">{t(row.labelKey)}</dt>
+ <dd className="text-right font-medium">{row.value}</dd>
+ </div>
+ ))}
+ </dl>
+ ),
+ },
+ ]}
+ />
+ );
+};
diff --git a/frontend/src/features/backups/physical/monitoring/ui/WalStreamMonitoringComponent.tsx b/frontend/src/features/backups/physical/monitoring/ui/WalStreamMonitoringComponent.tsx
new file mode 100644
index 00000000..ae3091ca
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/ui/WalStreamMonitoringComponent.tsx
@@ -0,0 +1,228 @@
+import { Alert, Button, Skeleton } from 'antd';
+import { type ReactNode, memo, useCallback, useEffect, useRef, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+
+import {
+ WalStreamHealthLevel,
+ WalStreamStatisticsPeriod,
+ physicalWalStreamStatisticsApi,
+} from '../../../../../entity/backups/physical';
+import type { Database } from '../../../../../entity/databases';
+import { translateApiError } from '../../../../../shared/i18n';
+import { getKeepingUpSentences } from '../lib/keepingUpSentences';
+import { getStageTones } from '../lib/pathStages';
+import { getNoticeTexts, getReasonTexts, sortReasonsBrokenFirst } from '../lib/reasonTexts';
+import { getVerdictBanner } from '../lib/verdictBanner';
+import {
+ type PageVisibility,
+ type RefreshTimer,
+ WalStreamRefreshController,
+} from '../lib/walStreamRefreshController';
+import type { WalStreamRefreshState } from '../model/WalStreamRefreshState';
+import { ChangesPathComponent } from './ChangesPathComponent';
+import { MonitoringReasonsComponent } from './MonitoringReasonsComponent';
+import { useCurrentTime } from './useCurrentTime';
+import { usePrefersReducedMotion } from './usePrefersReducedMotion';
+import { VerdictBannerComponent } from './VerdictBannerComponent';
+import { WalStreamChartsComponent } from './WalStreamChartsComponent';
+import { WalStreamDetailsComponent } from './WalStreamDetailsComponent';
+
+interface Props {
+ database: Database;
+ isDirectlyUnderTab: boolean;
+ onOpenConfig: () => void;
+}
+
+// eslint-disable-next-line i18next/no-literal-string -- DOM event name
+const VISIBILITY_CHANGE_EVENT = 'visibilitychange';
+
+const browserPageVisibility: PageVisibility = {
+ isVisible: () => document.visibilityState !== 'hidden',
+ subscribe: (onVisibilityChange) => {
+ document.addEventListener(VISIBILITY_CHANGE_EVENT, onVisibilityChange);
+
+ return () => document.removeEventListener(VISIBILITY_CHANGE_EVENT, onVisibilityChange);
+ },
+};
+
+const browserRefreshTimer: RefreshTimer = {
+ setInterval: (callback, intervalMs) => window.setInterval(callback, intervalMs),
+ clearInterval: (handle) => window.clearInterval(handle as number),
+};
+
+// Longer than any chart's entry animation, so the animation of a new period finishes before
+// later refreshes stop animating.
+const CHART_ENTRY_ANIMATION_MS = 1500;
+
+// The charts re-render only when a refresh brings a new series, not on every tick of the clock.
+const MemoizedWalStreamChartsComponent = memo(WalStreamChartsComponent);
+
+const MonitoringContainerComponent = ({
+ isDirectlyUnderTab,
+ children,
+}: {
+ isDirectlyUnderTab: boolean;
+ children: ReactNode;
+}) => (
+ <div
+ className={`w-full bg-white p-3 shadow md:p-5 dark:bg-gray-800 ${isDirectlyUnderTab ? 'rounded-tr-md rounded-br-md rounded-bl-md' : 'rounded-md'}`}
+ >
+ {children}
+ </div>
+);
+
+const MonitoringSkeletonComponent = () => (
+ <div className="space-y-4">
+ <div className="h-24 rounded-lg border border-gray-200 p-4 dark:border-gray-700">
+ <Skeleton active title paragraph={{ rows: 1 }} />
+ </div>
+ <div className="h-36 rounded-lg border border-gray-200 p-4 dark:border-gray-700">
+ <Skeleton active paragraph={{ rows: 3 }} />
+ </div>
+ <div className="h-[760px] rounded-lg border border-gray-200 p-4 dark:border-gray-700">
+ <Skeleton active paragraph={{ rows: 12 }} />
+ </div>
+ </div>
+);
+
+export const WalStreamMonitoringComponent = ({
+ database,
+ isDirectlyUnderTab,
+ onOpenConfig,
+}: Props) => {
+ const { t } = useTranslation();
+
+ const [period, setPeriod] = useState(WalStreamStatisticsPeriod.HOUR);
+ const [refreshState, setRefreshState] = useState<WalStreamRefreshState | null>(null);
+ const [lastAnimatedPeriod, setLastAnimatedPeriod] = useState<WalStreamStatisticsPeriod | null>(
+ null,
+ );
+
+ const controllerRef = useRef<WalStreamRefreshController | null>(null);
+
+ const retryFirstLoad = () => {
+ controllerRef.current?.retry();
+ };
+
+ const now = useCurrentTime();
+ const isReducedMotion = usePrefersReducedMotion();
+
+ const changePeriod = useCallback((newPeriod: WalStreamStatisticsPeriod) => {
+ setPeriod(newPeriod);
+ controllerRef.current?.setPeriod(newPeriod);
+ }, []);
+
+ useEffect(() => {
+ const controller = new WalStreamRefreshController(
+ {
+ fetchStatistics: (requestedPeriod) =>
+ physicalWalStreamStatisticsApi.getWalStreamStatistics(database.id, requestedPeriod),
+ timer: browserRefreshTimer,
+ pageVisibility: browserPageVisibility,
+ getNow: () => new Date(),
+ },
+ period,
+ setRefreshState,
+ );
+
+ controllerRef.current = controller;
+ controller.start();
+
+ return () => controller.stop();
+ }, [database.id]);
+
+ const shownSeriesPeriod = refreshState?.statistics?.series.period ?? null;
+
+ useEffect(() => {
+ if (shownSeriesPeriod === null || shownSeriesPeriod === lastAnimatedPeriod) {
+ return;
+ }
+
+ const timeout = setTimeout(
+ () => setLastAnimatedPeriod(shownSeriesPeriod),
+ CHART_ENTRY_ANIMATION_MS,
+ );
+
+ return () => clearTimeout(timeout);
+ }, [shownSeriesPeriod, lastAnimatedPeriod]);
+
+ const statistics = refreshState?.statistics ?? null;
+
+ if (!statistics || !refreshState?.lastSuccessfulRefreshAt) {
+ return (
+ <MonitoringContainerComponent isDirectlyUnderTab={isDirectlyUnderTab}>
+ {refreshState?.firstLoadError ? (
+ <Alert
+ type="error"
+ showIcon
+ message={translateApiError(refreshState.firstLoadError, t)}
+ action={
+ <Button size="small" onClick={retryFirstLoad}>
+ {t('common.actions.retry')}
+ </Button>
+ }
+ />
+ ) : (
+ <MonitoringSkeletonComponent />
+ )}
+ </MonitoringContainerComponent>
+ );
+ }
+
+ const { health } = statistics;
+ const banner = getVerdictBanner(health.level, refreshState.lastSuccessfulRefreshAt, now);
+ const isVerdictAboutStreaming =
+ health.level !== WalStreamHealthLevel.NOT_STREAMING &&
+ health.level !== WalStreamHealthLevel.COLLECTING_DATA;
+
+ const reasons = sortReasonsBrokenFirst(health.reasons).map((reason) =>
+ getReasonTexts(reason, health.isFullBackupPending, now),
+ );
+ const notices = health.notices.map((notice) => ({
+ code: notice.code,
+ texts: getNoticeTexts(notice),
+ }));
+
+ return (
+ <MonitoringContainerComponent isDirectlyUnderTab={isDirectlyUnderTab}>
+ <VerdictBannerComponent
+ banner={banner}
+ keepingUpSentences={
+ banner.isStateConfirmed && isVerdictAboutStreaming
+ ? getKeepingUpSentences(statistics.keepingUp)
+ : []
+ }
+ sampledAt={statistics.current?.sampledAt ?? null}
+ now={now}
+ isReducedMotion={isReducedMotion}
+ onOpenConfig={onOpenConfig}
+ />
+
+ {refreshState.isLastRefreshFailed && (
+ <Alert
+ className="mt-3"
+ type="warning"
+ showIcon
+ message={t('backups.physical.monitoring.banner.refreshFailed')}
+ />
+ )}
+
+ <MonitoringReasonsComponent reasons={reasons} notices={notices} />
+
+ <ChangesPathComponent
+ statistics={statistics}
+ stageTones={getStageTones(health.level, health.reasons)}
+ isReducedMotion={isReducedMotion}
+ />
+
+ <MemoizedWalStreamChartsComponent
+ series={statistics.series}
+ period={period}
+ onPeriodChange={changePeriod}
+ isAnimationActive={!isReducedMotion && statistics.series.period !== lastAnimatedPeriod}
+ />
+
+ <WalStreamDetailsComponent statistics={statistics} />
+ </MonitoringContainerComponent>
+ );
+};
diff --git a/frontend/src/features/backups/physical/monitoring/ui/WalStreamMonitoringSections.test.tsx b/frontend/src/features/backups/physical/monitoring/ui/WalStreamMonitoringSections.test.tsx
new file mode 100644
index 00000000..0da08911
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/ui/WalStreamMonitoringSections.test.tsx
@@ -0,0 +1,170 @@
+import type { JSX } from 'react';
+import { renderToStaticMarkup } from 'react-dom/server';
+import { describe, expect, it, vi } from 'vitest';
+
+import {
+ WalStreamHealthLevel,
+ WalStreamKeepingUpState,
+ WalStreamNoticeCode,
+ WalStreamReasonCode,
+ WalStreamReasonLevel,
+ WalStreamSourceState,
+ type WalStreamStatistics,
+ WalStreamStatisticsPeriod,
+ WalStreamStreamerState,
+} from '../../../../../entity/backups/physical';
+import { LocaleProvider } from '../../../../../shared/i18n';
+import { getKeepingUpSentences } from '../lib/keepingUpSentences';
+import { getStageTones } from '../lib/pathStages';
+import { getNoticeTexts, getReasonTexts } from '../lib/reasonTexts';
+import { getVerdictBanner } from '../lib/verdictBanner';
+import { ChangesPathComponent } from './ChangesPathComponent';
+import { MonitoringReasonsComponent } from './MonitoringReasonsComponent';
+import { VerdictBannerComponent } from './VerdictBannerComponent';
+import { WalStreamDetailsComponent } from './WalStreamDetailsComponent';
+
+// The entity's public API imports its API client, and the client's config module reads window at
+// import time. The tests run in node, so they supply window. LocaleProvider then reads
+// localStorage, which here reports no saved language.
+vi.hoisted(() => {
+ vi.stubGlobal('window', {});
+ vi.stubGlobal('localStorage', { getItem: () => null });
+ vi.stubGlobal('navigator', { language: 'en-US', languages: ['en-US'] });
+});
+
+const MiB = 1024 * 1024;
+const now = new Date('2026-10-18T12:00:00Z');
+
+const statistics: WalStreamStatistics = {
+ health: {
+ level: WalStreamHealthLevel.NEEDS_ATTENTION,
+ reasons: [
+ {
+ code: WalStreamReasonCode.LOCAL_QUEUE_FILLING,
+ level: WalStreamReasonLevel.NEEDS_ATTENTION,
+ valueBytes: 600 * MiB,
+ thresholdBytes: 512 * MiB,
+ },
+ ],
+ notices: [{ code: WalStreamNoticeCode.NO_SLOT_WAL_LIMIT, level: WalStreamReasonLevel.INFO }],
+ isFullBackupPending: false,
+ },
+ streamer: {
+ state: WalStreamStreamerState.RUNNING,
+ sessionStartedAt: '2026-10-18T10:00:00Z',
+ failingSince: null,
+ restartsLastHour: 1,
+ },
+ current: {
+ sampledAt: '2026-10-18T11:59:48Z',
+ lastArchivedWalAt: '2026-10-18T11:59:20Z',
+ sessionStartedAt: '2026-10-18T10:00:00Z',
+ sourceState: WalStreamSourceState.REACHABLE,
+ sourceUnreachableSince: null,
+ sourceCurrentLsn: '0/5000000',
+ archiveProgressLsn: '0/4F00000',
+ sourceWrittenBytes: 16 * MiB,
+ archivedSegments: 2,
+ archivedWalBytes: 32 * MiB,
+ archivedStoredBytes: 8 * MiB,
+ walSegmentSizeBytes: 16 * MiB,
+ highestConcurrentUploads: 3,
+ segmentWriteAvgMillis: 120,
+ segmentWriteMaxMillis: 300,
+ slotWalStatus: 'reserved',
+ slotExtraWalSince: null,
+ slotLagBytes: 2 * MiB,
+ slotSafeWalSizeBytes: null,
+ unarchivedWalBytes: 24 * MiB,
+ archiveStalledSince: null,
+ localQueueBytes: 600 * MiB,
+ localQueuePauseBytes: 512 * MiB,
+ receiverRestarts: 0,
+ receiverPauses: 0,
+ isReceiverRunning: false,
+ isForcedRotationDenied: false,
+ },
+ keepingUp: {
+ state: WalStreamKeepingUpState.KEEPING_UP,
+ sourceWriteBytesPerSecond: 10 * MiB,
+ archiveBytesPerSecond: 10 * MiB,
+ unarchivedWalBytes: 24 * MiB,
+ unarchivedWalChangeBytesPerSecond: 0,
+ slotLagBytes: 2 * MiB,
+ slotLagChangeBytesPerSecond: 0,
+ slotRebuildLagBytes: 256 * MiB,
+ estimatedCatchUpSeconds: null,
+ estimatedSlotRebuildSeconds: null,
+ estimatedSlotLimitSeconds: null,
+ },
+ series: {
+ period: WalStreamStatisticsPeriod.HOUR,
+ bucketSeconds: 60,
+ from: '2026-10-18T11:01:00Z',
+ to: '2026-10-18T12:01:00Z',
+ points: [],
+ },
+};
+
+const renderWithLocale = (element: JSX.Element) =>
+ renderToStaticMarkup(<LocaleProvider>{element}</LocaleProvider>);
+
+describe('the Monitoring tab sections', () => {
+ it('leads with the verdict, the keeping-up sentence and the age of the newest sample', () => {
+ const markup = renderWithLocale(
+ <VerdictBannerComponent
+ banner={getVerdictBanner(statistics.health.level, now, now)}
+ keepingUpSentences={getKeepingUpSentences(statistics.keepingUp)}
+ sampledAt={statistics.current?.sampledAt ?? null}
+ now={now}
+ isReducedMotion
+ onOpenConfig={() => {}}
+ />,
+ );
+
+ expect(markup).toContain('Something needs attention');
+ expect(markup).toContain('aria-live="polite"');
+ expect(markup).toContain(
+ 'Databasus saves changes as fast as the database writes them: 10 MB/s written',
+ );
+ expect(markup).toContain('Measured 12 s ago');
+ });
+
+ it('lists each reason with what happened and what to do, and names the setting in the notice', () => {
+ const markup = renderWithLocale(
+ <MonitoringReasonsComponent
+ reasons={statistics.health.reasons.map((reason) => getReasonTexts(reason, false, now))}
+ notices={statistics.health.notices.map((notice) => ({
+ code: notice.code,
+ texts: getNoticeTexts(notice),
+ }))}
+ />,
+ );
+
+ expect(markup).toContain('The Databasus buffer is filling up');
+ expect(markup).toContain('The Databasus buffer holds 600 MB of 512 MB.');
+ expect(markup).toContain('No limit on kept changes');
+ });
+
+ it('shows an amount above its limit as over the limit and the receiver as paused', () => {
+ const markup = renderWithLocale(
+ <ChangesPathComponent
+ statistics={statistics}
+ stageTones={getStageTones(statistics.health.level, statistics.health.reasons)}
+ isReducedMotion
+ />,
+ );
+
+ expect(markup).toContain('Over the limit');
+ expect(markup).toContain('Receiving paused');
+ expect(markup).toContain('Receiving pauses at 512 MB');
+ expect(markup).toContain('No limit on kept changes');
+ });
+
+ it('keeps the technical names in details that start collapsed', () => {
+ const markup = renderWithLocale(<WalStreamDetailsComponent statistics={statistics} />);
+
+ expect(markup).toContain('Details');
+ expect(markup).not.toContain('wal_status');
+ });
+});
diff --git a/frontend/src/features/backups/physical/monitoring/ui/useCurrentTime.ts b/frontend/src/features/backups/physical/monitoring/ui/useCurrentTime.ts
new file mode 100644
index 00000000..e7a91885
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/ui/useCurrentTime.ts
@@ -0,0 +1,15 @@
+import { useEffect, useState } from 'react';
+
+// The age of the newest sample is shown to the second, so the clock it is measured against ticks
+// every second.
+export const useCurrentTime = (): Date => {
+ const [currentTime, setCurrentTime] = useState(() => new Date());
+
+ useEffect(() => {
+ const interval = setInterval(() => setCurrentTime(new Date()), 1000);
+
+ return () => clearInterval(interval);
+ }, []);
+
+ return currentTime;
+};
diff --git a/frontend/src/features/backups/physical/monitoring/ui/useEasedTransition.ts b/frontend/src/features/backups/physical/monitoring/ui/useEasedTransition.ts
new file mode 100644
index 00000000..6902000a
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/ui/useEasedTransition.ts
@@ -0,0 +1,47 @@
+import { useEffect, useRef, useState } from 'react';
+
+const EASING_DURATION_MS = 400;
+
+// Shows a value moving from what is on screen to its new target over 400 ms, so a refresh reads as
+// a change. With reduced motion the new value appears at once.
+export const useEasedTransition = <TValue>(
+ target: TValue,
+ interpolate: (from: TValue, to: TValue, progress: number) => TValue,
+ isReducedMotion: boolean,
+): TValue => {
+ const [displayed, setDisplayed] = useState(target);
+
+ const displayedRef = useRef(target);
+ const targetKey = JSON.stringify(target);
+
+ useEffect(() => {
+ if (isReducedMotion) {
+ displayedRef.current = target;
+ setDisplayed(target);
+
+ return;
+ }
+
+ const from = displayedRef.current;
+ const startedAt = performance.now();
+ let animationFrame = 0;
+
+ const step = (frameTime: number) => {
+ const progress = Math.min(1, (frameTime - startedAt) / EASING_DURATION_MS);
+ const value = interpolate(from, target, progress);
+
+ displayedRef.current = value;
+ setDisplayed(value);
+
+ if (progress < 1) {
+ animationFrame = requestAnimationFrame(step);
+ }
+ };
+
+ animationFrame = requestAnimationFrame(step);
+
+ return () => cancelAnimationFrame(animationFrame);
+ }, [targetKey, isReducedMotion]);
+
+ return displayed;
+};
diff --git a/frontend/src/features/backups/physical/monitoring/ui/usePrefersReducedMotion.ts b/frontend/src/features/backups/physical/monitoring/ui/usePrefersReducedMotion.ts
new file mode 100644
index 00000000..217d69b0
--- /dev/null
+++ b/frontend/src/features/backups/physical/monitoring/ui/usePrefersReducedMotion.ts
@@ -0,0 +1,21 @@
+import { useEffect, useState } from 'react';
+
+// eslint-disable-next-line i18next/no-literal-string -- CSS media query
+const REDUCED_MOTION_QUERY = '(prefers-reduced-motion: reduce)';
+
+export const usePrefersReducedMotion = (): boolean => {
+ const [isReducedMotion, setIsReducedMotion] = useState(
+ () => typeof window !== 'undefined' && window.matchMedia(REDUCED_MOTION_QUERY).matches,
+ );
+
+ useEffect(() => {
+ const mediaQuery = window.matchMedia(REDUCED_MOTION_QUERY);
+ const handleChange = () => setIsReducedMotion(mediaQuery.matches);
+
+ mediaQuery.addEventListener('change', handleChange);
+
+ return () => mediaQuery.removeEventListener('change', handleChange);
+ }, []);
+
+ return isReducedMotion;
+};
diff --git a/frontend/src/features/databases/ui/DatabaseComponent.tsx b/frontend/src/features/databases/ui/DatabaseComponent.tsx
index 0d85b6ca..1d7db6da 100644
--- a/frontend/src/features/databases/ui/DatabaseComponent.tsx
+++ b/frontend/src/features/databases/ui/DatabaseComponent.tsx
@@ -4,7 +4,11 @@ import { useTranslation } from 'react-i18next';
import { type Database, DatabaseType, databaseApi } from '../../../entity/databases';
import { LogicalBackupsComponent } from '../../backups/logical';
-import { PhysicalBackupsComponent } from '../../backups/physical';
+import {
+ PhysicalBackupsComponent,
+ WalStreamMonitoringComponent,
+ isMonitoringTabShown,
+} from '../../backups/physical';
import { HealthckeckAttemptsComponent } from '../../healthcheck';
import { VerificationsComponent } from '../../verification/runs';
import { DatabaseConfigComponent } from './DatabaseConfigComponent';
@@ -25,7 +29,9 @@ export const DatabaseComponent = ({
isCanManageDBs,
}: Props) => {
const { t } = useTranslation();
- const [currentTab, setCurrentTab] = useState<'config' | 'backups' | 'verifications'>('backups');
+ const [currentTab, setCurrentTab] = useState<
+ 'config' | 'backups' | 'verifications' | 'monitoring'
+ >('backups');
const [database, setDatabase] = useState<Database | undefined>();
const [editDatabase, setEditDatabase] = useState<Database | undefined>();
@@ -40,6 +46,7 @@ export const DatabaseComponent = ({
const isLogicalDatabase = database?.type === DatabaseType.POSTGRES_LOGICAL;
const isPhysicalDatabase = database?.type === DatabaseType.POSTGRES_PHYSICAL;
+ const isMonitoringShown = database !== undefined && isMonitoringTabShown(database);
const loadSettings = () => {
setDatabase(undefined);
@@ -84,6 +91,15 @@ export const DatabaseComponent = ({
{t('databases.tabs.verifications')}
</div>
)}
+
+ {isMonitoringShown && (
+ <div
+ className={`mr-2 cursor-pointer rounded-tl-md rounded-tr-md px-6 py-2 ${currentTab === 'monitoring' ? 'bg-white dark:bg-gray-800' : 'bg-gray-200 dark:bg-gray-700'}`}
+ onClick={() => setCurrentTab('monitoring')}
+ >
+ {t('databases.tabs.monitoring')}
+ </div>
+ )}
</div>
{currentTab === 'config' && (
@@ -124,6 +140,14 @@ export const DatabaseComponent = ({
</>
)}
+ {currentTab === 'monitoring' && isMonitoringShown && (
+ <WalStreamMonitoringComponent
+ database={database}
+ isDirectlyUnderTab={true}
+ onOpenConfig={() => setCurrentTab('config')}
+ />
+ )}
+
{currentTab === 'verifications' && isLogicalDatabase && (
<VerificationsComponent
database={database}
diff --git a/frontend/src/shared/i18n/locales/en.ts b/frontend/src/shared/i18n/locales/en.ts
index 0e908cb8..b465d053 100644
--- a/frontend/src/shared/i18n/locales/en.ts
+++ b/frontend/src/shared/i18n/locales/en.ts
@@ -309,6 +309,7 @@ export const en = {
config: 'Config',
backups: 'Backups',
verifications: 'Verifications',
+ monitoring: 'Monitoring',
},
create: {
title: 'Add database for backup',
@@ -1662,6 +1663,250 @@ export const en = {
'A base backup copies only the data directory. If your source PostgreSQL keeps its configuration outside it - the Debian/Ubuntu layout, <configDir/> - then <code>postgresql.conf</code>, <code>pg_hba.conf</code> and <code>pg_ident.conf</code> are not in the backup and the server will not start. Ask the source for their real paths with <code>psql -Atc "SHOW config_file"</code>, copy all three in, then comment out <code>data_directory</code>, <code>hba_file</code>, <code>ident_file</code>, <code>external_pid_file</code> and the <code>ssl</code> / <code>ssl_*</code> lines, create <code>conf.d</code>, and set <code>listen_addresses = \'*\'</code> for a container. If those files are present and you still get the same error, it is the mounted directory that is wrong, not the configuration.',
},
},
+ monitoring: {
+ periods: {
+ hour: 'Last hour',
+ day: 'Last day',
+ week: 'Last week',
+ },
+ levels: {
+ notStreaming: 'Continuous saving of changes is off',
+ collectingData: 'Collecting data',
+ broken: 'Something is broken',
+ needsAttention: 'Something needs attention',
+ healthy: 'Everything is in order',
+ },
+ reasonLevels: {
+ broken: 'Broken',
+ needsAttention: 'Needs attention',
+ info: 'For your information',
+ },
+ keepingUpStates: {
+ unknown: 'Unknown',
+ fallingBehind: 'Falling behind',
+ catchingUp: 'Catching up',
+ idle: 'Quiet',
+ keepingUp: 'Keeping up',
+ },
+ banner: {
+ collectingDataDescription: 'Streaming started; the first numbers appear within a minute.',
+ notStreamingDescription:
+ 'Backups are disabled or no storage is selected. <configLink>Open the configuration</configLink> to turn it on.',
+ cannotConfirm: 'Cannot confirm the current state',
+ cannotConfirmDescription: 'Last known state: {{level}}, {{duration}} ago.',
+ measuredAgo: 'Measured {{duration}} ago',
+ notMeasuredYet: 'Not measured yet',
+ refreshFailed:
+ 'Could not refresh the data. The last received numbers stay on screen, and the next attempt runs in 30 seconds.',
+ },
+ keepingUp: {
+ keepingUp:
+ 'Databasus saves changes as fast as the database writes them: {{writeRate}} written, {{archiveRate}} saved.',
+ fallingBehind:
+ 'The database writes changes faster than Databasus saves them, by about {{differenceRate}}: {{writeRate}} against {{archiveRate}}.',
+ fallingBehindBacklog:
+ '{{unsavedAmount}} is not saved yet, and that grows by about {{growthPerHour}} an hour.',
+ catchingUp:
+ 'Databasus is saving a backlog: {{unsavedAmount}} is left, about {{duration}} to go.',
+ catchingUpWithoutEstimate: 'Databasus is saving a backlog: {{unsavedAmount}} is left.',
+ idle: 'The database writes almost nothing; new changes reach storage within minutes.',
+ unknown: 'No current numbers are available.',
+ rebuildConsequence:
+ 'At this pace Databasus will start a new backup chain from a full backup in about {{duration}}.',
+ limitConsequence:
+ 'At this pace the database will drop changes Databasus has not received yet in about {{duration}}.',
+ },
+ reasons: {
+ streamingStopped: {
+ title: 'Saving has stopped',
+ whatHappened: 'Databasus stopped receiving changes {{duration}} ago.',
+ whatToDo: 'Databasus retries by itself. If this lasts, check the Databasus logs.',
+ },
+ sourceUnreachable: {
+ title: 'The database is unreachable',
+ whatHappened: 'Databasus has not been able to reach the database for {{duration}}.',
+ whatToDo: 'Check that the database server is running and reachable from Databasus.',
+ },
+ walChainBroken: {
+ title: 'Changes were lost between backups',
+ whatHappened:
+ 'Some changes were lost between backups, so recent changes cannot be restored yet.',
+ whatToDo: 'Start a full backup on the Backups tab.',
+ whatToDoPending:
+ 'A new full backup is on its way. You can watch it on the Backups tab.',
+ },
+ slotMissing: {
+ title: 'The database stopped keeping changes',
+ whatHappened:
+ 'The database no longer keeps changes for Databasus: the reservation Databasus made on it is gone.',
+ whatToDo:
+ 'Databasus restores the reservation when streaming restarts, but the changes made in between are lost. Start a full backup on the Backups tab, and check that nobody removes the reservation on the database.',
+ whatToDoPending:
+ 'Databasus restores the reservation when streaming restarts, and a new full backup is on its way. Check that nobody removes the reservation on the database.',
+ },
+ slotLost: {
+ title: 'The database deleted unsaved changes',
+ whatHappened: 'The database deleted changes Databasus had not received yet.',
+ whatToDo:
+ 'Databasus starts over with a new full backup by itself. To avoid this next time, make saving faster or let the database keep more changes.',
+ },
+ archiveNotAdvancing: {
+ title: 'Changes do not reach storage',
+ whatHappened:
+ 'No new changes have reached storage for {{duration}} while the database keeps writing.',
+ whatToDo: 'Check that the storage is reachable and has free space.',
+ },
+ archiveSlowerThanSource: {
+ title: 'Saving is too slow',
+ whatHappened:
+ 'The database writes faster than Databasus saves; the backlog grew by {{amount}} in 10 minutes.',
+ whatToDo:
+ 'Check the network speed to the storage, or move the storage closer to Databasus.',
+ },
+ slotKeepsExtraWal: {
+ title: 'The database keeps extra changes',
+ whatHappened:
+ 'For {{duration}} the database has kept more of its change log than usual for Databasus, and the amount keeps growing.',
+ whatToDo: 'Make sure the database server has free disk space.',
+ },
+ slotLimitApproaching: {
+ title: 'Unsaved changes will soon be deleted',
+ whatHappened:
+ 'At this pace the database will delete changes Databasus has not received yet in about {{duration}}.',
+ whatToDo:
+ 'Make saving faster, or raise <code>max_slot_wal_keep_size</code> on the database.',
+ },
+ nearSlotRebuild: {
+ title: 'Close to starting over',
+ whatHappened:
+ 'Changes not received by Databasus yet reached {{amount}} of the {{limit}} at which Databasus starts over from a full backup.',
+ whatToDo: 'Make saving faster, or raise the limit in the backup settings.',
+ },
+ localQueueFilling: {
+ title: 'The Databasus buffer is filling up',
+ whatHappened: 'The Databasus buffer holds {{amount}} of {{limit}}.',
+ whatToDo:
+ 'Databasus pauses receiving by itself when the buffer is full. Check the speed of the storage.',
+ },
+ frequentRestarts: {
+ title: 'Receiving restarts often',
+ whatHappened: 'Restarts of receiving in the last hour: {{count}}.',
+ whatToDo: 'Check the Databasus logs for the cause.',
+ },
+ forcedRotationRefused: {
+ title: 'Quiet periods reach storage late',
+ whatHappened:
+ 'The database refuses to close its change file on time, so changes from quiet periods reach storage later.',
+ whatToDo: 'Grant the backup user permission to run <code>pg_switch_wal()</code>.',
+ },
+ },
+ notices: {
+ noSlotWalLimit: {
+ title: 'No limit on kept changes',
+ whatHappened: 'The database keeps changes for Databasus without a limit.',
+ whatToDo:
+ "A limit protects the database server's disk if Databasus stays down for long. The cost is a new full backup when the limit is hit. Set it with <code>max_slot_wal_keep_size</code>.",
+ },
+ slotKeepsExtraWalSteadily: {
+ title: 'The database keeps extra changes',
+ whatHappened: 'The database keeps more of its change log than usual for Databasus.',
+ whatToDo:
+ 'This is normal under heavy load. Nothing to do while the amount does not grow.',
+ },
+ noFullBackupYet: {
+ title: 'No full backup yet',
+ whatHappened:
+ 'The first full backup has not finished, so saved changes cannot be restored yet.',
+ whatToDo: 'Wait for it, or start it on the Backups tab.',
+ },
+ },
+ path: {
+ database: 'Database',
+ databasus: 'Databasus',
+ storage: 'Storage',
+ notReceivedYet: 'Not received by Databasus yet',
+ rebuildLimit: 'At {{amount}} Databasus starts a new chain from a full backup',
+ allowanceLeft: 'Room before the database drops changes: {{amount}}',
+ noAllowanceLimit: 'No limit on kept changes',
+ waitingInBuffer: 'Waiting in the buffer',
+ pauseLimit: 'Receiving pauses at {{amount}}',
+ receivingPaused: 'Receiving paused',
+ lastSaved: 'Last change saved {{relativeTime}}',
+ nothingSavedYet: 'Nothing saved yet',
+ overLimit: 'Over the limit',
+ writeRate: 'Writes {{rate}}',
+ saveRate: 'Saves {{rate}}',
+ unknownAmount: 'Unknown',
+ },
+ tooltips: {
+ notReceivedYet:
+ 'Changes the database keeps for Databasus because Databasus has not received them yet. Technical name: replication slot lag.',
+ rebuildLimit:
+ "Past this amount Databasus drops the replication slot and starts a new backup chain, so the database server's disk does not fill up. Technical name: WAL lag threshold.",
+ allowanceLeft:
+ 'How much more the database can write before it deletes changes Databasus has not received. Technical name: safe_wal_size, limited by max_slot_wal_keep_size.',
+ waitingInBuffer:
+ 'Changes Databasus has received but not saved to storage yet. Technical name: local WAL queue.',
+ pauseLimit:
+ 'When the buffer reaches this size, Databasus pauses receiving until it drains. Technical name: back pressure high watermark.',
+ lastSaved:
+ 'When the newest change file reached storage. Technical name: newest archived WAL segment.',
+ writeRate:
+ 'How fast the database writes changes, over the last 10 minutes. Technical name: WAL write rate.',
+ saveRate:
+ 'How fast Databasus saves changes to storage, over the last 10 minutes. Technical name: WAL archive rate.',
+ },
+ charts: {
+ keepingUpTitle: 'Is Databasus keeping up?',
+ unsavedTitle: 'How much is not saved yet?',
+ unsavedSubtitle: 'Not received by Databasus yet is part of it',
+ bufferTitle: 'Databasus buffer',
+ databaseWrites: 'Database writes',
+ savedToStorage: 'Saved to storage',
+ notSavedYet: 'Not saved to storage yet',
+ notReceivedYet: 'Not received by Databasus yet',
+ waitingInBuffer: 'Waiting in the buffer',
+ pauseLimit: 'Receiving pauses here',
+ restarts: 'Restarts',
+ pauses: 'Pauses',
+ noHistory: 'Charts appear after a few minutes of streaming.',
+ dataSince: 'Data since {{time}}',
+ perSecond: '{{amount}}/s',
+ },
+ details: {
+ title: 'Details',
+ slotStatus: 'Slot status (wal_status)',
+ allowance: 'Remaining allowance (safe_wal_size)',
+ noLimit: 'No limit',
+ averageSegmentWrite: 'Average segment write',
+ longestSegmentWrite: 'Longest segment write',
+ uploadConcurrency: 'Uploads at once',
+ walToStoredRatio: 'WAL to stored size ratio',
+ walSegmentSize: 'WAL segment size',
+ sessionStart: 'Streaming session start',
+ streamerState: 'Streamer state',
+ sourceState: 'Source state',
+ restartsLastHour: 'Restarts in the last hour',
+ milliseconds: '{{value}} ms',
+ notAvailable: 'Not available',
+ streamerStates: {
+ running: 'Running',
+ notRunning: 'Not running',
+ notStarted: 'Not started yet',
+ },
+ sourceStates: {
+ reachable: 'Reachable',
+ unreachable: 'Unreachable',
+ slotMissing: 'Reachable, replication slot missing',
+ },
+ },
+ durations: {
+ seconds: '{{seconds}} s',
+ minutes: '{{minutes}} min',
+ hoursMinutes: '{{hours}} h {{minutes}} min',
+ daysHours: '{{days}} d {{hours}} h',
+ },
+ },
},
},
storages: {
diff --git a/frontend/src/shared/i18n/locales/es.ts b/frontend/src/shared/i18n/locales/es.ts
index e9fd21eb..79359566 100644
--- a/frontend/src/shared/i18n/locales/es.ts
+++ b/frontend/src/shared/i18n/locales/es.ts
@@ -316,6 +316,7 @@ export const es: typeof en = {
config: 'Configuración',
backups: 'Copias de seguridad',
verifications: 'Verificaciones',
+ monitoring: 'Monitorización',
},
create: {
title: 'Añadir una base de datos para copias de seguridad',
@@ -1694,6 +1695,258 @@ export const es: typeof en = {
'Una copia base solo copia el directorio de datos. Si su PostgreSQL de origen guarda la configuración fuera de él (la estructura de Debian/Ubuntu, <configDir/>), <code>postgresql.conf</code>, <code>pg_hba.conf</code> y <code>pg_ident.conf</code> no están en la copia y el servidor no arrancará. Consulte sus rutas reales en el origen con <code>psql -Atc "SHOW config_file"</code>, copie los tres archivos, comente <code>data_directory</code>, <code>hba_file</code>, <code>ident_file</code>, <code>external_pid_file</code> y las líneas <code>ssl</code> / <code>ssl_*</code>, cree <code>conf.d</code> y, para un contenedor, establezca <code>listen_addresses = \'*\'</code>. Si esos archivos están y el error persiste, lo que está mal es el directorio montado, no la configuración.',
},
},
+ monitoring: {
+ periods: {
+ hour: 'Última hora',
+ day: 'Último día',
+ week: 'Última semana',
+ },
+ levels: {
+ notStreaming: 'El guardado continuo de cambios está desactivado',
+ collectingData: 'Recopilando datos',
+ broken: 'Algo está roto',
+ needsAttention: 'Algo requiere atención',
+ healthy: 'Todo está en orden',
+ },
+ reasonLevels: {
+ broken: 'Roto',
+ needsAttention: 'Requiere atención',
+ info: 'Para su información',
+ },
+ keepingUpStates: {
+ unknown: 'Desconocido',
+ fallingBehind: 'Se queda atrás',
+ catchingUp: 'Recuperando',
+ idle: 'Sin actividad',
+ keepingUp: 'Al día',
+ },
+ banner: {
+ collectingDataDescription:
+ 'El streaming ha comenzado; las primeras cifras aparecerán en menos de un minuto.',
+ notStreamingDescription:
+ 'Las copias de seguridad están desactivadas o no hay ningún almacenamiento seleccionado. Puede activarlo en la <configLink>configuración</configLink>.',
+ cannotConfirm: 'No se puede confirmar el estado actual',
+ cannotConfirmDescription: 'Último estado conocido: {{level}}, hace {{duration}}.',
+ measuredAgo: 'Medido hace {{duration}}',
+ notMeasuredYet: 'Aún no hay mediciones',
+ refreshFailed:
+ 'No se pudieron actualizar los datos. Se siguen mostrando las últimas cifras recibidas y el próximo intento será en 30 segundos.',
+ },
+ keepingUp: {
+ keepingUp:
+ 'Databasus guarda los cambios tan rápido como la base de datos los escribe: {{writeRate}} escritos, {{archiveRate}} guardados.',
+ fallingBehind:
+ 'La base de datos escribe cambios más rápido de lo que Databasus los guarda, unos {{differenceRate}} más: {{writeRate}} frente a {{archiveRate}}.',
+ fallingBehindBacklog:
+ 'Quedan {{unsavedAmount}} sin guardar, y esa cantidad crece unos {{growthPerHour}} por hora.',
+ catchingUp:
+ 'Databasus está guardando los cambios acumulados: quedan {{unsavedAmount}}, lo que llevará unos {{duration}}.',
+ catchingUpWithoutEstimate:
+ 'Databasus está guardando los cambios acumulados: quedan {{unsavedAmount}}.',
+ idle: 'La base de datos apenas escribe; los cambios nuevos llegan al almacenamiento en pocos minutos.',
+ unknown: 'No hay cifras actuales disponibles.',
+ rebuildConsequence:
+ 'A este ritmo, Databasus empezará una nueva cadena de copias de seguridad con una copia completa en unos {{duration}}.',
+ limitConsequence:
+ 'A este ritmo, la base de datos eliminará cambios que Databasus aún no ha recibido en unos {{duration}}.',
+ },
+ reasons: {
+ streamingStopped: {
+ title: 'El guardado se ha detenido',
+ whatHappened: 'Databasus dejó de recibir cambios hace {{duration}}.',
+ whatToDo:
+ 'Databasus lo vuelve a intentar por sí mismo. Si esto se prolonga, revise los registros de Databasus.',
+ },
+ sourceUnreachable: {
+ title: 'La base de datos no está accesible',
+ whatHappened: 'Databasus lleva {{duration}} sin poder conectarse a la base de datos.',
+ whatToDo:
+ 'Compruebe que el servidor de la base de datos está en marcha y es accesible desde Databasus.',
+ },
+ walChainBroken: {
+ title: 'Se perdieron cambios entre copias de seguridad',
+ whatHappened:
+ 'Se perdieron algunos cambios entre copias de seguridad, por lo que los cambios recientes aún no se pueden restaurar.',
+ whatToDo: 'Inicie una copia de seguridad completa en la pestaña Copias de seguridad.',
+ whatToDoPending:
+ 'Ya se está haciendo una nueva copia de seguridad completa. Puede seguirla en la pestaña Copias de seguridad.',
+ },
+ slotMissing: {
+ title: 'La base de datos dejó de conservar cambios',
+ whatHappened:
+ 'La base de datos ya no conserva cambios para Databasus: la reserva que Databasus creó en ella ha desaparecido.',
+ whatToDo:
+ 'Databasus restablece la reserva cuando el streaming se reinicia, pero los cambios hechos entretanto se pierden. Inicie una copia de seguridad completa en la pestaña Copias de seguridad y compruebe que nadie elimina la reserva en la base de datos.',
+ whatToDoPending:
+ 'Databasus restablece la reserva cuando el streaming se reinicia, y ya se está haciendo una nueva copia de seguridad completa. Compruebe que nadie elimina la reserva en la base de datos.',
+ },
+ slotLost: {
+ title: 'La base de datos eliminó cambios sin guardar',
+ whatHappened: 'La base de datos eliminó cambios que Databasus aún no había recibido.',
+ whatToDo:
+ 'Databasus vuelve a empezar por sí mismo con una nueva copia de seguridad completa. Para evitarlo la próxima vez, acelere el guardado o permita que la base de datos conserve más cambios.',
+ },
+ archiveNotAdvancing: {
+ title: 'Los cambios no llegan al almacenamiento',
+ whatHappened:
+ 'Hace {{duration}} que no llegan cambios nuevos al almacenamiento, aunque la base de datos sigue escribiendo.',
+ whatToDo: 'Compruebe que el almacenamiento es accesible y tiene espacio libre.',
+ },
+ archiveSlowerThanSource: {
+ title: 'El guardado es demasiado lento',
+ whatHappened:
+ 'La base de datos escribe más rápido de lo que Databasus guarda; lo pendiente creció {{amount}} en 10 minutos.',
+ whatToDo:
+ 'Compruebe la velocidad de la red hasta el almacenamiento o acerque el almacenamiento a Databasus.',
+ },
+ slotKeepsExtraWal: {
+ title: 'La base de datos conserva cambios de más',
+ whatHappened:
+ 'Desde hace {{duration}} la base de datos conserva para Databasus más registro de cambios de lo habitual, y la cantidad sigue creciendo.',
+ whatToDo:
+ 'Asegúrese de que el servidor de la base de datos tiene espacio libre en disco.',
+ },
+ slotLimitApproaching: {
+ title: 'Pronto se eliminarán cambios sin guardar',
+ whatHappened:
+ 'A este ritmo, la base de datos eliminará cambios que Databasus aún no ha recibido en unos {{duration}}.',
+ whatToDo:
+ 'Acelere el guardado o aumente <code>max_slot_wal_keep_size</code> en la base de datos.',
+ },
+ nearSlotRebuild: {
+ title: 'A punto de empezar de nuevo',
+ whatHappened:
+ 'Los cambios que Databasus aún no ha recibido alcanzan {{amount}} de los {{limit}} con los que Databasus vuelve a empezar desde una copia completa.',
+ whatToDo:
+ 'Acelere el guardado o aumente el límite en la configuración de copias de seguridad.',
+ },
+ localQueueFilling: {
+ title: 'El búfer de Databasus se está llenando',
+ whatHappened: 'El búfer de Databasus contiene {{amount}} de {{limit}}.',
+ whatToDo:
+ 'Databasus pausa la recepción por sí mismo cuando el búfer se llena. Compruebe la velocidad del almacenamiento.',
+ },
+ frequentRestarts: {
+ title: 'La recepción se reinicia a menudo',
+ whatHappened: 'Reinicios de la recepción en la última hora: {{count}}.',
+ whatToDo: 'Busque la causa en los registros de Databasus.',
+ },
+ forcedRotationRefused: {
+ title: 'Los periodos tranquilos llegan tarde al almacenamiento',
+ whatHappened:
+ 'La base de datos se niega a cerrar a tiempo su archivo de cambios, así que los cambios de los periodos tranquilos llegan más tarde al almacenamiento.',
+ whatToDo:
+ 'Conceda al usuario de las copias de seguridad permiso para ejecutar <code>pg_switch_wal()</code>.',
+ },
+ },
+ notices: {
+ noSlotWalLimit: {
+ title: 'Sin límite para los cambios conservados',
+ whatHappened: 'La base de datos conserva cambios para Databasus sin límite.',
+ whatToDo:
+ 'Un límite protege el disco del servidor de la base de datos si Databasus se detiene durante mucho tiempo. A cambio, al alcanzarlo se necesita una nueva copia de seguridad completa. Se configura con <code>max_slot_wal_keep_size</code>.',
+ },
+ slotKeepsExtraWalSteadily: {
+ title: 'La base de datos conserva cambios de más',
+ whatHappened:
+ 'La base de datos conserva para Databasus más registro de cambios de lo habitual.',
+ whatToDo:
+ 'Es normal con carga alta. No hace falta hacer nada mientras la cantidad no crezca.',
+ },
+ noFullBackupYet: {
+ title: 'Aún no hay copia de seguridad completa',
+ whatHappened:
+ 'La primera copia de seguridad completa aún no ha terminado, así que los cambios guardados todavía no se pueden restaurar.',
+ whatToDo: 'Espere a que termine o iníciela en la pestaña Copias de seguridad.',
+ },
+ },
+ path: {
+ database: 'Base de datos',
+ databasus: 'Databasus',
+ storage: 'Almacenamiento',
+ notReceivedYet: 'Aún no recibido por Databasus',
+ rebuildLimit: 'Con {{amount}}, Databasus empieza una nueva cadena con una copia completa',
+ allowanceLeft: 'Margen antes de que la base de datos elimine cambios: {{amount}}',
+ noAllowanceLimit: 'Sin límite para los cambios conservados',
+ waitingInBuffer: 'En espera en el búfer',
+ pauseLimit: 'La recepción se pausa con {{amount}}',
+ receivingPaused: 'Recepción en pausa',
+ lastSaved: 'Último cambio guardado {{relativeTime}}',
+ nothingSavedYet: 'Aún no se ha guardado nada',
+ overLimit: 'Por encima del límite',
+ writeRate: 'Escribe {{rate}}',
+ saveRate: 'Guarda {{rate}}',
+ unknownAmount: 'Desconocido',
+ },
+ tooltips: {
+ notReceivedYet:
+ 'Cambios que la base de datos conserva para Databasus porque Databasus aún no los ha recibido. Nombre técnico: retraso del slot de replicación.',
+ rebuildLimit:
+ 'Al superar esta cantidad, Databasus elimina el slot de replicación y empieza una nueva cadena de copias de seguridad para que no se llene el disco del servidor de la base de datos. Nombre técnico: umbral de retraso de WAL.',
+ allowanceLeft:
+ 'Cuánto más puede escribir la base de datos antes de eliminar cambios que Databasus no ha recibido. Nombre técnico: safe_wal_size, limitado por max_slot_wal_keep_size.',
+ waitingInBuffer:
+ 'Cambios que Databasus ha recibido pero aún no ha guardado en el almacenamiento. Nombre técnico: cola local de WAL.',
+ pauseLimit:
+ 'Cuando el búfer alcanza este tamaño, Databasus pausa la recepción hasta que se vacía. Nombre técnico: límite superior de contrapresión.',
+ lastSaved:
+ 'Cuándo llegó al almacenamiento el archivo de cambios más reciente. Nombre técnico: último segmento de WAL archivado.',
+ writeRate:
+ 'A qué velocidad escribe cambios la base de datos, de media en los últimos 10 minutos. Nombre técnico: velocidad de escritura de WAL.',
+ saveRate:
+ 'A qué velocidad guarda Databasus los cambios en el almacenamiento, de media en los últimos 10 minutos. Nombre técnico: velocidad de archivado de WAL.',
+ },
+ charts: {
+ keepingUpTitle: '¿Databasus va al día?',
+ unsavedTitle: '¿Cuánto queda sin guardar?',
+ unsavedSubtitle: 'Incluye lo que Databasus aún no ha recibido',
+ bufferTitle: 'Búfer de Databasus',
+ databaseWrites: 'Escritura de la base de datos',
+ savedToStorage: 'Guardado en el almacenamiento',
+ notSavedYet: 'Aún no en el almacenamiento',
+ notReceivedYet: 'Aún no recibido por Databasus',
+ waitingInBuffer: 'En espera en el búfer',
+ pauseLimit: 'Aquí se pausa la recepción',
+ restarts: 'Reinicios',
+ pauses: 'Pausas',
+ noHistory: 'Los gráficos aparecerán tras unos minutos de streaming.',
+ dataSince: 'Datos desde {{time}}',
+ perSecond: '{{amount}}/s',
+ },
+ details: {
+ title: 'Detalles',
+ slotStatus: 'Estado del slot (wal_status)',
+ allowance: 'Margen restante (safe_wal_size)',
+ noLimit: 'Sin límite',
+ averageSegmentWrite: 'Escritura media de un segmento',
+ longestSegmentWrite: 'Escritura más larga de un segmento',
+ uploadConcurrency: 'Subidas simultáneas',
+ walToStoredRatio: 'Relación entre WAL y tamaño guardado',
+ walSegmentSize: 'Tamaño del segmento de WAL',
+ sessionStart: 'Inicio de la sesión de streaming',
+ streamerState: 'Estado del streamer',
+ sourceState: 'Estado del origen',
+ restartsLastHour: 'Reinicios en la última hora',
+ milliseconds: '{{value}} ms',
+ notAvailable: 'No disponible',
+ streamerStates: {
+ running: 'En marcha',
+ notRunning: 'Detenido',
+ notStarted: 'Aún no iniciado',
+ },
+ sourceStates: {
+ reachable: 'Accesible',
+ unreachable: 'No accesible',
+ slotMissing: 'Accesible, sin slot de replicación',
+ },
+ },
+ durations: {
+ seconds: '{{seconds}} s',
+ minutes: '{{minutes}} min',
+ hoursMinutes: '{{hours}} h {{minutes}} min',
+ daysHours: '{{days}} d {{hours}} h',
+ },
+ },
},
},
storages: {
diff --git a/frontend/src/shared/i18n/locales/fr.ts b/frontend/src/shared/i18n/locales/fr.ts
index b2f0b745..a6b65dfc 100644
--- a/frontend/src/shared/i18n/locales/fr.ts
+++ b/frontend/src/shared/i18n/locales/fr.ts
@@ -316,6 +316,7 @@ export const fr: typeof en = {
config: 'Configuration',
backups: 'Sauvegardes',
verifications: 'Vérifications',
+ monitoring: 'Surveillance',
},
create: {
title: 'Ajouter une base de données à sauvegarder',
@@ -1704,6 +1705,264 @@ export const fr: typeof en = {
"Une sauvegarde de base ne copie que le répertoire de données. Si votre PostgreSQL source garde sa configuration en dehors (disposition Debian/Ubuntu, <configDir/>), alors <code>postgresql.conf</code>, <code>pg_hba.conf</code> et <code>pg_ident.conf</code> ne sont pas dans la sauvegarde et le serveur ne démarrera pas. Demandez leurs chemins réels à la source avec <code>psql -Atc \"SHOW config_file\"</code>, copiez les trois fichiers, puis commentez <code>data_directory</code>, <code>hba_file</code>, <code>ident_file</code>, <code>external_pid_file</code> et les lignes <code>ssl</code> / <code>ssl_*</code>, créez <code>conf.d</code> et, pour un conteneur, définissez <code>listen_addresses = '*'</code>. Si ces fichiers sont présents et que vous obtenez toujours la même erreur, c'est le répertoire monté qui est en cause, pas la configuration.",
},
},
+ monitoring: {
+ periods: {
+ hour: 'Dernière heure',
+ day: 'Dernier jour',
+ week: 'Dernière semaine',
+ },
+ levels: {
+ notStreaming: "L'enregistrement continu des modifications est désactivé",
+ collectingData: 'Collecte des données',
+ broken: 'Quelque chose ne fonctionne plus',
+ needsAttention: 'Quelque chose demande votre attention',
+ healthy: 'Tout va bien',
+ },
+ reasonLevels: {
+ broken: 'En panne',
+ needsAttention: 'Demande votre attention',
+ info: 'Pour information',
+ },
+ keepingUpStates: {
+ unknown: 'Inconnu',
+ fallingBehind: 'Prend du retard',
+ catchingUp: 'Rattrape son retard',
+ idle: 'Calme',
+ keepingUp: 'Suit le rythme',
+ },
+ banner: {
+ collectingDataDescription:
+ "Le streaming a démarré ; les premiers chiffres arrivent d'ici une minute.",
+ notStreamingDescription:
+ "Les sauvegardes sont désactivées ou aucun stockage n'est sélectionné. Vous pouvez l'activer dans la <configLink>configuration</configLink>.",
+ cannotConfirm: "Impossible de confirmer l'état actuel",
+ cannotConfirmDescription: 'Dernier état connu : {{level}}, il y a {{duration}}.',
+ measuredAgo: 'Mesuré il y a {{duration}}',
+ notMeasuredYet: 'Pas encore de mesure',
+ refreshFailed:
+ "Impossible d'actualiser les données. Les derniers chiffres reçus restent affichés, et la prochaine tentative a lieu dans 30 secondes.",
+ },
+ keepingUp: {
+ keepingUp:
+ 'Databasus enregistre les modifications aussi vite que la base de données les écrit : {{writeRate}} écrits, {{archiveRate}} enregistrés.',
+ fallingBehind:
+ "La base de données écrit des modifications plus vite que Databasus ne les enregistre, d'environ {{differenceRate}} : {{writeRate}} contre {{archiveRate}}.",
+ fallingBehindBacklog:
+ "{{unsavedAmount}} ne sont pas encore enregistrés, et ce volume augmente d'environ {{growthPerHour}} par heure.",
+ catchingUp:
+ 'Databasus enregistre le retard accumulé : il reste {{unsavedAmount}}, soit environ {{duration}}.',
+ catchingUpWithoutEstimate:
+ 'Databasus enregistre le retard accumulé : il reste {{unsavedAmount}}.',
+ idle: "La base de données n'écrit presque rien ; les nouvelles modifications arrivent dans le stockage en quelques minutes.",
+ unknown: "Aucun chiffre actuel n'est disponible.",
+ rebuildConsequence:
+ "À ce rythme, Databasus commencera une nouvelle chaîne de sauvegardes à partir d'une sauvegarde complète dans environ {{duration}}.",
+ limitConsequence:
+ "À ce rythme, la base de données supprimera des modifications que Databasus n'a pas encore reçues dans environ {{duration}}.",
+ },
+ reasons: {
+ streamingStopped: {
+ title: "L'enregistrement s'est arrêté",
+ whatHappened: 'Databasus a cessé de recevoir des modifications il y a {{duration}}.',
+ whatToDo:
+ 'Databasus réessaie de lui-même. Si cela dure, consultez les journaux de Databasus.',
+ },
+ sourceUnreachable: {
+ title: 'La base de données est injoignable',
+ whatHappened:
+ "Databasus n'arrive pas à joindre la base de données depuis {{duration}}.",
+ whatToDo:
+ "Vérifiez que le serveur de la base de données fonctionne et qu'il est joignable depuis Databasus.",
+ },
+ walChainBroken: {
+ title: 'Des modifications ont été perdues entre les sauvegardes',
+ whatHappened:
+ 'Des modifications ont été perdues entre les sauvegardes, les modifications récentes ne peuvent donc pas encore être restaurées.',
+ whatToDo: "Lancez une sauvegarde complète dans l'onglet Sauvegardes.",
+ whatToDoPending:
+ "Une nouvelle sauvegarde complète est en cours. Vous pouvez la suivre dans l'onglet Sauvegardes.",
+ },
+ slotMissing: {
+ title: 'La base de données ne conserve plus les modifications',
+ whatHappened:
+ 'La base de données ne conserve plus de modifications pour Databasus : la réservation que Databasus y avait créée a disparu.',
+ whatToDo:
+ "Databasus recrée la réservation au redémarrage du streaming, mais les modifications faites entre-temps sont perdues. Lancez une sauvegarde complète dans l'onglet Sauvegardes et vérifiez que personne ne supprime la réservation sur la base de données.",
+ whatToDoPending:
+ 'Databasus recrée la réservation au redémarrage du streaming, et une nouvelle sauvegarde complète est en cours. Vérifiez que personne ne supprime la réservation sur la base de données.',
+ },
+ slotLost: {
+ title: 'La base de données a supprimé des modifications non enregistrées',
+ whatHappened:
+ "La base de données a supprimé des modifications que Databasus n'avait pas encore reçues.",
+ whatToDo:
+ "Databasus repart de lui-même avec une nouvelle sauvegarde complète. Pour l'éviter la prochaine fois, accélérez l'enregistrement ou laissez la base de données conserver plus de modifications.",
+ },
+ archiveNotAdvancing: {
+ title: "Les modifications n'arrivent pas dans le stockage",
+ whatHappened:
+ "Aucune nouvelle modification n'est arrivée dans le stockage depuis {{duration}}, alors que la base de données continue d'écrire.",
+ whatToDo:
+ "Vérifiez que le stockage est joignable et qu'il lui reste de l'espace libre.",
+ },
+ archiveSlowerThanSource: {
+ title: "L'enregistrement est trop lent",
+ whatHappened:
+ "La base de données écrit plus vite que Databasus n'enregistre ; le volume en attente a augmenté de {{amount}} en 10 minutes.",
+ whatToDo:
+ 'Vérifiez le débit réseau vers le stockage, ou rapprochez le stockage de Databasus.',
+ },
+ slotKeepsExtraWal: {
+ title: 'La base de données conserve trop de modifications',
+ whatHappened:
+ "Depuis {{duration}}, la base de données conserve pour Databasus plus de journal de modifications que d'habitude, et le volume continue d'augmenter.",
+ whatToDo:
+ "Vérifiez qu'il reste de l'espace disque sur le serveur de la base de données.",
+ },
+ slotLimitApproaching: {
+ title: 'Des modifications non enregistrées seront bientôt supprimées',
+ whatHappened:
+ "À ce rythme, la base de données supprimera des modifications que Databasus n'a pas encore reçues dans environ {{duration}}.",
+ whatToDo:
+ "Accélérez l'enregistrement, ou augmentez <code>max_slot_wal_keep_size</code> sur la base de données.",
+ },
+ nearSlotRebuild: {
+ title: 'Bientôt un nouveau départ',
+ whatHappened:
+ "Les modifications pas encore reçues par Databasus atteignent {{amount}} sur les {{limit}} à partir desquels Databasus repart d'une sauvegarde complète.",
+ whatToDo:
+ "Accélérez l'enregistrement, ou relevez la limite dans les paramètres de sauvegarde.",
+ },
+ localQueueFilling: {
+ title: 'Le tampon de Databasus se remplit',
+ whatHappened: 'Le tampon de Databasus contient {{amount}} sur {{limit}}.',
+ whatToDo:
+ 'Databasus suspend la réception de lui-même quand le tampon est plein. Vérifiez le débit du stockage.',
+ },
+ frequentRestarts: {
+ title: 'La réception redémarre souvent',
+ whatHappened: 'Redémarrages de la réception au cours de la dernière heure : {{count}}.',
+ whatToDo: 'Cherchez la cause dans les journaux de Databasus.',
+ },
+ forcedRotationRefused: {
+ title: 'Les périodes calmes arrivent en retard dans le stockage',
+ whatHappened:
+ 'La base de données refuse de fermer son fichier de modifications à temps, donc les modifications des périodes calmes arrivent plus tard dans le stockage.',
+ whatToDo:
+ "Accordez à l'utilisateur des sauvegardes le droit d'exécuter <code>pg_switch_wal()</code>.",
+ },
+ },
+ notices: {
+ noSlotWalLimit: {
+ title: 'Aucune limite sur les modifications conservées',
+ whatHappened:
+ 'La base de données conserve des modifications pour Databasus sans limite.',
+ whatToDo:
+ 'Une limite protège le disque du serveur de la base de données si Databasus reste arrêté longtemps. En contrepartie, une nouvelle sauvegarde complète est nécessaire quand la limite est atteinte. Elle se règle avec <code>max_slot_wal_keep_size</code>.',
+ },
+ slotKeepsExtraWalSteadily: {
+ title: 'La base de données conserve trop de modifications',
+ whatHappened:
+ "La base de données conserve pour Databasus plus de journal de modifications que d'habitude.",
+ whatToDo:
+ "C'est normal sous forte charge. Rien à faire tant que le volume n'augmente pas.",
+ },
+ noFullBackupYet: {
+ title: 'Pas encore de sauvegarde complète',
+ whatHappened:
+ "La première sauvegarde complète n'est pas terminée, les modifications enregistrées ne peuvent donc pas encore être restaurées.",
+ whatToDo: "Attendez-la, ou lancez-la dans l'onglet Sauvegardes.",
+ },
+ },
+ path: {
+ database: 'Base de données',
+ databasus: 'Databasus',
+ storage: 'Stockage',
+ notReceivedYet: 'Pas encore reçu par Databasus',
+ rebuildLimit:
+ "À {{amount}}, Databasus commence une nouvelle chaîne à partir d'une sauvegarde complète",
+ allowanceLeft:
+ 'Marge avant que la base de données ne supprime des modifications : {{amount}}',
+ noAllowanceLimit: 'Aucune limite sur les modifications conservées',
+ waitingInBuffer: 'En attente dans le tampon',
+ pauseLimit: 'La réception est suspendue à {{amount}}',
+ receivingPaused: 'Réception suspendue',
+ lastSaved: 'Dernière modification enregistrée {{relativeTime}}',
+ nothingSavedYet: "Rien n'est encore enregistré",
+ overLimit: 'Au-delà de la limite',
+ writeRate: 'Écrit {{rate}}',
+ saveRate: 'Enregistre {{rate}}',
+ unknownAmount: 'Inconnu',
+ },
+ tooltips: {
+ notReceivedYet:
+ 'Modifications que la base de données conserve pour Databasus parce que Databasus ne les a pas encore reçues. Nom technique : retard du slot de réplication.',
+ rebuildLimit:
+ 'Au-delà de ce volume, Databasus supprime le slot de réplication et commence une nouvelle chaîne de sauvegardes, pour que le disque du serveur de la base de données ne se remplisse pas. Nom technique : seuil de retard du WAL.',
+ allowanceLeft:
+ "Ce que la base de données peut encore écrire avant de supprimer des modifications que Databasus n'a pas reçues. Nom technique : safe_wal_size, limité par max_slot_wal_keep_size.",
+ waitingInBuffer:
+ 'Modifications reçues par Databasus mais pas encore enregistrées dans le stockage. Nom technique : file locale du WAL.',
+ pauseLimit:
+ "Quand le tampon atteint cette taille, Databasus suspend la réception jusqu'à ce qu'il se vide. Nom technique : seuil haut de la contre-pression.",
+ lastSaved:
+ 'Quand le fichier de modifications le plus récent est arrivé dans le stockage. Nom technique : dernier segment WAL archivé.',
+ writeRate:
+ "La vitesse à laquelle la base de données écrit des modifications, en moyenne sur les 10 dernières minutes. Nom technique : débit d'écriture du WAL.",
+ saveRate:
+ "La vitesse à laquelle Databasus enregistre les modifications dans le stockage, en moyenne sur les 10 dernières minutes. Nom technique : débit d'archivage du WAL.",
+ },
+ charts: {
+ keepingUpTitle: 'Databasus suit-il le rythme ?',
+ unsavedTitle: "Qu'est-ce qui n'est pas encore enregistré ?",
+ unsavedSubtitle: "Inclut ce que Databasus n'a pas encore reçu",
+ bufferTitle: 'Tampon de Databasus',
+ databaseWrites: 'Écriture de la base de données',
+ savedToStorage: 'Enregistré dans le stockage',
+ notSavedYet: 'Pas encore dans le stockage',
+ notReceivedYet: 'Pas encore reçu par Databasus',
+ waitingInBuffer: 'En attente dans le tampon',
+ pauseLimit: 'La réception est suspendue ici',
+ restarts: 'Redémarrages',
+ pauses: 'Pauses',
+ noHistory: 'Les graphiques apparaissent après quelques minutes de streaming.',
+ dataSince: 'Données depuis {{time}}',
+ perSecond: '{{amount}}/s',
+ },
+ details: {
+ title: 'Détails',
+ slotStatus: 'Statut du slot (wal_status)',
+ allowance: 'Marge restante (safe_wal_size)',
+ noLimit: 'Aucune limite',
+ averageSegmentWrite: "Écriture moyenne d'un segment",
+ longestSegmentWrite: "Écriture la plus longue d'un segment",
+ uploadConcurrency: 'Envois simultanés',
+ walToStoredRatio: 'Rapport entre WAL et taille enregistrée',
+ walSegmentSize: 'Taille du segment WAL',
+ sessionStart: 'Début de la session de streaming',
+ streamerState: 'État du streamer',
+ sourceState: 'État de la source',
+ restartsLastHour: 'Redémarrages au cours de la dernière heure',
+ milliseconds: '{{value}} ms',
+ notAvailable: 'Non disponible',
+ streamerStates: {
+ running: 'En marche',
+ notRunning: 'Arrêté',
+ notStarted: 'Pas encore démarré',
+ },
+ sourceStates: {
+ reachable: 'Joignable',
+ unreachable: 'Injoignable',
+ slotMissing: 'Joignable, slot de réplication absent',
+ },
+ },
+ durations: {
+ seconds: '{{seconds}} s',
+ minutes: '{{minutes}} min',
+ hoursMinutes: '{{hours}} h {{minutes}} min',
+ daysHours: '{{days}} j {{hours}} h',
+ },
+ },
},
},
storages: {
diff --git a/frontend/src/shared/i18n/locales/pt.ts b/frontend/src/shared/i18n/locales/pt.ts
index f4850e40..a2114688 100644
--- a/frontend/src/shared/i18n/locales/pt.ts
+++ b/frontend/src/shared/i18n/locales/pt.ts
@@ -315,6 +315,7 @@ export const pt: typeof en = {
config: 'Configuração',
backups: 'Backups',
verifications: 'Verificações',
+ monitoring: 'Monitoramento',
},
create: {
title: 'Adicionar banco de dados para backup',
@@ -1687,6 +1688,258 @@ export const pt: typeof en = {
'Um backup base copia só o diretório de dados. Se o PostgreSQL de origem guarda a configuração fora dele (é o layout do Debian/Ubuntu, <configDir/>), então <code>postgresql.conf</code>, <code>pg_hba.conf</code> e <code>pg_ident.conf</code> não estão no backup e o servidor não vai iniciar. Consulte os caminhos reais na origem com <code>psql -Atc "SHOW config_file"</code>, copie os três arquivos e depois comente <code>data_directory</code>, <code>hba_file</code>, <code>ident_file</code>, <code>external_pid_file</code> e as linhas <code>ssl</code> / <code>ssl_*</code>, crie <code>conf.d</code> e defina <code>listen_addresses = \'*\'</code> para um container. Se esses arquivos estão presentes e o mesmo erro continua, o problema é o diretório montado, não a configuração.',
},
},
+ monitoring: {
+ periods: {
+ hour: 'Última hora',
+ day: 'Último dia',
+ week: 'Última semana',
+ },
+ levels: {
+ notStreaming: 'O salvamento contínuo de alterações está desligado',
+ collectingData: 'Coletando dados',
+ broken: 'Algo está quebrado',
+ needsAttention: 'Algo precisa de atenção',
+ healthy: 'Tudo em ordem',
+ },
+ reasonLevels: {
+ broken: 'Quebrado',
+ needsAttention: 'Precisa de atenção',
+ info: 'Para sua informação',
+ },
+ keepingUpStates: {
+ unknown: 'Desconhecido',
+ fallingBehind: 'Ficando para trás',
+ catchingUp: 'Recuperando o atraso',
+ idle: 'Sem atividade',
+ keepingUp: 'Em dia',
+ },
+ banner: {
+ collectingDataDescription:
+ 'O streaming começou; os primeiros números aparecem em menos de um minuto.',
+ notStreamingDescription:
+ 'Os backups estão desativados ou nenhum armazenamento foi selecionado. Você pode ativá-lo na <configLink>configuração</configLink>.',
+ cannotConfirm: 'Não é possível confirmar o estado atual',
+ cannotConfirmDescription: 'Último estado conhecido: {{level}}, há {{duration}}.',
+ measuredAgo: 'Medido há {{duration}}',
+ notMeasuredYet: 'Ainda sem medições',
+ refreshFailed:
+ 'Não foi possível atualizar os dados. Os últimos números recebidos continuam na tela, e a próxima tentativa acontece em 30 segundos.',
+ },
+ keepingUp: {
+ keepingUp:
+ 'O Databasus salva as alterações tão rápido quanto o banco de dados as grava: {{writeRate}} gravados, {{archiveRate}} salvos.',
+ fallingBehind:
+ 'O banco de dados grava alterações mais rápido do que o Databasus consegue salvá-las, cerca de {{differenceRate}} a mais: {{writeRate}} contra {{archiveRate}}.',
+ fallingBehindBacklog:
+ 'Ainda faltam salvar {{unsavedAmount}}, e esse volume cresce cerca de {{growthPerHour}} por hora.',
+ catchingUp:
+ 'O Databasus está salvando as alterações acumuladas: faltam {{unsavedAmount}}, o que leva cerca de {{duration}}.',
+ catchingUpWithoutEstimate:
+ 'O Databasus está salvando as alterações acumuladas: faltam {{unsavedAmount}}.',
+ idle: 'O banco de dados quase não grava nada; as novas alterações chegam ao armazenamento em poucos minutos.',
+ unknown: 'Não há números atuais disponíveis.',
+ rebuildConsequence:
+ 'Nesse ritmo, o Databasus vai começar uma nova cadeia de backups a partir de um backup completo em cerca de {{duration}}.',
+ limitConsequence:
+ 'Nesse ritmo, o banco de dados vai apagar alterações que o Databasus ainda não recebeu em cerca de {{duration}}.',
+ },
+ reasons: {
+ streamingStopped: {
+ title: 'O salvamento parou',
+ whatHappened: 'O Databasus parou de receber alterações há {{duration}}.',
+ whatToDo:
+ 'O Databasus tenta de novo sozinho. Se isso continuar, verifique os logs do Databasus.',
+ },
+ sourceUnreachable: {
+ title: 'O banco de dados está inacessível',
+ whatHappened: 'O Databasus não consegue se conectar ao banco de dados há {{duration}}.',
+ whatToDo:
+ 'Verifique se o servidor do banco de dados está em execução e acessível a partir do Databasus.',
+ },
+ walChainBroken: {
+ title: 'Alterações se perderam entre os backups',
+ whatHappened:
+ 'Algumas alterações se perderam entre os backups, por isso as alterações recentes ainda não podem ser restauradas.',
+ whatToDo: 'Inicie um backup completo na aba Backups.',
+ whatToDoPending:
+ 'Um novo backup completo já está a caminho. Você pode acompanhá-lo na aba Backups.',
+ },
+ slotMissing: {
+ title: 'O banco de dados parou de guardar alterações',
+ whatHappened:
+ 'O banco de dados não guarda mais alterações para o Databasus: a reserva que o Databasus criou nele sumiu.',
+ whatToDo:
+ 'O Databasus recria a reserva quando o streaming reinicia, mas as alterações feitas nesse meio-tempo se perdem. Inicie um backup completo na aba Backups e verifique se ninguém remove a reserva no banco de dados.',
+ whatToDoPending:
+ 'O Databasus recria a reserva quando o streaming reinicia, e um novo backup completo já está a caminho. Verifique se ninguém remove a reserva no banco de dados.',
+ },
+ slotLost: {
+ title: 'O banco de dados apagou alterações não salvas',
+ whatHappened:
+ 'O banco de dados apagou alterações que o Databasus ainda não tinha recebido.',
+ whatToDo:
+ 'O Databasus recomeça sozinho com um novo backup completo. Para evitar isso da próxima vez, acelere o salvamento ou deixe o banco de dados guardar mais alterações.',
+ },
+ archiveNotAdvancing: {
+ title: 'As alterações não chegam ao armazenamento',
+ whatHappened:
+ 'Há {{duration}} nenhuma alteração nova chega ao armazenamento, embora o banco de dados continue gravando.',
+ whatToDo: 'Verifique se o armazenamento está acessível e tem espaço livre.',
+ },
+ archiveSlowerThanSource: {
+ title: 'O salvamento está lento demais',
+ whatHappened:
+ 'O banco de dados grava mais rápido do que o Databasus salva; o volume pendente cresceu {{amount}} em 10 minutos.',
+ whatToDo:
+ 'Verifique a velocidade da rede até o armazenamento ou aproxime o armazenamento do Databasus.',
+ },
+ slotKeepsExtraWal: {
+ title: 'O banco de dados guarda alterações a mais',
+ whatHappened:
+ 'Há {{duration}} o banco de dados guarda para o Databasus mais registro de alterações do que o normal, e o volume continua crescendo.',
+ whatToDo: 'Verifique se o servidor do banco de dados tem espaço livre em disco.',
+ },
+ slotLimitApproaching: {
+ title: 'Alterações não salvas serão apagadas em breve',
+ whatHappened:
+ 'Nesse ritmo, o banco de dados vai apagar alterações que o Databasus ainda não recebeu em cerca de {{duration}}.',
+ whatToDo:
+ 'Acelere o salvamento ou aumente <code>max_slot_wal_keep_size</code> no banco de dados.',
+ },
+ nearSlotRebuild: {
+ title: 'Perto de recomeçar',
+ whatHappened:
+ 'As alterações ainda não recebidas pelo Databasus chegaram a {{amount}} dos {{limit}} em que o Databasus recomeça a partir de um backup completo.',
+ whatToDo: 'Acelere o salvamento ou aumente o limite nas configurações de backup.',
+ },
+ localQueueFilling: {
+ title: 'O buffer do Databasus está enchendo',
+ whatHappened: 'O buffer do Databasus tem {{amount}} de {{limit}}.',
+ whatToDo:
+ 'O Databasus pausa o recebimento sozinho quando o buffer enche. Verifique a velocidade do armazenamento.',
+ },
+ frequentRestarts: {
+ title: 'O recebimento reinicia com frequência',
+ whatHappened: 'Reinícios do recebimento na última hora: {{count}}.',
+ whatToDo: 'Procure a causa nos logs do Databasus.',
+ },
+ forcedRotationRefused: {
+ title: 'Períodos tranquilos chegam atrasados ao armazenamento',
+ whatHappened:
+ 'O banco de dados se recusa a fechar o arquivo de alterações na hora certa, por isso as alterações de períodos tranquilos chegam mais tarde ao armazenamento.',
+ whatToDo:
+ 'Dê ao usuário de backup permissão para executar <code>pg_switch_wal()</code>.',
+ },
+ },
+ notices: {
+ noSlotWalLimit: {
+ title: 'Sem limite para alterações guardadas',
+ whatHappened: 'O banco de dados guarda alterações para o Databasus sem limite.',
+ whatToDo:
+ 'Um limite protege o disco do servidor do banco de dados se o Databasus ficar parado por muito tempo. O custo é um novo backup completo quando o limite é atingido. Ele é definido por <code>max_slot_wal_keep_size</code>.',
+ },
+ slotKeepsExtraWalSteadily: {
+ title: 'O banco de dados guarda alterações a mais',
+ whatHappened:
+ 'O banco de dados guarda para o Databasus mais registro de alterações do que o normal.',
+ whatToDo:
+ 'Com carga alta isso é normal. Não é preciso fazer nada enquanto o volume não cresce.',
+ },
+ noFullBackupYet: {
+ title: 'Ainda não há backup completo',
+ whatHappened:
+ 'O primeiro backup completo ainda não terminou, por isso as alterações salvas ainda não podem ser restauradas.',
+ whatToDo: 'Aguarde ou inicie-o na aba Backups.',
+ },
+ },
+ path: {
+ database: 'Banco de dados',
+ databasus: 'Databasus',
+ storage: 'Armazenamento',
+ notReceivedYet: 'Ainda não recebido pelo Databasus',
+ rebuildLimit:
+ 'Com {{amount}}, o Databasus começa uma nova cadeia a partir de um backup completo',
+ allowanceLeft: 'Margem até o banco de dados apagar alterações: {{amount}}',
+ noAllowanceLimit: 'Sem limite para alterações guardadas',
+ waitingInBuffer: 'Aguardando no buffer',
+ pauseLimit: 'O recebimento pausa com {{amount}}',
+ receivingPaused: 'Recebimento pausado',
+ lastSaved: 'Última alteração salva {{relativeTime}}',
+ nothingSavedYet: 'Nada salvo ainda',
+ overLimit: 'Acima do limite',
+ writeRate: 'Grava {{rate}}',
+ saveRate: 'Salva {{rate}}',
+ unknownAmount: 'Desconhecido',
+ },
+ tooltips: {
+ notReceivedYet:
+ 'Alterações que o banco de dados guarda para o Databasus porque o Databasus ainda não as recebeu. Nome técnico: atraso do slot de replicação.',
+ rebuildLimit:
+ 'Acima desse volume, o Databasus remove o slot de replicação e começa uma nova cadeia de backups, para que o disco do servidor do banco de dados não encha. Nome técnico: limite de atraso do WAL.',
+ allowanceLeft:
+ 'Quanto o banco de dados ainda pode gravar antes de apagar alterações que o Databasus não recebeu. Nome técnico: safe_wal_size, limitado por max_slot_wal_keep_size.',
+ waitingInBuffer:
+ 'Alterações que o Databasus recebeu, mas ainda não salvou no armazenamento. Nome técnico: fila local de WAL.',
+ pauseLimit:
+ 'Quando o buffer chega a esse tamanho, o Databasus pausa o recebimento até ele esvaziar. Nome técnico: limite superior de contrapressão.',
+ lastSaved:
+ 'Quando o arquivo de alterações mais recente chegou ao armazenamento. Nome técnico: último segmento de WAL arquivado.',
+ writeRate:
+ 'A velocidade com que o banco de dados grava alterações, em média nos últimos 10 minutos. Nome técnico: taxa de gravação do WAL.',
+ saveRate:
+ 'A velocidade com que o Databasus salva alterações no armazenamento, em média nos últimos 10 minutos. Nome técnico: taxa de arquivamento do WAL.',
+ },
+ charts: {
+ keepingUpTitle: 'O Databasus está dando conta?',
+ unsavedTitle: 'Quanto ainda não foi salvo?',
+ unsavedSubtitle: 'Inclui o que o Databasus ainda não recebeu',
+ bufferTitle: 'Buffer do Databasus',
+ databaseWrites: 'Gravação do banco de dados',
+ savedToStorage: 'Salvo no armazenamento',
+ notSavedYet: 'Ainda não no armazenamento',
+ notReceivedYet: 'Ainda não recebido pelo Databasus',
+ waitingInBuffer: 'Aguardando no buffer',
+ pauseLimit: 'Aqui o recebimento pausa',
+ restarts: 'Reinícios',
+ pauses: 'Pausas',
+ noHistory: 'Os gráficos aparecem após alguns minutos de streaming.',
+ dataSince: 'Dados desde {{time}}',
+ perSecond: '{{amount}}/s',
+ },
+ details: {
+ title: 'Detalhes',
+ slotStatus: 'Status do slot (wal_status)',
+ allowance: 'Margem restante (safe_wal_size)',
+ noLimit: 'Sem limite',
+ averageSegmentWrite: 'Gravação média de um segmento',
+ longestSegmentWrite: 'Gravação mais longa de um segmento',
+ uploadConcurrency: 'Envios simultâneos',
+ walToStoredRatio: 'Proporção entre WAL e tamanho salvo',
+ walSegmentSize: 'Tamanho do segmento de WAL',
+ sessionStart: 'Início da sessão de streaming',
+ streamerState: 'Estado do streamer',
+ sourceState: 'Estado da origem',
+ restartsLastHour: 'Reinícios na última hora',
+ milliseconds: '{{value}} ms',
+ notAvailable: 'Indisponível',
+ streamerStates: {
+ running: 'Em execução',
+ notRunning: 'Parado',
+ notStarted: 'Ainda não iniciado',
+ },
+ sourceStates: {
+ reachable: 'Acessível',
+ unreachable: 'Inacessível',
+ slotMissing: 'Acessível, sem slot de replicação',
+ },
+ },
+ durations: {
+ seconds: '{{seconds}} s',
+ minutes: '{{minutes}} min',
+ hoursMinutes: '{{hours}} h {{minutes}} min',
+ daysHours: '{{days}} d {{hours}} h',
+ },
+ },
},
},
storages: {
diff --git a/frontend/src/shared/i18n/locales/ru.ts b/frontend/src/shared/i18n/locales/ru.ts
index 2ef3e63c..94a2a9d4 100644
--- a/frontend/src/shared/i18n/locales/ru.ts
+++ b/frontend/src/shared/i18n/locales/ru.ts
@@ -310,6 +310,7 @@ export const ru: typeof en = {
config: 'Настройки',
backups: 'Бекапы',
verifications: 'Проверки',
+ monitoring: 'Мониторинг',
},
create: {
title: 'Добавить базу данных для бекапа',
@@ -1668,6 +1669,252 @@ export const ru: typeof en = {
'Базовый бекап копирует только каталог данных. Если исходный PostgreSQL хранит конфигурацию вне его (так устроены Debian и Ubuntu: <configDir/>), то <code>postgresql.conf</code>, <code>pg_hba.conf</code> и <code>pg_ident.conf</code> в бекап не попадают и сервер не запустится. Узнайте их настоящие пути на исходном сервере командой <code>psql -Atc "SHOW config_file"</code>, скопируйте все три файла, затем закомментируйте <code>data_directory</code>, <code>hba_file</code>, <code>ident_file</code>, <code>external_pid_file</code> и строки <code>ssl</code> / <code>ssl_*</code>, создайте <code>conf.d</code>, а для контейнера задайте <code>listen_addresses = \'*\'</code>. Если эти файлы на месте, а ошибка та же, значит, неверно смонтирован каталог, а не конфигурация.',
},
},
+ monitoring: {
+ periods: {
+ hour: 'Последний час',
+ day: 'Последние сутки',
+ week: 'Последняя неделя',
+ },
+ levels: {
+ notStreaming: 'Непрерывное сохранение изменений выключено',
+ collectingData: 'Собираем данные',
+ broken: 'Что-то сломано',
+ needsAttention: 'Что-то требует внимания',
+ healthy: 'Все в порядке',
+ },
+ reasonLevels: {
+ broken: 'Сломано',
+ needsAttention: 'Требует внимания',
+ info: 'Для информации',
+ },
+ keepingUpStates: {
+ unknown: 'Неизвестно',
+ fallingBehind: 'Отстает',
+ catchingUp: 'Догоняет',
+ idle: 'Тишина',
+ keepingUp: 'Успевает',
+ },
+ banner: {
+ collectingDataDescription: 'Стриминг запущен, первые цифры появятся в течение минуты.',
+ notStreamingDescription:
+ 'Бекапы выключены или не выбрано хранилище. Включить можно в <configLink>настройках</configLink>.',
+ cannotConfirm: 'Не удается подтвердить текущее состояние',
+ cannotConfirmDescription: 'Последнее известное состояние: {{level}}, {{duration}} назад.',
+ measuredAgo: 'Замер {{duration}} назад',
+ notMeasuredYet: 'Замеров еще нет',
+ refreshFailed:
+ 'Не удалось обновить данные. На экране остаются последние полученные цифры, следующая попытка через 30 секунд.',
+ },
+ keepingUp: {
+ keepingUp:
+ 'Databasus сохраняет изменения так же быстро, как база их пишет: запись {{writeRate}}, сохранение {{archiveRate}}.',
+ fallingBehind:
+ 'База пишет изменения быстрее, чем Databasus успевает их сохранять, примерно на {{differenceRate}}: {{writeRate}} против {{archiveRate}}.',
+ fallingBehindBacklog:
+ 'Еще не сохранено {{unsavedAmount}}, и этот объем растет примерно на {{growthPerHour}} в час.',
+ catchingUp:
+ 'Databasus сохраняет накопившиеся изменения: осталось {{unsavedAmount}}, это займет около {{duration}}.',
+ catchingUpWithoutEstimate:
+ 'Databasus сохраняет накопившиеся изменения: осталось {{unsavedAmount}}.',
+ idle: 'База почти ничего не пишет, новые изменения попадают в хранилище в течение нескольких минут.',
+ unknown: 'Текущих цифр пока нет.',
+ rebuildConsequence:
+ 'Если так пойдет дальше, примерно через {{duration}} Databasus начнет новую цепочку бекапов с полного бекапа.',
+ limitConsequence:
+ 'Если так пойдет дальше, примерно через {{duration}} база удалит изменения, которые Databasus еще не получил.',
+ },
+ reasons: {
+ streamingStopped: {
+ title: 'Сохранение остановилось',
+ whatHappened: 'Databasus перестал получать изменения {{duration}} назад.',
+ whatToDo:
+ 'Databasus сам повторяет попытки. Если это затянется, посмотрите логи Databasus.',
+ },
+ sourceUnreachable: {
+ title: 'База недоступна',
+ whatHappened: 'Databasus не может подключиться к базе уже {{duration}}.',
+ whatToDo: 'Проверьте, что сервер базы работает и доступен из Databasus.',
+ },
+ walChainBroken: {
+ title: 'Между бекапами потерялись изменения',
+ whatHappened:
+ 'Часть изменений между бекапами потерялась, поэтому последние изменения пока нельзя восстановить.',
+ whatToDo: 'Запустите полный бекап на вкладке «Бекапы».',
+ whatToDoPending:
+ 'Новый полный бекап уже в работе, за ним можно следить на вкладке «Бекапы».',
+ },
+ slotMissing: {
+ title: 'База перестала хранить изменения',
+ whatHappened:
+ 'База больше не хранит изменения для Databasus: резерв, который Databasus создал на ней, пропал.',
+ whatToDo:
+ 'Databasus восстановит резерв, когда стриминг перезапустится, но изменения за это время потеряны. Запустите полный бекап на вкладке «Бекапы» и проверьте, что резерв на базе никто не удаляет.',
+ whatToDoPending:
+ 'Databasus восстановит резерв, когда стриминг перезапустится, а новый полный бекап уже в работе. Проверьте, что резерв на базе никто не удаляет.',
+ },
+ slotLost: {
+ title: 'База удалила несохраненные изменения',
+ whatHappened: 'База удалила изменения, которые Databasus еще не получил.',
+ whatToDo:
+ 'Databasus сам начнет заново с нового полного бекапа. Чтобы это не повторилось, ускорьте сохранение или разрешите базе хранить больше изменений.',
+ },
+ archiveNotAdvancing: {
+ title: 'Изменения не доходят до хранилища',
+ whatHappened:
+ 'Новые изменения не попадают в хранилище уже {{duration}}, хотя база продолжает писать.',
+ whatToDo: 'Проверьте, что хранилище доступно и на нем есть свободное место.',
+ },
+ archiveSlowerThanSource: {
+ title: 'Сохранение слишком медленное',
+ whatHappened:
+ 'База пишет быстрее, чем Databasus сохраняет: за 10 минут несохраненный объем вырос на {{amount}}.',
+ whatToDo:
+ 'Проверьте скорость сети до хранилища или перенесите хранилище ближе к Databasus.',
+ },
+ slotKeepsExtraWal: {
+ title: 'База хранит лишние изменения',
+ whatHappened:
+ 'Уже {{duration}} база держит для Databasus больше журнала изменений, чем обычно, и объем продолжает расти.',
+ whatToDo: 'Убедитесь, что на сервере базы есть свободное место на диске.',
+ },
+ slotLimitApproaching: {
+ title: 'Скоро база удалит несохраненные изменения',
+ whatHappened:
+ 'Если так пойдет дальше, примерно через {{duration}} база удалит изменения, которые Databasus еще не получил.',
+ whatToDo:
+ 'Ускорьте сохранение или увеличьте <code>max_slot_wal_keep_size</code> на базе.',
+ },
+ nearSlotRebuild: {
+ title: 'Скоро придется начать заново',
+ whatHappened:
+ 'Изменений, еще не полученных Databasus, накопилось {{amount}} из {{limit}}. При этом объеме Databasus начинает заново с полного бекапа.',
+ whatToDo: 'Ускорьте сохранение или поднимите порог в настройках бекапов.',
+ },
+ localQueueFilling: {
+ title: 'Буфер Databasus заполняется',
+ whatHappened: 'В буфере Databasus {{amount}} из {{limit}}.',
+ whatToDo:
+ 'Когда буфер заполнится, Databasus сам приостановит прием. Проверьте скорость хранилища.',
+ },
+ frequentRestarts: {
+ title: 'Прием часто перезапускается',
+ whatHappened: 'Перезапусков приема за последний час: {{count}}.',
+ whatToDo: 'Причину ищите в логах Databasus.',
+ },
+ forcedRotationRefused: {
+ title: 'Изменения из тихих периодов приходят с опозданием',
+ whatHappened:
+ 'База отказывается вовремя закрывать файл изменений, поэтому в тихие периоды изменения попадают в хранилище позже.',
+ whatToDo: 'Дайте пользователю бекапов право запускать <code>pg_switch_wal()</code>.',
+ },
+ },
+ notices: {
+ noSlotWalLimit: {
+ title: 'Нет предела для хранимых изменений',
+ whatHappened: 'База хранит изменения для Databasus без ограничения.',
+ whatToDo:
+ 'Предел защищает диск сервера базы, если Databasus надолго остановится. Цена — новый полный бекап, когда предел будет достигнут. Задается параметром <code>max_slot_wal_keep_size</code>.',
+ },
+ slotKeepsExtraWalSteadily: {
+ title: 'База хранит лишние изменения',
+ whatHappened: 'База держит для Databasus больше журнала изменений, чем обычно.',
+ whatToDo:
+ 'При высокой нагрузке это нормально. Пока объем не растет, делать ничего не нужно.',
+ },
+ noFullBackupYet: {
+ title: 'Полного бекапа еще нет',
+ whatHappened:
+ 'Первый полный бекап еще не завершился, поэтому сохраненные изменения пока нельзя восстановить.',
+ whatToDo: 'Дождитесь его или запустите на вкладке «Бекапы».',
+ },
+ },
+ path: {
+ database: 'База',
+ databasus: 'Databasus',
+ storage: 'Хранилище',
+ notReceivedYet: 'Еще не получено Databasus',
+ rebuildLimit: 'При {{amount}} Databasus начнет новую цепочку с полного бекапа',
+ allowanceLeft: 'Запас, пока база не начнет удалять изменения: {{amount}}',
+ noAllowanceLimit: 'Предела для хранимых изменений нет',
+ waitingInBuffer: 'Ждет в буфере',
+ pauseLimit: 'Прием приостанавливается при {{amount}}',
+ receivingPaused: 'Прием приостановлен',
+ lastSaved: 'Последнее изменение сохранено {{relativeTime}}',
+ nothingSavedYet: 'Пока ничего не сохранено',
+ overLimit: 'Сверх предела',
+ writeRate: 'Запись {{rate}}',
+ saveRate: 'Сохранение {{rate}}',
+ unknownAmount: 'Неизвестно',
+ },
+ tooltips: {
+ notReceivedYet:
+ 'Изменения, которые база держит для Databasus, потому что Databasus их еще не получил. Технически это отставание слота репликации.',
+ rebuildLimit:
+ 'Когда объем превысит этот порог, Databasus удалит слот репликации и начнет новую цепочку бекапов, чтобы не переполнить диск сервера базы. Технически это порог WAL lag.',
+ allowanceLeft:
+ 'Сколько база еще может записать, прежде чем удалит изменения, которые Databasus не получил. Технически это safe_wal_size, его ограничивает max_slot_wal_keep_size.',
+ waitingInBuffer:
+ 'Изменения, которые Databasus получил, но еще не сохранил в хранилище. Технически это локальная очередь WAL.',
+ pauseLimit:
+ 'Когда буфер дорастает до этого размера, Databasus приостанавливает прием, пока буфер не освободится. Технически это верхняя граница back pressure.',
+ lastSaved:
+ 'Когда в хранилище попал самый свежий файл изменений. Технически это последний заархивированный сегмент WAL.',
+ writeRate:
+ 'С какой скоростью база пишет изменения, в среднем за последние 10 минут. Технически это скорость записи WAL.',
+ saveRate:
+ 'С какой скоростью Databasus сохраняет изменения в хранилище, в среднем за последние 10 минут. Технически это скорость архивации WAL.',
+ },
+ charts: {
+ keepingUpTitle: 'Успевает ли Databasus?',
+ unsavedTitle: 'Сколько еще не сохранено?',
+ unsavedSubtitle: 'Сюда входит и то, что Databasus еще не получил',
+ bufferTitle: 'Буфер Databasus',
+ databaseWrites: 'Запись в базе',
+ savedToStorage: 'Сохранено в хранилище',
+ notSavedYet: 'Еще не в хранилище',
+ notReceivedYet: 'Еще не получено Databasus',
+ waitingInBuffer: 'Ждет в буфере',
+ pauseLimit: 'Здесь прием приостанавливается',
+ restarts: 'Перезапуски',
+ pauses: 'Паузы',
+ noHistory: 'Графики появятся через несколько минут стриминга.',
+ dataSince: 'Данные с {{time}}',
+ perSecond: '{{amount}}/с',
+ },
+ details: {
+ title: 'Подробности',
+ slotStatus: 'Статус слота (wal_status)',
+ allowance: 'Оставшийся запас (safe_wal_size)',
+ noLimit: 'Без предела',
+ averageSegmentWrite: 'Средняя запись сегмента',
+ longestSegmentWrite: 'Самая долгая запись сегмента',
+ uploadConcurrency: 'Загрузок одновременно',
+ walToStoredRatio: 'Отношение WAL к сохраненному объему',
+ walSegmentSize: 'Размер сегмента WAL',
+ sessionStart: 'Начало сессии стриминга',
+ streamerState: 'Состояние стримера',
+ sourceState: 'Состояние источника',
+ restartsLastHour: 'Перезапуски за последний час',
+ milliseconds: '{{value}} мс',
+ notAvailable: 'Нет данных',
+ streamerStates: {
+ running: 'Работает',
+ notRunning: 'Не работает',
+ notStarted: 'Еще не запускался',
+ },
+ sourceStates: {
+ reachable: 'Доступен',
+ unreachable: 'Недоступен',
+ slotMissing: 'Доступен, но слота репликации нет',
+ },
+ },
+ durations: {
+ seconds: '{{seconds}} с',
+ minutes: '{{minutes}} мин',
+ hoursMinutes: '{{hours}} ч {{minutes}} мин',
+ daysHours: '{{days}} дн. {{hours}} ч',
+ },
+ },
},
},
storages: {
diff --git a/frontend/src/shared/i18n/locales/zh.ts b/frontend/src/shared/i18n/locales/zh.ts
index 603006b2..c0bfcd19 100644
--- a/frontend/src/shared/i18n/locales/zh.ts
+++ b/frontend/src/shared/i18n/locales/zh.ts
@@ -301,6 +301,7 @@ export const zh: typeof en = {
config: '配置',
backups: '备份',
verifications: '恢复验证',
+ monitoring: '监控',
},
create: {
title: '添加要备份的数据库',
@@ -1598,6 +1599,234 @@ export const zh: typeof en = {
'基础备份只复制数据目录。如果源 PostgreSQL 把配置放在数据目录之外(Debian/Ubuntu 的布局,<configDir/>),那么 <code>postgresql.conf</code>、<code>pg_hba.conf</code> 和 <code>pg_ident.conf</code> 不在备份中,服务器将无法启动。在源服务器上用 <code>psql -Atc "SHOW config_file"</code> 查出它们的实际路径,把三个文件都复制过来,然后注释掉 <code>data_directory</code>、<code>hba_file</code>、<code>ident_file</code>、<code>external_pid_file</code> 以及 <code>ssl</code> / <code>ssl_*</code> 这些行,创建 <code>conf.d</code>;如果在容器中运行,还要设置 <code>listen_addresses = \'*\'</code>。如果这些文件都在,却仍然出现同样的错误,那么出错的是挂载的目录,而不是配置。',
},
},
+ monitoring: {
+ periods: {
+ hour: '最近 1 小时',
+ day: '最近 1 天',
+ week: '最近 1 周',
+ },
+ levels: {
+ notStreaming: '变更的持续保存已关闭',
+ collectingData: '正在收集数据',
+ broken: '出现故障',
+ needsAttention: '有情况需要注意',
+ healthy: '一切正常',
+ },
+ reasonLevels: {
+ broken: '故障',
+ needsAttention: '需要注意',
+ info: '提示',
+ },
+ keepingUpStates: {
+ unknown: '未知',
+ fallingBehind: '跟不上',
+ catchingUp: '正在追赶',
+ idle: '空闲',
+ keepingUp: '跟得上',
+ },
+ banner: {
+ collectingDataDescription: '流式传输已开始,一分钟内会出现第一批数据。',
+ notStreamingDescription:
+ '备份已停用或未选择存储。可在<configLink>配置</configLink>中开启。',
+ cannotConfirm: '无法确认当前状态',
+ cannotConfirmDescription: '最后已知状态:{{level}},{{duration}}前。',
+ measuredAgo: '{{duration}}前测量',
+ notMeasuredYet: '尚无测量',
+ refreshFailed: '数据刷新失败。屏幕上保留最后收到的数据,30 秒后会再次尝试。',
+ },
+ keepingUp: {
+ keepingUp:
+ 'Databasus 保存变更的速度与数据库写入一样快:写入 {{writeRate}},保存 {{archiveRate}}。',
+ fallingBehind:
+ '数据库写入变更的速度比 Databasus 保存得快,大约快 {{differenceRate}}:{{writeRate}} 对 {{archiveRate}}。',
+ fallingBehindBacklog:
+ '还有 {{unsavedAmount}} 未保存,并且每小时大约增加 {{growthPerHour}}。',
+ catchingUp:
+ 'Databasus 正在保存积压的变更:还剩 {{unsavedAmount}},大约需要 {{duration}}。',
+ catchingUpWithoutEstimate: 'Databasus 正在保存积压的变更:还剩 {{unsavedAmount}}。',
+ idle: '数据库几乎没有写入,新的变更会在几分钟内进入存储。',
+ unknown: '暂无当前数据。',
+ rebuildConsequence:
+ '按这个速度,大约 {{duration}}后 Databasus 将从完整备份开始新的备份链。',
+ limitConsequence:
+ '按这个速度,大约 {{duration}}后数据库将删除 Databasus 尚未接收的变更。',
+ },
+ reasons: {
+ streamingStopped: {
+ title: '保存已停止',
+ whatHappened: 'Databasus 在 {{duration}}前停止接收变更。',
+ whatToDo: 'Databasus 会自动重试。如果持续这样,请查看 Databasus 的日志。',
+ },
+ sourceUnreachable: {
+ title: '无法连接数据库',
+ whatHappened: 'Databasus 已有 {{duration}}无法连接数据库。',
+ whatToDo: '请检查数据库服务器是否在运行,以及 Databasus 能否访问它。',
+ },
+ walChainBroken: {
+ title: '备份之间丢失了变更',
+ whatHappened: '备份之间丢失了部分变更,因此最近的变更暂时无法恢复。',
+ whatToDo: '请在“备份”标签页启动一次完整备份。',
+ whatToDoPending: '新的完整备份已在进行中,可以在“备份”标签页查看进度。',
+ },
+ slotMissing: {
+ title: '数据库不再保留变更',
+ whatHappened: '数据库不再为 Databasus 保留变更:Databasus 在数据库上创建的预留已消失。',
+ whatToDo:
+ '流式传输重启时 Databasus 会重新创建预留,但这期间的变更已经丢失。请在“备份”标签页启动一次完整备份,并检查是否有人删除数据库上的预留。',
+ whatToDoPending:
+ '流式传输重启时 Databasus 会重新创建预留,新的完整备份也已在进行中。请检查是否有人删除数据库上的预留。',
+ },
+ slotLost: {
+ title: '数据库删除了未保存的变更',
+ whatHappened: '数据库删除了 Databasus 尚未接收的变更。',
+ whatToDo:
+ 'Databasus 会自动从新的完整备份重新开始。为避免再次发生,请加快保存,或让数据库保留更多变更。',
+ },
+ archiveNotAdvancing: {
+ title: '变更没有进入存储',
+ whatHappened: '已有 {{duration}}没有新的变更进入存储,而数据库仍在写入。',
+ whatToDo: '请检查存储是否可以访问,以及是否还有可用空间。',
+ },
+ archiveSlowerThanSource: {
+ title: '保存太慢',
+ whatHappened: '数据库写入比 Databasus 保存得快,10 分钟内积压增加了 {{amount}}。',
+ whatToDo: '请检查到存储的网络速度,或把存储放得离 Databasus 更近。',
+ },
+ slotKeepsExtraWal: {
+ title: '数据库保留了额外的变更',
+ whatHappened:
+ '已有 {{duration}},数据库为 Databasus 保留的变更日志比平时多,而且还在增加。',
+ whatToDo: '请确认数据库服务器有足够的可用磁盘空间。',
+ },
+ slotLimitApproaching: {
+ title: '未保存的变更即将被删除',
+ whatHappened: '按这个速度,大约 {{duration}}后数据库将删除 Databasus 尚未接收的变更。',
+ whatToDo: '请加快保存,或在数据库上调大 <code>max_slot_wal_keep_size</code>。',
+ },
+ nearSlotRebuild: {
+ title: '即将重新开始',
+ whatHappened:
+ 'Databasus 尚未接收的变更已达到 {{amount}},达到 {{limit}} 时 Databasus 将从完整备份重新开始。',
+ whatToDo: '请加快保存,或在备份设置中调高这个上限。',
+ },
+ localQueueFilling: {
+ title: 'Databasus 缓冲区正在填满',
+ whatHappened: 'Databasus 缓冲区已用 {{amount}},上限 {{limit}}。',
+ whatToDo: '缓冲区满时 Databasus 会自动暂停接收。请检查存储的速度。',
+ },
+ frequentRestarts: {
+ title: '接收频繁重启',
+ whatHappened: '最近 1 小时内接收重启次数:{{count}}。',
+ whatToDo: '请在 Databasus 的日志中查找原因。',
+ },
+ forcedRotationRefused: {
+ title: '空闲时段的变更会晚到存储',
+ whatHappened: '数据库拒绝按时关闭变更文件,因此空闲时段的变更会更晚进入存储。',
+ whatToDo: '请授予备份用户执行 <code>pg_switch_wal()</code> 的权限。',
+ },
+ },
+ notices: {
+ noSlotWalLimit: {
+ title: '保留的变更没有上限',
+ whatHappened: '数据库为 Databasus 保留变更时没有上限。',
+ whatToDo:
+ '如果 Databasus 长时间停止,上限可以保护数据库服务器的磁盘。代价是达到上限时需要新的完整备份。可通过 <code>max_slot_wal_keep_size</code> 设置。',
+ },
+ slotKeepsExtraWalSteadily: {
+ title: '数据库保留了额外的变更',
+ whatHappened: '数据库为 Databasus 保留的变更日志比平时多。',
+ whatToDo: '高负载下这很正常。只要数量不再增加,就无需处理。',
+ },
+ noFullBackupYet: {
+ title: '还没有完整备份',
+ whatHappened: '第一次完整备份尚未完成,因此已保存的变更暂时无法恢复。',
+ whatToDo: '请等待它完成,或在“备份”标签页启动。',
+ },
+ },
+ path: {
+ database: '数据库',
+ databasus: 'Databasus',
+ storage: '存储',
+ notReceivedYet: 'Databasus 尚未接收',
+ rebuildLimit: '达到 {{amount}} 时 Databasus 将从完整备份开始新的链',
+ allowanceLeft: '数据库开始删除变更前的余量:{{amount}}',
+ noAllowanceLimit: '保留的变更没有上限',
+ waitingInBuffer: '缓冲区中等待',
+ pauseLimit: '达到 {{amount}} 时暂停接收',
+ receivingPaused: '接收已暂停',
+ lastSaved: '最后一次保存变更:{{relativeTime}}',
+ nothingSavedYet: '尚未保存任何内容',
+ overLimit: '超出上限',
+ writeRate: '写入 {{rate}}',
+ saveRate: '保存 {{rate}}',
+ unknownAmount: '未知',
+ },
+ tooltips: {
+ notReceivedYet:
+ '数据库为 Databasus 保留的、Databasus 尚未接收的变更。技术名称:复制槽延迟。',
+ rebuildLimit:
+ '超过这个数量后,Databasus 会删除复制槽并开始新的备份链,以免数据库服务器磁盘被占满。技术名称:WAL 延迟阈值。',
+ allowanceLeft:
+ '数据库在删除 Databasus 尚未接收的变更前还能写入多少。技术名称:safe_wal_size,受 max_slot_wal_keep_size 限制。',
+ waitingInBuffer: 'Databasus 已接收但尚未保存到存储的变更。技术名称:本地 WAL 队列。',
+ pauseLimit:
+ '缓冲区达到这个大小时,Databasus 会暂停接收,直到缓冲区清空。技术名称:背压高水位。',
+ lastSaved: '最新的变更文件进入存储的时间。技术名称:最新归档的 WAL 段。',
+ writeRate: '数据库写入变更的速度,按最近 10 分钟平均。技术名称:WAL 写入速率。',
+ saveRate:
+ 'Databasus 把变更保存到存储的速度,按最近 10 分钟平均。技术名称:WAL 归档速率。',
+ },
+ charts: {
+ keepingUpTitle: 'Databasus 跟得上吗?',
+ unsavedTitle: '还有多少未保存?',
+ unsavedSubtitle: '其中包括 Databasus 尚未接收的部分',
+ bufferTitle: 'Databasus 缓冲区',
+ databaseWrites: '数据库写入',
+ savedToStorage: '已保存到存储',
+ notSavedYet: '尚未进入存储',
+ notReceivedYet: 'Databasus 尚未接收',
+ waitingInBuffer: '缓冲区中等待',
+ pauseLimit: '在此暂停接收',
+ restarts: '重启',
+ pauses: '暂停',
+ noHistory: '流式传输几分钟后会显示图表。',
+ dataSince: '数据起始于 {{time}}',
+ perSecond: '{{amount}}/秒',
+ },
+ details: {
+ title: '详细信息',
+ slotStatus: '槽状态(wal_status)',
+ allowance: '剩余余量(safe_wal_size)',
+ noLimit: '无上限',
+ averageSegmentWrite: '段的平均写入时间',
+ longestSegmentWrite: '段的最长写入时间',
+ uploadConcurrency: '同时上传数',
+ walToStoredRatio: 'WAL 与保存大小之比',
+ walSegmentSize: 'WAL 段大小',
+ sessionStart: '流式传输会话开始时间',
+ streamerState: '流式传输进程状态',
+ sourceState: '源状态',
+ restartsLastHour: '最近 1 小时重启次数',
+ milliseconds: '{{value}} 毫秒',
+ notAvailable: '不可用',
+ streamerStates: {
+ running: '运行中',
+ notRunning: '未运行',
+ notStarted: '尚未启动',
+ },
+ sourceStates: {
+ reachable: '可访问',
+ unreachable: '不可访问',
+ slotMissing: '可访问,但复制槽不存在',
+ },
+ },
+ durations: {
+ seconds: '{{seconds}} 秒',
+ minutes: '{{minutes}} 分钟',
+ hoursMinutes: '{{hours}} 小时 {{minutes}} 分钟',
+ daysHours: '{{days}} 天 {{hours}} 小时',
+ },
+ },
},
},
storages: {
diff --git a/frontend/src/shared/lib/formatBytes.test.ts b/frontend/src/shared/lib/formatBytes.test.ts
new file mode 100644
index 00000000..c6defd96
--- /dev/null
+++ b/frontend/src/shared/lib/formatBytes.test.ts
@@ -0,0 +1,35 @@
+import { describe, expect, it } from 'vitest';
+
+import { formatNumber } from '../i18n/formatNumber';
+import { formatBytes } from './formatBytes';
+
+const formatEnglishNumber = (value: number, options?: Intl.NumberFormatOptions) =>
+ formatNumber(value, 'en', options);
+
+describe('formatBytes', () => {
+ it.each([
+ [0, '0 B'],
+ [1023, '1,023 B'],
+ [1024, '1 KB'],
+ [1536, '1.5 KB'],
+ [1024 ** 2 - 1, '1,024 KB'],
+ [1024 ** 2, '1 MB'],
+ [16 * 1024 ** 2, '16 MB'],
+ [1024 ** 3, '1 GB'],
+ [3.4 * 1024 ** 3, '3.4 GB'],
+ [1024 ** 4, '1 TB'],
+ [2048 * 1024 ** 4, '2,048 TB'],
+ ])('formats %d bytes as %s', (bytes, expected) => {
+ expect(formatBytes(bytes, formatEnglishNumber)).toBe(expected);
+ });
+
+ it('keeps the sign of a shrinking amount', () => {
+ expect(formatBytes(-9 * 1024 ** 2, formatEnglishNumber)).toBe('-9 MB');
+ });
+
+ it('groups digits by the selected language', () => {
+ const frenchBytes = formatBytes(1023, (value, options) => formatNumber(value, 'fr', options));
+
+ expect(frenchBytes.replace(/\s/g, ' ')).toBe('1 023 B');
+ });
+});
diff --git a/frontend/src/shared/lib/formatBytes.ts b/frontend/src/shared/lib/formatBytes.ts
new file mode 100644
index 00000000..247eb98f
--- /dev/null
+++ b/frontend/src/shared/lib/formatBytes.ts
@@ -0,0 +1,23 @@
+/* eslint-disable i18next/no-literal-string -- unit symbols stay English in every language, like the backup lists */
+const BYTE_UNITS = ['B', 'KB', 'MB', 'GB', 'TB'] as const;
+
+const BYTES_PER_UNIT_STEP = 1024;
+
+type NumberFormatter = (value: number, options?: Intl.NumberFormatOptions) => string;
+
+// Base 1024, the way PostgreSQL and the backup lists count, so 16 MB is one WAL segment. The
+// number goes through the formatter from useLocale(), so digits are grouped by the selected
+// language.
+export const formatBytes = (bytes: number, formatNumber: NumberFormatter): string => {
+ let unitIndex = 0;
+ let unitValue = Math.abs(bytes);
+
+ while (unitValue >= BYTES_PER_UNIT_STEP && unitIndex < BYTE_UNITS.length - 1) {
+ unitValue /= BYTES_PER_UNIT_STEP;
+ unitIndex++;
+ }
+
+ const signedValue = bytes < 0 ? -unitValue : unitValue;
+
+ return `${formatNumber(signedValue, { maximumFractionDigits: 1 })} ${BYTE_UNITS[unitIndex]}`;
+};
diff --git a/frontend/src/shared/lib/index.ts b/frontend/src/shared/lib/index.ts
index b30ecfc4..3732ce82 100644
--- a/frontend/src/shared/lib/index.ts
+++ b/frontend/src/shared/lib/index.ts
@@ -1,3 +1,4 @@
+export { formatBytes } from './formatBytes';
export { formatDuration } from './formatDuration';
export { FormValidator } from './FormValidator';
export { type PagedListPage, appendNextPage, mergeRefreshedFirstPage } from './pagedList';
diff --git a/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/.openspec.yaml b/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/.openspec.yaml
new file mode 100644
index 00000000..5c881ba7
--- /dev/null
+++ b/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/.openspec.yaml
@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-10-18
diff --git a/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/design.md b/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/design.md
new file mode 100644
index 00000000..9c5b1453
--- /dev/null
+++ b/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/design.md
@@ -0,0 +1,341 @@
+## Context
+
+See proposal.md for the motivation. Backend paths below are under `backend/internal/features/`, frontend paths under `frontend/src/`, and `<pg>` stands for `backups/backups/usecases/physical/postgresql`.
+
+The change has two consumers in mind. The Monitoring tab is the first. The second is anyone reading the authenticated API, so the statistics endpoint is a contract in its own right, not the tab's private feed.
+
+What exists today:
+
+- Every 30 seconds the WAL streamer's lag monitor (`<pg>/lag_monitor.go:121`) probes the slot through `GetSlotStateIfReachable` (`<pg>/slot_lsn_watcher.go:74`), which calls `InspectSlot` (`<pg>/slot.go:29`). The query returns `active`, `wal_status`, `restart_lsn` and the lag of `restart_lsn` behind `pg_current_wal_lsn()`, but neither `safe_wal_size` nor the current position itself. When the probe fails, the tick does nothing else (`lag_monitor.go:135-138`). The probe's connection has no timeout: the backend sets no `connect_timeout` anywhere (`backuping/physical/constants.go:64-67`), so a silent source holds the tick.
+- The source's current position is derived as `restart_lsn + lag` (`lag_monitor.go:219`). For a `lost` slot `InspectSlot` substitutes `0/0` and `0` (`slot.go:38-42`), so that sum becomes 0. The archive progress point is 0 while nothing has been archived (`<pg>/wal_staleness.go:86-90`).
+- The probe returns `(nil, false)` both for an unreachable source and for a source without the slot (`slot_lsn_watcher.go:93-98`). Its other callers, the slot-LSN watcher (`slot_lsn_watcher.go:125`) and the resume path (`<pg>/wal_resume.go:108`), treat both the same way.
+- The same tick logs the archiving rollup from one upload snapshot (`lag_monitor.go:175-180`). The snapshot holds the archived segment count, their compressed size, the per-step upload timings and the highest concurrency (`<pg>/wal_upload_stats.go:35-45`). Taking it resets the counters (`wal_upload_stats.go:71-78`, through `<pg>/wal_upload.go:190`).
+- The tick feeds two trackers that decide notifications. The slot classifier remembers since when the slot's status has been `extended` or `unreserved` and alerts after `warningSlotStatusHoldPeriod`, 5 minutes (`lag_monitor.go:18`, `:60-114`). The staleness tracker remembers since when the archive progress point has stood still while the source writes, keeps that clock through quiet ticks, and alerts after `DefaultArchiveStalenessThreshold`, 30 minutes (`wal_staleness.go:13`, `:29-82`). For a `lost` slot, or a slot lag past `WalLagThresholdBytes` while the receiver runs and the status is neither `extended` nor `unreserved`, the classifier asks for a rebuild, which drops and recreates the slot in the same tick (`lag_monitor.go:88-114`, `:160-167`) and requests a full backup (`backuping/physical/wal_supervisor.go:370-394`). Session start also recreates a missing slot silently (`databases/databases/postgresql/physical/model.go:347`, called from `<pg>/wal_stream.go:273`). Both leave a WAL gap in the chain; the slot looks healthy right after.
+- The forced WAL rotation loop stops for the rest of the session when the source refuses `pg_switch_wal()` and sends a `WAL_ROTATION_DENIED` alert (`<pg>/wal_rotation.go:100-113`).
+- The local queue size is computed on demand by `backlogBytes` (`<pg>/wal_backpressure.go:71`). The receiver is paused at `highWatermarkBytes`, at least 512 MiB and at least eight segments (`<pg>/wal_stream.go:245`). Back pressure and a stalled slot both stop the receiver through `receiverInternalRestart` (`<pg>/receivewal_process.go:122`); before respawning, the supervision loop waits for the queue to drain (`wal_stream.go:340`). Nothing counts restarts or pauses.
+- A receiver that keeps exiting quickly escalates after five rapid failures and ends the session with an error (`wal_stream.go:401-412`). With the source down, that takes under a minute, and the next session fails in `VerifyWalSlot` before its lag monitor starts.
+- Streamer ownership lives in `physical_wal_streamers` (`backups/backups/core/physical/models/wal_streamer.go`): status `RUNNING` or `FAILED`, `started_at` and a heartbeat. The supervisor ticks every 15 seconds and reclaims a `FAILED` row or one whose heartbeat is older than 90 seconds, resetting `started_at` (`backuping/physical/constants.go:48`, `:54`; `backups/backups/core/physical/repositories/wal_streamer_repository.go:42-76`). Session errors and failed starts mark the row `FAILED` (`wal_supervisor.go:307-321`, `:588-592`). A `FAILED` row therefore lasts at most one tick, and a `RUNNING` row can belong to a process that is gone.
+- A WAL streaming config always has a rebuild lag: `WalLagThresholdBytes` defaults to 256 MiB (`backups/config/physical/constants.go:6`, `model.go:150-153`). Whether streaming should run is `isWalStreamCandidate`, applied to configs with backups enabled (`wal_supervisor.go:142`, `:175-182`).
+- WAL gaps in a chain are computed by `ChainViewService.FindWalGapsInChain` from the committed segments within the chain's span (`backups/backups/core/physical/chain_view/service.go:83-123`). The newest completed full backup comes from `FindCompletedNewestFirstByDatabase` (`chain_view/service.go:25-45`).
+- The retention cleaner ticks every minute (`backuping/physical/cleaner.go:40-51`, `constants.go:30`).
+- The only periodic-sample table today is `healthcheck_attempts`, with an index on `(database_id, created_at)` (`backend/migrations/20250704095930_add_healthckeck.sql:40`).
+- The physical backup endpoints authorize through `databaseService.GetDatabase(ctx, user, databaseID)` (`backups/backups/services/physical_service.go:85`) and answer a refusal with 400 and the error text (`backups/backups/controllers/physical/controller.go:91-94`). `PhysicalBackupService` holds the backup config service (`physical_service.go:58`).
+- Packages under `<pg>` and `backuping/physical` import `backups/backups/core/physical`; nothing in `core/physical` imports them.
+- The database page builds its tabs by hand (`features/databases/ui/DatabaseComponent.tsx:28`, headers at `:65-86`); the Verifications tab is conditional on the database type (`:79`). `recharts` 3 is in `package.json` but nothing imports it since monitoring was removed in `fcc894d1`. The healthcheck component polls only its "today" view (`features/healthcheck/ui/HealthckeckAttemptsComponent.tsx:105-110`). Translation keys are typed against `en.ts`, and every other dictionary is typed `typeof en`, so a key exists for `pnpm typecheck` only once it is in all six dictionaries.
+
+Constraints:
+
+- `backend/AGENTS.md`:
+ - Migrations: TIMESTAMPTZ columns, with the table, constraints and indexes as separate statements.
+ - Controllers and services: one controller per feature, Swagger annotations on every endpoint, no injecting another feature's repository.
+ - Testing: controller tests preferred; unit tests only for complex model logic with no API surface.
+ - Time: `time.Now().UTC()`.
+ - Logging: lowercase messages, IDs as key-value pairs.
+- `frontend/AGENTS.md`:
+ - AntD, `@ant-design/icons` and Tailwind only.
+ - Component member order.
+ - No literal user-facing strings; every key in all six dictionaries; counts phrased so one string fits every number.
+ - Enum labels through `*_LABEL_KEYS` tables; code outside components holds keys, not text.
+ - `useLocale()` for `formatNumber` and `formatRelativeTime`.
+ - API errors through `translateApiError`.
+ - FSD imports only downward; one type per file in `model/`.
+- ADR-0008: physical backups run on PostgreSQL 17 and 18, so `safe_wal_size` (added in 13) is always available.
+- The `single-instance-runtime` spec runs one application process. Samples still have to outlive a restart, because the hour before a restart is what an operator wants to see.
+
+## Goals / Non-Goals
+
+**Goals:**
+
+- One verdict a junior operator understands without knowing what a replication slot is.
+- When the verdict is not healthy, the reason, its size, its age and what to do.
+- One statistics API other consumers can rely on, with units and codes instead of display text.
+- The verdict agrees with the existing notifications about when a condition began.
+- The verdict stays red while the backup chain is actually broken, not only while the slot looks bad.
+- No extra queries against the source beyond widening the existing slot query.
+
+**Non-Goals:**
+
+- New notifications or alerting on sample values.
+- Sub-30-second resolution.
+- Configurable thresholds.
+
+## Decisions
+
+### D1. Samples persisted in a new table
+
+New table `physical_wal_stream_samples`, one row per streaming database every 30 seconds:
+
+| Column | Type | Null means |
+|---|---|---|
+| `id` | UUID, `gen_random_uuid()` | |
+| `database_id` | UUID, FK to `databases(id)` ON DELETE CASCADE | |
+| `sampled_at` | TIMESTAMPTZ | |
+| `session_started_at` | TIMESTAMPTZ | |
+| `source_state` | TEXT: `REACHABLE`, `UNREACHABLE`, `SLOT_MISSING` | |
+| `source_unreachable_since` | TIMESTAMPTZ | source reachable |
+| `source_current_lsn` | PG_LSN | no source-side values |
+| `archive_progress_lsn` | PG_LSN | nothing archived yet |
+| `source_written_bytes` | BIGINT | see D4 |
+| `archived_segments` | INTEGER | |
+| `archived_wal_bytes` | BIGINT | |
+| `archived_stored_bytes` | BIGINT | |
+| `wal_segment_size_bytes` | BIGINT | |
+| `highest_concurrent_uploads` | INTEGER | |
+| `segment_write_avg_millis` | INTEGER | nothing archived since the previous sample |
+| `segment_write_max_millis` | INTEGER | nothing archived since the previous sample |
+| `slot_wal_status` | TEXT | no source-side values |
+| `slot_extra_wal_since` | TIMESTAMPTZ | the classifier holds no start time |
+| `slot_lag_bytes` | BIGINT | no source-side values |
+| `slot_safe_wal_size_bytes` | BIGINT | no limit, or no source-side values |
+| `unarchived_wal_bytes` | BIGINT | no source-side values, or nothing archived yet |
+| `archive_stalled_since` | TIMESTAMPTZ | the staleness tracker holds no start time |
+| `local_queue_bytes` | BIGINT | |
+| `local_queue_pause_bytes` | BIGINT | |
+| `receiver_restarts` | INTEGER | |
+| `receiver_pauses` | INTEGER | |
+| `is_receiver_running` | BOOLEAN | |
+| `is_forced_rotation_denied` | BOOLEAN | |
+
+"No source-side values" means `source_state` is not `REACHABLE`. A NULL `slot_safe_wal_size_bytes` with a reachable source therefore means no limit. The LSN columns use `PG_LSN`, like `physical_wal_segments`.
+
+There are two indexes: `(database_id, sampled_at)` for reads, and `(sampled_at)` for the global expiry delete in D7.
+
+`archived_wal_bytes` is the archived segment count times `wal_segment_size_bytes`, so it compares directly with `source_written_bytes`. `archived_stored_bytes` is the compressed and encrypted size the snapshot already counts, so the ratio of the two shows compression. `wal_segment_size_bytes` and `local_queue_pause_bytes` depend on the source's segment size, which only the streamer knows. Storing them per sample costs 16 bytes and keeps the read path free of streamer internals.
+
+A row is about 250 bytes with its index entries. At 2,880 rows a day and 7 days that is about 5 MB per streaming database.
+
+Alternatives rejected:
+
+- *An in-memory ring buffer in the streamer.* No migration, but a restart or a crash loop erases the history that explains it.
+- *Deriving throughput from `physical_wal_segments`.* It has no source-side values, its timestamps are set before the upload, and retention deletes the rows the history needs.
+- *A boolean `is_source_reachable`.* It cannot tell a missing slot from an unreachable source. A missing slot means the chain is broken, while an unreachable source may be a network blip.
+
+### D2. Samples are taken on the lag monitor's tick
+
+The lag monitor already wakes every 30 seconds, probes the slot and takes the upload snapshot. The tick's body becomes a method that takes `now`, probes, classifies, records the sample and then acts on the classification. Tests call that method directly with a real test source instead of waiting 30 seconds per sample. The sample itself is assembled by a pure function from the tick's inputs: the probe outcome, the snapshot, the previous position, the tracker times and `now`. Edge cases a test source cannot produce on demand, such as a `lost` slot or a timeline switch, are tested through that function. The sample goes through a repository passed in `WalStreamSpec`, the same way `WalSegmentRepo` and `HistoryRepo` are passed today (`wal_supervisor.go:283-284`).
+
+Order within a tick:
+
+1. Probe the source with a 10-second timeout on the probe's context.
+2. Feed the classifier and, when reachable, the staleness tracker, as today.
+3. Record the sample. Its slot values describe the slot as probed, before any rebuild in step 4.
+4. Act on the classification: alert or rebuild, as today.
+
+When the probe fails, the tick still records a sample with the source-side columns NULL instead of skipping. The trackers keep their last state on such ticks, so the "since" columns carry over. The verdict ignores reasons read from them while the source is unreachable (D8).
+
+The probe changes to return one of three outcomes: reachable with a slot state, reachable without the slot, unreachable. The slot-LSN watcher and the resume path map "without the slot" to their current behavior; only the lag monitor uses the difference.
+
+`source_unreachable_since` is kept by the lag monitor. A new session starts it from the newest stored sample of the database when that sample was unreachable, so the duration survives the crash loop a down source causes.
+
+The upload snapshot is taken once per tick and used by both the log line and the sample, because it resets on read and two readers would each see half the data.
+
+Alternatives rejected:
+
+- *A separate sampler goroutine.* It would probe the slot a second time every 30 seconds and need its own coordination for the reset-on-read counters.
+- *Making the poll interval a `WalStreamSpec` field for tests.* Tests would still wait real time, only less of it; calling the tick directly makes them deterministic.
+
+### D3. The slot query reads the WAL allowance and the current position
+
+`InspectSlot` adds `s.safe_wal_size` and `pg_current_wal_lsn()` to its select list. `safe_wal_size` is NULL when `max_slot_wal_keep_size` is `-1` and when the slot is `lost`, so `SlotState` carries it as a nullable value; together with `wal_status`, that tells "no limit" from "already lost". The current position replaces `restart_lsn + lag` in the staleness tracker, in D4 and in `unarchived_wal_bytes`, which then stay correct for a lost slot.
+
+`unarchived_wal_bytes` is the current position minus the archive progress point, NULL while the progress point is 0 because nothing has been archived yet.
+
+Alternative rejected: *reading `current_setting('max_slot_wal_keep_size')` as well.* `safe_wal_size` already says how much room is left and whether a limit exists. The setting adds a value nobody shows and GUC unit parsing.
+
+### D4. Source write volume from consecutive positions
+
+The lag monitor keeps the previous tick's current position in memory. `source_written_bytes` is the difference to the current position. It is NULL on the first tick of a session, after a tick without source-side values, and when the position moved backwards, which happens on a timeline switch to a promoted replica.
+
+Alternative rejected: *storing only the raw position and differencing at read time.* It moves the same logic to the read path and makes bucketing harder, without being more accurate. The raw position is stored anyway for the chain gap rule in D8.
+
+### D5. Restarts, pauses and repeated failures
+
+An atomic counter on the streamer grows each time the supervision loop respawns `pg_receivewal`. A respawn that followed a wait for the local queue to drain counts as a pause in `receiver_pauses`; any other counts as a restart in `receiver_restarts`. Back pressure is Databasus working as designed, so it must not look like a crash. The lag monitor takes and resets both counters with each sample.
+
+A whole-streamer restart starts a new session. Each sample carries `session_started_at`, the time the current `WalStreamSupervisor` started running, so the read path counts session starts by `session_started_at` within a window, not by `sampled_at`.
+
+A session that fails before its first tick leaves no sample. To see that, `physical_wal_streamers` gains a nullable `failing_since` TIMESTAMPTZ. `MarkFailed` sets it with `COALESCE(failing_since, NOW())`, so a crash loop keeps the first failure. The lag monitor clears it when it inserts the first sample of a session. `MarkStaleRunningFailed` at startup sets it the same way; a healthy restart clears it within one tick.
+
+The rotation loop sets an atomic flag when the source refuses `pg_switch_wal()`. Each sample copies it to `is_forced_rotation_denied`.
+
+Alternatives rejected:
+
+- *Counting session starts in `physical_wal_streamers`.* That row is overwritten on each claim and has no history.
+- *Storing the failure's error text.* Connection errors carry host names and user names, and the spec forbids connection details in the statistics.
+
+### D6. Condition start times come from the notification trackers
+
+The classifier's `warningStatusSince` and the staleness tracker's `fallingBehindSince` are written into `slot_extra_wal_since` and `archive_stalled_since` on every tick. The thresholds that turn these times into reasons live in one place, see D8.
+
+The verdict then agrees with the notifications about start times by construction. A user who gets an "archiving fell behind" message and opens the tab sees the same reason at the broken level, with the same start time. The levels differ in one case by design: a steadily `extended` slot is a notice in the tab (D8), while the existing notification still fires for it.
+
+Alternative rejected: *recomputing the conditions from sample history at read time.* It duplicates the tracker logic, and the two copies would drift.
+
+### D7. Retention by one delete per cleaner tick
+
+The retention cleaner, which ticks every minute, deletes all samples older than 7 days in one statement on the `(sampled_at)` index. This also expires the samples of a database that stopped streaming, which a delete on insert would never reach. Database removal is covered by ON DELETE CASCADE.
+
+Alternatives rejected:
+
+- *A delete after each insert, as `healthcheck_attempts` does.* It never runs for a database that stopped streaming, so its samples would outlive the 7 days.
+- *Raw samples for one day plus 5-minute rollups for a week.* It saves about 4 MB per database, but adds a second table, a rollup job and two read paths for the week view.
+
+### D8. The verdict is computed in the backend
+
+A new package `backups/backups/core/physical/wal_health` holds the verdict as a pure function of its inputs: the samples of the last hour with the newest one, the streamer row, the rebuild lag, whether streaming is expected, whether a full backup has completed, whether the newest chain has a permanent gap, whether a full backup is pending, and `now`. A full backup is pending when the config's `ForceFullRequestedAt` is set or a full backup is in flight. It returns the level, the reasons and the notices.
+
+The package also holds every threshold as a named constant. `warningSlotStatusHoldPeriod`, `DefaultArchiveStalenessThreshold` and the 90-second takeover age (`streamerHeartbeatStaleness`) move here, and `<pg>` and `backuping/physical` import them, so the notification trackers, the supervisor and the verdict share one value. The import direction already allows that. `isWalStreamCandidate` becomes `PhysicalBackupConfig.IsWalStreamingExpected()`, which also checks `IsBackupsEnabled`, so the supervisor and the read path share it.
+
+Levels, first match wins: `NOT_STREAMING`, `COLLECTING_DATA`, `BROKEN`, `NEEDS_ATTENTION`, `HEALTHY`. The spec defines them. The streamer state is `RUNNING` only when the row says `RUNNING` and the heartbeat is younger than the takeover age, `NOT_RUNNING` otherwise, `NOT_STARTED` without a row.
+
+Reasons. "2 samples" marks the rule that the condition must hold in the two newest samples. "Slot" marks reasons dropped while the newest sample has no source-side values. Every reason except `WAL_CHAIN_BROKEN`, `STREAMING_STOPPED` and `SOURCE_UNREACHABLE` is dropped when the newest sample is older than 2 minutes.
+
+| Code | `NEEDS_ATTENTION` | `BROKEN` | Carries |
+|---|---|---|---|
+| `STREAMING_STOPPED` | | `failing_since` 1 min or older, or newest sample older than 2 min; not when the newest sample is `UNREACHABLE` | `since` = newest `sampled_at` or `failing_since` |
+| `SOURCE_UNREACHABLE` | newest sample `UNREACHABLE`, `source_unreachable_since` 1 to 5 min ago | the same, 5 min or longer | `since` |
+| `WAL_CHAIN_BROKEN` | | the newest completed full backup's chain has a gap ending at or below the newest `archive_progress_lsn` | |
+| `SLOT_MISSING` | | newest `source_state` is `SLOT_MISSING` | |
+| `SLOT_LOST` | | newest `slot_wal_status` is `lost` (slot) | |
+| `ARCHIVE_NOT_ADVANCING` | `archive_stalled_since` 10 min or older (slot) | 30 min or older | `since` |
+| `ARCHIVE_SLOWER_THAN_SOURCE` | growth over 10 min at least the backlog threshold (slot, 2 samples) | | `valueBytes`, `thresholdBytes` |
+| `SLOT_KEEPS_EXTRA_WAL` | `slot_extra_wal_since` older than 5 min, and status `unreserved` or slot lag growing (slot) | | `since` |
+| `SLOT_LIMIT_APPROACHING` | `slot_safe_wal_size_bytes` over the slot lag's growth rate under 1 h (slot, 2 samples) | | `valueBytes`, `estimatedSeconds` |
+| `NEAR_SLOT_REBUILD` | `slot_lag_bytes` at least 80% of `WalLagThresholdBytes`, status neither `extended` nor `unreserved` (slot, 2 samples) | | `valueBytes`, `thresholdBytes` |
+| `LOCAL_QUEUE_FILLING` | `local_queue_bytes` at least half of `local_queue_pause_bytes` (2 samples) | | `valueBytes`, `thresholdBytes` |
+| `FREQUENT_RESTARTS` | restarts plus session starts in the last hour, 3 or more | | `count` |
+| `FORCED_ROTATION_REFUSED` | newest `is_forced_rotation_denied` | | |
+
+Notices, level `INFO`, never raise the level: `NO_SLOT_WAL_LIMIT`, `SLOT_KEEPS_EXTRA_WAL_STEADILY` and `NO_FULL_BACKUP_YET`, with the conditions in the spec.
+
+The backlog threshold is `max(64 MiB, 4 × wal_segment_size_bytes)`. Growth is the lowest unarchived value in the window's last 2.5 minutes minus the lowest in its first 2.5 minutes. The lowest values strip the sawtooth of in-flight uploads, which reaches four segments plus the open one in normal operation.
+
+`WAL_CHAIN_BROKEN` reads durable state, not the slot, because a rebuild makes the slot look healthy within one tick. A gap counts only when it ends at or below the archive progress point: that point never passes a pending segment, so a hole left by out-of-order parallel uploads is still above it. The reason clears when a newer full backup completes and starts a new chain. Databasus requests that backup itself only after a rebuild in the lag monitor (`wal_supervisor.go:370-394`); the silent recreate at session start requests none. The reason's "what to do" sentence therefore depends on whether a full backup is pending. A new `ChainViewService.GetNewestCompletedFullWithGaps(databaseID)` returns the newest completed full backup, whatever its extendability, with the `FindWalGapsInChain` result for its chain, or nil without one. `FindLastExtendableChainByDatabase` is not used, because it returns nil when an incremental broke the chain, which would hide both the gap and the existing full backup.
+
+`NEAR_SLOT_REBUILD` stays listed while the receiver is paused. The classifier rebuilds on the first tick the receiver runs again with the lag above the threshold, so a paused receiver above it is exactly the risky case.
+
+The thresholds sit just ahead of what already acts. Half the pause size warns before back pressure stops the receiver. 80% of the rebuild lag warns before Databasus sacrifices the chain. Three restarts an hour tolerate a deploy and one blip.
+
+Alternatives rejected:
+
+- *Computing the verdict in the browser.* Most thresholds live in the backend, the API would have no verdict for other consumers, and two copies of the rules would drift.
+- *Making the missing slot limit or a steadily `extended` slot orange.* PostgreSQL's default is no limit, and `extended` is routine on a busy cluster (`lag_monitor.go:64-67`). Either would show "needs attention" permanently and teach the reader to ignore orange.
+- *Recording slot rebuilds in samples and clearing on the next full backup.* It misses the silent recreate at session start and any other gap; the chain itself is the authority.
+
+### D9. Keeping-up summary
+
+The same package computes the keeping-up summary from the samples of the last 10 minutes:
+
+- `sourceWriteBytesPerSecond` and `archiveBytesPerSecond`: the sums of `source_written_bytes` and `archived_wal_bytes` over the samples that hold a write value, both divided by the seconds those samples cover, so a hole in the window lowers neither rate alone;
+- `unarchivedWalBytes` and `unarchivedWalChangeBytesPerSecond`, from the lowest values at both ends of the window as in D8;
+- `slotLagBytes` and `slotLagChangeBytesPerSecond`, the same way;
+- `slotRebuildLagBytes`, the config's `WalLagThresholdBytes`;
+- `estimatedCatchUpSeconds`, `estimatedSlotRebuildSeconds` and `estimatedSlotLimitSeconds`, under the conditions in the spec;
+- `state`: `UNKNOWN`, `FALLING_BEHIND`, `CATCHING_UP`, `IDLE` or `KEEPING_UP`, first match wins.
+
+The window is 10 minutes because a 30-second rate swings with every upload batch, while an hour hides a backlog that started five minutes ago.
+
+Alternative rejected: *letting the frontend derive rates from the series.* The series' bucket length differs by period, so the "now" numbers would change with the selected period.
+
+### D10. One statistics endpoint with server-side bucketing
+
+`GET /backups/physical/database/:id/wal-stream/statistics?period=HOUR|DAY|WEEK` on the existing `PhysicalBackupController`. `PhysicalBackupService.GetWalStreamStatistics` authorizes through `databaseService.GetDatabase` like `GetBackups`, refuses a database that is not PostgreSQL physical, reads the backup config through its config service, asks the chain view for the newest completed full backup and its gaps, reads whether a full backup is pending, and calls a new read method on the physical core service. That service owns the samples repository and the streamer repository and calls `wal_health`. Refusals return 400 with the error text, as `GetBackups` does. An unknown ID and a database in another workspace therefore get different messages. The spec accepts that, because every other physical backup endpoint behaves the same way and changing it for one endpoint would protect nothing.
+
+Response:
+
+- `health`: `level`, `reasons`, `notices` and `isFullBackupPending`. Each entry has `code`, `level` and, when known, `since`, `valueBytes`, `thresholdBytes`, `count` and `estimatedSeconds`.
+- `streamer`: `state`, `sessionStartedAt`, `failingSince`, `restartsLastHour`.
+- `current`: the newest sample's values under the column names in camelCase, plus `sampledAt` and `lastArchivedWalAt`, the `received_at` of the newest committed segment, which the "archiving fell behind" notification also reports.
+- `keepingUp`: D9.
+- `series`: `period`, `bucketSeconds`, `from`, `to` and `points`. Each point has `startedAt`, `sourceWriteBytesPerSecond`, `archiveBytesPerSecond`, `archivedStoredBytesPerSecond`, `maxSlotLagBytes`, `maxUnarchivedWalBytes`, `maxLocalQueueBytes`, `localQueuePauseBytes`, `restarts`, `pauses`, `sessionStarts`, `highestConcurrentUploads`, `segmentWriteAvgMillis` and `segmentWriteMaxMillis`.
+
+Enums are Go string types with Swagger enum annotations, so clients get the code lists from the API docs.
+
+Buckets are 1 minute for an hour (60 points), 4 minutes for a day (360 points) and 28 minutes for a week (360 points). Buckets align to `date_bin(stride, ts, '2000-01-01')`. The series starts one stride after `date_bin` of `now - period`, so it holds exactly period ÷ stride buckets and the last one contains `now`. A `generate_series` of bucket starts is left-joined to the binned samples, so empty buckets come back with NULL values. Rates divide by the seconds the bucket's samples cover, so the partial last bucket is not understated. The hour view uses 1-minute buckets because 30-second buckets would hold zero or two samples depending on the sampling phase, which draws false gaps and doubled rates.
+
+Alternatives rejected:
+
+- *Bucketing in the browser.* A week is about 20,000 rows, about 2 MB of JSON per refresh, into a chart that cannot draw more points than it has pixels.
+- *A separate status endpoint.* The tab needs both parts on every refresh, and one request keeps them from the same moment.
+- *Naming the endpoint after the tab (`/wal-stream-monitoring`).* The endpoint serves any consumer.
+
+### D11. The tab's layout answers the question first
+
+`DatabaseComponent` gains a `'monitoring'` tab when `database.type` is `POSTGRES_PHYSICAL` and the physical backup type is `FULL_INCREMENTAL_WAL_STREAM`. The tab is a new slice, `features/backups/physical/monitoring`, exported through `features/backups/physical`, which `DatabaseComponent` already imports. It reads through `physicalWalStreamStatisticsApi` in `entity/backups/physical/api`. Top to bottom:
+
+1. **Verdict banner.** Full width, background and border from AntD tokens through `theme.useToken()`: `colorSuccessBg`/`colorSuccessBorder`, `colorWarningBg`/`colorWarningBorder`, `colorErrorBg`/`colorErrorBorder`, `colorFillTertiary` for gray. The tokens follow the dark theme by themselves. A large icon per level (`CheckCircleFilled`, `ExclamationCircleFilled`, `CloseCircleFilled`, `ClockCircleOutlined`, `PauseCircleFilled`, `QuestionCircleFilled` for "cannot confirm") and the headline in an `aria-live="polite"` region. Under the headline, the keeping-up sentence. On the right, the live indicator: a dot that pulses while the newest sample is younger than 2 minutes, and "measured 12 s ago" from `current.sampledAt`. Below `md`, the indicator moves under the headline.
+2. **Reasons.** A list under the banner, broken first, each with its level icon, the "what happened" sentence and the "what to do" sentence from the table below. Notices follow in a blue `Alert` group. The `NO_SLOT_WAL_LIMIT` notice is collapsible and names `max_slot_wal_keep_size` through `InlineCodeComponent`.
+3. **Path of changes.** Three cards, Source, Databasus and Storage, joined by arrows labeled with the write rate and the archive rate. Source shows "not received by Databasus yet" with an AntD `Progress` against the rebuild lag, labeled "at 256 MB Databasus starts a new chain from a full backup", plus the remaining allowance or "no limit". Databasus shows the local queue against its pause size and "receiving paused" while the receiver waits. Storage shows "last change saved 40 s ago". An amount over its limit shows the bar full in the error color with "over the limit". A card's border takes the color of the most severe reason mapped to its stage. Each amount has a `Tooltip` on a focusable element with its plain meaning and its technical name. Below `md` the cards stack and the arrows point down.
+4. **Charts.** A `Segmented` control for hour, day and week, then three `recharts` charts of fixed height, about 220 px, sharing one `syncId`. Titles are questions: "Is Databasus keeping up?", "How much is not saved yet?" with the subtitle "Not received by Databasus yet is part of it", and "Databasus buffer". The Y axes format through `formatBytes`, with "/s" for rates. The X axis shows `HH:mm` for hour and day and weekday plus `HH:mm` for week, in local time. Tooltips show formatted values. Legends sit on top. Restarts and pauses are markers on the buffer chart without a second axis, and the pause size is a labeled dashed reference line. When the series has no point with values, the charts are replaced by "Charts appear after a few minutes of streaming". When data starts inside the period, a caption says "Data since HH:mm".
+5. **Details.** A `Collapse`, closed by default, with the raw values under their technical names: slot status, remaining allowance (`safe_wal_size`), average and longest segment write, upload concurrency, WAL-to-stored ratio, WAL segment size, session start.
+
+Reason and notice texts, stage and action. The exact wording is in the dictionaries; the table fixes the meaning.
+
+| Code | Stage | What happened | What to do |
+|---|---|---|---|
+| `STREAMING_STOPPED` | Databasus | Databasus stopped receiving changes N min ago | Databasus retries by itself; if this lasts, check the Databasus logs |
+| `SOURCE_UNREACHABLE` | Source | Databasus cannot reach the database for N min | Check that the database server is running and reachable from Databasus |
+| `WAL_CHAIN_BROKEN` | Storage | Some changes were lost between backups, so recent changes cannot be restored yet | With a full backup pending: a new full backup is on its way, watch it on the Backups tab. Without: start a full backup on the Backups tab |
+| `SLOT_MISSING` | Source | The database no longer keeps changes for Databasus | Databasus recreates it when streaming restarts, but changes in between are lost: start a full backup on the Backups tab unless one is pending, and check that nobody removes it |
+| `SLOT_LOST` | Source | The database deleted changes Databasus had not received | Databasus starts over with a new full backup by itself; to avoid this, see the speed advice below |
+| `ARCHIVE_NOT_ADVANCING` | Storage | No new changes reached storage for N min while the database keeps writing | Check that the storage is reachable and has free space |
+| `ARCHIVE_SLOWER_THAN_SOURCE` | Storage | The database writes faster than Databasus saves; the backlog grew by X in 10 min | Check the network speed to the storage, or move the storage closer |
+| `SLOT_KEEPS_EXTRA_WAL` | Source | The database keeps more of its change log than usual for Databasus, and it grows | Make sure the database server has free disk space |
+| `SLOT_LIMIT_APPROACHING` | Source | At this pace the database will delete unsaved changes in about N min | Speed up saving, or raise `max_slot_wal_keep_size` |
+| `NEAR_SLOT_REBUILD` | Source | Unreceived changes reached X of the Y at which Databasus starts over | Speed up saving, or raise the limit in the backup settings |
+| `LOCAL_QUEUE_FILLING` | Databasus | Databasus's buffer is X of Y full | Databasus pauses receiving by itself when it is full; check the storage speed |
+| `FREQUENT_RESTARTS` | Databasus | Receiving restarted N times in the last hour | Check the Databasus logs for the cause |
+| `FORCED_ROTATION_REFUSED` | Source | The database refuses to close its change file on time, so quiet periods reach storage later | Grant the backup user permission to run `pg_switch_wal()` |
+| `NO_SLOT_WAL_LIMIT` | Source | The database keeps changes for Databasus without a limit | A limit protects the server's disk if Databasus stays down long; the cost is a new full backup when it is hit |
+| `SLOT_KEEPS_EXTRA_WAL_STEADILY` | Source | The database keeps more of its change log than usual for Databasus | Normal under heavy load; nothing to do while it does not grow |
+| `NO_FULL_BACKUP_YET` | Storage | The first full backup has not finished, so saved changes cannot be restored yet | Wait for it, or start it on the Backups tab |
+
+`NOT_STREAMING` says "Continuous saving of changes is off because backups are disabled or no storage is selected" with a link that switches to the Config tab. `COLLECTING_DATA` says "Streaming started; the first numbers appear within a minute".
+
+The keeping-up sentence for falling behind reads like "The database writes changes faster than Databasus saves them: 21 MB/s against 12 MB/s. 3.4 GB is not saved yet, and that grows by about 32 GB an hour." With an estimate under a day it adds the consequence: "At this pace Databasus will start a new backup chain in about 40 minutes." Idle reads like "The database writes almost nothing; new changes reach storage within minutes."
+
+Chart lines differ in line style as well as hue, so they survive color blindness and grayscale. Light and dark variants: source writes blue `#1677ff` / `#4096ff` solid, archiving orange `#fa8c16` / `#ffa940` dashed, not saved yet purple `#722ed1` / `#9254de` solid, not received yet cyan `#13c2c2` / `#36cfc9` dashed, local queue purple solid with the pause size as a gray dashed line, restart markers red `#f5222d` / `#ff4d4f`, pause markers gray. Axis, grid and tooltip colors come from the AntD tokens, because recharts' default grid gray nearly vanishes on the dark background. Gaps stay gaps (`connectNulls={false}`).
+
+Alternatives rejected:
+
+- *A row of AntD `Statistic` items as the top of the tab.* It shows numbers without saying whether they are good, which is the question the tab exists to answer.
+- *AntD charts or another library.* AntD 5 ships no chart components, `recharts` is already a dependency, and `frontend/AGENTS.md` forbids other UI kits.
+
+### D12. Live updates without flicker
+
+The tab fetches on open, on period change, every 30 seconds in every period, and at once when the browser tab becomes visible again. It does not fetch while `document.visibilityState` is `hidden`. The healthcheck component's "poll only today" pattern is not followed, because the banner must stay current whatever period the charts show. A day refresh reads about 2,900 rows into 360 buckets on one index, which is cheap.
+
+A refresh keeps the previous response rendered until the next one arrives, so nothing collapses to a spinner. The first load renders a skeleton at the final heights, so the layout does not jump; a failed first load shows the error through `translateApiError` and a Retry button. A later failed refresh keeps the last response, shows a small inline warning, and the next scheduled refresh retries. When the last successful refresh is older than 2 minutes, the banner switches to the gray "cannot confirm the current state" with the last known level and its age.
+
+Charts animate on first render and on period change only; refreshes pass `isAnimationActive={false}`, otherwise every 30 seconds each line would redraw from zero. Numbers in the banner and the path cards ease to their new value over about 400 ms, and the banner's colors transition over 300 ms. Under `prefers-reduced-motion: reduce` there are no transitions, no pulse and no chart animation. The "measured N s ago" label ticks every second.
+
+Alternative rejected: *server-sent events or a WebSocket.* Samples arrive every 30 seconds, so polling at the same rate shows the same data without a new transport.
+
+### D13. Display rules as pure functions
+
+The rules that decide what the tab shows live in pure functions in the slice's `lib/`, so vitest covers them without rendering:
+
+- whether to show the Monitoring tab;
+- the banner's tone and icon per level, the "cannot confirm" override, and the order of reasons;
+- the "what happened" and "what to do" keys and parameters per reason and notice code, as `LocalizedText`, with the pending-full variant for `WAL_CHAIN_BROKEN` and `SLOT_MISSING`;
+- the keeping-up sentence per state, with the consequence sentence when an estimate is under a day;
+- the stage per reason and the color of each stage;
+- the mapper from series points to chart rows, keeping NULL as a gap;
+- rounding of durations and estimates for reading;
+- the refresh schedule, as a small controller with the timer and the page visibility injected.
+
+A new `shared/lib/formatBytes.ts` formats byte amounts with base 1024 and the labels B, KB, MB, GB and TB through `formatNumber`, matching how the backup lists already label megabytes. The existing local copies on the backup lists take megabytes and stay as they are, because this change does not touch those components.
+
+## Risks / Trade-offs
+
+- [Samples add about 2,900 inserts per streaming database per day, and one expiry delete a minute, to the internal PostgreSQL.] → Single-row inserts on two indexes, and a delete that touches about one row per streaming database per tick in steady state.
+- [Thresholds are guesses until real installations report.] → They are named constants in one package, tested by value, and sit ahead of the existing notifications rather than replacing them.
+- [A streamer that crash-loops with a session that takes longer than 2 minutes to fail shows "collecting data" until its first failure.] → `failing_since` turns it red one minute after that failure, and the Backups tab's healthcheck shows the source state meanwhile.
+- [The first sample after a restart has no source write rate.] → The chart shows one missing point. Inventing a rate from the slot lag would mislead.
+- [A timeline switch makes the source position jump backwards.] → D4 treats that tick as unknown instead of recording a negative write volume.
+- [Changing the probe's return touches two other callers.] → They map the new outcome to their current behavior, and their existing tests must keep passing.
+- [Moving four thresholds and one predicate touches the streamer, the supervisor and the config.] → Pure moves with no value change; the existing tests of those packages pin the behavior.
+
+## Migration Plan
+
+One goose migration creates the samples table, its foreign key and its two indexes, and adds the nullable `failing_since` column to `physical_wal_streamers`. Its `Down` section drops the column, the indexes, the constraint and the table, as recent migrations do (`backend/migrations/20260920215925_add_two_factor_auth.sql`). The migration is additive, so the previous binary runs fine with it applied, and a rollback of the binary alone loses nothing but the samples it stops writing. Samples start appearing on the first lag-monitor tick after deploy, so the tab shows "collecting data" for up to 30 seconds per database.
diff --git a/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/proposal.md b/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/proposal.md
new file mode 100644
index 00000000..8ac5d193
--- /dev/null
+++ b/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/proposal.md
@@ -0,0 +1,65 @@
+## Why
+
+An operator whose WAL streaming falls behind has nothing in Databasus to look at. A "WAL chain at risk" notification says the slot is behind by some number of bytes. Nothing shows whether the gap is growing or shrinking, how fast the source writes WAL compared with how fast Databasus archives it, or whether the source has a limit on how much WAL the slot may hold. The only evidence today is one log line every 30 seconds, and most operators running Databasus in a container never read it. A user with a busy production database asked for exactly these numbers on the day they enabled WAL streaming.
+
+Raw numbers alone would not help most users either. "Slot lag: 3.2 GB" means nothing to someone who does not know what a replication slot is. The first thing the screen has to say is whether everything is fine, and if not, what is wrong, how urgent it is and what to do.
+
+## What Changes
+
+- Databasus records a sample of each WAL-streaming database's archiving health every 30 seconds and keeps the samples for 7 days. Besides the slot and archive values, a sample records upload timing and concurrency, receiver restarts and pauses, and the source's state: reachable, unreachable, or reachable without the slot. It also records when each tracked problem began. Those times come from the same tracking that sends the existing notifications.
+- A new authenticated API endpoint returns WAL stream statistics for one database. It is a documented contract with units in field names and codes instead of display text:
+ - one health verdict, healthy, needs attention, broken, collecting data or not streaming, with the reasons behind it and notices that never change it;
+ - a broken verdict while the backup chain has a gap, until a newer full backup completes, even when the slot already looks healthy again;
+ - a keeping-up summary for the last 10 minutes: write rate against archive rate, how much is not archived yet and whether that grows, and estimates of when the backlog clears or when the source or Databasus will act on it;
+ - the streamer's state, which accounts for its heartbeat and for repeated failures before the first sample;
+ - a chart series for the last hour, day or week.
+- The database page gets a **Monitoring** tab, shown only for PostgreSQL physical databases in the full + incremental + WAL streaming mode. From top to bottom:
+ - a colored verdict banner in plain words, with the keeping-up state as a sentence a junior understands, for example "the database writes changes faster than Databasus saves them, by 9 MB/s";
+ - the reasons, each with what happened and what to do;
+ - the path of changes from source through Databasus to storage, showing where unsaved changes are;
+ - charts for the selected period;
+ - a collapsed details section with the raw values.
+- The tab refreshes every 30 seconds in every period without flicker, and shows how fresh its data is.
+- A source without a limit on WAL kept for the slot gets a notice that names `max_slot_wal_keep_size`. It does not change the verdict.
+
+## Capabilities
+
+### New Capabilities
+
+- `physical-backups/wal-stream-monitoring`: what Databasus records about a WAL-streaming database's archiving health, for how long, who can read it, how the verdict is decided, what the statistics API returns and what the Monitoring tab shows.
+
+### Modified Capabilities
+
+None. The existing notifications keep their behavior. Their thresholds move to a shared place so the verdict uses the same values, and the staleness tracker reads the source's real current position instead of deriving it from the slot.
+
+## Impact
+
+- Module rules: `AGENTS.md`, `backend/AGENTS.md` and `frontend/AGENTS.md`.
+- Builds on `parallel-wal-segment-uploads`, which is applied: the unarchived WAL is measured from its archive progress point, and the archive counters come from its upload snapshot.
+- Backend:
+ - a migration for the samples table and a `failing_since` column on the streamer table;
+ - sampling in the WAL streamer's lag monitor, which already probes the slot every 30 seconds;
+ - the slot probe tells a missing slot from an unreachable source and gives up after 10 seconds;
+ - the slot query also reads `safe_wal_size` and the source's current position, which keeps a lost slot from reporting position 0;
+ - receiver restart and pause counters, and a flag for a refused forced WAL switch;
+ - expiry of old samples on the retention cleaner's tick;
+ - a new package with the verdict, the keeping-up summary and the thresholds. Three thresholds move there from the streamer and the supervisor, and the "should streaming run" test moves onto the backup config, so both sides share one value;
+ - a new endpoint on the physical backup controller and regenerated Swagger docs.
+- Frontend:
+ - a new tab in the database page and a monitoring slice under the physical backups feature;
+ - API and model types under the physical backups entity;
+ - charts drawn with `recharts`, already a dependency;
+ - a shared byte formatter;
+ - new strings in all six interface dictionaries.
+- Database load: one row per streaming database every 30 seconds, about 20,000 rows and 5 MB per database at the 7-day limit, plus one expiry delete a minute.
+
+## Out of scope
+
+- General database monitoring such as CPU, memory or IO of the source. It was removed in `fcc894d1` and stays removed.
+- A Prometheus or OpenTelemetry metrics endpoint. The samples make one easy to add later.
+- New notifications or alerts based on the verdict. The existing notifications stay as they are.
+- Settings for the sampling interval, the retention period or the verdict thresholds.
+- Metrics for full and incremental backups, logical backups, or physical databases outside WAL streaming mode.
+- Free disk space on the Databasus host or on the source.
+- A verdict summary in the database list.
+- Adding the workspace name to notifications.
diff --git a/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/specs/physical-backups/wal-stream-monitoring/spec.md b/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/specs/physical-backups/wal-stream-monitoring/spec.md
new file mode 100644
index 00000000..c364a113
--- /dev/null
+++ b/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/specs/physical-backups/wal-stream-monitoring/spec.md
@@ -0,0 +1,566 @@
+## Purpose
+
+Lets an operator of any experience see at a glance whether WAL archiving of a streaming PostgreSQL database is healthy, and if it is not, what is wrong, how urgent it is and where the unsaved changes are. The same statistics are available through the authenticated API for other consumers.
+
+## ADDED Requirements
+
+### Requirement: A streaming database is sampled every 30 seconds
+
+While Databasus streams WAL for a database, it SHALL record one sample of that database's archiving health every 30 seconds. Each sample SHALL hold:
+
+- when it was taken, and when the current streaming session started;
+- the source's state: reachable, unreachable, or reachable but without Databasus's replication slot;
+- since when the source has been unreachable, while it is, counted across streaming sessions;
+- the source's current WAL position and the archive progress point;
+- the WAL bytes the source wrote since the previous sample;
+- the number of segments archived since the previous sample, their WAL size, and the bytes written to storage for them;
+- the source's WAL segment size;
+- the highest number of segment uploads running at the same time since the previous sample;
+- the average and the longest time to write one segment to storage since the previous sample;
+- the slot's status as the source reports it;
+- since when the slot has kept extra WAL, meaning status `extended` or `unreserved`, while it does;
+- the slot lag: how much WAL the source keeps for Databasus because Databasus has not received it yet;
+- how much WAL the source can still write before the slot loses WAL, or that the source sets no limit;
+- the unarchived WAL: everything the source wrote after the archive progress point, which is not in storage yet;
+- since when the archive progress point has stood still while the source keeps writing, while it does;
+- the size of the local queue in bytes, and the size at which the receiver is paused;
+- the number of receiver restarts since the previous sample, not counting pauses for a full local queue;
+- the number of receiver pauses for a full local queue since the previous sample;
+- whether the receiver was running;
+- whether the source refused the forced WAL switch in this session.
+
+The two "since" times about the slot and the archive SHALL come from the tracking that decides the existing "WAL chain at risk" and "archiving fell behind" notifications. A sample and a notification then agree about when a condition began.
+
+Source-side values SHALL be absent from a sample taken while the source was unreachable or without the slot. The unarchived WAL SHALL also be absent while nothing has been archived for the database yet. The other values SHALL still be recorded.
+
+The WAL bytes the source wrote since the previous sample SHALL also be absent in three more cases:
+
+- the first sample of a streaming session;
+- the first sample after one without source-side values;
+- a sample whose source position is behind the previous sample's position, as happens when the source switches to a new timeline.
+
+Probing the source SHALL give up after 10 seconds, so a source that accepts connections but does not answer still yields a sample on time.
+
+#### Scenario: Healthy streaming
+
+- **WHEN** four consecutive sampling ticks run for a streaming database with the source reachable
+- **THEN** four samples exist for it, all with the source reachable
+- **AND** every sample after the first holds a source write value
+- **AND** every sample holds either a remaining WAL allowance or the marker that the source sets no limit
+- **AND** every sample holds the slot lag, the unarchived WAL, the local queue size and its pause size
+
+#### Scenario: Source unreachable
+
+- **WHEN** the source stops answering while the streamer keeps running
+- **THEN** samples keep being recorded with the source unreachable
+- **AND** their slot status, slot lag, remaining WAL allowance, unarchived WAL and source write values are absent
+- **AND** they all carry the same time since which the source has been unreachable
+
+#### Scenario: Unreachable across a session restart
+
+- **WHEN** the newest sample found the source unreachable and a new streaming session takes its first sample with the source still unreachable
+- **THEN** that sample carries the same unreachable-since time as the newest sample before it
+
+#### Scenario: Silent source
+
+- **WHEN** the source accepts connections but never answers the probe
+- **THEN** a sample with the source unreachable is recorded no later than 10 seconds after its tick
+
+#### Scenario: Slot removed on the source
+
+- **WHEN** the source answers but Databasus's replication slot no longer exists there
+- **THEN** the sample records the source as reachable without the slot, not as unreachable
+
+#### Scenario: Slot lost on the source
+
+- **WHEN** the source reports the slot's status as `lost`
+- **THEN** the sample still holds the source's real current WAL position
+- **AND** no unarchived WAL value in it is negative
+
+#### Scenario: Source write volume after a restart
+
+- **WHEN** the streamer starts and takes its first sample
+- **THEN** that sample has no value for WAL written since the previous sample
+- **AND** the second sample has one
+
+#### Scenario: Source reachable again
+
+- **WHEN** a sample is taken while the source is unreachable and the next one is taken with the source reachable again
+- **THEN** the next sample has no value for WAL written since the previous sample
+- **AND** the sample after it has one
+
+#### Scenario: Source position moves backwards
+
+- **WHEN** the source's current WAL position in a sample is lower than in the previous sample
+- **THEN** that sample has no value for WAL written since the previous sample
+- **AND** no sample holds a negative write value
+
+#### Scenario: Pause for a full local queue
+
+- **WHEN** the local queue reaches the pause size and the receiver is stopped and started again once it drains
+- **THEN** the next sample counts one pause and no restart
+
+#### Scenario: Database not streaming
+
+- **WHEN** a physical database in full and incremental mode is reconciled alongside a WAL streaming database
+- **AND** the WAL streaming database has its first sample
+- **THEN** no sample exists for the full and incremental database
+
+### Requirement: Samples are kept for 7 days
+
+Databasus SHALL delete samples older than 7 days at least once an hour, whether or not the database still streams. Deleting a database SHALL delete its samples.
+
+#### Scenario: Expired samples
+
+- **WHEN** the hourly cleanup runs and a database has samples older than 7 days
+- **THEN** those samples no longer exist and the newer ones remain
+
+#### Scenario: Streaming switched off
+
+- **WHEN** a database stopped streaming 8 days ago
+- **THEN** none of its samples remain
+
+#### Scenario: Database removed
+
+- **WHEN** a database with samples is deleted
+- **THEN** none of its samples remain
+
+### Requirement: Only users who can view the database can read its WAL stream statistics
+
+The WAL stream statistics of a database SHALL be returned only to an authenticated user who can view that database. Any other request SHALL be refused, the same way the other physical backup endpoints refuse it. The statistics SHALL contain no connection details, credentials, error messages from the source or WAL contents.
+
+#### Scenario: Member of the database's workspace
+
+- **WHEN** a member of the workspace that owns the database requests its statistics
+- **THEN** the statistics are returned
+
+#### Scenario: User from another workspace
+
+- **WHEN** a user without access to the database's workspace requests its statistics
+- **THEN** the request is refused
+
+#### Scenario: Unauthenticated request
+
+- **WHEN** a request without valid authentication asks for statistics
+- **THEN** the request is refused with an authentication error
+
+#### Scenario: Nonexistent database
+
+- **WHEN** an authenticated user requests statistics for a database ID that does not exist
+- **THEN** the request is refused
+- **AND** no statistics are returned
+
+### Requirement: The statistics API is a documented, unit-explicit contract
+
+The statistics SHALL be served by an authenticated API endpoint documented with the rest of the API. Every amount SHALL carry its unit in its name, in bytes, bytes per second, seconds or milliseconds. Every time SHALL be an absolute UTC time. Every verdict, state, reason and notice SHALL be a stable code from a documented list, not display text, so other consumers can rely on it.
+
+A response SHALL hold:
+
+- the health verdict, its reasons and its notices;
+- whether a full backup is requested or running;
+- the streamer's state;
+- the current values from the newest sample, the time of that sample and the time the newest WAL segment was archived;
+- the keeping-up summary;
+- a chart series for the requested period.
+
+A request for a database that is not a PostgreSQL physical database SHALL be refused as invalid. A PostgreSQL physical database outside WAL streaming mode SHALL get the not streaming verdict and whatever samples it still has.
+
+#### Scenario: Codes, not text
+
+- **WHEN** a client reads a response for a database that needs attention
+- **THEN** the verdict, each reason and each notice are codes from the documented lists
+- **AND** no field holds a sentence meant for display
+
+#### Scenario: Logical database
+
+- **WHEN** a user requests statistics for a logical database they can view
+- **THEN** the request is refused as invalid
+
+#### Scenario: Switched out of WAL streaming
+
+- **WHEN** a physical database switched from WAL streaming to full and incremental mode yesterday
+- **THEN** the verdict is not streaming
+- **AND** the series holds the samples recorded before the switch
+
+### Requirement: The streamer's state accounts for its heartbeat and repeated failures
+
+The streamer's state SHALL be running only while Databasus marks it running and its heartbeat is younger than the age after which another tick may take the streamer over. It SHALL be not running when it is marked failed or its heartbeat is older. It SHALL be not started when no streamer has ever run for the database.
+
+Databasus SHALL remember since when the streamer has been failing: from its first failure after the last session that recorded a sample, until a session records a sample again. A failing streamer that is taken over and fails again SHALL keep the first failure time.
+
+#### Scenario: Stale heartbeat
+
+- **WHEN** the streamer is marked running but its last heartbeat is older than the takeover age
+- **THEN** the state is not running
+
+#### Scenario: Streamer never started
+
+- **WHEN** no streamer has ever run for the database
+- **THEN** the state is not started
+
+#### Scenario: Crash loop before the first sample
+
+- **WHEN** every new session fails before its first sample for five minutes
+- **THEN** the failing-since time is the time of the first of those failures
+
+### Requirement: The statistics state one health verdict with reasons
+
+The statistics SHALL state one verdict:
+
+- **not streaming**: WAL streaming is not expected to run, because the database is not in WAL streaming mode, its backups are disabled or it has no storage;
+- **collecting data**: the streamer is not failing, no sample is newer than the current session's start, and that session started less than 2 minutes ago, or no streamer has started yet;
+- **broken**: at least one reason applies at the broken level;
+- **needs attention**: at least one reason applies at the needs-attention level and none at the broken level;
+- **healthy**: no reason applies.
+
+The first matching verdict in this order wins.
+
+Each reason SHALL carry its code and its level. It SHALL also carry the time the condition began when that time is known, and the measured amount and the threshold it was compared with when the rule compares an amount. A needs-attention reason that is read from samples SHALL appear only when its condition holds in the two newest samples, so a single unusual sample does not change the verdict. Reasons whose rule already waits for a duration are exempt.
+
+| Reason | Needs attention | Broken |
+|---|---|---|
+| Streaming stopped | - | the streamer has been failing for 1 minute or longer, or the newest sample is older than 2 minutes, and the newest sample did not find the source unreachable |
+| Source unreachable | the newest sample found the source unreachable, and the source has been unreachable for 1 to 5 minutes since | the same for 5 minutes or longer, also when streaming stopped after that sample |
+| WAL chain broken | - | the newest completed full backup's chain has a WAL gap below the archive progress point; it clears when a newer full backup completes |
+| Slot missing | - | the newest sample found the source without the slot |
+| Slot lost | - | the newest sample reports the slot's status as `lost` |
+| Archive not advancing | the archive progress point has stood still for 10 minutes or longer while the source keeps writing | the same for the archive staleness threshold of 30 minutes or longer, which is when the "archiving fell behind" notification fires |
+| Archive slower than source | the unarchived WAL grew by at least the backlog threshold over the last 10 minutes | - |
+| Slot keeps extra WAL | the slot's status has been `unreserved` for longer than 5 minutes, or `extended` for longer than 5 minutes while the slot lag grows | - |
+| Slot limit approaching | the source sets a limit, and the remaining WAL allowance would run out in under 1 hour if the slot lag keeps growing at its rate of the last 10 minutes | - |
+| Near slot rebuild | the slot lag is at least 80% of the configured lag at which Databasus rebuilds the slot, and the slot's status is neither `extended` nor `unreserved` | - |
+| Local queue filling | the local queue is at least half the size at which the receiver is paused | - |
+| Frequent restarts | receiver restarts and streaming session starts in the last hour add up to 3 or more | - |
+| Forced WAL switch refused | the newest sample says the source refused the forced WAL switch | - |
+
+The backlog threshold is 64 MiB or four WAL segments, whichever is larger. Growth of the unarchived WAL SHALL be measured between the lowest value in the first 2.5 minutes of the window and the lowest value in its last 2.5 minutes, so normal upload batches do not count as growth.
+
+When the newest sample is more than 2 minutes old, reasons read from that sample SHALL be dropped, except the source unreachable reason described in the table. The WAL chain broken reason does not depend on samples and stays.
+
+Reasons read from the slot or the archive progress point SHALL be dropped while the newest sample has no source-side values.
+
+The statistics SHALL also list notices, which never change the verdict:
+
+- **no slot WAL limit**: the newest sample was taken with the source reachable, the slot's status is not `lost`, and the source sets no limit on WAL kept for the slot;
+- **slot keeps extra WAL steadily**: the slot's status has been `extended` for longer than 5 minutes while the slot lag does not grow;
+- **no full backup yet**: no full backup of the database has completed, so archived WAL cannot be restored yet.
+
+#### Scenario: Everything in order
+
+- **WHEN** the source is reachable, archiving keeps pace and no threshold above is crossed
+- **THEN** the verdict is healthy and no reason is listed
+
+#### Scenario: Single unusual sample
+
+- **WHEN** only the newest sample shows the local queue above half its pause size
+- **THEN** the local queue filling reason is not listed
+
+#### Scenario: Short network blip
+
+- **WHEN** the source stopped answering 30 seconds ago
+- **THEN** the source unreachable reason is not listed
+
+#### Scenario: Source unreachable for a few minutes
+
+- **WHEN** the source has been unreachable for 3 minutes and the streamer keeps sampling
+- **THEN** the verdict is needs attention with the source unreachable reason and the time it began
+
+#### Scenario: Source down for a while
+
+- **WHEN** the newest sample found the source unreachable, the streamer then stopped, and the source has been unreachable for 6 minutes
+- **THEN** the verdict is broken with the source unreachable reason, not the streaming stopped reason
+
+#### Scenario: Archive stuck
+
+- **WHEN** the source keeps writing and the archive progress point has not moved for 12 minutes
+- **THEN** the verdict is needs attention with the archive not advancing reason
+- **AND** after 30 minutes without movement the same reason is listed at the broken level
+
+#### Scenario: Streamer stopped
+
+- **WHEN** the streamer failed and the newest sample, taken with the source reachable, is ten minutes old
+- **THEN** the verdict is broken with the streaming stopped reason
+- **AND** the response carries the time of that newest sample
+- **AND** no other reason read from that sample is listed
+
+#### Scenario: Restart after a deploy
+
+- **WHEN** Databasus restarted 20 seconds ago after 10 minutes of downtime, and the new session has not taken a sample yet
+- **THEN** the verdict is collecting data, not broken
+
+#### Scenario: Crash loop on a new database
+
+- **WHEN** WAL streaming was enabled five minutes ago and every session has failed before its first sample
+- **THEN** the verdict is broken with the streaming stopped reason
+
+#### Scenario: Slot rebuilt
+
+- **WHEN** Databasus rebuilt the slot after it was lost and the requested full backup has not completed yet
+- **THEN** the verdict is broken with the WAL chain broken reason, even though the new slot is healthy
+- **AND** once a newer full backup completes, the reason is no longer listed
+
+#### Scenario: Slot removed on the source
+
+- **WHEN** the newest sample found the source without the slot
+- **THEN** the verdict is broken with the slot missing reason, not the source unreachable reason
+
+#### Scenario: Just enabled
+
+- **WHEN** WAL streaming was just enabled and no sample exists yet
+- **THEN** the verdict is collecting data
+
+#### Scenario: Backups disabled
+
+- **WHEN** a database in WAL streaming mode has its backups disabled
+- **THEN** the verdict is not streaming
+- **AND** the series still holds the samples recorded before
+
+#### Scenario: Several problems at once
+
+- **WHEN** the local queue has been half full for two samples and the slot is missing
+- **THEN** the verdict is broken
+- **AND** both reasons are listed, each with its own level
+
+#### Scenario: Busy cluster with steady extra WAL
+
+- **WHEN** the slot's status has been `extended` for an hour while the slot lag stays flat
+- **THEN** the slot keeps extra WAL reason is not listed
+- **AND** the slot keeps extra WAL steadily notice is listed
+
+#### Scenario: Paused receiver above the rebuild lag
+
+- **WHEN** the receiver is paused for a full local queue and the slot lag is at 90% of the rebuild lag with status `reserved`
+- **THEN** the near slot rebuild reason is listed
+
+#### Scenario: No limit on the source
+
+- **WHEN** the source's `max_slot_wal_keep_size` is `-1` and nothing else is wrong
+- **THEN** the verdict is healthy
+- **AND** the no slot WAL limit notice is listed
+
+#### Scenario: Slot already lost
+
+- **WHEN** the newest sample reports the slot's status as `lost`
+- **THEN** the no slot WAL limit notice is not listed
+
+### Requirement: The statistics say whether archiving keeps up with the source
+
+The keeping-up summary SHALL cover the samples of the last 10 minutes and hold:
+
+- the source's write rate and the archive rate in WAL bytes per second, both over the seconds covered by samples that hold a source write value;
+- the unarchived WAL now and how fast it changes, in bytes per second, negative while it shrinks;
+- the slot lag now and how fast it changes;
+- the configured slot lag at which Databasus rebuilds the slot;
+- an estimate of the time until the unarchived WAL is archived, while it shrinks;
+- an estimate of the time until the slot lag reaches the rebuild lag, while the slot lag grows and the slot's status is neither `extended` nor `unreserved`;
+- an estimate of the time until the remaining WAL allowance runs out, while the slot lag grows and the source sets a limit;
+- one keeping-up state.
+
+The keeping-up state SHALL be the first of these that matches:
+
+1. **unknown**: the window holds no sample with a source write value;
+2. **falling behind**: the archive slower than source reason applies, ignoring its two-sample rule;
+3. **catching up**: the unarchived WAL is at least the backlog threshold and its lowest value in the last 2.5 minutes is below its lowest value in the first 2.5 minutes;
+4. **idle**: the source wrote less than 1 MiB over the window and the unarchived WAL is at most one segment;
+5. **keeping up**: anything else.
+
+#### Scenario: Archiving keeps up
+
+- **WHEN** the source writes 10 MB/s and Databasus archives 10 MB/s for ten minutes
+- **THEN** the keeping-up state is keeping up
+- **AND** both rates are about 10 MB/s
+
+#### Scenario: Archiving falls behind
+
+- **WHEN** the source writes 21 MB/s and Databasus archives 12 MB/s for ten minutes
+- **THEN** the keeping-up state is falling behind
+- **AND** the unarchived WAL grows by about 9 MB/s
+
+#### Scenario: Catching up after a burst
+
+- **WHEN** 3 GiB of WAL is unarchived and it shrank by 600 MiB over the last 10 minutes
+- **THEN** the keeping-up state is catching up
+- **AND** the estimate until it is archived is about 50 minutes
+
+#### Scenario: Running out of allowance
+
+- **WHEN** the source sets a limit, 5 GiB of allowance remains and the slot lag grew by 1 GiB over the last 10 minutes
+- **THEN** the estimate until the allowance runs out is about 50 minutes
+
+#### Scenario: Quiet database
+
+- **WHEN** the source wrote 200 KiB over the last 10 minutes and 100 KiB is unarchived
+- **THEN** the keeping-up state is idle
+
+#### Scenario: Quiet database with a stuck backlog
+
+- **WHEN** the source wrote 200 KiB over the last 10 minutes and 2 GiB is unarchived and not shrinking
+- **THEN** the keeping-up state is not idle
+
+### Requirement: The statistics cover the last hour, day or week
+
+A request SHALL name a period of the last hour, the last day or the last week. The series SHALL hold at most 360 points of equal length and state that length: 1 minute for an hour, 4 minutes for a day and 28 minutes for a week. The last point covers the interval that contains the current moment. In each point:
+
+- write and archive volumes become rates in bytes per second over the seconds its samples cover;
+- slot lag, unarchived WAL and local queue size take the highest value in the interval;
+- receiver restarts, receiver pauses and streaming session starts are summed;
+- upload concurrency and the longest segment write take the highest value, and the average segment write is weighted by the segments archived.
+
+An interval with no samples SHALL appear as a point without values, not as zero.
+
+#### Scenario: Week view
+
+- **WHEN** a user requests the last week for a database that streamed the whole week
+- **THEN** the series has at most 360 points
+- **AND** each point's archive rate equals the WAL bytes archived in its interval divided by the seconds its samples cover
+
+#### Scenario: Streamer was down for an hour
+
+- **WHEN** the streamer recorded no samples for one hour of the requested day
+- **THEN** the points covering that hour carry no values
+
+#### Scenario: Unknown period
+
+- **WHEN** a request names a period other than hour, day or week
+- **THEN** the request is refused as invalid
+
+### Requirement: The Monitoring tab appears only for WAL streaming databases
+
+The database page SHALL show a Monitoring tab for a PostgreSQL physical database whose backup mode is full + incremental + WAL streaming, and for no other database.
+
+#### Scenario: WAL streaming database
+
+- **WHEN** a user opens a PostgreSQL physical database in full + incremental + WAL streaming mode
+- **THEN** the page offers Config, Backups and Monitoring tabs
+
+#### Scenario: Other databases
+
+- **WHEN** a user opens a logical database or a physical database in a mode without WAL streaming
+- **THEN** the page offers no Monitoring tab
+
+### Requirement: The Monitoring tab speaks plain language
+
+Every label, headline, sentence, legend and chart title the tab shows by default SHALL avoid the terms "WAL", "slot", "LSN" and "lag". The tab SHALL call the unarchived WAL "not saved to storage yet" and the slot lag "not received by Databasus yet", and SHALL say that the second is part of the first. Technical names SHALL appear only in explanations shown on hover or keyboard focus, in the details section, and in a "what to do" sentence that names a setting or permission to change, such as `max_slot_wal_keep_size`.
+
+Each reason and each notice SHALL have one sentence about what happened, with its amounts and, where known, how long it has lasted, and one sentence about what to do. When Databasus resolves a condition by itself, the second sentence SHALL say so, and only then. For a broken chain or a missing slot, the second sentence SHALL ask the user to start a full backup unless one is requested or running. Durations SHALL be rounded for reading: minutes under an hour, hours and minutes under a day, days and hours beyond.
+
+#### Scenario: Plain names
+
+- **WHEN** a user opens the Monitoring tab of a healthy database
+- **THEN** no default label, legend or chart title contains "WAL", "slot", "LSN" or "lag"
+
+#### Scenario: Broken chain without a pending full backup
+
+- **WHEN** the WAL chain broken reason is shown and no full backup is requested or running
+- **THEN** the tab asks the user to start a full backup
+- **AND** does not say that Databasus handles it by itself
+
+#### Scenario: Every reason has an action
+
+- **WHEN** any reason or notice is shown
+- **THEN** it has both a sentence about what happened and a sentence about what to do
+
+### Requirement: The Monitoring tab leads with the verdict
+
+The tab SHALL open with the verdict as a headline: all is well, something needs attention, something is broken, data is being collected, or streaming is off. The headline SHALL use the verdict's color, green, orange, red or neutral gray, together with an icon and words, so color is never the only signal. Screen readers SHALL announce a change of the headline.
+
+The not streaming headline SHALL say how to turn streaming on and link to the database's configuration. Below the headline, the tab SHALL list the reasons, broken ones first. Notices SHALL appear after the reasons in a neutral informational style, and the no slot WAL limit notice SHALL explain in plain words what the limit protects and what it costs.
+
+#### Scenario: Healthy database
+
+- **WHEN** the verdict is healthy
+- **THEN** the headline is green and says that everything is in order
+- **AND** no reason list is shown
+
+#### Scenario: Problems listed
+
+- **WHEN** the verdict is broken because the slot is missing and the local queue is filling
+- **THEN** the headline is red
+- **AND** the slot missing reason is listed before the local queue reason
+
+### Requirement: The Monitoring tab explains whether archiving keeps up
+
+Right below the headline, the tab SHALL describe the keeping-up state in one or two plain sentences:
+
+- keeping up: Databasus saves changes as fast as the database writes them, with both rates;
+- falling behind: the database writes changes faster than Databasus saves them, by how much per second, how much is not saved yet and how fast that amount grows;
+- catching up: Databasus is saving a backlog, how much is left and about how long it takes;
+- idle: the database writes almost nothing, and new changes reach storage within minutes;
+- unknown: no current numbers are available.
+
+When an estimate until the rebuild lag or until the WAL allowance runs out exists and is under one day, the tab SHALL add what happens then: Databasus will start a new backup chain from a full backup, or the source will drop changes that Databasus has not received yet.
+
+#### Scenario: Falling behind explained
+
+- **WHEN** the source writes 21 MB/s, Databasus archives 12 MB/s and 3.4 GB is unarchived
+- **THEN** the tab says that the database writes changes faster than Databasus saves them, by about 9 MB/s
+- **AND** says that 3.4 GB is not saved yet and grows by about 32 GB an hour
+
+#### Scenario: Consequence only when near
+
+- **WHEN** the estimate until the rebuild lag is three days
+- **THEN** the tab adds no sentence about a new backup chain
+
+### Requirement: The Monitoring tab shows where unsaved changes are
+
+The tab SHALL show the path of changes as three stages, source, Databasus and storage, with:
+
+- at the source: what is not received by Databasus yet, against the rebuild lag and, when the source sets a limit, the remaining allowance;
+- in Databasus: the local queue, against the size at which the receiver is paused, and whether receiving is paused;
+- in storage: how long ago the newest WAL segment was archived;
+- between the stages: the source's write rate and the archive rate.
+
+An amount above its limit SHALL show as over the limit, not as a full bar. Each stage SHALL take the color of the most severe reason about it. Each amount on a stage SHALL offer a short explanation on hover or keyboard focus. On a narrow screen the stages SHALL stack vertically.
+
+#### Scenario: Backlog in the local queue
+
+- **WHEN** the slot lag is small and the local queue holds 300 MB of its 512 MB pause size for two samples
+- **THEN** the Databasus stage shows 300 MB of 512 MB in the needs-attention color
+- **AND** the source stage stays in the healthy color
+
+### Requirement: The Monitoring tab shows charts for the selected period
+
+The tab SHALL offer the last hour, day and week, and show for the selected period:
+
+- the source write rate and the archive rate on one chart;
+- what is not saved to storage yet and what is not received by Databasus yet on one chart;
+- the local queue size with its pause size, and receiver restarts and pauses.
+
+Axes SHALL show formatted amounts and local times. Hovering a time on one chart SHALL mark the same time on the others and show the formatted values. Intervals without samples SHALL show as gaps. When the series holds no point with values, the tab SHALL say that charts appear after a few minutes of streaming instead of showing empty charts. When the data starts inside the period, the tab SHALL say since when it has data.
+
+A details section, collapsed by default, SHALL show the raw values with their technical names: the slot's status, the remaining allowance, the average and longest segment write, the upload concurrency, the ratio of WAL to stored bytes, the WAL segment size and the streaming session's start.
+
+#### Scenario: Archiving falls behind on the chart
+
+- **WHEN** every point of the hour holds a source write rate of 21 MB/s and an archive rate of 12 MB/s, and a rising unarchived amount
+- **THEN** the archive series lies below the source series at every point
+- **AND** the not-saved-yet series rises across the hour
+
+#### Scenario: No history yet
+
+- **WHEN** the verdict is collecting data and the series holds no point with values
+- **THEN** the tab shows the explanation instead of the charts
+
+### Requirement: The Monitoring tab stays current without disturbing the reader
+
+The tab SHALL refresh every 30 seconds in every period while it is open and the browser tab is visible. It SHALL refresh at once when the browser tab becomes visible again and when the period changes.
+
+The tab SHALL show the age of the newest sample, updated every second, and mark it as live only while that sample is younger than 2 minutes. When the last successful refresh is older than 2 minutes, the headline SHALL turn neutral gray and say that the current state cannot be confirmed, with the last known verdict and its age.
+
+A refresh SHALL keep the previous content on screen until the new content arrives, keep the scroll position and the selected period, and not replay the charts' entry animation. Changed numbers and the headline color SHALL change with a short transition. When the reader's system asks for reduced motion, there SHALL be no transitions, no pulsing and no chart animation. A failed refresh SHALL keep the last content, say that refreshing failed, and keep retrying on the same schedule. A failed first load SHALL show the error and a retry button.
+
+#### Scenario: Status changes while open
+
+- **WHEN** the tab is open on the day view and the verdict changes from healthy to needs attention
+- **THEN** the headline changes within 30 seconds without reloading the page
+
+#### Scenario: Browser tab hidden
+
+- **WHEN** the browser tab is hidden for ten minutes and then shown again
+- **THEN** no refresh happened while it was hidden
+- **AND** a refresh happens as soon as it is shown
+
+#### Scenario: Refresh keeps failing
+
+- **WHEN** refreshes have failed for 3 minutes after a healthy response
+- **THEN** the headline is gray and says the state cannot be confirmed, last known healthy 3 minutes ago
diff --git a/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/tasks.md b/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/tasks.md
new file mode 100644
index 00000000..00e7a5f9
--- /dev/null
+++ b/openspec/changes/archive/2026-10-19-wal-stream-monitoring-tab/tasks.md
@@ -0,0 +1,113 @@
+This change builds on the applied `parallel-wal-segment-uploads`: it reads that change's archive progress point and upload snapshot. Backend commands run in `backend/` unless a task says otherwise, frontend commands in `frontend/`. `<pg>` stands for `./internal/features/backups/backups/usecases/physical/postgresql/...`, `<core>` for `./internal/features/backups/backups/core/physical/...` and `<backuping>` for `./internal/features/backups/backups/backuping/physical/...`.
+
+Every frontend task that introduces a translation key adds it to all six dictionaries in the same task, written in each language rather than word for word, because the keys are typed against `en.ts` and every other dictionary is typed `typeof en`.
+
+## 1. Storage
+
+- [x] 1.1 Create the migration from the repository root with `goose -dir backend/migrations create add_physical_wal_stream_samples sql` and fill it per design D1 and D5: the samples table, its foreign key to `databases` with ON DELETE CASCADE, the `(database_id, sampled_at)` and `(sampled_at)` indexes, the nullable `failing_since` TIMESTAMPTZ column on `physical_wal_streamers`, and a `Down` section that drops them. From the repository root, with `.env` exported by `set -a; source .env; set +a`:
+ - run `goose -dir backend/migrations up`, then `down`, then `up` against the dev database;
+ - apply it to every test slot database, because `just test-package` does not migrate them: `for ((i = 0; i < ${TEST_PARALLEL_WORKERS:-8}; i++)); do GOOSE_DRIVER=postgres GOOSE_DBSTRING="$(sed -E "s#/([^/?]+)\?#/\1_w$i?#" <<<"$GOOSE_TEST_DBSTRING")" goose -dir backend/migrations up; done`, the loop from `scripts/test.sh:49-54`.
+
+ Verify that every goose command exits with status 0.
+- [x] 1.2 Add the `PhysicalWalStreamSample` model in `core/physical/models` and its repository in `core/physical/repositories`: insert, delete older than a time across all databases, newest for a database, samples since a time, session starts since a time (by `session_started_at`), and the bucketed series (D10). Add `failing_since` to the streamer model, make `MarkFailed` and `MarkStaleRunningFailed` set it with `COALESCE`, and add a method that clears it. The controller tests in section 5 pin the series query. Verify with `just lint` in `backend/` and with `just test-package <core>` passing, the existing streamer repository tests included.
+
+## 2. Shared health rules
+
+- [x] 2.1 Create the `core/physical/wal_health` package (D8). Move `warningSlotStatusHoldPeriod`, `DefaultArchiveStalenessThreshold` and `streamerHeartbeatStaleness` into it as exported constants, and have `<pg>` and `<backuping>` use them. Replace `isWalStreamCandidate` with `PhysicalBackupConfig.IsWalStreamingExpected()`, which also checks `IsBackupsEnabled`, and use it in the supervisor. Verify with `just test-package <pg> <backuping> ./internal/features/backups/config/physical/...` passing, the existing classifier, staleness and supervisor tests included.
+- [x] 2.2 Add the streamer state rule and the verdict function (D8) as pure functions. The verdict is complex rule logic, and testing each threshold through HTTP would need a seeded database per case, so it gets a table-driven unit test in `wal_health`. Verify with `just test-package <core>` passing, covering:
+ - every reason at each of its levels, with values just below and just above each threshold;
+ - the two-sample rule: "Single unusual sample";
+ - the spec scenarios "Everything in order", "Short network blip", "Source unreachable for a few minutes", "Source down for a while", "Archive stuck", "Streamer stopped", "Restart after a deploy", "Crash loop on a new database", "Slot removed on the source", "Just enabled", "Backups disabled", "Several problems at once", "Busy cluster with steady extra WAL", "Paused receiver above the rebuild lag", "No limit on the source" and "Slot already lost";
+ - the gap input set for "Slot rebuilt" gives `WAL_CHAIN_BROKEN`, and the same input without a gap gives none;
+ - `NO_FULL_BACKUP_YET` listed without a completed full backup and the verdict unchanged by it;
+ - "Stale heartbeat" and "Streamer never started" for the streamer state;
+ - a growth of four segments plus one in the unarchived WAL's sawtooth stays below the backlog threshold.
+- [x] 2.3 Add the keeping-up summary (D9) to the same package. Verify with a table-driven test in `wal_health`, run through `just test-package <core>`, covering the spec scenarios "Archiving keeps up", "Archiving falls behind", "Catching up after a burst", "Running out of allowance", "Quiet database" and "Quiet database with a stuck backlog", the unknown state for a window without write values, the first-match order for a shrinking backlog on a quiet source, both rates sharing one denominator when the window has a hole, and no rebuild estimate while the slot's status is `extended`.
+
+## 3. Sampling inputs in the streamer
+
+- [x] 3.1 Add `safe_wal_size` and `pg_current_wal_lsn()` to `InspectSlot` (D3). Use the real current position in the staleness tracker. Verify with tests against the test PostgreSQL container, run through `just test-package <pg>`: the allowance is NULL with `max_slot_wal_keep_size = -1` and positive with a limit set, and the returned position is not below the position read just before the call. The existing staleness tests must still pass.
+- [x] 3.2 Make the slot probe return reachable with a slot state, reachable without the slot, or unreachable, with a 10-second timeout on its context (D2). Map "without the slot" to the current behavior in the slot-LSN watcher and the resume path. Verify with tests in `slot_lsn_watcher_test.go`, run through `just test-package <pg>`:
+ - "Slot removed on the source": dropping the slot on a reachable source gives the "without the slot" outcome;
+ - "Silent source": a probe against a local TCP listener that accepts and never answers returns unreachable within 11 seconds;
+ - the existing watcher and resume tests still pass.
+- [x] 3.3 Count receiver restarts and pauses with atomic counters that the sampler takes and resets (D5). Verify with a test in `wal_stream_test.go`, run through `just test-package <pg>`: one back-pressure pause and one receiver exit of its own give one pause and one restart, and the next take gives zero of each ("Pause for a full local queue").
+- [x] 3.4 Set an atomic flag when the rotation loop gets the insufficient-privilege refusal (D5). Verify with a test in `wal_rotation_test.go`, run through `just test-package <pg>`, against a test source whose backup user lacks `EXECUTE` on `pg_switch_wal()`: the flag is set after the refusal.
+
+## 4. Sampling in the streamer
+
+- [x] 4.1 Add the pure sample builder (D2, D3, D4). Verify with unit tests in `lag_monitor_test.go`, run through `just test-package <pg>`, covering:
+ - "Slot lost on the source": a `lost` state keeps the real current position and gives no negative unarchived value;
+ - "Source position moves backwards": no write value and no negative one;
+ - "Source write volume after a restart" and "Source reachable again";
+ - no unarchived value while the archive progress point is 0;
+ - source-side columns NULL for the unreachable and the without-slot outcomes, other columns filled.
+- [x] 4.2 Turn the lag monitor's tick body into a method that takes `now`, records a sample built by 4.1 after classifying and before acting (D2), carries the trackers' times, keeps `source_unreachable_since` and seeds it from the newest stored sample at session start, and clears `failing_since` on a session's first sample. Pass the repository through `WalStreamSpec`. Verify with tests in `lag_monitor_test.go` that call the tick method against the test source, run through `just test-package <pg>`, covering:
+ - "Healthy streaming": four ticks give four reachable samples with the listed values;
+ - "Source unreachable": ticks against a stopped source give unreachable samples sharing one unreachable-since time;
+ - "Unreachable across a session restart";
+ - a slot held in `extended` gets `slot_extra_wal_since` equal to the time the classifier started its hold;
+ - a rebuild in a tick leaves that tick's sample with the slot values probed before the rebuild;
+ - the first sample of a session clears `failing_since`.
+- [x] 4.3 Wire the repository in `backuping/physical/wal_supervisor.go`. Verify with tests in `wal_supervisor_test.go`, run through `just test-package <backuping>`, covering:
+ - "Database not streaming": reconcile a full + incremental database alongside a WAL streaming control database, wait until the control database has its first sample, and assert that the full + incremental database has none;
+ - "Crash loop before the first sample": sessions that fail at start keep the first `failing_since`.
+- [x] 4.4 Delete samples older than 7 days on each retention cleaner tick (D7). Verify with tests in `cleaner_test.go`, run through `just test-package <backuping>`, covering "Expired samples" and "Streaming switched off".
+
+## 5. Statistics API
+
+- [x] 5.1 Add the read method on the physical core service and `GetWalStreamStatistics` on `PhysicalBackupService` with the current values and the series, authorizing through `databaseService.GetDatabase` and refusing a database that is not PostgreSQL physical (D10). Add the `GET /backups/physical/database/:id/wal-stream/statistics` route with Swagger annotations. Seed samples through a helper in `core/physical/testing`. Verify with controller tests in `controllers/physical/controller_test.go`, run through `just test-package ./internal/features/backups/backups/controllers/physical/...`, covering:
+ - "Member of the database's workspace" gets current values and series;
+ - "User from another workspace", "Unauthenticated request", "Nonexistent database" and "Logical database" are refused, the nonexistent one with no statistics in the body;
+ - "Unknown period" returns 400;
+ - "Week view": at most 360 points, each archive rate equal to the seeded WAL bytes of its interval divided by the seconds its samples cover;
+ - the hour view has 60 one-minute points and the last point contains the current moment;
+ - "Streamer was down for an hour": the points covering the hole carry NULL values;
+ - "Database removed": deleting the database leaves no samples;
+ - `lastArchivedWalAt` equals the newest committed segment's `received_at`.
+- [x] 5.2 Add `ChainViewService.GetNewestCompletedFullWithGaps` (D8). Add `health`, `streamer` and `keepingUp` to the response, with the full backup and chain gap inputs from that method, `isFullBackupPending` from the config's `ForceFullRequestedAt` and the in-flight backup, and the rebuild lag from the backup config. Add Swagger enum annotations for every code. Verify with controller tests in the same file, run through the same command, covering:
+ - "Codes, not text": a needs-attention database returns a level, reasons and notices whose codes are all in the documented lists, and no string field outside the codes, times and IDs;
+ - "Streamer stopped": a `FAILED` row and a newest sample ten minutes old give `BROKEN` with `STREAMING_STOPPED` and that sample's time;
+ - "Slot rebuilt": seeded segments with a gap below the progress point after the newest full give `WAL_CHAIN_BROKEN`, and seeding a newer completed full clears it;
+ - the same gap with an incremental of the newest full marked chain-broken still gives `WAL_CHAIN_BROKEN` and no `NO_FULL_BACKUP_YET`;
+ - `isFullBackupPending` is true with `ForceFullRequestedAt` set and false without it or an in-flight full;
+ - "Backups disabled" and "Switched out of WAL streaming": `NOT_STREAMING` with the seeded series still returned;
+ - "Stale heartbeat": a `RUNNING` row with an old heartbeat gives the not running state.
+- [x] 5.3 Regenerate the API docs with `swag init -d . -g cmd/main.go -o swagger` in `backend/`. Verify that `grep -c 'wal-stream/statistics' swagger/swagger.json` prints at least 1, and that `grep -c 'WAL_CHAIN_BROKEN' swagger/swagger.json` prints at least 1.
+
+## 6. Frontend
+
+- [x] 6.1 Add `shared/lib/formatBytes.ts` (D13). Verify with a vitest test next to it covering the B, KB, MB, GB and TB boundaries with base 1024, then `pnpm test` and `pnpm typecheck` passing.
+- [x] 6.2 Add the response types, one per file under `entity/backups/physical/model`, the label key tables for every code enum with their keys, and `physicalWalStreamStatisticsApi` under `entity/backups/physical/api`, exported through the slice's `index.ts`. Verify with `pnpm typecheck` and `pnpm test` passing; `dictionaries.test.ts` checks the placeholders.
+- [x] 6.3 Add the pure display rules from design D13, except the refresh schedule, to the monitoring slice's `lib/`, with the reason, notice, keeping-up and banner sentence keys from the D11 table. Verify with vitest tests next to them, run through `pnpm test`, covering:
+ - the tab rules for a logical database, a physical database without WAL streaming and a WAL streaming database;
+ - "Healthy database" and "Problems listed": the banner tone per level, broken reasons first;
+ - "Every reason has an action": every reason and notice code maps to both a "what happened" and a "what to do" key;
+ - "Broken chain without a pending full backup": `WAL_CHAIN_BROKEN` and `SLOT_MISSING` map to the "start a full backup" action without a pending full and to the "on its way" action with one;
+ - "Plain names": no default label or sentence in the English dictionary for this slice contains "WAL", "slot", "LSN" or "lag", except "what to do" sentences that name a setting;
+ - "Falling behind explained": parameters of about 9 MB/s, 3.4 GB and about 32 GB an hour;
+ - "Consequence only when near": the consequence sentence present for an estimate under a day and absent for three days;
+ - "Backlog in the local queue": the Databasus stage in the needs-attention color and the source stage in the healthy color;
+ - an amount above its limit marked as over the limit;
+ - "Archiving falls behind on the chart": rows mapped from 21 and 12 MB/s points put the archive series below the source series at every point and keep the not-saved-yet series rising;
+ - NULL points staying gaps, and "No history yet" when no point has values;
+ - "Refresh keeps failing": a last successful refresh older than 2 minutes gives the "cannot confirm" banner with the last known level;
+ - durations rounded to minutes under an hour, hours and minutes under a day, days and hours beyond.
+- [x] 6.4 Add the refresh schedule (D12) as a small controller in the slice's `lib/` with the timer and the page visibility injected. Verify with vitest tests using fake timers, run through `pnpm test`, covering: a refresh every 30 seconds in every period ("Status changes while open"); none while hidden and one at once when visible again ("Browser tab hidden"); one at once on a period change; a failed refresh keeps the previous response and the next one still runs.
+- [x] 6.5 Build the monitoring slice's UI per design D11 and D12: the verdict banner with its live indicator and `aria-live` headline, the reasons and notices, the path of changes, the period selector, the three synchronized charts with formatted axes and the empty state, the collapsed details, the first-load skeleton and error with Retry, the token-based colors and the reduced-motion rules. Take every display decision from 6.3 and the schedule from 6.4. Verify with `pnpm lint`, `pnpm typecheck` and `pnpm test` passing.
+- [x] 6.6 Add the `'monitoring'` tab to `DatabaseComponent` behind the predicate from 6.3, and let the not streaming banner's link switch to the Config tab. Verify with `pnpm typecheck` and `pnpm test` passing.
+
+## 7. Wrap-up
+
+- [x] 7.1 Run the formatters and linters for both modules. Verify with `just format` and `just lint` at the repository root finishing clean.
+
+## Change verification
+
+```sh
+just test-package ./internal/features/backups/backups/core/physical/...
+just test-package ./internal/features/backups/backups/usecases/physical/postgresql/...
+just test-package ./internal/features/backups/backups/backuping/physical/...
+just test-package ./internal/features/backups/config/physical/...
+just test-package ./internal/features/backups/backups/controllers/physical/...
+pnpm --dir frontend test
+```
diff --git a/openspec/specs/physical-backups/wal-stream-monitoring/spec.md b/openspec/specs/physical-backups/wal-stream-monitoring/spec.md
new file mode 100644
index 00000000..00253a57
--- /dev/null
+++ b/openspec/specs/physical-backups/wal-stream-monitoring/spec.md
@@ -0,0 +1,568 @@
+# WAL Stream Monitoring Specification
+
+## Purpose
+
+Lets an operator of any experience see at a glance whether WAL archiving of a streaming PostgreSQL database is healthy, and if it is not, what is wrong, how urgent it is and where the unsaved changes are. The same statistics are available through the authenticated API for other consumers.
+
+## Requirements
+
+### Requirement: A streaming database is sampled every 30 seconds
+
+While Databasus streams WAL for a database, it SHALL record one sample of that database's archiving health every 30 seconds. Each sample SHALL hold:
+
+- when it was taken, and when the current streaming session started;
+- the source's state: reachable, unreachable, or reachable but without Databasus's replication slot;
+- since when the source has been unreachable, while it is, counted across streaming sessions;
+- the source's current WAL position and the archive progress point;
+- the WAL bytes the source wrote since the previous sample;
+- the number of segments archived since the previous sample, their WAL size, and the bytes written to storage for them;
+- the source's WAL segment size;
+- the highest number of segment uploads running at the same time since the previous sample;
+- the average and the longest time to write one segment to storage since the previous sample;
+- the slot's status as the source reports it;
+- since when the slot has kept extra WAL, meaning status `extended` or `unreserved`, while it does;
+- the slot lag: how much WAL the source keeps for Databasus because Databasus has not received it yet;
+- how much WAL the source can still write before the slot loses WAL, or that the source sets no limit;
+- the unarchived WAL: everything the source wrote after the archive progress point, which is not in storage yet;
+- since when the archive progress point has stood still while the source keeps writing, while it does;
+- the size of the local queue in bytes, and the size at which the receiver is paused;
+- the number of receiver restarts since the previous sample, not counting pauses for a full local queue;
+- the number of receiver pauses for a full local queue since the previous sample;
+- whether the receiver was running;
+- whether the source refused the forced WAL switch in this session.
+
+The two "since" times about the slot and the archive SHALL come from the tracking that decides the existing "WAL chain at risk" and "archiving fell behind" notifications. A sample and a notification then agree about when a condition began.
+
+Source-side values SHALL be absent from a sample taken while the source was unreachable or without the slot. The unarchived WAL SHALL also be absent while nothing has been archived for the database yet. The other values SHALL still be recorded.
+
+The WAL bytes the source wrote since the previous sample SHALL also be absent in three more cases:
+
+- the first sample of a streaming session;
+- the first sample after one without source-side values;
+- a sample whose source position is behind the previous sample's position, as happens when the source switches to a new timeline.
+
+Probing the source SHALL give up after 10 seconds, so a source that accepts connections but does not answer still yields a sample on time.
+
+#### Scenario: Healthy streaming
+
+- **WHEN** four consecutive sampling ticks run for a streaming database with the source reachable
+- **THEN** four samples exist for it, all with the source reachable
+- **AND** every sample after the first holds a source write value
+- **AND** every sample holds either a remaining WAL allowance or the marker that the source sets no limit
+- **AND** every sample holds the slot lag, the unarchived WAL, the local queue size and its pause size
+
+#### Scenario: Source unreachable
+
+- **WHEN** the source stops answering while the streamer keeps running
+- **THEN** samples keep being recorded with the source unreachable
+- **AND** their slot status, slot lag, remaining WAL allowance, unarchived WAL and source write values are absent
+- **AND** they all carry the same time since which the source has been unreachable
+
+#### Scenario: Unreachable across a session restart
+
+- **WHEN** the newest sample found the source unreachable and a new streaming session takes its first sample with the source still unreachable
+- **THEN** that sample carries the same unreachable-since time as the newest sample before it
+
+#### Scenario: Silent source
+
+- **WHEN** the source accepts connections but never answers the probe
+- **THEN** a sample with the source unreachable is recorded no later than 10 seconds after its tick
+
+#### Scenario: Slot removed on the source
+
+- **WHEN** the source answers but Databasus's replication slot no longer exists there
+- **THEN** the sample records the source as reachable without the slot, not as unreachable
+
+#### Scenario: Slot lost on the source
+
+- **WHEN** the source reports the slot's status as `lost`
+- **THEN** the sample still holds the source's real current WAL position
+- **AND** no unarchived WAL value in it is negative
+
+#### Scenario: Source write volume after a restart
+
+- **WHEN** the streamer starts and takes its first sample
+- **THEN** that sample has no value for WAL written since the previous sample
+- **AND** the second sample has one
+
+#### Scenario: Source reachable again
+
+- **WHEN** a sample is taken while the source is unreachable and the next one is taken with the source reachable again
+- **THEN** the next sample has no value for WAL written since the previous sample
+- **AND** the sample after it has one
+
+#### Scenario: Source position moves backwards
+
+- **WHEN** the source's current WAL position in a sample is lower than in the previous sample
+- **THEN** that sample has no value for WAL written since the previous sample
+- **AND** no sample holds a negative write value
+
+#### Scenario: Pause for a full local queue
+
+- **WHEN** the local queue reaches the pause size and the receiver is stopped and started again once it drains
+- **THEN** the next sample counts one pause and no restart
+
+#### Scenario: Database not streaming
+
+- **WHEN** a physical database in full and incremental mode is reconciled alongside a WAL streaming database
+- **AND** the WAL streaming database has its first sample
+- **THEN** no sample exists for the full and incremental database
+
+### Requirement: Samples are kept for 7 days
+
+Databasus SHALL delete samples older than 7 days at least once an hour, whether or not the database still streams. Deleting a database SHALL delete its samples.
+
+#### Scenario: Expired samples
+
+- **WHEN** the hourly cleanup runs and a database has samples older than 7 days
+- **THEN** those samples no longer exist and the newer ones remain
+
+#### Scenario: Streaming switched off
+
+- **WHEN** a database stopped streaming 8 days ago
+- **THEN** none of its samples remain
+
+#### Scenario: Database removed
+
+- **WHEN** a database with samples is deleted
+- **THEN** none of its samples remain
+
+### Requirement: Only users who can view the database can read its WAL stream statistics
+
+The WAL stream statistics of a database SHALL be returned only to an authenticated user who can view that database. Any other request SHALL be refused, the same way the other physical backup endpoints refuse it. The statistics SHALL contain no connection details, credentials, error messages from the source or WAL contents.
+
+#### Scenario: Member of the database's workspace
+
+- **WHEN** a member of the workspace that owns the database requests its statistics
+- **THEN** the statistics are returned
+
+#### Scenario: User from another workspace
+
+- **WHEN** a user without access to the database's workspace requests its statistics
+- **THEN** the request is refused
+
+#### Scenario: Unauthenticated request
+
+- **WHEN** a request without valid authentication asks for statistics
+- **THEN** the request is refused with an authentication error
+
+#### Scenario: Nonexistent database
+
+- **WHEN** an authenticated user requests statistics for a database ID that does not exist
+- **THEN** the request is refused
+- **AND** no statistics are returned
+
+### Requirement: The statistics API is a documented, unit-explicit contract
+
+The statistics SHALL be served by an authenticated API endpoint documented with the rest of the API. Every amount SHALL carry its unit in its name, in bytes, bytes per second, seconds or milliseconds. Every time SHALL be an absolute UTC time. Every verdict, state, reason and notice SHALL be a stable code from a documented list, not display text, so other consumers can rely on it.
+
+A response SHALL hold:
+
+- the health verdict, its reasons and its notices;
+- whether a full backup is requested or running;
+- the streamer's state;
+- the current values from the newest sample, the time of that sample and the time the newest WAL segment was archived;
+- the keeping-up summary;
+- a chart series for the requested period.
+
+A request for a database that is not a PostgreSQL physical database SHALL be refused as invalid. A PostgreSQL physical database outside WAL streaming mode SHALL get the not streaming verdict and whatever samples it still has.
+
+#### Scenario: Codes, not text
+
+- **WHEN** a client reads a response for a database that needs attention
+- **THEN** the verdict, each reason and each notice are codes from the documented lists
+- **AND** no field holds a sentence meant for display
+
+#### Scenario: Logical database
+
+- **WHEN** a user requests statistics for a logical database they can view
+- **THEN** the request is refused as invalid
+
+#### Scenario: Switched out of WAL streaming
+
+- **WHEN** a physical database switched from WAL streaming to full and incremental mode yesterday
+- **THEN** the verdict is not streaming
+- **AND** the series holds the samples recorded before the switch
+
+### Requirement: The streamer's state accounts for its heartbeat and repeated failures
+
+The streamer's state SHALL be running only while Databasus marks it running and its heartbeat is younger than the age after which another tick may take the streamer over. It SHALL be not running when it is marked failed or its heartbeat is older. It SHALL be not started when no streamer has ever run for the database.
+
+Databasus SHALL remember since when the streamer has been failing: from its first failure after the last session that recorded a sample, until a session records a sample again. A failing streamer that is taken over and fails again SHALL keep the first failure time.
+
+#### Scenario: Stale heartbeat
+
+- **WHEN** the streamer is marked running but its last heartbeat is older than the takeover age
+- **THEN** the state is not running
+
+#### Scenario: Streamer never started
+
+- **WHEN** no streamer has ever run for the database
+- **THEN** the state is not started
+
+#### Scenario: Crash loop before the first sample
+
+- **WHEN** every new session fails before its first sample for five minutes
+- **THEN** the failing-since time is the time of the first of those failures
+
+### Requirement: The statistics state one health verdict with reasons
+
+The statistics SHALL state one verdict:
+
+- **not streaming**: WAL streaming is not expected to run, because the database is not in WAL streaming mode, its backups are disabled or it has no storage;
+- **collecting data**: the streamer is not failing, no sample is newer than the current session's start, and that session started less than 2 minutes ago, or no streamer has started yet;
+- **broken**: at least one reason applies at the broken level;
+- **needs attention**: at least one reason applies at the needs-attention level and none at the broken level;
+- **healthy**: no reason applies.
+
+The first matching verdict in this order wins.
+
+Each reason SHALL carry its code and its level. It SHALL also carry the time the condition began when that time is known, and the measured amount and the threshold it was compared with when the rule compares an amount. A needs-attention reason that is read from samples SHALL appear only when its condition holds in the two newest samples, so a single unusual sample does not change the verdict. Reasons whose rule already waits for a duration are exempt.
+
+| Reason | Needs attention | Broken |
+|---|---|---|
+| Streaming stopped | - | the streamer has been failing for 1 minute or longer, or the newest sample is older than 2 minutes, and the newest sample did not find the source unreachable |
+| Source unreachable | the newest sample found the source unreachable, and the source has been unreachable for 1 to 5 minutes since | the same for 5 minutes or longer, also when streaming stopped after that sample |
+| WAL chain broken | - | the newest completed full backup's chain has a WAL gap below the archive progress point; it clears when a newer full backup completes |
+| Slot missing | - | the newest sample found the source without the slot |
+| Slot lost | - | the newest sample reports the slot's status as `lost` |
+| Archive not advancing | the archive progress point has stood still for 10 minutes or longer while the source keeps writing | the same for the archive staleness threshold of 30 minutes or longer, which is when the "archiving fell behind" notification fires |
+| Archive slower than source | the unarchived WAL grew by at least the backlog threshold over the last 10 minutes | - |
+| Slot keeps extra WAL | the slot's status has been `unreserved` for longer than 5 minutes, or `extended` for longer than 5 minutes while the slot lag grows | - |
+| Slot limit approaching | the source sets a limit, and the remaining WAL allowance would run out in under 1 hour if the slot lag keeps growing at its rate of the last 10 minutes | - |
+| Near slot rebuild | the slot lag is at least 80% of the configured lag at which Databasus rebuilds the slot, and the slot's status is neither `extended` nor `unreserved` | - |
+| Local queue filling | the local queue is at least half the size at which the receiver is paused | - |
+| Frequent restarts | receiver restarts and streaming session starts in the last hour add up to 3 or more | - |
+| Forced WAL switch refused | the newest sample says the source refused the forced WAL switch | - |
+
+The backlog threshold is 64 MiB or four WAL segments, whichever is larger. Growth of the unarchived WAL SHALL be measured between the lowest value in the first 2.5 minutes of the window and the lowest value in its last 2.5 minutes, so normal upload batches do not count as growth.
+
+When the newest sample is more than 2 minutes old, reasons read from that sample SHALL be dropped, except the source unreachable reason described in the table. The WAL chain broken reason does not depend on samples and stays.
+
+Reasons read from the slot or the archive progress point SHALL be dropped while the newest sample has no source-side values.
+
+The statistics SHALL also list notices, which never change the verdict:
+
+- **no slot WAL limit**: the newest sample was taken with the source reachable, the slot's status is not `lost`, and the source sets no limit on WAL kept for the slot;
+- **slot keeps extra WAL steadily**: the slot's status has been `extended` for longer than 5 minutes while the slot lag does not grow;
+- **no full backup yet**: no full backup of the database has completed, so archived WAL cannot be restored yet.
+
+#### Scenario: Everything in order
+
+- **WHEN** the source is reachable, archiving keeps pace and no threshold above is crossed
+- **THEN** the verdict is healthy and no reason is listed
+
+#### Scenario: Single unusual sample
+
+- **WHEN** only the newest sample shows the local queue above half its pause size
+- **THEN** the local queue filling reason is not listed
+
+#### Scenario: Short network blip
+
+- **WHEN** the source stopped answering 30 seconds ago
+- **THEN** the source unreachable reason is not listed
+
+#### Scenario: Source unreachable for a few minutes
+
+- **WHEN** the source has been unreachable for 3 minutes and the streamer keeps sampling
+- **THEN** the verdict is needs attention with the source unreachable reason and the time it began
+
+#### Scenario: Source down for a while
+
+- **WHEN** the newest sample found the source unreachable, the streamer then stopped, and the source has been unreachable for 6 minutes
+- **THEN** the verdict is broken with the source unreachable reason, not the streaming stopped reason
+
+#### Scenario: Archive stuck
+
+- **WHEN** the source keeps writing and the archive progress point has not moved for 12 minutes
+- **THEN** the verdict is needs attention with the archive not advancing reason
+- **AND** after 30 minutes without movement the same reason is listed at the broken level
+
+#### Scenario: Streamer stopped
+
+- **WHEN** the streamer failed and the newest sample, taken with the source reachable, is ten minutes old
+- **THEN** the verdict is broken with the streaming stopped reason
+- **AND** the response carries the time of that newest sample
+- **AND** no other reason read from that sample is listed
+
+#### Scenario: Restart after a deploy
+
+- **WHEN** Databasus restarted 20 seconds ago after 10 minutes of downtime, and the new session has not taken a sample yet
+- **THEN** the verdict is collecting data, not broken
+
+#### Scenario: Crash loop on a new database
+
+- **WHEN** WAL streaming was enabled five minutes ago and every session has failed before its first sample
+- **THEN** the verdict is broken with the streaming stopped reason
+
+#### Scenario: Slot rebuilt
+
+- **WHEN** Databasus rebuilt the slot after it was lost and the requested full backup has not completed yet
+- **THEN** the verdict is broken with the WAL chain broken reason, even though the new slot is healthy
+- **AND** once a newer full backup completes, the reason is no longer listed
+
+#### Scenario: Slot removed on the source
+
+- **WHEN** the newest sample found the source without the slot
+- **THEN** the verdict is broken with the slot missing reason, not the source unreachable reason
+
+#### Scenario: Just enabled
+
+- **WHEN** WAL streaming was just enabled and no sample exists yet
+- **THEN** the verdict is collecting data
+
+#### Scenario: Backups disabled
+
+- **WHEN** a database in WAL streaming mode has its backups disabled
+- **THEN** the verdict is not streaming
+- **AND** the series still holds the samples recorded before
+
+#### Scenario: Several problems at once
+
+- **WHEN** the local queue has been half full for two samples and the slot is missing
+- **THEN** the verdict is broken
+- **AND** both reasons are listed, each with its own level
+
+#### Scenario: Busy cluster with steady extra WAL
+
+- **WHEN** the slot's status has been `extended` for an hour while the slot lag stays flat
+- **THEN** the slot keeps extra WAL reason is not listed
+- **AND** the slot keeps extra WAL steadily notice is listed
+
+#### Scenario: Paused receiver above the rebuild lag
+
+- **WHEN** the receiver is paused for a full local queue and the slot lag is at 90% of the rebuild lag with status `reserved`
+- **THEN** the near slot rebuild reason is listed
+
+#### Scenario: No limit on the source
+
+- **WHEN** the source's `max_slot_wal_keep_size` is `-1` and nothing else is wrong
+- **THEN** the verdict is healthy
+- **AND** the no slot WAL limit notice is listed
+
+#### Scenario: Slot already lost
+
+- **WHEN** the newest sample reports the slot's status as `lost`
+- **THEN** the no slot WAL limit notice is not listed
+
+### Requirement: The statistics say whether archiving keeps up with the source
+
+The keeping-up summary SHALL cover the samples of the last 10 minutes and hold:
+
+- the source's write rate and the archive rate in WAL bytes per second, both over the seconds covered by samples that hold a source write value;
+- the unarchived WAL now and how fast it changes, in bytes per second, negative while it shrinks;
+- the slot lag now and how fast it changes;
+- the configured slot lag at which Databasus rebuilds the slot;
+- an estimate of the time until the unarchived WAL is archived, while it shrinks;
+- an estimate of the time until the slot lag reaches the rebuild lag, while the slot lag grows and the slot's status is neither `extended` nor `unreserved`;
+- an estimate of the time until the remaining WAL allowance runs out, while the slot lag grows and the source sets a limit;
+- one keeping-up state.
+
+The keeping-up state SHALL be the first of these that matches:
+
+1. **unknown**: the window holds no sample with a source write value;
+2. **falling behind**: the archive slower than source reason applies, ignoring its two-sample rule;
+3. **catching up**: the unarchived WAL is at least the backlog threshold and its lowest value in the last 2.5 minutes is below its lowest value in the first 2.5 minutes;
+4. **idle**: the source wrote less than 1 MiB over the window and the unarchived WAL is at most one segment;
+5. **keeping up**: anything else.
+
+#### Scenario: Archiving keeps up
+
+- **WHEN** the source writes 10 MB/s and Databasus archives 10 MB/s for ten minutes
+- **THEN** the keeping-up state is keeping up
+- **AND** both rates are about 10 MB/s
+
+#### Scenario: Archiving falls behind
+
+- **WHEN** the source writes 21 MB/s and Databasus archives 12 MB/s for ten minutes
+- **THEN** the keeping-up state is falling behind
+- **AND** the unarchived WAL grows by about 9 MB/s
+
+#### Scenario: Catching up after a burst
+
+- **WHEN** 3 GiB of WAL is unarchived and it shrank by 600 MiB over the last 10 minutes
+- **THEN** the keeping-up state is catching up
+- **AND** the estimate until it is archived is about 50 minutes
+
+#### Scenario: Running out of allowance
+
+- **WHEN** the source sets a limit, 5 GiB of allowance remains and the slot lag grew by 1 GiB over the last 10 minutes
+- **THEN** the estimate until the allowance runs out is about 50 minutes
+
+#### Scenario: Quiet database
+
+- **WHEN** the source wrote 200 KiB over the last 10 minutes and 100 KiB is unarchived
+- **THEN** the keeping-up state is idle
+
+#### Scenario: Quiet database with a stuck backlog
+
+- **WHEN** the source wrote 200 KiB over the last 10 minutes and 2 GiB is unarchived and not shrinking
+- **THEN** the keeping-up state is not idle
+
+### Requirement: The statistics cover the last hour, day or week
+
+A request SHALL name a period of the last hour, the last day or the last week. The series SHALL hold at most 360 points of equal length and state that length: 1 minute for an hour, 4 minutes for a day and 28 minutes for a week. The last point covers the interval that contains the current moment. In each point:
+
+- write and archive volumes become rates in bytes per second over the seconds its samples cover;
+- slot lag, unarchived WAL and local queue size take the highest value in the interval;
+- receiver restarts, receiver pauses and streaming session starts are summed;
+- upload concurrency and the longest segment write take the highest value, and the average segment write is weighted by the segments archived.
+
+An interval with no samples SHALL appear as a point without values, not as zero.
+
+#### Scenario: Week view
+
+- **WHEN** a user requests the last week for a database that streamed the whole week
+- **THEN** the series has at most 360 points
+- **AND** each point's archive rate equals the WAL bytes archived in its interval divided by the seconds its samples cover
+
+#### Scenario: Streamer was down for an hour
+
+- **WHEN** the streamer recorded no samples for one hour of the requested day
+- **THEN** the points covering that hour carry no values
+
+#### Scenario: Unknown period
+
+- **WHEN** a request names a period other than hour, day or week
+- **THEN** the request is refused as invalid
+
+### Requirement: The Monitoring tab appears only for WAL streaming databases
+
+The database page SHALL show a Monitoring tab for a PostgreSQL physical database whose backup mode is full + incremental + WAL streaming, and for no other database.
+
+#### Scenario: WAL streaming database
+
+- **WHEN** a user opens a PostgreSQL physical database in full + incremental + WAL streaming mode
+- **THEN** the page offers Config, Backups and Monitoring tabs
+
+#### Scenario: Other databases
+
+- **WHEN** a user opens a logical database or a physical database in a mode without WAL streaming
+- **THEN** the page offers no Monitoring tab
+
+### Requirement: The Monitoring tab speaks plain language
+
+Every label, headline, sentence, legend and chart title the tab shows by default SHALL avoid the terms "WAL", "slot", "LSN" and "lag". The tab SHALL call the unarchived WAL "not saved to storage yet" and the slot lag "not received by Databasus yet", and SHALL say that the second is part of the first. Technical names SHALL appear only in explanations shown on hover or keyboard focus, in the details section, and in a "what to do" sentence that names a setting or permission to change, such as `max_slot_wal_keep_size`.
+
+Each reason and each notice SHALL have one sentence about what happened, with its amounts and, where known, how long it has lasted, and one sentence about what to do. When Databasus resolves a condition by itself, the second sentence SHALL say so, and only then. For a broken chain or a missing slot, the second sentence SHALL ask the user to start a full backup unless one is requested or running. Durations SHALL be rounded for reading: minutes under an hour, hours and minutes under a day, days and hours beyond.
+
+#### Scenario: Plain names
+
+- **WHEN** a user opens the Monitoring tab of a healthy database
+- **THEN** no default label, legend or chart title contains "WAL", "slot", "LSN" or "lag"
+
+#### Scenario: Broken chain without a pending full backup
+
+- **WHEN** the WAL chain broken reason is shown and no full backup is requested or running
+- **THEN** the tab asks the user to start a full backup
+- **AND** does not say that Databasus handles it by itself
+
+#### Scenario: Every reason has an action
+
+- **WHEN** any reason or notice is shown
+- **THEN** it has both a sentence about what happened and a sentence about what to do
+
+### Requirement: The Monitoring tab leads with the verdict
+
+The tab SHALL open with the verdict as a headline: all is well, something needs attention, something is broken, data is being collected, or streaming is off. The headline SHALL use the verdict's color, green, orange, red or neutral gray, together with an icon and words, so color is never the only signal. Screen readers SHALL announce a change of the headline.
+
+The not streaming headline SHALL say how to turn streaming on and link to the database's configuration. Below the headline, the tab SHALL list the reasons, broken ones first. Notices SHALL appear after the reasons in a neutral informational style, and the no slot WAL limit notice SHALL explain in plain words what the limit protects and what it costs.
+
+#### Scenario: Healthy database
+
+- **WHEN** the verdict is healthy
+- **THEN** the headline is green and says that everything is in order
+- **AND** no reason list is shown
+
+#### Scenario: Problems listed
+
+- **WHEN** the verdict is broken because the slot is missing and the local queue is filling
+- **THEN** the headline is red
+- **AND** the slot missing reason is listed before the local queue reason
+
+### Requirement: The Monitoring tab explains whether archiving keeps up
+
+Right below the headline, the tab SHALL describe the keeping-up state in one or two plain sentences:
+
+- keeping up: Databasus saves changes as fast as the database writes them, with both rates;
+- falling behind: the database writes changes faster than Databasus saves them, by how much per second, how much is not saved yet and how fast that amount grows;
+- catching up: Databasus is saving a backlog, how much is left and about how long it takes;
+- idle: the database writes almost nothing, and new changes reach storage within minutes;
+- unknown: no current numbers are available.
+
+When an estimate until the rebuild lag or until the WAL allowance runs out exists and is under one day, the tab SHALL add what happens then: Databasus will start a new backup chain from a full backup, or the source will drop changes that Databasus has not received yet.
+
+#### Scenario: Falling behind explained
+
+- **WHEN** the source writes 21 MB/s, Databasus archives 12 MB/s and 3.4 GB is unarchived
+- **THEN** the tab says that the database writes changes faster than Databasus saves them, by about 9 MB/s
+- **AND** says that 3.4 GB is not saved yet and grows by about 32 GB an hour
+
+#### Scenario: Consequence only when near
+
+- **WHEN** the estimate until the rebuild lag is three days
+- **THEN** the tab adds no sentence about a new backup chain
+
+### Requirement: The Monitoring tab shows where unsaved changes are
+
+The tab SHALL show the path of changes as three stages, source, Databasus and storage, with:
+
+- at the source: what is not received by Databasus yet, against the rebuild lag and, when the source sets a limit, the remaining allowance;
+- in Databasus: the local queue, against the size at which the receiver is paused, and whether receiving is paused;
+- in storage: how long ago the newest WAL segment was archived;
+- between the stages: the source's write rate and the archive rate.
+
+An amount above its limit SHALL show as over the limit, not as a full bar. Each stage SHALL take the color of the most severe reason about it. Each amount on a stage SHALL offer a short explanation on hover or keyboard focus. On a narrow screen the stages SHALL stack vertically.
+
+#### Scenario: Backlog in the local queue
+
+- **WHEN** the slot lag is small and the local queue holds 300 MB of its 512 MB pause size for two samples
+- **THEN** the Databasus stage shows 300 MB of 512 MB in the needs-attention color
+- **AND** the source stage stays in the healthy color
+
+### Requirement: The Monitoring tab shows charts for the selected period
+
+The tab SHALL offer the last hour, day and week, and show for the selected period:
+
+- the source write rate and the archive rate on one chart;
+- what is not saved to storage yet and what is not received by Databasus yet on one chart;
+- the local queue size with its pause size, and receiver restarts and pauses.
+
+Axes SHALL show formatted amounts and local times. Hovering a time on one chart SHALL mark the same time on the others and show the formatted values. Intervals without samples SHALL show as gaps. When the series holds no point with values, the tab SHALL say that charts appear after a few minutes of streaming instead of showing empty charts. When the data starts inside the period, the tab SHALL say since when it has data.
+
+A details section, collapsed by default, SHALL show the raw values with their technical names: the slot's status, the remaining allowance, the average and longest segment write, the upload concurrency, the ratio of WAL to stored bytes, the WAL segment size and the streaming session's start.
+
+#### Scenario: Archiving falls behind on the chart
+
+- **WHEN** every point of the hour holds a source write rate of 21 MB/s and an archive rate of 12 MB/s, and a rising unarchived amount
+- **THEN** the archive series lies below the source series at every point
+- **AND** the not-saved-yet series rises across the hour
+
+#### Scenario: No history yet
+
+- **WHEN** the verdict is collecting data and the series holds no point with values
+- **THEN** the tab shows the explanation instead of the charts
+
+### Requirement: The Monitoring tab stays current without disturbing the reader
+
+The tab SHALL refresh every 30 seconds in every period while it is open and the browser tab is visible. It SHALL refresh at once when the browser tab becomes visible again and when the period changes.
+
+The tab SHALL show the age of the newest sample, updated every second, and mark it as live only while that sample is younger than 2 minutes. When the last successful refresh is older than 2 minutes, the headline SHALL turn neutral gray and say that the current state cannot be confirmed, with the last known verdict and its age.
+
+A refresh SHALL keep the previous content on screen until the new content arrives, keep the scroll position and the selected period, and not replay the charts' entry animation. Changed numbers and the headline color SHALL change with a short transition. When the reader's system asks for reduced motion, there SHALL be no transitions, no pulsing and no chart animation. A failed refresh SHALL keep the last content, say that refreshing failed, and keep retrying on the same schedule. A failed first load SHALL show the error and a retry button.
+
+#### Scenario: Status changes while open
+
+- **WHEN** the tab is open on the day view and the verdict changes from healthy to needs attention
+- **THEN** the headline changes within 30 seconds without reloading the page
+
+#### Scenario: Browser tab hidden
+
+- **WHEN** the browser tab is hidden for ten minutes and then shown again
+- **THEN** no refresh happened while it was hidden
+- **AND** a refresh happens as soon as it is shown
+
+#### Scenario: Refresh keeps failing
+
+- **WHEN** refreshes have failed for 3 minutes after a healthy response
+- **THEN** the headline is gray and says the state cannot be confirmed, last known healthy 3 minutes ago