PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FEATURE (wal): Upload up to four WAL segments at once per database and reuse storage connections between writes, so archiving keeps up with a busy source writing to a distant storage instead of stopping the receiver
Rostislav Dugin026050923698
diff --git a/backend/Justfile b/backend/Justfileindex 7fbc617f..24a1bf45 100644--- a/backend/Justfile+++ b/backend/Justfile@@ -27,6 +27,6 @@ clean-testcontainers:# physical tests isn't mistaken for an image tag; every real image tag there is version-like. A# namespaced name (owner/image) cannot be confused with anything, so its tag is matched as written.pull-testcontainers:- grep -rhoE '"(mysql|mariadb|mongo|postgres):[0-9][a-zA-Z0-9._-]*"|"pgsty/minio:[a-zA-Z0-9._-]+"' ./internal \+ grep -rhoE '"(mysql|mariadb|mongo|postgres):[0-9][a-zA-Z0-9._-]*"|"pgsty/minio:[a-zA-Z0-9._-]+"|"nicolaka/netshoot:[a-zA-Z0-9._-]+"' ./internal \| tr -d '"' | sort -u \| xargs -P 4 -I {} sh -c 'for n in 1 2 3; do docker pull {} && exit 0; echo "retry {} ($n)"; sleep 3; done; echo "FAILED to pull {} after 3 attempts"; exit 1'diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.goindex a395066a..57e40ba0 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go@@ -169,24 +169,41 @@ func (s *WalStreamSupervisor) runLagMonitor(ctx context.Context, logger *slog.Lo// Archiving throughput and lag are only visible today once something has already broken, which// leaves no history to read after an incident. This rides the existing poll rather than adding a-// ticker, and stays at INFO only when segments actually moved.+// ticker, and stays at INFO only when segments actually moved. The step timings tell latency-bound+// archiving from bandwidth-bound: a segment write near its size divided by the link speed means+// more uploads at once will not help, a much longer one means the time goes to round trips.func (s *WalStreamSupervisor) reportArchivingProgress(ctx context.Context,logger *slog.Logger,state *SlotState,) {- archivedSegments, archivedBytes := s.uploader.TakeArchivedSinceLastReport()+ snapshot := s.uploader.TakeArchivingSnapshot()- if archivedSegments == 0 {+ if snapshot.ArchivedSegments == 0 {logger.DebugContext(ctx, fmt.Sprintf("no wal archived since the last report, %d bytes behind",state.LagBytes), "wal_status", state.WalStatus)return}- logger.InfoContext(ctx, fmt.Sprintf("archived %d wal segments (%.1f MB), %d bytes behind",- archivedSegments, float64(archivedBytes)/(1024*1024), state.LagBytes),- "wal_status", state.WalStatus)+ logger.InfoContext(ctx, fmt.Sprintf(+ "archived %d wal segments (%.1f MB), up to %d uploads at once; "++ "claim %s, segment write %s, sidecar write %s, commit %s; %d bytes behind",+ snapshot.ArchivedSegments,+ float64(snapshot.ArchivedBytes)/(1024*1024),+ snapshot.HighestConcurrentUploads,+ formatUploadStepTiming(snapshot.Claim, snapshot.ArchivedSegments),+ formatUploadStepTiming(snapshot.SegmentWrite, snapshot.ArchivedSegments),+ formatUploadStepTiming(snapshot.SidecarWrite, snapshot.ArchivedSegments),+ formatUploadStepTiming(snapshot.Commit, snapshot.ArchivedSegments),+ state.LagBytes,+ ), "wal_status", state.WalStatus)+}++func formatUploadStepTiming(timing uploadStepTiming, archivedSegments int64) string {+ return fmt.Sprintf("avg %s max %s",+ timing.GetAverage(archivedSegments).Round(time.Millisecond),+ timing.Longest.Round(time.Millisecond))}// The slot's restart_lsn plus its lag is where the source currently is, so the@@ -196,13 +213,13 @@ func (s *WalStreamSupervisor) reportArchiveStalenessIfDue(stalenessTracker *walArchiveStalenessTracker,state *SlotState,) {- lastCommittedWalLSN, lastCommittedWalAt := s.getLastCommittedWal(logger)+ newestCommitted, lastCommittedWalAt := s.getLastCommittedWal(logger)isArchiveStale := stalenessTracker.recordSampleAndDetectStaleness(walArchiveSample{- SourceCurrentLSN: state.RestartLSN + walmath.LSN(state.LagBytes),- LastCommittedWalLSN: lastCommittedWalLSN,- ObservedAt: time.Now().UTC(),- StalenessThreshold: s.spec.ArchiveStalenessThreshold,+ SourceCurrentLSN: state.RestartLSN + walmath.LSN(state.LagBytes),+ ArchiveProgressPoint: getArchiveProgressPoint(s.getLowestQueuedSegment(), newestCommitted),+ ObservedAt: time.Now().UTC(),+ StalenessThreshold: s.spec.ArchiveStalenessThreshold,})if !isArchiveStale {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.goindex b04d36b7..7d06b0ba 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go@@ -1,6 +1,7 @@package usecases_physical_postgresqlimport (+ "log/slog""testing""time"@@ -143,3 +144,51 @@ func Test_RecordSampleAndClassifyBreak_WhenSlotHealthy_DoesNotRebuildAndClearsWarequire.Equal(t, slotBreakActionNone, actionAfterRecovery,"a recovered slot restarts the hold window instead of inheriting the previous one")}++func newSupervisorReportingTo(log *capturedLog) (*WalStreamSupervisor, *slog.Logger) {+ return &WalStreamSupervisor{uploader: NewWalUploader(WalUploadDeps{})},+ slog.New(slog.NewTextHandler(log, &slog.HandlerOptions{Level: slog.LevelDebug}))+}++func Test_ReportArchivingProgress_AfterBusyInterval_LogsCountsStepTimingsAndConcurrency(t *testing.T) {+ log := &capturedLog{}+ supervisor, supervisorLogger := newSupervisorReportingTo(log)++ supervisor.uploader.stats.recordArchived(3*1024*1024, segmentUploadTimings{+ Claim: 4 * time.Millisecond,+ SegmentWrite: 300 * time.Millisecond,+ SidecarWrite: 60 * time.Millisecond,+ Commit: 8 * time.Millisecond,+ })+ supervisor.uploader.stats.recordArchived(1024*1024, segmentUploadTimings{+ Claim: 2 * time.Millisecond,+ SegmentWrite: 100 * time.Millisecond,+ SidecarWrite: 40 * time.Millisecond,+ Commit: 4 * time.Millisecond,+ })+ supervisor.uploader.RecordConcurrentUploads(4)++ supervisor.reportArchivingProgress(t.Context(), supervisorLogger, &SlotState{LagBytes: 2048, WalStatus: "reserved"})++ logLine := log.String()+ require.Contains(t, logLine, "level=INFO")+ require.Contains(t, logLine, "archived 2 wal segments (4.0 MB)")+ require.Contains(t, logLine, "up to 4 uploads at once")+ require.Contains(t, logLine, "claim avg 3ms max 4ms")+ require.Contains(t, logLine, "segment write avg 200ms max 300ms")+ require.Contains(t, logLine, "sidecar write avg 50ms max 60ms")+ require.Contains(t, logLine, "commit avg 6ms max 8ms")+ require.Contains(t, logLine, "2048 bytes behind")+}++func Test_ReportArchivingProgress_AfterQuietInterval_LogsOnlyBytesBehindAtDebug(t *testing.T) {+ log := &capturedLog{}+ supervisor, supervisorLogger := newSupervisorReportingTo(log)++ supervisor.reportArchivingProgress(t.Context(), supervisorLogger, &SlotState{LagBytes: 4096, WalStatus: "reserved"})++ logLine := log.String()+ require.Contains(t, logLine, "level=DEBUG")+ require.Contains(t, logLine, "no wal archived since the last report, 4096 bytes behind")+ require.NotContains(t, logLine, "avg")+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.goindex deadbc3e..a5558d89 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go@@ -8,6 +8,7 @@ import ("log/slog""os""path/filepath"+ "slices""strings""sync""sync/atomic"@@ -44,6 +45,97 @@ type mockWalStorage struct {blockOnce sync.Oncestarted chan struct{}release chan struct{}++ // Holds every segment artifact write, sidecars aside, until the test lets it through.+ artifactGate *artifactWriteGate+}++// Records which segment artifacts started writing, in order, and how many wrote at once.+type artifactWriteGate struct {+ mutex sync.Mutex+ startedObjectNames []string+ activeWriteCount int+ highestActiveWriteCount int++ releases chan struct{}+ releaseAllOnce sync.Once+ openAtWriteCount int+}++func newArtifactWriteGate() *artifactWriteGate {+ return &artifactWriteGate{releases: make(chan struct{})}+}++// The gate stays closed until this many artifacts write at once, then lets every write through, so a+// test can prove writes overlapped without releasing them one by one.+func newArtifactWriteGateOpeningAt(activeWriteCount int) *artifactWriteGate {+ gate := newArtifactWriteGate()+ gate.openAtWriteCount = activeWriteCount++ return gate+}++func (g *artifactWriteGate) passThrough(objectName string) {+ g.mutex.Lock()+ g.startedObjectNames = append(g.startedObjectNames, objectName)+ g.activeWriteCount+++ g.highestActiveWriteCount = max(g.highestActiveWriteCount, g.activeWriteCount)+ isOpeningCountReached := g.openAtWriteCount > 0 && g.activeWriteCount >= g.openAtWriteCount+ g.mutex.Unlock()++ if isOpeningCountReached {+ g.releaseAll()+ }++ <-g.releases++ g.mutex.Lock()+ g.activeWriteCount--+ g.mutex.Unlock()+}++func (g *artifactWriteGate) releaseOne() {+ g.releases <- struct{}{}+}++func (g *artifactWriteGate) releaseAll() {+ g.releaseAllOnce.Do(func() { close(g.releases) })+}++func (g *artifactWriteGate) getActiveWriteCount() int {+ g.mutex.Lock()+ defer g.mutex.Unlock()++ return g.activeWriteCount+}++func (g *artifactWriteGate) getHighestActiveWriteCount() int {+ g.mutex.Lock()+ defer g.mutex.Unlock()++ return g.highestActiveWriteCount+}++func (g *artifactWriteGate) getStartedObjectNames() []string {+ g.mutex.Lock()+ defer g.mutex.Unlock()++ return slices.Clone(g.startedObjectNames)+}++func (g *artifactWriteGate) countStartedWritesFor(walFilename string) int {+ g.mutex.Lock()+ defer g.mutex.Unlock()++ count := 0++ for _, objectName := range g.startedObjectNames {+ if strings.Contains(objectName, walFilename) {+ count+++ }+ }++ return count}// hasObjectFor reports whether any stored object belongs to the given segment.@@ -119,6 +211,10 @@ func (m *mockWalStorage) SaveFile(body, _ := io.ReadAll(file)+ if m.artifactGate != nil && !strings.HasSuffix(fileName, metadataSuffix) {+ m.artifactGate.passThrough(fileName)+ }+// The sidecar shares the artifact's name, so the block is scoped to the// artifact and fires once.if m.blockOn != "" && strings.Contains(fileName, m.blockOn) && !strings.HasSuffix(fileName, metadataSuffix) {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_local_queue.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_local_queue.gonew file mode 100644index 00000000..2630f1a7--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_local_queue.go@@ -0,0 +1,79 @@+package usecases_physical_postgresql++import (+ "cmp"+ "os"+ "path/filepath"+ "slices"++ "databasus-backend/internal/util/walmath"+)++// A finalized segment waiting in the watch directory or in pending-upload/, including one whose+// upload is running: its file stays in place until the upload commits.+type queuedWalSegment struct {+ walFilename string+ localPath string+ timelineID int+ startLSN walmath.LSN+ endLSN walmath.LSN+ isComplete bool+}++// The watch directory is listed before pending-upload/. realignResumePath only moves files from the+// first into the second, so a file renamed between the two reads is still found in the second.+func listQueuedWalSegments(watchDir string, segmentSizeBytes int64) []queuedWalSegment {+ queuedSegments := listQueuedWalSegmentsIn(watchDir, segmentSizeBytes)++ return append(+ queuedSegments,+ listQueuedWalSegmentsIn(filepath.Join(watchDir, pendingUploadDirName), segmentSizeBytes)...)+}++func listQueuedWalSegmentsIn(directory string, segmentSizeBytes int64) []queuedWalSegment {+ entries, err := os.ReadDir(directory)+ if err != nil {+ return nil+ }++ queuedSegments := make([]queuedWalSegment, 0, len(entries))++ for _, entry := range entries {+ if entry.IsDir() || !walmath.IsWalFilename(entry.Name()) {+ continue+ }++ timelineID, startLSN, endLSN, err := segmentBounds(entry.Name(), segmentSizeBytes)+ if err != nil {+ continue+ }++ info, err := entry.Info()+ if err != nil {+ continue+ }++ queuedSegments = append(queuedSegments, queuedWalSegment{+ walFilename: entry.Name(),+ localPath: filepath.Join(directory, entry.Name()),+ timelineID: timelineID,+ startLSN: startLSN,+ endLSN: endLSN,+ isComplete: info.Size() == segmentSizeBytes,+ })+ }++ return queuedSegments+}++// Timeline first, then position: the same order FindLatestCommittedSegment uses, so the oldest WAL+// drains first and the archive progress point moves forward.+func compareWalOrder(firstTimelineID int, firstLSN walmath.LSN, secondTimelineID int, secondLSN walmath.LSN) int {+ return cmp.Or(cmp.Compare(firstTimelineID, secondTimelineID), cmp.Compare(firstLSN, secondLSN))+}++func sortQueuedWalSegments(queuedSegments []queuedWalSegment) {+ slices.SortStableFunc(queuedSegments, func(first, second queuedWalSegment) int {+ return compareWalOrder(first.timelineID, first.startLSN, second.timelineID, second.startLSN)+ })+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_staleness.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_staleness.goindex f10642e9..6dda11fe 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_staleness.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_staleness.go@@ -13,15 +13,15 @@ import (const DefaultArchiveStalenessThreshold = 30 * time.Minutetype walArchiveSample struct {- SourceCurrentLSN walmath.LSN- LastCommittedWalLSN walmath.LSN- ObservedAt time.Time- StalenessThreshold time.Duration+ SourceCurrentLSN walmath.LSN+ ArchiveProgressPoint walmath.LSN+ ObservedAt time.Time+ StalenessThreshold time.Duration}type walArchiveStalenessTracker struct {previousSourceCurrentLSN walmath.LSN- lastCommittedWalLSN walmath.LSN+ archiveProgressPoint walmath.LSNhasSample boolfallingBehindSince time.Time@@ -42,10 +42,10 @@ func (t *walArchiveStalenessTracker) recordSampleAndDetectStaleness(sample walAr}isSourceWriting := t.hasSample && sample.SourceCurrentLSN > t.previousSourceCurrentLSN- isArchiveProgressing := sample.LastCommittedWalLSN > t.lastCommittedWalLSN+ isArchiveProgressing := sample.ArchiveProgressPoint > t.archiveProgressPointt.previousSourceCurrentLSN = sample.SourceCurrentLSN- t.lastCommittedWalLSN = max(t.lastCommittedWalLSN, sample.LastCommittedWalLSN)+ t.archiveProgressPoint = max(t.archiveProgressPoint, sample.ArchiveProgressPoint)t.hasSample = trueif isArchiveProgressing {@@ -74,19 +74,73 @@ func (t *walArchiveStalenessTracker) recordSampleAndDetectStaleness(sample walArreturn true}-func (s *WalStreamSupervisor) getLastCommittedWal(logger *slog.Logger) (endLSN walmath.LSN, committedAt *time.Time) {+// A WAL position as restore reads it: timeline first, then the position within it.+type walPosition struct {+ TimelineID int+ LSN walmath.LSN+}++// The archive progress point is where WAL is archived without holes. Newer segments that finish+// while an older one is still pending do not move it, so one stuck segment shows up as staleness even+// while later segments keep arriving. A pending segment on an older timeline counts as older.+func getArchiveProgressPoint(lowestQueuedStart, newestCommittedEnd *walPosition) walmath.LSN {+ if lowestQueuedStart == nil {+ if newestCommittedEnd == nil {+ return 0+ }++ return newestCommittedEnd.LSN+ }++ if newestCommittedEnd == nil ||+ compareWalOrder(+ lowestQueuedStart.TimelineID, lowestQueuedStart.LSN,+ newestCommittedEnd.TimelineID, newestCommittedEnd.LSN,+ ) < 0 {+ return lowestQueuedStart.LSN+ }++ return newestCommittedEnd.LSN+}++func (s *WalStreamSupervisor) getLastCommittedWal(+ logger *slog.Logger,+) (newestCommittedEnd *walPosition, committedAt *time.Time) {segment, err := s.spec.WalSegmentRepo.FindLatestCommittedSegment(s.spec.DatabaseID)if err != nil {logger.Error("failed to read the last committed wal segment", "error", err)- return 0, nil+ return nil, nil}if segment == nil {- return 0, nil+ return nil, nil}receivedAt := segment.ReceivedAt- return segment.EndLSN, &receivedAt+ return &walPosition{TimelineID: segment.TimelineID, LSN: segment.EndLSN}, &receivedAt+}++// The dispatcher stores the lowest segment of each listing, short files included: restore cannot pass+// that position until the segment is archived, so the progress point must not pass it either.+// Between listings the stored value can only be lower than the truth, which errs towards reporting+// staleness rather than hiding it.+func (s *WalStreamSupervisor) setLowestQueuedSegment(sortedSegments []queuedWalSegment) {+ s.lowestQueuedSegmentMutex.Lock()+ defer s.lowestQueuedSegmentMutex.Unlock()++ if len(sortedSegments) == 0 {+ s.lowestQueuedSegment = nil+ return+ }++ s.lowestQueuedSegment = &walPosition{TimelineID: sortedSegments[0].timelineID, LSN: sortedSegments[0].startLSN}+}++func (s *WalStreamSupervisor) getLowestQueuedSegment() *walPosition {+ s.lowestQueuedSegmentMutex.Lock()+ defer s.lowestQueuedSegmentMutex.Unlock()++ return s.lowestQueuedSegment}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_staleness_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_staleness_test.goindex f44583a7..ed6ba6e7 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_staleness_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_staleness_test.go@@ -14,10 +14,10 @@ const testStalenessThreshold = 30 * time.Minute// The source keeps writing while the newest restorable segment stays where it is.func fallingBehindSample(sourceCurrentLSN walmath.LSN, observedAt time.Time) walArchiveSample {return walArchiveSample{- SourceCurrentLSN: sourceCurrentLSN,- LastCommittedWalLSN: walmath.LSN(testWalSegmentSize),- ObservedAt: observedAt,- StalenessThreshold: testStalenessThreshold,+ SourceCurrentLSN: sourceCurrentLSN,+ ArchiveProgressPoint: walmath.LSN(testWalSegmentSize),+ ObservedAt: observedAt,+ StalenessThreshold: testStalenessThreshold,}}@@ -72,17 +72,17 @@ func Test_RecordSampleAndDetectStaleness_WhenSegmentCommits_ReArmsClock(t *testi))require.False(t, stalenessTracker.recordSampleAndDetectStaleness(walArchiveSample{- SourceCurrentLSN: writtenLSN + 8192,- LastCommittedWalLSN: walmath.LSN(2 * testWalSegmentSize),- ObservedAt: base.Add(testStalenessThreshold / 2),- StalenessThreshold: testStalenessThreshold,+ SourceCurrentLSN: writtenLSN + 8192,+ ArchiveProgressPoint: walmath.LSN(2 * testWalSegmentSize),+ ObservedAt: base.Add(testStalenessThreshold / 2),+ StalenessThreshold: testStalenessThreshold,}), "a segment reached storage")require.False(t, stalenessTracker.recordSampleAndDetectStaleness(walArchiveSample{- SourceCurrentLSN: writtenLSN + 12288,- LastCommittedWalLSN: walmath.LSN(2 * testWalSegmentSize),- ObservedAt: base.Add(testStalenessThreshold + time.Minute),- StalenessThreshold: testStalenessThreshold,+ SourceCurrentLSN: writtenLSN + 12288,+ ArchiveProgressPoint: walmath.LSN(2 * testWalSegmentSize),+ ObservedAt: base.Add(testStalenessThreshold + time.Minute),+ StalenessThreshold: testStalenessThreshold,}), "a slow but progressing archive must not accumulate staleness across the segments it did deliver")}@@ -137,3 +137,59 @@ func Test_RecordSampleAndDetectStaleness_WhenThresholdDisabled_DoesNotDetectStalrequire.False(t, stalenessTracker.recordSampleAndDetectStaleness(disabledSample))}++func segmentStart(timelineID int, segmentNo uint64) *walPosition {+ return &walPosition{TimelineID: timelineID, LSN: walmath.LSN(segmentNo * uint64(testWalSegmentSize))}+}++func segmentEnd(timelineID int, segmentNo uint64) *walPosition {+ return segmentStart(timelineID, segmentNo+1)+}++func Test_GetArchiveProgressPoint_WhenOneSegmentKeepsFailingWhileOthersSucceed_HoldsAndDetectsStaleness(t *testing.T) {+ var stalenessTracker walArchiveStalenessTracker++ base := time.Now().UTC()+ sourceLSN := walmath.LSN(100 * uint64(testWalSegmentSize))+ stuckSegmentStart := segmentStart(1, 50)++ for minute, newestCommittedSegmentNo := range []uint64{51, 60, 70, 80, 90} {+ progressPoint := getArchiveProgressPoint(stuckSegmentStart, segmentEnd(1, newestCommittedSegmentNo))++ require.Equal(t, stuckSegmentStart.LSN, progressPoint,+ "segments archived after a stuck one must not move the gap-free archive")++ require.False(t, stalenessTracker.recordSampleAndDetectStaleness(walArchiveSample{+ SourceCurrentLSN: sourceLSN + walmath.LSN(minute*4096),+ ArchiveProgressPoint: progressPoint,+ ObservedAt: base.Add(time.Duration(minute) * time.Minute),+ StalenessThreshold: testStalenessThreshold,+ }))+ }++ require.True(t, stalenessTracker.recordSampleAndDetectStaleness(walArchiveSample{+ SourceCurrentLSN: sourceLSN + 1024*1024,+ ArchiveProgressPoint: getArchiveProgressPoint(stuckSegmentStart, segmentEnd(1, 90)),+ ObservedAt: base.Add(testStalenessThreshold + 5*time.Minute),+ StalenessThreshold: testStalenessThreshold,+ }), "one stuck segment must trip the alert even while later segments keep arriving")+}++func Test_GetArchiveProgressPoint_WhenNothingIsPendingBelowNewestCommitted_ReturnsNewestCommittedEnd(t *testing.T) {+ require.Equal(t, segmentEnd(1, 12).LSN, getArchiveProgressPoint(segmentStart(1, 13), segmentEnd(1, 12)),+ "the next segment waiting right behind the archive does not hold the point back")+ require.Equal(t, segmentEnd(1, 12).LSN, getArchiveProgressPoint(segmentStart(1, 20), segmentEnd(1, 12)))+}++func Test_GetArchiveProgressPoint_WhenSourceIsIdle_ReturnsNewestCommittedEnd(t *testing.T) {+ require.Equal(t, segmentEnd(1, 12).LSN, getArchiveProgressPoint(nil, segmentEnd(1, 12)))+ require.Zero(t, getArchiveProgressPoint(nil, nil))+}++func Test_GetArchiveProgressPoint_WhenPendingSegmentIsOnOlderTimeline_HoldsPoint(t *testing.T) {+ olderTimelinePending := segmentStart(1, 40)+ newerTimelineCommitted := segmentEnd(2, 30)++ require.Equal(t, olderTimelinePending.LSN, getArchiveProgressPoint(olderTimelinePending, newerTimelineCommitted),+ "restore cannot reach the newer timeline past WAL the older one still owes")+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.goindex 3b59ed1d..23a80abc 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go@@ -212,6 +212,10 @@ type WalStreamSupervisor struct {// physical_wal_streamers heartbeat claim), so this is the only guard needed.rebuildMu sync.MutexrebuildTimestamps []time.Time++ // Written by the uploader dispatcher, read by the lag monitor (wal_staleness.go).+ lowestQueuedSegmentMutex sync.Mutex+ lowestQueuedSegment *walPosition}func NewWalStreamSupervisor(spec WalStreamSpec) *WalStreamSupervisor {@@ -234,6 +238,7 @@ func NewWalStreamSupervisor(spec WalStreamSpec) *WalStreamSupervisor {WalSegmentRepo: spec.WalSegmentRepo,WalSegmentSizeBytes: walSegmentSizeBytes(spec.SourceDB),Logger: scopedLogger,+ WatchDir: watchDir,OnGapDetected: spec.OnGapDetected,})diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.goindex 6e9ca975..6ee08201 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go@@ -8,8 +8,8 @@ import ("io""log/slog""os"+ "slices""sync"- "sync/atomic""time""github.com/google/uuid"@@ -48,8 +48,12 @@ type WalUploadDeps struct {WalSegmentSizeBytes int64Logger *slog.Logger- // OnGapDetected fires at most once per newly-observed WAL gap (de-duplicated- // by lastNotifiedGapEnd inside the uploader). nil disables notification.+ // The local queue: segments waiting here, or in its pending-upload/, are still coming, so a hole+ // in front of a committed segment is not a gap while its predecessor sits here.+ WatchDir string++ // OnGapDetected fires once per WAL gap: only the first segment after a hole reports it, and that+ // segment commits once. nil disables notification.OnGapDetected func(gapStart, gapEnd walmath.LSN)}@@ -60,15 +64,13 @@ type WalUploadDeps struct {type WalUploader struct {deps WalUploadDeps- mu sync.Mutex- lastNotifiedGapEnd walmath.LSN+ mu sync.Mutex// The uploader loop retries every second, so a torn segment that never gets// re-streamed would otherwise warn once per second forever.warnedIncompleteSegments map[string]struct{}- archivedSegments atomic.Int64- archivedBytes atomic.Int64+ stats archivingStats}func NewWalUploader(deps WalUploadDeps) *WalUploader {@@ -106,6 +108,8 @@ func (u *WalUploader) ProcessSegment(ctx context.Context, localPath, walFilenameClaimedAt: time.Now().UTC(),}+ claimStartedAt := time.Now()+inserted, err := u.deps.WalSegmentRepo.ClaimInsert(claim)if err != nil {return fmt.Errorf("claim wal segment: %w", err)@@ -115,7 +119,7 @@ func (u *WalUploader) ProcessSegment(ctx context.Context, localPath, walFilenamereturn u.handleClaimConflict(localPath, claim)}- return u.uploadAndCommit(ctx, localPath, claim)+ return u.uploadAndCommit(ctx, localPath, claim, time.Since(claimStartedAt))}// RecoverSegment runs the insert-first flow for a leftover finalized segment@@ -152,13 +156,15 @@ func (u *WalUploader) RecoverSegment(ctx context.Context, localPath, walFilenameClaimedAt: time.Now().UTC(),}+ claimStartedAt := time.Now()+inserted, err := u.deps.WalSegmentRepo.ClaimInsert(claim)if err != nil {return fmt.Errorf("claim wal segment: %w", err)}if inserted {- return u.uploadAndCommit(ctx, localPath, claim)+ return u.uploadAndCommit(ctx, localPath, claim, time.Since(claimStartedAt))}existing, err := u.deps.WalSegmentRepo.FindByChainKey(claim.DatabaseID, claim.TimelineID, claim.StartLSN)@@ -177,12 +183,18 @@ func (u *WalUploader) RecoverSegment(ctx context.Context, localPath, walFilename}// Own pre-crash NULL claim: take it over by uploading against the existing row.- return u.uploadAndCommit(ctx, localPath, existing)+ return u.uploadAndCommit(ctx, localPath, existing, time.Since(claimStartedAt))+}++// One rollup line per interval, so the snapshot is consumed rather than read.+func (u *WalUploader) TakeArchivingSnapshot() ArchivingSnapshot {+ return u.stats.takeSnapshot()}-// One rollup line per interval, so the counters are consumed rather than read.-func (u *WalUploader) TakeArchivedSinceLastReport() (int64, int64) {- return u.archivedSegments.Swap(0), u.archivedBytes.Swap(0)+// The dispatcher knows how many uploads run, the uploader knows what they cost; reporting the count+// here keeps both in the one snapshot the rollup takes.+func (u *WalUploader) RecordConcurrentUploads(count int) {+ u.stats.recordConcurrentUploads(count)}// handleClaimConflict resolves a lost ClaimInsert race by probing the existing@@ -230,7 +242,10 @@ func (u *WalUploader) uploadAndCommit(ctx context.Context,localPath string,claim *physical_models.PhysicalWalSegment,+ claimDuration time.Duration,) error {+ timings := segmentUploadTimings{Claim: claimDuration}+// One name per upload attempt, minted here and passed down: a retry under the// same segment identity must not address the object a pending cleanup owns.objectName := walSegmentObjectName(claim.DatabaseID, claim.TimelineID, claim.WalFilename, uuid.New())@@ -245,6 +260,8 @@ func (u *WalUploader) uploadAndCommit(saveCtx, cancel := context.WithTimeout(ctx, walSegmentUploadTimeout)defer cancel()+ segmentWriteStartedAt := time.Now()+receipt, err := u.deps.FileStore.WriteFile(saveCtx, u.reference(objectName), artifact)if err != nil {artifact.abort(err)@@ -263,6 +280,9 @@ func (u *WalUploader) uploadAndCommit(return fmt.Errorf("stream wal segment artifact: %w", err)}+ timings.SegmentWrite = time.Since(segmentWriteStartedAt)+ sidecarWriteStartedAt := time.Now()+sidecarReceipt, err := u.uploadSidecar(saveCtx, claim, objectName, salt, iv, compressedSizeBytes)if err != nil {u.discardAttempt(ctx, objectName)@@ -271,6 +291,9 @@ func (u *WalUploader) uploadAndCommit(return fmt.Errorf("upload wal segment sidecar: %w", err)}+ timings.SidecarWrite = time.Since(sidecarWriteStartedAt)+ commitStartedAt := time.Now()+committed, err := u.commitSegment(ctx, commitSegmentSpec{Claim: claim,ObjectName: objectName,@@ -296,8 +319,6 @@ func (u *WalUploader) uploadAndCommit(return nil}- u.recordArchived(compressedSizeBytes)-// Per segment rather than per archive: a busy cluster finalizes one every few seconds, which// would swamp INFO. The periodic rollup carries the same information at a readable rate.u.deps.Logger.DebugContext(ctx, fmt.Sprintf("archived wal segment %s (%.2f MB)",@@ -305,22 +326,20 @@ func (u *WalUploader) uploadAndCommit("wal_segment_id", claim.ID, "timeline_id", claim.TimelineID)u.probeChainGap(claim)++ timings.Commit = time.Since(commitStartedAt)+ u.stats.recordArchived(compressedSizeBytes, timings)+u.removeLocal(localPath, claim.WalFilename)return nil}-// recordArchived accumulates what the periodic rollup reports, so that INFO can answer "is WAL-// reaching storage" without a line per segment.-func (u *WalUploader) recordArchived(compressedSizeBytes int64) {- u.archivedSegments.Add(1)- u.archivedBytes.Add(compressedSizeBytes)-}--// probeChainGap emits one notification when the just-committed segment is not-// LSN-contiguous with the previous committed segment on its timeline. It is a-// read-only check (no catalog row is written for the gap) de-duplicated by-// lastNotifiedGapEnd so a backlog drained after a storage outage notifies once.+// probeChainGap reports a gap in front of the just-committed segment only when the segment directly+// before it will never arrive. Under concurrent uploads a later segment often commits first, so a+// missing predecessor that still waits in the local queue, or that commits meanwhile, is not a gap.+// Only the first segment after a real hole reports it, and that segment commits once, which keeps+// each gap to one notification without any bookkeeping.func (u *WalUploader) probeChainGap(claim *physical_models.PhysicalWalSegment) {prevEnd, err := u.deps.WalSegmentRepo.FindLatestCommittedBefore(claim.DatabaseID, claim.TimelineID, claim.StartLSN)if err != nil {@@ -333,22 +352,30 @@ func (u *WalUploader) probeChainGap(claim *physical_models.PhysicalWalSegment) {return}- u.mu.Lock()- alreadyNotified := claim.StartLSN <= u.lastNotifiedGapEnd- if !alreadyNotified {- u.lastNotifiedGapEnd = claim.StartLSN+ // The local check runs before the catalog check: a predecessor that commits concurrently writes+ // its row before it removes its file, so at least one of the two sees it.+ queuedSegments := listQueuedWalSegments(u.deps.WatchDir, u.deps.WalSegmentSizeBytes)+ if isPredecessorQueued(queuedSegments, claim) {+ return}- u.mu.Unlock()- // Log and notify once per outage: a drained backlog re-enters this path for- // the first segment past the gap only (later segments are contiguous and- // returned above), and the lastNotifiedGapEnd guard covers re-processing.- if alreadyNotified {+ predecessorStartLSN := claim.StartLSN - walmath.LSN(u.deps.WalSegmentSizeBytes)++ predecessor, err := u.deps.WalSegmentRepo.FindByChainKey(claim.DatabaseID, claim.TimelineID, predecessorStartLSN)+ if err != nil {+ u.deps.Logger.Error("wal gap probe query failed", "wal_filename", claim.WalFilename, "error", err)+return}+ if predecessor != nil {+ return+ }++ gapStart := max(*prevEnd, getHighestQueuedEndBelow(queuedSegments, claim))+u.deps.Logger.Warn(- fmt.Sprintf("wal_gap_detected_post_upload: gap [%s, %s)", prevEnd.String(), claim.StartLSN.String()),+ fmt.Sprintf("wal_gap_detected_post_upload: gap [%s, %s)", gapStart.String(), claim.StartLSN.String()),"database_id", claim.DatabaseID,"timeline_id", claim.TimelineID,)@@ -357,7 +384,33 @@ func (u *WalUploader) probeChainGap(claim *physical_models.PhysicalWalSegment) {return}- u.deps.OnGapDetected(*prevEnd, claim.StartLSN)+ u.deps.OnGapDetected(gapStart, claim.StartLSN)+}++// A short file never counts: pg_receivewal resumes after the highest complete segment and never+// streams a segment below that point again, so counting a torn file would hide its hole forever.+// Such a file surfaces through the archive staleness alert instead.+func isPredecessorQueued(queuedSegments []queuedWalSegment, claim *physical_models.PhysicalWalSegment) bool {+ return slices.ContainsFunc(queuedSegments, func(queued queuedWalSegment) bool {+ return queued.isComplete && queued.timelineID == claim.TimelineID && queued.endLSN == claim.StartLSN+ })+}++// The newest committed predecessor can lag behind a segment that is still uploading, which would make+// the reported hole wider than the real one.+func getHighestQueuedEndBelow(+ queuedSegments []queuedWalSegment,+ claim *physical_models.PhysicalWalSegment,+) walmath.LSN {+ var highestEnd walmath.LSN++ for _, queued := range queuedSegments {+ if queued.isComplete && queued.timelineID == claim.TimelineID && queued.endLSN <= claim.StartLSN {+ highestEnd = max(highestEnd, queued.endLSN)+ }+ }++ return highestEnd}// uploadSidecar writes the `<object>.metadata` JSON describing the segment so a@@ -594,7 +647,15 @@ func buildWalSegmentArtifactReader(}func streamWalSegmentArtifact(source io.Reader, output io.Writer, encryptionWriter io.Closer) error {- encoder, err := zstd.NewWriter(output, zstd.WithEncoderLevel(zstd.SpeedDefault))+ // Up to four segments compress at once per database. One goroutine per encoder and the+ // lower-memory history buffer take each encoder from about 18 MB to about 9.5 MB, and neither+ // changes the compressed format or the window.+ encoder, err := zstd.NewWriter(+ output,+ zstd.WithEncoderLevel(zstd.SpeedDefault),+ zstd.WithEncoderConcurrency(1),+ zstd.WithLowerEncoderMem(true),+ )if err != nil {return fmt.Errorf("init zstd writer: %w", err)}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_dispatcher.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_dispatcher.gonew file mode 100644index 00000000..160ef05e--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_dispatcher.go@@ -0,0 +1,98 @@+package usecases_physical_postgresql++import (+ "context"+ "log/slog"+ "sync"+)++// Four uploads per database reach the rate of a busy source against a storage in another region,+// while staying under common per-user connection limits of FTP and SFTP servers and bounding the+// encoder and write-buffer memory every streaming database holds.+const maxConcurrentSegmentUploads = 4++type segmentUpload func(ctx context.Context, localPath, walFilename string) error++// Only the goroutine that owns the dispatcher touches the in-flight set; workers report back through+// finishedUploads. The set is keyed by WAL file name rather than path, because realignResumePath can+// move a file into pending-upload/ while its upload runs, and the moved copy must not start again.+type segmentUploadDispatcher struct {+ upload segmentUpload+ recordConcurrentUploads func(count int)++ inFlightFilenames map[string]struct{}+ finishedUploads chan string+ workers sync.WaitGroup+}++func newSegmentUploadDispatcher(+ upload segmentUpload,+ recordConcurrentUploads func(count int),+) *segmentUploadDispatcher {+ return &segmentUploadDispatcher{+ upload: upload,+ recordConcurrentUploads: recordConcurrentUploads,+ inFlightFilenames: make(map[string]struct{}, maxConcurrentSegmentUploads),+ // Room for every worker's report, so a worker never blocks on a dispatcher that stopped+ // reading.+ finishedUploads: make(chan string, maxConcurrentSegmentUploads),+ }+}++// Starts segments in the given order while slots are free and returns the ones it could not start+// yet. A segment already in flight is skipped rather than returned.+func (d *segmentUploadDispatcher) startUploads(+ ctx context.Context,+ logger *slog.Logger,+ sortedSegments []queuedWalSegment,+) []queuedWalSegment {+ for index, segment := range sortedSegments {+ if _, isInFlight := d.inFlightFilenames[segment.walFilename]; isInFlight {+ continue+ }++ if len(d.inFlightFilenames) >= maxConcurrentSegmentUploads || ctx.Err() != nil {+ return d.dropInFlight(sortedSegments[index:])+ }++ d.inFlightFilenames[segment.walFilename] = struct{}{}+ d.recordConcurrentUploads(len(d.inFlightFilenames))++ d.workers.Go(func() {+ if err := d.upload(ctx, segment.localPath, segment.walFilename); err != nil {+ logger.WarnContext(ctx, "wal segment upload failed; will retry on a later pass",+ "wal_filename", segment.walFilename,+ "error", err,+ )+ }++ d.finishedUploads <- segment.walFilename+ })+ }++ return nil+}++func (d *segmentUploadDispatcher) dropInFlight(segments []queuedWalSegment) []queuedWalSegment {+ notStarted := make([]queuedWalSegment, 0, len(segments))++ for _, segment := range segments {+ if _, isInFlight := d.inFlightFilenames[segment.walFilename]; !isInFlight {+ notStarted = append(notStarted, segment)+ }+ }++ return notStarted+}++func (d *segmentUploadDispatcher) markFinished(walFilename string) {+ delete(d.inFlightFilenames, walFilename)+}++func (d *segmentUploadDispatcher) getInFlightCount() int {+ return len(d.inFlightFilenames)+}++func (d *segmentUploadDispatcher) waitForUploads() {+ d.workers.Wait()+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_stats.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_stats.gonew file mode 100644index 00000000..6418b9b9--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_stats.go@@ -0,0 +1,79 @@+package usecases_physical_postgresql++import (+ "sync"+ "time"+)++type uploadStepTiming struct {+ Total time.Duration+ Longest time.Duration+}++func (t uploadStepTiming) GetAverage(count int64) time.Duration {+ if count == 0 {+ return 0+ }++ return t.Total / time.Duration(count)+}++func (t *uploadStepTiming) record(duration time.Duration) {+ t.Total += duration+ t.Longest = max(t.Longest, duration)+}++type segmentUploadTimings struct {+ Claim time.Duration+ SegmentWrite time.Duration+ SidecarWrite time.Duration+ Commit time.Duration+}++// One snapshot describes one rollup interval: the step timings belong to the segments it counts, and+// the concurrency is the highest seen in the same interval.+type ArchivingSnapshot struct {+ ArchivedSegments int64+ ArchivedBytes int64++ Claim uploadStepTiming+ SegmentWrite uploadStepTiming+ SidecarWrite uploadStepTiming+ Commit uploadStepTiming++ HighestConcurrentUploads int+}++type archivingStats struct {+ mutex sync.Mutex+ snapshot ArchivingSnapshot+}++func (s *archivingStats) recordArchived(compressedSizeBytes int64, timings segmentUploadTimings) {+ s.mutex.Lock()+ defer s.mutex.Unlock()++ s.snapshot.ArchivedSegments+++ s.snapshot.ArchivedBytes += compressedSizeBytes+ s.snapshot.Claim.record(timings.Claim)+ s.snapshot.SegmentWrite.record(timings.SegmentWrite)+ s.snapshot.SidecarWrite.record(timings.SidecarWrite)+ s.snapshot.Commit.record(timings.Commit)+}++func (s *archivingStats) recordConcurrentUploads(count int) {+ s.mutex.Lock()+ defer s.mutex.Unlock()++ s.snapshot.HighestConcurrentUploads = max(s.snapshot.HighestConcurrentUploads, count)+}++func (s *archivingStats) takeSnapshot() ArchivingSnapshot {+ s.mutex.Lock()+ defer s.mutex.Unlock()++ taken := s.snapshot+ s.snapshot = ArchivingSnapshot{}++ return taken+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.goindex 335778f3..7e12707e 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go@@ -6,6 +6,8 @@ import ("io""os""path/filepath"+ "slices"+ "sync""sync/atomic""testing"@@ -22,21 +24,27 @@ import ()// newTestUploader wires a WalUploader against the real catalog repo and a mock-// storage, using the fixture's database/storage IDs for FK integrity.+// storage, using the fixture's database/storage IDs for FK integrity. Tests write+// the segments they upload into its watch directory, which the gap probe reads.func newTestUploader(+ t *testing.T,fixture *PhysicalDBFixture,store *mockWalStorage,onGap func(walmath.LSN, walmath.LSN),) *WalUploader {- return newTestUploaderWithStore(fixture, store, newMockWalStoreFor(store), onGap)+ t.Helper()++ return newTestUploaderWithStore(t, fixture, newMockWalStoreFor(store), onGap)}func newTestUploaderWithStore(+ t *testing.T,fixture *PhysicalDBFixture,- store *mockWalStorage,fileStore *storage_files.Store,onGap func(walmath.LSN, walmath.LSN),) *WalUploader {+ t.Helper()+return NewWalUploader(WalUploadDeps{DatabaseID: fixture.DB.ID,StorageID: fixture.Storage.ID,@@ -46,6 +54,7 @@ func newTestUploaderWithStore(WalSegmentRepo: physical_repositories.GetWalSegmentRepository(),WalSegmentSizeBytes: testWalSegmentSize,Logger: logger.GetLogger(),+ WatchDir: t.TempDir(),OnGapDetected: onGap,})}@@ -68,9 +77,9 @@ func Test_WalUpload_ClaimSucceeds_SaveFileLands_FileNameUpdatedFromNullToNonNullfixture := SetupPhysicalDBForBackup(t)store := newMockWalStorage()- uploader := newTestUploader(fixture, store, nil)+ uploader := newTestUploader(t, fixture, store, nil)- dir := t.TempDir()+ dir := uploader.deps.WatchDirname := walName(1, 5)localPath := writeWalFile(t, dir, name)@@ -95,9 +104,9 @@ func Test_WalUpload_SaveFileFails_ClaimRowDeletedLocalRetained_NextTickSucceeds(store := newMockWalStorage()store.failSaveTimes = 1- uploader := newTestUploader(fixture, store, nil)+ uploader := newTestUploader(t, fixture, store, nil)- dir := t.TempDir()+ dir := uploader.deps.WatchDirname := walName(1, 9)localPath := writeWalFile(t, dir, name)@@ -121,9 +130,9 @@ func Test_WalUpload_WhenSegmentFileIsShort_SkipsClaimAndRetainsFile(t *testing.Tfixture := SetupPhysicalDBForBackup(t)store := newMockWalStorage()- uploader := newTestUploader(fixture, store, nil)+ uploader := newTestUploader(t, fixture, store, nil)- dir := t.TempDir()+ dir := uploader.deps.WatchDirname := walName(1, 21)tornSegmentPath := writeSegmentOfSize(t, dir, name, testWalSegmentSize/2)@@ -138,9 +147,9 @@ func Test_WalUpload_WhenSegmentFileIsShort_WritesNothingToStorage(t *testing.T)fixture := SetupPhysicalDBForBackup(t)store := newMockWalStorage()- uploader := newTestUploader(fixture, store, nil)+ uploader := newTestUploader(t, fixture, store, nil)- dir := t.TempDir()+ dir := uploader.deps.WatchDirname := walName(1, 22)require.NoError(t, uploader.ProcessSegment(@@ -157,9 +166,9 @@ func Test_WalUpload_DuplicateClaim_CompletedRow_DropsLocalNoStorageWrite(t *testfixture := SetupPhysicalDBForBackup(t)store := newMockWalStorage()- uploader := newTestUploader(fixture, store, nil)+ uploader := newTestUploader(t, fixture, store, nil)- dir := t.TempDir()+ dir := uploader.deps.WatchDirname := walName(1, 11)// First uploader commits the segment.@@ -182,7 +191,7 @@ func Test_WalUpload_DuplicateClaim_InFlightClaim_RetainsLocalNoStorageWrite(t *tfixture := SetupPhysicalDBForBackup(t)store := newMockWalStorage()- uploader := newTestUploader(fixture, store, nil)+ uploader := newTestUploader(t, fixture, store, nil)name := walName(1, 13)startLSN := walmath.LSN(13 * uint64(testWalSegmentSize))@@ -202,7 +211,7 @@ func Test_WalUpload_DuplicateClaim_InFlightClaim_RetainsLocalNoStorageWrite(t *trequire.NoError(t, err)require.True(t, inserted)- dir := t.TempDir()+ dir := uploader.deps.WatchDirlocal := writeWalFile(t, dir, name)require.NoError(t, uploader.ProcessSegment(context.Background(), local, name))@@ -226,9 +235,9 @@ func Test_WalUpload_DeleteCascadesIntoNullClaim_UploadCompletes_DeleteFileCleansstore.blockOn = namestore.started = make(chan struct{})store.release = make(chan struct{})- uploader := newTestUploaderWithStore(fixture, store, walStore, nil)+ uploader := newTestUploaderWithStore(t, fixture, walStore, nil)- dir := t.TempDir()+ dir := uploader.deps.WatchDirlocal := writeWalFile(t, dir, name)uploadErr := make(chan error, 1)@@ -270,12 +279,12 @@ func Test_WalUpload_PostUpdateGapProbe_EmitsSingleNotificationPerOutage(t *testivar gapStart, gapEnd walmath.LSNstore := newMockWalStorage()- uploader := newTestUploader(fixture, store, func(start, end walmath.LSN) {+ uploader := newTestUploader(t, fixture, store, func(start, end walmath.LSN) {gapCount.Add(1)gapStart, gapEnd = start, end})- dir := t.TempDir()+ dir := uploader.deps.WatchDir// Commit segment 20, then a discontiguous backlog 22, 23, 24 (segment 21 was// lost to a storage outage). Only the first post-gap segment (22) is@@ -296,9 +305,9 @@ func Test_WalUpload_ContiguousSegments_NoGapNotification(t *testing.T) {var gapCount atomic.Int64store := newMockWalStorage()- uploader := newTestUploader(fixture, store, func(_, _ walmath.LSN) { gapCount.Add(1) })+ uploader := newTestUploader(t, fixture, store, func(_, _ walmath.LSN) { gapCount.Add(1) })- dir := t.TempDir()+ dir := uploader.deps.WatchDirfor _, segNo := range []uint64{30, 31, 32} {name := walName(1, segNo)@@ -327,7 +336,7 @@ func Test_WalUpload_RecoverSegment_TakesOverOwnNullClaim_CommitsExistingRow(t *tfixture := SetupPhysicalDBForBackup(t)store := newMockWalStorage()- uploader := newTestUploader(fixture, store, nil)+ uploader := newTestUploader(t, fixture, store, nil)name := walName(1, 23)startLSN := walmath.LSN(23 * uint64(testWalSegmentSize))@@ -351,7 +360,7 @@ func Test_WalUpload_RecoverSegment_TakesOverOwnNullClaim_CommitsExistingRow(t *trequire.NotNil(t, claim)require.Nil(t, claim.FileName)- dir := t.TempDir()+ dir := uploader.deps.WatchDirlocal := writeWalFile(t, dir, name)require.NoError(t, uploader.RecoverSegment(context.Background(), local, name))@@ -369,12 +378,12 @@ func Test_WalUpload_RecoverSegment_WhenAlreadyCommitted_DropsLocalNoReupload(t *fixture := SetupPhysicalDBForBackup(t)store := newMockWalStorage()- uploader := newTestUploader(fixture, store, nil)+ uploader := newTestUploader(t, fixture, store, nil)name := walName(1, 29)// A normal upload commits the segment (the crash landed after the commit).- dir1 := t.TempDir()+ dir1 := uploader.deps.WatchDirrequire.NoError(t, uploader.ProcessSegment(context.Background(), writeWalFile(t, dir1, name), name))savesAfterCommit := store.saveCount.Load()@@ -462,3 +471,171 @@ func Test_WalUpload_ConcurrentClaimSameSegment_OnlyWinnerInserts(t *testing.T) {require.Equal(t, 1, winners, "exactly one concurrent claim may win the (db, tl, start_lsn) slot")}++func Test_TakeArchivingSnapshot_AfterTwoSegmentsArchived_ReportsCountsTimingsAndConcurrencyOnce(t *testing.T) {+ fixture := SetupPhysicalDBForBackup(t)++ store := newMockWalStorage()+ uploader := newTestUploader(t, fixture, store, nil)++ for _, segmentNo := range []uint64{60, 61} {+ name := walName(1, segmentNo)+ require.NoError(t, uploader.ProcessSegment(t.Context(), writeWalFile(t, uploader.deps.WatchDir, name), name))+ }++ uploader.RecordConcurrentUploads(3)+ uploader.RecordConcurrentUploads(2)++ snapshot := uploader.TakeArchivingSnapshot()++ require.Equal(t, int64(2), snapshot.ArchivedSegments)+ require.Positive(t, snapshot.ArchivedBytes)+ require.Positive(t, snapshot.Claim.Longest)+ require.Positive(t, snapshot.SegmentWrite.Longest)+ require.Positive(t, snapshot.SidecarWrite.Longest)+ require.Positive(t, snapshot.Commit.Longest)+ require.Equal(t, 3, snapshot.HighestConcurrentUploads)++ require.Equal(t, ArchivingSnapshot{}, uploader.TakeArchivingSnapshot(),+ "each rollup interval reports only what happened since the previous one")+}++type recordedGap struct {+ start walmath.LSN+ end walmath.LSN+}++type gapRecorder struct {+ mutex sync.Mutex+ gaps []recordedGap+}++func (r *gapRecorder) record(start, end walmath.LSN) {+ r.mutex.Lock()+ defer r.mutex.Unlock()++ r.gaps = append(r.gaps, recordedGap{start: start, end: end})+}++func (r *gapRecorder) getGaps() []recordedGap {+ r.mutex.Lock()+ defer r.mutex.Unlock()++ return slices.Clone(r.gaps)+}++func newGapProbeUploader(t *testing.T) (*WalUploader, *mockWalStorage, *gapRecorder) {+ t.Helper()++ fixture := SetupPhysicalDBForBackup(t)+ store := newMockWalStorage()+ recorder := &gapRecorder{}++ return newTestUploader(t, fixture, store, recorder.record), store, recorder+}++func archiveSegment(t *testing.T, uploader *WalUploader, segmentNo uint64) {+ t.Helper()++ name := walName(1, segmentNo)+ require.NoError(t, uploader.ProcessSegment(t.Context(), writeWalFile(t, uploader.deps.WatchDir, name), name))+}++func segmentStartLSN(segmentNo uint64) walmath.LSN {+ return walmath.LSN(segmentNo * uint64(testWalSegmentSize))+}++func Test_WalUpload_WhenLaterSegmentFinishesFirst_ReportsNoGap(t *testing.T) {+ uploader, _, recorder := newGapProbeUploader(t)++ archiveSegment(t, uploader, 9)++ earlierPath := writeWalFile(t, uploader.deps.WatchDir, walName(1, 10))+ archiveSegment(t, uploader, 11)+ require.NoError(t, uploader.ProcessSegment(t.Context(), earlierPath, walName(1, 10)))++ require.Empty(t, recorder.getGaps(), "a segment still uploading is not a gap, before or after it finishes")+}++func Test_WalUpload_WhenEarlierSegmentFailedAndWaitsForRetry_ReportsNoGap(t *testing.T) {+ uploader, store, recorder := newGapProbeUploader(t)++ archiveSegment(t, uploader, 9)++ store.failSaveTimes = 1+ failedPath := writeWalFile(t, uploader.deps.WatchDir, walName(1, 10))+ require.Error(t, uploader.ProcessSegment(t.Context(), failedPath, walName(1, 10)))++ archiveSegment(t, uploader, 11)++ require.Empty(t, recorder.getGaps(), "a failed segment retried on a later pass is still coming")+}++func Test_WalUpload_WhenSegmentWasNeverReceived_ReportsGapFromNewestArchived(t *testing.T) {+ uploader, _, recorder := newGapProbeUploader(t)++ archiveSegment(t, uploader, 18)+ archiveSegment(t, uploader, 20)++ require.Equal(t, []recordedGap{{start: segmentStartLSN(19), end: segmentStartLSN(20)}}, recorder.getGaps())+}++func Test_WalUpload_WhenSegmentBeforeHoleIsStillUploading_StartsGapAfterIt(t *testing.T) {+ uploader, _, recorder := newGapProbeUploader(t)++ archiveSegment(t, uploader, 17)+ writeWalFile(t, uploader.deps.WatchDir, walName(1, 18))+ archiveSegment(t, uploader, 20)++ require.Equal(t, []recordedGap{{start: segmentStartLSN(19), end: segmentStartLSN(20)}}, recorder.getGaps(),+ "the hole starts after the segment that is still uploading, not after the newest archived one")+}++func Test_WalUpload_WhenHoleSpansSeveralSegments_ReportsOneGap(t *testing.T) {+ uploader, _, recorder := newGapProbeUploader(t)++ archiveSegment(t, uploader, 27)+ archiveSegment(t, uploader, 30)++ require.Equal(t, []recordedGap{{start: segmentStartLSN(28), end: segmentStartLSN(30)}}, recorder.getGaps())+}++func Test_WalUpload_WhenTwoGapsFinishOutOfOrder_ReportsBoth(t *testing.T) {+ uploader, _, recorder := newGapProbeUploader(t)++ archiveSegment(t, uploader, 28)++ lowerSegmentPath := writeWalFile(t, uploader.deps.WatchDir, walName(1, 30))+ archiveSegment(t, uploader, 40)+ require.NoError(t, uploader.ProcessSegment(t.Context(), lowerSegmentPath, walName(1, 30)))++ require.Equal(t, []recordedGap{+ {start: segmentStartLSN(31), end: segmentStartLSN(40)},+ {start: segmentStartLSN(29), end: segmentStartLSN(30)},+ }, recorder.getGaps(), "a lower gap found after a higher one is a separate gap")+}++func Test_WalUpload_WhenPredecessorWaitsInPendingUpload_ReportsNoGap(t *testing.T) {+ uploader, _, recorder := newGapProbeUploader(t)++ archiveSegment(t, uploader, 9)++ pendingUploadDir := filepath.Join(uploader.deps.WatchDir, pendingUploadDirName)+ require.NoError(t, os.MkdirAll(pendingUploadDir, 0o700))+ writeWalFile(t, pendingUploadDir, walName(1, 10))++ archiveSegment(t, uploader, 11)++ require.Empty(t, recorder.getGaps())+}++func Test_WalUpload_WhenSegmentBeforeGapLeftOnlyShortFile_ReportsGap(t *testing.T) {+ uploader, _, recorder := newGapProbeUploader(t)++ archiveSegment(t, uploader, 19)+ writeSegmentOfSize(t, uploader.deps.WatchDir, walName(1, 20), testWalSegmentSize/2)+ archiveSegment(t, uploader, 21)++ require.Equal(t, []recordedGap{{start: segmentStartLSN(20), end: segmentStartLSN(21)}}, recorder.getGaps(),+ "the receiver never streams a segment below its resume point again, so a short file is a hole")+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.goindex 075b38a1..7756ea9c 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go@@ -9,8 +9,6 @@ import ("strconv""strings""time"-- "databasus-backend/internal/util/walmath")// Segments rotate at the source write rate; a tight loop keeps local dwell time@@ -31,50 +29,58 @@ func (s *WalStreamSupervisor) runUploaderLoop(ctx context.Context, logger *slog.ticker := time.NewTicker(uploaderPollInterval)defer ticker.Stop()+ dispatcher := newSegmentUploadDispatcher(s.uploader.ProcessSegment, s.uploader.RecordConcurrentUploads)+ defer dispatcher.waitForUploads()+for {+ s.archiveTimelineHistoryFiles(ctx, logger)+ dispatcher.startUploads(ctx, logger, s.listAndRecordQueuedSegments())+select {case <-ctx.Done():returncase <-ticker.C:- s.scanAndUpload(ctx, logger)++ case finishedFilename := <-dispatcher.finishedUploads:+ dispatcher.markFinished(finishedFilename)}}}-// Uses the uploader's takeover path so a segment whose pre-crash claim row is-// still file_name NULL gets finished rather than left for the cleaner.+// Uses the uploader's takeover path so a segment whose pre-crash claim row is still file_name NULL+// gets finished rather than left for the cleaner. It is one pass over the files found at startup and+// returns once their uploads have finished, whatever the outcome: a failed segment waits for the live+// loop, so a storage outage at startup never keeps the receiver from spawning.func (s *WalStreamSupervisor) recoverLocalSegmentsOnStartup(ctx context.Context, logger *slog.Logger) {- entries, err := os.ReadDir(s.watchDir)- if err != nil {- return- }+ s.archiveTimelineHistoryFiles(ctx, logger)- for _, entry := range entries {- if entry.IsDir() {- continue- }+ dispatcher := newSegmentUploadDispatcher(s.uploader.RecoverSegment, s.uploader.RecordConcurrentUploads)+ defer dispatcher.waitForUploads()- name := entry.Name()+ notStartedSegments := s.listAndRecordQueuedSegments()- switch {- case walmath.IsWalFilename(name):- if err := s.uploader.RecoverSegment(ctx, filepath.Join(s.watchDir, name), name); err != nil {- logger.WarnContext(ctx, "startup wal recovery failed; live loop will retry",- "wal_filename", name,- "error", err,- )- }+ for ctx.Err() == nil {+ notStartedSegments = dispatcher.startUploads(ctx, logger, notStartedSegments)- case strings.HasSuffix(name, ".history"):- s.archiveTimelineHistoryFile(ctx, logger, name)+ if dispatcher.getInFlightCount() == 0 {+ return}++ dispatcher.markFinished(<-dispatcher.finishedUploads)}+}++func (s *WalStreamSupervisor) listAndRecordQueuedSegments() []queuedWalSegment {+ queuedSegments := listQueuedWalSegments(s.watchDir, walSegmentSizeBytes(s.spec.SourceDB))+ sortQueuedWalSegments(queuedSegments)+ s.setLowestQueuedSegment(queuedSegments)- s.sweepPendingUploads(ctx, logger, s.uploader.RecoverSegment)+ return queuedSegments}-func (s *WalStreamSupervisor) scanAndUpload(ctx context.Context, logger *slog.Logger) {+// History files are rare and need a source connection, so they stay on the dispatching goroutine.+func (s *WalStreamSupervisor) archiveTimelineHistoryFiles(ctx context.Context, logger *slog.Logger) {entries, err := os.ReadDir(s.watchDir)if err != nil {logger.ErrorContext(ctx, "read wal watch dir", "error", err)@@ -83,51 +89,8 @@ func (s *WalStreamSupervisor) scanAndUpload(ctx context.Context, logger *slog.Lo}for _, entry := range entries {- if entry.IsDir() {- continue- }-- name := entry.Name()-- switch {- case walmath.IsWalFilename(name):- if err := s.uploader.ProcessSegment(ctx, filepath.Join(s.watchDir, name), name); err != nil {- logger.WarnContext(ctx, "wal segment upload failed; will retry next tick",- "wal_filename", name,- "error", err,- )- }-- case strings.HasSuffix(name, ".history"):- s.archiveTimelineHistoryFile(ctx, logger, name)- }- }-- s.sweepPendingUploads(ctx, logger, s.uploader.ProcessSegment)-}--// Segments moved out of pg_receivewal's resume path (wal_resume.go) are valid-// WAL of the older chain, so they keep flowing to storage from here.-func (s *WalStreamSupervisor) sweepPendingUploads(- ctx context.Context,- logger *slog.Logger,- upload func(ctx context.Context, localPath, walFilename string) error,-) {- pendingUploadDir := filepath.Join(s.watchDir, pendingUploadDirName)-- entries, err := os.ReadDir(pendingUploadDir)- if err != nil {- return- }-- for _, entry := range entries {- if entry.IsDir() || !walmath.IsWalFilename(entry.Name()) {- continue- }-- if err := upload(ctx, filepath.Join(pendingUploadDir, entry.Name()), entry.Name()); err != nil {- logger.WarnContext(ctx, "pending wal segment upload failed; will retry next tick",- "wal_filename", entry.Name(), "error", err)+ if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".history") {+ s.archiveTimelineHistoryFile(ctx, logger, entry.Name())}}}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir_test.goindex fc02930f..0d1e8906 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir_test.go@@ -1,27 +1,57 @@package usecases_physical_postgresqlimport (+ "bytes"+ "context"+ "log/slog""os""path/filepath"+ "slices"+ "sync""testing"+ "time"+ "github.com/stretchr/testify/assert""github.com/stretchr/testify/require"backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories""databasus-backend/internal/util/encryption""databasus-backend/internal/util/logger"+ "databasus-backend/internal/util/walmath")-func createSupervisorWithStagedSegment(+const dispatcherConditionWaitLimit = 10 * time.Second++type capturedLog struct {+ mutex sync.Mutex+ buffer bytes.Buffer+}++func (c *capturedLog) Write(p []byte) (int, error) {+ c.mutex.Lock()+ defer c.mutex.Unlock()++ return c.buffer.Write(p)+}++func (c *capturedLog) String() string {+ c.mutex.Lock()+ defer c.mutex.Unlock()++ return c.buffer.String()+}++func createTestSupervisor(t *testing.T,fixture *PhysicalDBFixture,store *mockWalStorage,- segmentNo uint64,-) (supervisor *WalStreamSupervisor, stagedSegmentPath string) {+ supervisorLogger *slog.Logger,+) *WalStreamSupervisor {t.Helper()- supervisor = NewWalStreamSupervisor(WalStreamSpec{+ supervisor := NewWalStreamSupervisor(WalStreamSpec{DatabaseID: fixture.DB.ID,SourceDB: fixture.DB.PostgresqlPhysical,StorageID: fixture.Storage.ID,@@ -31,43 +61,351 @@ func createSupervisorWithStagedSegment(WalSegmentRepo: physical_repositories.GetWalSegmentRepository(),HistoryRepo: physical_repositories.GetWalHistoryRepository(),WatchDirRoot: t.TempDir(),- Logger: logger.GetLogger(),+ Logger: supervisorLogger,})- pendingUploadDir := filepath.Join(supervisor.watchDir, pendingUploadDirName)- require.NoError(t, os.MkdirAll(pendingUploadDir, 0o700))+ require.NoError(t, os.MkdirAll(filepath.Join(supervisor.watchDir, pendingUploadDirName), 0o700))- return supervisor, writeSegmentOfSize(t, pendingUploadDir, walName(1, segmentNo), testWalSegmentSize)+ return supervisor}-func Test_SweepPendingUploads_WhenUploadSucceeds_RemovesSegmentFromStagingDir(t *testing.T) {+func getPendingUploadDir(supervisor *WalStreamSupervisor) string {+ return filepath.Join(supervisor.watchDir, pendingUploadDirName)+}++// The gate is opened before the loop is stopped: the mock's writes ignore the context, and the loop+// waits for its uploads on the way out.+func runUploaderLoopInBackground(t *testing.T, supervisor *WalStreamSupervisor, store *mockWalStorage) {+ t.Helper()++ ctx, cancel := context.WithCancel(t.Context())+ loopStopped := make(chan struct{})++ go func() {+ supervisor.runUploaderLoop(ctx, supervisor.logger)+ close(loopStopped)+ }()++ t.Cleanup(func() {+ if store.artifactGate != nil {+ store.artifactGate.releaseAll()+ }++ cancel()+ <-loopStopped+ })+}++func isSegmentCommitted(t *testing.T, fixture *PhysicalDBFixture, segmentNo uint64) bool {+ t.Helper()++ row := findWalSegment(t, fixture.DB.ID, 1, walmath.LSN(segmentNo*uint64(testWalSegmentSize)))++ return row != nil && row.FileName != nil+}++func waitForSegmentCommitted(t *testing.T, fixture *PhysicalDBFixture, segmentNo uint64) {+ t.Helper()++ require.Eventually(t, func() bool {+ return isSegmentCommitted(t, fixture, segmentNo)+ }, dispatcherConditionWaitLimit, 20*time.Millisecond, "segment %d was not committed", segmentNo)+}++func Test_UploaderLoop_WhenEightSegmentsAreQueued_UploadsFourAtOnceAndNeverFive(t *testing.T) {fixture := SetupPhysicalDBForBackup(t)store := newMockWalStorage()- supervisor, stagedSegmentPath := createSupervisorWithStagedSegment(t, fixture, store, 70)+ store.artifactGate = newArtifactWriteGate()+ supervisor := createTestSupervisor(t, fixture, store, logger.GetLogger())++ for segmentNo := uint64(100); segmentNo < 108; segmentNo++ {+ writeWalFile(t, supervisor.watchDir, walName(1, segmentNo))+ }++ runUploaderLoopInBackground(t, supervisor, store)++ require.Eventually(t, func() bool {+ return store.artifactGate.getActiveWriteCount() == maxConcurrentSegmentUploads+ }, dispatcherConditionWaitLimit, 10*time.Millisecond, "four segments must upload at once")++ store.artifactGate.releaseOne()++ require.Eventually(t, func() bool {+ return len(store.artifactGate.getStartedObjectNames()) == maxConcurrentSegmentUploads+1+ }, dispatcherConditionWaitLimit, 10*time.Millisecond, "a freed slot must start the next segment")++ store.artifactGate.releaseAll()++ for segmentNo := uint64(100); segmentNo < 108; segmentNo++ {+ waitForSegmentCommitted(t, fixture, segmentNo)+ }++ assert.Equal(t, maxConcurrentSegmentUploads, store.artifactGate.getHighestActiveWriteCount(),+ "no more than four segments of one database may upload at once")+}++func Test_UploaderLoop_WhenSlotFreesUp_StartsPendingUploadSegmentInWalOrder(t *testing.T) {+ fixture := SetupPhysicalDBForBackup(t)++ store := newMockWalStorage()+ store.artifactGate = newArtifactWriteGate()+ supervisor := createTestSupervisor(t, fixture, store, logger.GetLogger())++ for _, segmentNo := range []uint64{110, 111, 112, 113, 115} {+ writeWalFile(t, supervisor.watchDir, walName(1, segmentNo))+ }++ writeWalFile(t, getPendingUploadDir(supervisor), walName(1, 114))++ runUploaderLoopInBackground(t, supervisor, store)++ require.Eventually(t, func() bool {+ return store.artifactGate.getActiveWriteCount() == maxConcurrentSegmentUploads+ }, dispatcherConditionWaitLimit, 10*time.Millisecond)++ store.artifactGate.releaseOne()++ require.Eventually(t, func() bool {+ return len(store.artifactGate.getStartedObjectNames()) == maxConcurrentSegmentUploads+1+ }, dispatcherConditionWaitLimit, 10*time.Millisecond)++ fifthStarted := store.artifactGate.getStartedObjectNames()[maxConcurrentSegmentUploads]+ assert.Contains(t, fifthStarted, walName(1, 114), "segment 14 must start before segment 15")+}++func Test_UploaderLoop_WhenSegmentWriteStaysBlocked_DoesNotStartItAgainOrWarn(t *testing.T) {+ fixture := SetupPhysicalDBForBackup(t)++ log := &capturedLog{}+ blockedSegmentName := walName(1, 120)++ store := newMockWalStorage()+ store.blockOn = blockedSegmentName+ store.started = make(chan struct{})+ store.release = make(chan struct{})+ supervisor := createTestSupervisor(t, fixture, store, slog.New(slog.NewTextHandler(log, nil)))++ writeWalFile(t, supervisor.watchDir, blockedSegmentName)+ runUploaderLoopInBackground(t, supervisor, store)+ t.Cleanup(func() { close(store.release) })++ <-store.started++ // A later segment committing proves that passes kept running while the first write was blocked.+ writeWalFile(t, supervisor.watchDir, walName(1, 121))+ waitForSegmentCommitted(t, fixture, 121)++ writeWalFile(t, supervisor.watchDir, walName(1, 122))+ waitForSegmentCommitted(t, fixture, 122)++ assert.Equal(t, int64(5), store.saveCount.Load(),+ "the blocked artifact plus two artifacts and two sidecars; a second start would add more")+ assert.NotContains(t, log.String(), "held by another in-flight claim")+}++func Test_RecoverLocalSegmentsOnStartup_WhenSegmentsHaveOwnNullClaims_CommitsAllConcurrently(t *testing.T) {+ fixture := SetupPhysicalDBForBackup(t)++ store := newMockWalStorage()+ store.artifactGate = newArtifactWriteGateOpeningAt(2)+ supervisor := createTestSupervisor(t, fixture, store, logger.GetLogger())++ recoveredSegmentNos := []uint64{130, 131, 132, 133, 134, 135}- supervisor.sweepPendingUploads(t.Context(), logger.GetLogger(), supervisor.uploader.ProcessSegment)+ for _, segmentNo := range recoveredSegmentNos {+ seedNullClaim(t, fixture, segmentNo)+ writeWalFile(t, supervisor.watchDir, walName(1, segmentNo))+ }++ supervisor.recoverLocalSegmentsOnStartup(t.Context(), supervisor.logger)++ for _, segmentNo := range recoveredSegmentNos {+ assert.True(t, isSegmentCommitted(t, fixture, segmentNo), "segment %d must be committed", segmentNo)+ assert.NoFileExists(t, filepath.Join(supervisor.watchDir, walName(1, segmentNo)))+ }++ assert.Greater(t, store.artifactGate.getHighestActiveWriteCount(), 1,+ "recovery must use the same concurrent slots as the live loop")+}++func Test_RecoverLocalSegmentsOnStartup_WhenStorageFails_ReturnsAfterOneAttemptPerSegment(t *testing.T) {+ fixture := SetupPhysicalDBForBackup(t)++ store := newMockWalStorage()+ store.startFailingSaves()+ supervisor := createTestSupervisor(t, fixture, store, logger.GetLogger())++ recoveredSegmentNos := []uint64{140, 141, 142, 143, 144}++ for _, segmentNo := range recoveredSegmentNos {+ writeWalFile(t, supervisor.watchDir, walName(1, segmentNo))+ }++ supervisor.recoverLocalSegmentsOnStartup(t.Context(), supervisor.logger)++ assert.Equal(t, int64(len(recoveredSegmentNos)), store.saveCount.Load(),+ "recovery is one pass, so a storage outage at startup cannot keep the receiver from spawning")++ for _, segmentNo := range recoveredSegmentNos {+ assert.FileExists(t, filepath.Join(supervisor.watchDir, walName(1, segmentNo)))+ }+}++func Test_UploaderLoop_WhenPendingUploadSucceeds_RemovesSegmentFromStagingDir(t *testing.T) {+ fixture := SetupPhysicalDBForBackup(t)++ store := newMockWalStorage()+ supervisor := createTestSupervisor(t, fixture, store, logger.GetLogger())+ stagedSegmentPath := writeWalFile(t, getPendingUploadDir(supervisor), walName(1, 70))++ runUploaderLoopInBackground(t, supervisor, store)+ waitForSegmentCommitted(t, fixture, 70)require.True(t, store.hasObjectFor(fixture.DB.ID, 1, walName(1, 70)),"a segment staged out of the resume path is valid WAL of the older chain and must still reach storage")- require.NoFileExists(t, stagedSegmentPath, "an uploaded segment must not keep occupying the local queue")+ require.Eventually(t, func() bool {+ _, err := os.Stat(stagedSegmentPath)+ return os.IsNotExist(err)+ }, dispatcherConditionWaitLimit, 20*time.Millisecond, "an uploaded segment must not keep occupying the local queue")}-func Test_SweepPendingUploads_WhenUploadFails_RetainsSegmentForNextTick(t *testing.T) {+func Test_UploaderLoop_WhenPendingUploadFails_RetainsSegmentForNextPass(t *testing.T) {fixture := SetupPhysicalDBForBackup(t)store := newMockWalStorage()store.startFailingSaves()+ supervisor := createTestSupervisor(t, fixture, store, logger.GetLogger())+ stagedSegmentPath := writeWalFile(t, getPendingUploadDir(supervisor), walName(1, 71))- supervisor, stagedSegmentPath := createSupervisorWithStagedSegment(t, fixture, store, 71)-- supervisor.sweepPendingUploads(t.Context(), logger.GetLogger(), supervisor.uploader.ProcessSegment)+ runUploaderLoopInBackground(t, supervisor, store)+ require.Eventually(t, func() bool {+ return store.saveCount.Load() >= 1+ }, dispatcherConditionWaitLimit, 20*time.Millisecond)require.FileExists(t, stagedSegmentPath, "staged WAL must survive a storage outage")store.stopFailingSaves()- supervisor.sweepPendingUploads(t.Context(), logger.GetLogger(), supervisor.uploader.ProcessSegment)+ waitForSegmentCommitted(t, fixture, 71)require.True(t, store.hasObjectFor(fixture.DB.ID, 1, walName(1, 71)))- require.NoFileExists(t, stagedSegmentPath)+}++func Test_UploaderLoop_WhenSegmentMovesToPendingUploadMidUpload_WritesItOnce(t *testing.T) {+ fixture := SetupPhysicalDBForBackup(t)++ movedSegmentName := walName(1, 150)++ store := newMockWalStorage()+ store.blockOn = movedSegmentName+ store.started = make(chan struct{})+ store.release = make(chan struct{})+ supervisor := createTestSupervisor(t, fixture, store, logger.GetLogger())++ watchDirPath := writeWalFile(t, supervisor.watchDir, movedSegmentName)+ runUploaderLoopInBackground(t, supervisor, store)++ releaseBlockedWrite := sync.OnceFunc(func() { close(store.release) })+ t.Cleanup(releaseBlockedWrite)++ <-store.started++ movedPath := filepath.Join(getPendingUploadDir(supervisor), movedSegmentName)+ require.NoError(t, os.Rename(watchDirPath, movedPath))++ // A later segment committing proves that passes saw the moved copy while its upload was running.+ writeWalFile(t, supervisor.watchDir, walName(1, 151))+ waitForSegmentCommitted(t, fixture, 151)++ releaseBlockedWrite()++ waitForSegmentCommitted(t, fixture, 150)+ require.Eventually(t, func() bool {+ _, err := os.Stat(movedPath)+ return os.IsNotExist(err)+ }, dispatcherConditionWaitLimit, 20*time.Millisecond, "the moved copy of a committed segment must be dropped")++ assert.Len(t, store.objectNamesFor(fixture.DB.ID, 1, movedSegmentName), 2,+ "one artifact and one sidecar: the moved copy must not be uploaded again")+}++func Test_UploaderLoop_WhenOlderSegmentWriteIsBlocked_RecordsItAsLowestQueuedSegment(t *testing.T) {+ fixture := SetupPhysicalDBForBackup(t)++ blockedSegmentName := walName(1, 50)++ store := newMockWalStorage()+ store.blockOn = blockedSegmentName+ store.started = make(chan struct{})+ store.release = make(chan struct{})+ supervisor := createTestSupervisor(t, fixture, store, logger.GetLogger())++ for segmentNo := uint64(50); segmentNo <= 53; segmentNo++ {+ writeWalFile(t, supervisor.watchDir, walName(1, segmentNo))+ }++ runUploaderLoopInBackground(t, supervisor, store)+ t.Cleanup(func() { close(store.release) })++ for segmentNo := uint64(51); segmentNo <= 53; segmentNo++ {+ waitForSegmentCommitted(t, fixture, segmentNo)+ }++ // The listing after the last commit no longer holds 53, so the value read below was stored after+ // the newer segments finished.+ require.Eventually(t, func() bool {+ return !slices.Contains(listQueuedFilenames(supervisor), walName(1, 53))+ }, dispatcherConditionWaitLimit, 20*time.Millisecond)++ assert.Equal(t, segmentStart(1, 50), supervisor.getLowestQueuedSegment(),+ "newer segments finishing must not move the lowest queued segment past one still uploading")+}++func Test_UploaderLoop_WhenShortSegmentFileStaysQueued_RecordsItAsLowestQueuedSegment(t *testing.T) {+ fixture := SetupPhysicalDBForBackup(t)++ store := newMockWalStorage()+ supervisor := createTestSupervisor(t, fixture, store, logger.GetLogger())++ writeSegmentOfSize(t, supervisor.watchDir, walName(1, 60), testWalSegmentSize/2)+ writeWalFile(t, supervisor.watchDir, walName(1, 61))++ runUploaderLoopInBackground(t, supervisor, store)+ waitForSegmentCommitted(t, fixture, 61)++ require.Eventually(t, func() bool {+ lowestQueuedSegment := supervisor.getLowestQueuedSegment()+ return lowestQueuedSegment != nil && *lowestQueuedSegment == *segmentStart(1, 60) &&+ len(listQueuedFilenames(supervisor)) == 1+ }, dispatcherConditionWaitLimit, 20*time.Millisecond,+ "a short file holds the archive progress point, because restore cannot pass it either")+}++func listQueuedFilenames(supervisor *WalStreamSupervisor) []string {+ queuedSegments := listQueuedWalSegments(supervisor.watchDir, testWalSegmentSize)+ filenames := make([]string, 0, len(queuedSegments))++ for _, queued := range queuedSegments {+ filenames = append(filenames, queued.walFilename)+ }++ return filenames+}++func seedNullClaim(t *testing.T, fixture *PhysicalDBFixture, segmentNo uint64) {+ t.Helper()++ startLSN := walmath.LSN(segmentNo * uint64(testWalSegmentSize))++ inserted, err := physical_repositories.GetWalSegmentRepository().ClaimInsert(&physical_models.PhysicalWalSegment{+ DatabaseID: fixture.DB.ID,+ StorageID: fixture.Storage.ID,+ TimelineID: 1,+ WalFilename: walName(1, segmentNo),+ StartLSN: startLSN,+ EndLSN: startLSN + walmath.LSN(testWalSegmentSize),+ Encryption: backups_core_enums.BackupEncryptionNone,+ })+ require.NoError(t, err)+ require.True(t, inserted)}diff --git a/backend/internal/features/storages/connections/fingerprint.go b/backend/internal/features/storages/connections/fingerprint.gonew file mode 100644index 00000000..7840626e--- /dev/null+++ b/backend/internal/features/storages/connections/fingerprint.go@@ -0,0 +1,28 @@+package storage_connections++import (+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"++ "github.com/google/uuid"+)++// A session authenticates with the settings it was dialed with, so the key carries a fingerprint of+// those settings: once they change, the next checkout lands under a new key and never reuses a+// session opened with the old ones.+type Key struct {+ StorageID uuid.UUID+ Fingerprint string+}++func Fingerprint(parts ...any) string {+ encodedParts, err := json.Marshal(parts)+ if err != nil {+ panic("storage fingerprint parts must be JSON-encodable: " + err.Error())+ }++ digest := sha256.Sum256(encodedParts)++ return hex.EncodeToString(digest[:])+}diff --git a/backend/internal/features/storages/connections/session_pool.go b/backend/internal/features/storages/connections/session_pool.gonew file mode 100644index 00000000..062d1a0d--- /dev/null+++ b/backend/internal/features/storages/connections/session_pool.go@@ -0,0 +1,257 @@+package storage_connections++import (+ "context"+ "sync"+ "time"++ "github.com/google/uuid"+)++// Four matches the WAL uploader's concurrent writes per database, so a busy database keeps every+// session it needs between segments.+const (+ defaultIdleCap = 4+ defaultProbeAfter = 30 * time.Second+ defaultProbeTimeout = 5 * time.Second+ defaultIdleTimeout = 5 * time.Minute+)++type PoolSettings struct {+ IdleCap int++ // A server may drop a quiet session without telling us, so a session idle longer than this is+ // probed before a write trusts it.+ ProbeAfter time.Duration+ ProbeTimeout time.Duration++ // Frees server-side connection slots for storages nobody writes to, deleted ones included.+ IdleTimeout time.Duration+}++func GetDefaultPoolSettings() PoolSettings {+ return PoolSettings{+ IdleCap: defaultIdleCap,+ ProbeAfter: defaultProbeAfter,+ ProbeTimeout: defaultProbeTimeout,+ IdleTimeout: defaultIdleTimeout,+ }+}++type SessionOperations[T any] struct {+ Close func(session T)+ Probe func(ctx context.Context, session T) error+}++// Each checkout gets a session nobody else uses until it is released, so one slow transfer never+// makes another wait behind it on the same connection, and one dead connection fails one write.+type SessionPool[T any] struct {+ operations SessionOperations[T]+ settings PoolSettings++ mutex sync.Mutex+ currentFingerprints map[uuid.UUID]string+ idleSessions map[Key][]*idleSession[T]+}++type idleSession[T any] struct {+ session T+ idleSince time.Time+ idleTimer *time.Timer+}++func NewSessionPool[T any](operations SessionOperations[T], settings PoolSettings) *SessionPool[T] {+ return &SessionPool[T]{+ operations: operations,+ settings: settings,+ currentFingerprints: make(map[uuid.UUID]string),+ idleSessions: make(map[Key][]*idleSession[T]),+ }+}++// The dial comes with each checkout because only the caller holds the decrypted settings the key was+// fingerprinted from.+func (p *SessionPool[T]) Checkout(+ ctx context.Context,+ key Key,+ dial func(ctx context.Context) (T, error),+) (*Lease[T], error) {+ p.closeSessions(p.retireOtherFingerprints(key))++ for {+ candidate, isFound := p.takeIdleSession(key)+ if !isFound {+ break+ }++ if p.isSessionAlive(ctx, candidate) {+ return p.lease(ctx, key, candidate.session), nil+ }++ p.operations.Close(candidate.session)+ }++ session, err := dial(ctx)+ if err != nil {+ return nil, err+ }++ return p.lease(ctx, key, session), nil+}++func (p *SessionPool[T]) retireOtherFingerprints(key Key) []T {+ p.mutex.Lock()+ defer p.mutex.Unlock()++ previousFingerprint, isKnown := p.currentFingerprints[key.StorageID]+ p.currentFingerprints[key.StorageID] = key.Fingerprint++ if !isKnown || previousFingerprint == key.Fingerprint {+ return nil+ }++ previousKey := Key{StorageID: key.StorageID, Fingerprint: previousFingerprint}+ retiredSessions := make([]T, 0, len(p.idleSessions[previousKey]))++ for _, retired := range p.idleSessions[previousKey] {+ retired.idleTimer.Stop()+ retiredSessions = append(retiredSessions, retired.session)+ }++ delete(p.idleSessions, previousKey)++ return retiredSessions+}++func (p *SessionPool[T]) takeIdleSession(key Key) (*idleSession[T], bool) {+ p.mutex.Lock()+ defer p.mutex.Unlock()++ sessions := p.idleSessions[key]+ if len(sessions) == 0 {+ return nil, false+ }++ newest := sessions[len(sessions)-1]+ p.setIdleSessions(key, sessions[:len(sessions)-1])+ newest.idleTimer.Stop()++ return newest, true+}++func (p *SessionPool[T]) isSessionAlive(ctx context.Context, candidate *idleSession[T]) bool {+ if time.Since(candidate.idleSince) < p.settings.ProbeAfter {+ return true+ }++ probeCtx, cancel := context.WithTimeout(ctx, p.settings.ProbeTimeout)+ defer cancel()++ return p.operations.Probe(probeCtx, candidate.session) == nil+}++func (p *SessionPool[T]) lease(ctx context.Context, key Key, session T) *Lease[T] {+ lease := &Lease[T]{pool: p, key: key, session: session}++ // Neither the SSH nor the SMB library honors a context once a request is on the wire. Closing the+ // session is what makes a call blocked on a silent server return.+ lease.stopCloseOnCancel = context.AfterFunc(ctx, func() {+ p.operations.Close(session)+ })++ return lease+}++func (p *SessionPool[T]) giveBack(key Key, session T) {+ p.mutex.Lock()++ isStale := p.currentFingerprints[key.StorageID] != key.Fingerprint+ isFull := len(p.idleSessions[key]) >= p.settings.IdleCap++ if isStale || isFull {+ p.mutex.Unlock()+ p.operations.Close(session)++ return+ }++ returned := &idleSession[T]{session: session, idleSince: time.Now()}+ returned.idleTimer = time.AfterFunc(p.settings.IdleTimeout, func() {+ p.expireIdleSession(key, returned)+ })++ p.idleSessions[key] = append(p.idleSessions[key], returned)+ p.mutex.Unlock()+}++// A checkout that took the session first removed it under the same mutex, so a timer that fires+// late finds nothing and leaves the session to its new holder.+func (p *SessionPool[T]) expireIdleSession(key Key, expired *idleSession[T]) {+ p.mutex.Lock()++ sessions := p.idleSessions[key]+ expiredIndex := -1++ for index, candidate := range sessions {+ if candidate == expired {+ expiredIndex = index+ break+ }+ }++ if expiredIndex < 0 {+ p.mutex.Unlock()+ return+ }++ p.setIdleSessions(key, append(sessions[:expiredIndex:expiredIndex], sessions[expiredIndex+1:]...))+ p.mutex.Unlock()++ p.operations.Close(expired.session)+}++func (p *SessionPool[T]) setIdleSessions(key Key, sessions []*idleSession[T]) {+ if len(sessions) == 0 {+ delete(p.idleSessions, key)+ return+ }++ p.idleSessions[key] = sessions+}++func (p *SessionPool[T]) closeSessions(sessions []T) {+ for _, session := range sessions {+ p.operations.Close(session)+ }+}++type Lease[T any] struct {+ pool *SessionPool[T]+ key Key+ session T+ stopCloseOnCancel func() bool++ releaseOnce sync.Once+}++func (l *Lease[T]) GetSession() T {+ return l.session+}++// A failed call may leave the protocol mid-exchange, so a session released with an error is closed+// rather than handed to the next write.+func (l *Lease[T]) Release(err error) {+ l.releaseOnce.Do(func() {+ isClosedByCancel := !l.stopCloseOnCancel()+ if isClosedByCancel {+ return+ }++ if err != nil {+ l.pool.operations.Close(l.session)+ return+ }++ l.pool.giveBack(l.key, l.session)+ })+}diff --git a/backend/internal/features/storages/connections/session_pool_test.go b/backend/internal/features/storages/connections/session_pool_test.gonew file mode 100644index 00000000..eb52841e--- /dev/null+++ b/backend/internal/features/storages/connections/session_pool_test.go@@ -0,0 +1,252 @@+package storage_connections++import (+ "context"+ "errors"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/google/uuid"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++const conditionWaitLimit = 5 * time.Second++type fakeSession struct {+ dialNumber int+ closed chan struct{}+ closeOnce sync.Once+ isBroken atomic.Bool+ probeCount atomic.Int32+}++func (s *fakeSession) isClosed() bool {+ select {+ case <-s.closed:+ return true+ default:+ return false+ }+}++type fakeServer struct {+ dialCount atomic.Int32+}++func (f *fakeServer) dial(_ context.Context) (*fakeSession, error) {+ return &fakeSession{dialNumber: int(f.dialCount.Add(1)), closed: make(chan struct{})}, nil+}++func newTestPool(settings PoolSettings) *SessionPool[*fakeSession] {+ return NewSessionPool(SessionOperations[*fakeSession]{+ Close: func(session *fakeSession) {+ session.closeOnce.Do(func() { close(session.closed) })+ },+ Probe: func(_ context.Context, session *fakeSession) error {+ session.probeCount.Add(1)++ if session.isBroken.Load() {+ return errors.New("session is broken")+ }++ return nil+ },+ }, settings)+}++func newTestKey(storageID uuid.UUID, fingerprintParts ...any) Key {+ return Key{StorageID: storageID, Fingerprint: Fingerprint(fingerprintParts...)}+}++func checkoutSession(+ t *testing.T,+ pool *SessionPool[*fakeSession],+ server *fakeServer,+ key Key,+) *Lease[*fakeSession] {+ t.Helper()++ lease, err := pool.Checkout(t.Context(), key, server.dial)+ require.NoError(t, err)++ return lease+}++func Test_Fingerprint_WhenArgumentsMatchOrDiffer_ReturnsSameOrDifferentValue(t *testing.T) {+ baseline := Fingerprint("host", 22, "user", "encrypted-password")++ assert.Equal(t, baseline, Fingerprint("host", 22, "user", "encrypted-password"))+ assert.NotEqual(t, baseline, Fingerprint("other-host", 22, "user", "encrypted-password"))+ assert.NotEqual(t, baseline, Fingerprint("host", 2222, "user", "encrypted-password"))+ assert.NotEqual(t, baseline, Fingerprint("host", 22, "user", "changed-password"))+}++func Test_Checkout_WhenCalledSequentially_ReusesOneSession(t *testing.T) {+ pool := newTestPool(GetDefaultPoolSettings())+ server := &fakeServer{}+ key := newTestKey(uuid.New(), "settings")++ firstLease := checkoutSession(t, pool, server, key)+ firstSession := firstLease.GetSession()+ firstLease.Release(nil)++ secondLease := checkoutSession(t, pool, server, key)+ defer secondLease.Release(nil)++ assert.Same(t, firstSession, secondLease.GetSession())+ assert.EqualValues(t, 1, server.dialCount.Load())+}++func Test_Checkout_WhenTwoLeasesAreHeldAtOnce_HandsOutDifferentSessions(t *testing.T) {+ pool := newTestPool(GetDefaultPoolSettings())+ server := &fakeServer{}+ key := newTestKey(uuid.New(), "settings")++ firstLease := checkoutSession(t, pool, server, key)+ defer firstLease.Release(nil)++ secondLease := checkoutSession(t, pool, server, key)+ defer secondLease.Release(nil)++ assert.NotSame(t, firstLease.GetSession(), secondLease.GetSession())+ assert.EqualValues(t, 2, server.dialCount.Load())+}++func Test_Release_WhenCallFailed_ClosesSessionAndNextCheckoutDials(t *testing.T) {+ pool := newTestPool(GetDefaultPoolSettings())+ server := &fakeServer{}+ key := newTestKey(uuid.New(), "settings")++ failedLease := checkoutSession(t, pool, server, key)+ failedSession := failedLease.GetSession()+ failedLease.Release(errors.New("write failed"))++ nextLease := checkoutSession(t, pool, server, key)+ defer nextLease.Release(nil)++ assert.True(t, failedSession.isClosed())+ assert.NotSame(t, failedSession, nextLease.GetSession())+ assert.EqualValues(t, 2, server.dialCount.Load())+}++func Test_Release_WhenIdleCapIsReached_ClosesFifthSession(t *testing.T) {+ pool := newTestPool(GetDefaultPoolSettings())+ server := &fakeServer{}+ key := newTestKey(uuid.New(), "settings")++ leases := make([]*Lease[*fakeSession], 0, 5)+ for range 5 {+ leases = append(leases, checkoutSession(t, pool, server, key))+ }++ for _, lease := range leases {+ lease.Release(nil)+ }++ for _, lease := range leases[:4] {+ assert.False(t, lease.GetSession().isClosed())+ }++ assert.True(t, leases[4].GetSession().isClosed())+}++func Test_Checkout_WhenIdleSessionFailsProbe_DialsNewSession(t *testing.T) {+ settings := GetDefaultPoolSettings()+ settings.ProbeAfter = time.Millisecond+ pool := newTestPool(settings)+ server := &fakeServer{}+ key := newTestKey(uuid.New(), "settings")++ staleLease := checkoutSession(t, pool, server, key)+ staleSession := staleLease.GetSession()+ staleLease.Release(nil)+ releasedAt := time.Now()++ staleSession.isBroken.Store(true)+ require.Eventually(t, func() bool {+ return time.Since(releasedAt) > settings.ProbeAfter+ }, conditionWaitLimit, time.Millisecond)++ freshLease := checkoutSession(t, pool, server, key)+ defer freshLease.Release(nil)++ assert.EqualValues(t, 1, staleSession.probeCount.Load())+ assert.True(t, staleSession.isClosed())+ assert.NotSame(t, staleSession, freshLease.GetSession())+}++func Test_Checkout_WhenContextEndsWhileHolderIsBlocked_ClosesSessionAndCallReturns(t *testing.T) {+ pool := newTestPool(GetDefaultPoolSettings())+ server := &fakeServer{}+ key := newTestKey(uuid.New(), "settings")++ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()++ lease, err := pool.Checkout(ctx, key, server.dial)+ require.NoError(t, err)++ blockedCallReturned := make(chan struct{})+ go func() {+ <-lease.GetSession().closed+ close(blockedCallReturned)+ }()++ cancel()++ select {+ case <-blockedCallReturned:+ case <-time.After(conditionWaitLimit):+ t.Fatal("blocked call did not return after the lease context was cancelled")+ }++ lease.Release(errors.New("call failed after its session was closed"))++ nextLease := checkoutSession(t, pool, server, key)+ defer nextLease.Release(nil)++ assert.NotSame(t, lease.GetSession(), nextLease.GetSession())+}++func Test_Checkout_WhenFingerprintChanges_ClosesOldSessionsAndStaleLeaseOnRelease(t *testing.T) {+ pool := newTestPool(GetDefaultPoolSettings())+ server := &fakeServer{}+ storageID := uuid.New()+ oldKey := newTestKey(storageID, "old-password")+ newKey := newTestKey(storageID, "new-password")++ idleOldLease := checkoutSession(t, pool, server, oldKey)+ heldOldLease := checkoutSession(t, pool, server, oldKey)+ idleOldLease.Release(nil)++ newLease := checkoutSession(t, pool, server, newKey)+ defer newLease.Release(nil)++ assert.True(t, idleOldLease.GetSession().isClosed())+ assert.NotSame(t, idleOldLease.GetSession(), newLease.GetSession())++ heldOldLease.Release(nil)++ assert.True(t, heldOldLease.GetSession().isClosed())+}++func Test_Release_WhenSessionStaysIdlePastIdleTimeout_ClosesSession(t *testing.T) {+ settings := GetDefaultPoolSettings()+ settings.IdleTimeout = 50 * time.Millisecond+ pool := newTestPool(settings)+ server := &fakeServer{}+ key := newTestKey(uuid.New(), "settings")++ lease := checkoutSession(t, pool, server, key)+ lease.Release(nil)++ require.Eventually(t, lease.GetSession().isClosed, conditionWaitLimit, 10*time.Millisecond)++ nextLease := checkoutSession(t, pool, server, key)+ defer nextLease.Release(nil)++ assert.NotSame(t, lease.GetSession(), nextLease.GetSession())+}diff --git a/backend/internal/features/storages/model_test.go b/backend/internal/features/storages/model_test.goindex 336815e3..b875a7b9 100644--- a/backend/internal/features/storages/model_test.go+++ b/backend/internal/features/storages/model_test.go@@ -3,20 +3,28 @@ package storagesimport ("bytes""context"+ "crypto/rand"+ "encoding/base64""fmt""io""os""path/filepath"+ "regexp"+ "slices"+ "sync""testing""time""github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror""github.com/google/uuid""github.com/minio/minio-go/v7""github.com/minio/minio-go/v7/pkg/credentials""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "gorm.io/gorm"+ storage_files "databasus-backend/internal/features/storages/files"azure_blob_storage "databasus-backend/internal/features/storages/models/azure_blob"ftp_storage "databasus-backend/internal/features/storages/models/ftp"local_storage "databasus-backend/internal/features/storages/models/local"@@ -24,6 +32,10 @@ import (rclone_storage "databasus-backend/internal/features/storages/models/rclone"s3_storage "databasus-backend/internal/features/storages/models/s3"sftp_storage "databasus-backend/internal/features/storages/models/sftp"+ users_enums "databasus-backend/internal/features/users/enums"+ users_testing "databasus-backend/internal/features/users/testing"+ workspaces_testing "databasus-backend/internal/features/workspaces/testing"+ db "databasus-backend/internal/storage""databasus-backend/internal/util/encryption""databasus-backend/internal/util/logger""databasus-backend/internal/util/testing/containers"@@ -37,136 +49,66 @@ type S3Container struct {region string}-type AzuriteContainer struct {- endpoint string- accountName string- accountKey string- containerNameKey string- containerNameStr string- connectionString string-}-func Test_Storage_BasicOperations(t *testing.T) {- ctx := t.Context()-- minioEndpoint := containers.StartMinio(t)- azuriteEndpoint := containers.StartAzurite(t)- nasEndpoint := containers.StartSamba(t)- ftpEndpoint := containers.StartFtp(t)- sftpEndpoint := containers.StartSftp(t)-- s3Container, err := setupS3Container(ctx, addressOf(minioEndpoint))- require.NoError(t, err, "Failed to setup S3 container")-- azuriteContainer, err := setupAzuriteContainer(ctx, addressOf(azuriteEndpoint))- require.NoError(t, err, "Failed to setup Azurite container")+ minio := containers.StartMinioContainer(t)+ azurite := containers.StartAzuriteContainer(t)+ samba := containers.StartSambaContainer(t)+ ftp := containers.StartFtpContainer(t)+ sftp := containers.StartSftpContainer(t)testFilePath, err := setupTestFile()require.NoError(t, err, "Failed to setup test file")defer os.Remove(testFilePath)+ s3WithStorageClass := buildTestStorage(t, StorageTypeS3, minio)+ s3WithStorageClass.S3Storage.S3StorageClass = s3_storage.S3StorageClassStandard++ azureWithConnectionString := buildTestStorage(t, StorageTypeAzureBlob, azurite)+ azureWithConnectionString.AzureBlobStorage = &azure_blob_storage.AzureBlobStorage{+ AuthMethod: azure_blob_storage.AuthMethodConnectionString,+ ConnectionString: getAzuriteConnectionString(addressOf(azurite.Endpoint)),+ ContainerName: azureWithConnectionString.AzureBlobStorage.ContainerName,+ }+testCases := []struct {name stringstorage StorageFileSaver}{{name: "LocalStorage",- storage: &local_storage.LocalStorage{StorageID: uuid.New()},+ storage: createTestStorageOfType(t, StorageTypeLocal, containers.ContainerHandle{}),},{- name: "S3Storage",- storage: &s3_storage.S3Storage{- StorageID: uuid.New(),- S3Bucket: s3Container.bucketName,- S3Region: s3Container.region,- S3AccessKey: s3Container.accessKey,- S3SecretKey: s3Container.secretKey,- S3Endpoint: "http://" + s3Container.endpoint,- },+ name: "S3Storage",+ storage: createTestStorageOfType(t, StorageTypeS3, minio),},{- name: "S3Storage_WithStorageClass",- storage: &s3_storage.S3Storage{- StorageID: uuid.New(),- S3Bucket: s3Container.bucketName,- S3Region: s3Container.region,- S3AccessKey: s3Container.accessKey,- S3SecretKey: s3Container.secretKey,- S3Endpoint: "http://" + s3Container.endpoint,- S3StorageClass: s3_storage.S3StorageClassStandard,- },+ name: "S3Storage_WithStorageClass",+ storage: saveTestStorage(t, s3WithStorageClass),},{- name: "NASStorage",- storage: &nas_storage.NASStorage{- StorageID: uuid.New(),- Host: nasEndpoint.Host,- Port: nasEndpoint.Port,- Share: containers.SambaShare,- Username: containers.SambaUsername,- Password: containers.SambaPassword,- UseSSL: false,- Domain: "",- Path: "test-files",- },+ name: "NASStorage",+ storage: createTestStorageOfType(t, StorageTypeNAS, samba),},{- name: "AzureBlobStorage_AccountKey",- storage: &azure_blob_storage.AzureBlobStorage{- StorageID: uuid.New(),- AuthMethod: azure_blob_storage.AuthMethodAccountKey,- AccountName: azuriteContainer.accountName,- AccountKey: azuriteContainer.accountKey,- ContainerName: azuriteContainer.containerNameKey,- Endpoint: azuriteContainer.endpoint,- },+ name: "AzureBlobStorage_AccountKey",+ storage: createTestStorageOfType(t, StorageTypeAzureBlob, azurite),},{- name: "AzureBlobStorage_ConnectionString",- storage: &azure_blob_storage.AzureBlobStorage{- StorageID: uuid.New(),- AuthMethod: azure_blob_storage.AuthMethodConnectionString,- ConnectionString: azuriteContainer.connectionString,- ContainerName: azuriteContainer.containerNameStr,- },+ name: "AzureBlobStorage_ConnectionString",+ storage: saveTestStorage(t, azureWithConnectionString),},{- name: "FTPStorage",- storage: &ftp_storage.FTPStorage{- StorageID: uuid.New(),- Host: ftpEndpoint.Host,- Port: ftpEndpoint.Port,- Username: containers.FtpUsername,- Password: containers.FtpPassword,- UseSSL: false,- Path: "test-files",- },+ name: "FTPStorage",+ storage: createTestStorageOfType(t, StorageTypeFTP, ftp),},{- name: "SFTPStorage",- storage: &sftp_storage.SFTPStorage{- StorageID: uuid.New(),- Host: sftpEndpoint.Host,- Port: sftpEndpoint.Port,- Username: containers.SftpUsername,- Password: containers.SftpPassword,- SkipHostKeyVerify: true,- Path: containers.SftpUploadDir,- },+ name: "SFTPStorage",+ storage: createTestStorageOfType(t, StorageTypeSFTP, sftp),},{- name: "RcloneStorage",- storage: &rclone_storage.RcloneStorage{- StorageID: uuid.New(),- ConfigContent: fmt.Sprintf(`[minio]-type = s3-provider = Other-access_key_id = %s-secret_access_key = %s-endpoint = http://%s-acl = private`, s3Container.accessKey, s3Container.secretKey, s3Container.endpoint),- RemotePath: s3Container.bucketName,- },+ name: "RcloneStorage",+ storage: createTestStorageOfType(t, StorageTypeRclone, minio),},}@@ -275,6 +217,412 @@ acl = private`, s3Container.accessKey, s3Container.secretKey, s3Container.endpoi}}+// WAL archiving writes a 16 MiB segment and a 1 KiB metadata sidecar per segment, and a busy source+// needs about 1.3 segments per second. Two pairs per second against a storage 50 ms away leaves room+// for a loaded CI runner while still catching a write path that pays a round trip per block or a+// handshake per file.+const (+ throughputEgressDelay = 50 * time.Millisecond+ throughputPairCount = 24+ throughputWriterCount = 4+ throughputTimeLimit = 12 * time.Second+ segmentSizedFileBytes = 16 * 1024 * 1024+ metadataSizedFileBytes = 1024+ writeFailureReportLimit = 3+)++type remoteStorageTarget struct {+ name string+ storageType StorageType+ startContainer func(t *testing.T) containers.ContainerHandle+}++func getRemoteStorageTargets() []remoteStorageTarget {+ return []remoteStorageTarget{+ {"S3", StorageTypeS3, containers.StartMinioContainer},+ {"AzureBlob", StorageTypeAzureBlob, containers.StartAzuriteContainer},+ {"SFTP", StorageTypeSFTP, containers.StartSftpContainer},+ {"FTP", StorageTypeFTP, containers.StartFtpContainer},+ {"NAS", StorageTypeNAS, containers.StartSambaContainer},+ {"Rclone", StorageTypeRclone, containers.StartMinioContainer},+ }+}++func Test_WriteFile_WhenStorageAnswers50msLate_SustainsTwoSegmentsPerSecond(t *testing.T) {+ targets := append(getRemoteStorageTargets(), remoteStorageTarget{"Local", StorageTypeLocal, nil})++ for _, target := range targets {+ t.Run(target.name, func(t *testing.T) {+ t.Parallel()++ var container containers.ContainerHandle+ if target.startContainer != nil {+ container = target.startContainer(t)+ containers.AddEgressDelay(t, container, throughputEgressDelay)+ }++ storage := createTestStorageOfType(t, target.storageType, container)++ segmentContent := generateRandomContent(t, segmentSizedFileBytes)+ metadataContent := generateRandomContent(t, metadataSizedFileBytes)++ written := writeSegmentPairsConcurrently(t, storage.ID, segmentContent, metadataContent)++ if target.startContainer != nil {+ t.Cleanup(func() { containers.ClearEgressShaping(t, container) })+ }++ require.Empty(t, written.failures, "writes failed after %d pairs in %s", written.pairCount, written.elapsed)+ require.Equal(+ t,+ throughputPairCount,+ written.pairCount,+ "only %d of %d pairs were written within %s, the writers stopped after %s",+ written.pairCount,+ throughputPairCount,+ throughputTimeLimit,+ written.elapsed,+ )++ t.Logf("%d pairs written in %s", written.pairCount, written.elapsed)+ })+ }+}++type writtenPairs struct {+ pairCount int+ elapsed time.Duration+ failures []error+}++// Writers stop taking jobs once the time limit passes, and the call waits for the writes already in+// flight, so every file that reached the storage is handed to cleanup.+func writeSegmentPairsConcurrently(+ t *testing.T,+ storageID uuid.UUID,+ segmentContent []byte,+ metadataContent []byte,+) writtenPairs {+ t.Helper()++ var (+ resultMutex sync.Mutex+ result writtenPairs+ writersWaitGroup sync.WaitGroup+ )++ writtenFiles := trackWrittenFiles(t, storageID)++ pairIndexes := make(chan int, throughputPairCount)+ for pairIndex := range throughputPairCount {+ pairIndexes <- pairIndex+ }+ close(pairIndexes)++ startedAt := time.Now()++ writePair := func(pairIndex int) error {+ pairName := fmt.Sprintf("throughput-%s-%d", uuid.New(), pairIndex)++ if err := writtenFiles.write(t.Context(), pairName, segmentContent); err != nil {+ return err+ }++ return writtenFiles.write(t.Context(), pairName+".metadata", metadataContent)+ }++ for range throughputWriterCount {+ writersWaitGroup.Go(func() {+ for pairIndex := range pairIndexes {+ if time.Since(startedAt) > throughputTimeLimit {+ return+ }++ err := writePair(pairIndex)+ isWithinLimit := time.Since(startedAt) <= throughputTimeLimit++ resultMutex.Lock()+ if err != nil && len(result.failures) < writeFailureReportLimit {+ result.failures = append(result.failures, err)+ }++ if err == nil && isWithinLimit {+ result.pairCount+++ result.elapsed = time.Since(startedAt)+ }+ resultMutex.Unlock()++ if err != nil {+ return+ }+ }+ })+ }++ writersWaitGroup.Wait()++ if result.pairCount < throughputPairCount {+ result.elapsed = time.Since(startedAt)+ }++ return result+}++// Every file a test writes, the failed attempts included, is handed to the deletion worker at the+// end of the test, the way a backup that never publishes its files hands them back.+type writtenFileTracker struct {+ storageID uuid.UUID++ mutex sync.Mutex+ references []storage_files.StoredFileReference+}++func trackWrittenFiles(t *testing.T, storageID uuid.UUID) *writtenFileTracker {+ t.Helper()++ tracker := &writtenFileTracker{storageID: storageID}+ t.Cleanup(tracker.requestDeletionAndDrainFunc(t))++ return tracker+}++func (w *writtenFileTracker) write(ctx context.Context, fileName string, content []byte) error {+ reference := storage_files.StoredFileReference{StorageID: w.storageID, FileName: fileName}++ w.mutex.Lock()+ w.references = append(w.references, reference)+ w.mutex.Unlock()++ _, err := GetStorageFileStore().WriteFile(ctx, reference, bytes.NewReader(content))++ return err+}++func (w *writtenFileTracker) writeUniqueFile(ctx context.Context, content []byte) error {+ return w.write(ctx, "test-file-"+uuid.New().String(), content)+}++func (w *writtenFileTracker) requestDeletionAndDrainFunc(t *testing.T) func() {+ return func() {+ w.mutex.Lock()+ references := slices.Clone(w.references)+ w.references = nil+ w.mutex.Unlock()++ requestDeletionAndDrain(t, references)+ }+}++func requestDeletionAndDrain(t *testing.T, references []storage_files.StoredFileReference) {+ t.Helper()++ if len(references) == 0 {+ return+ }++ ctx := context.WithoutCancel(t.Context())++ err := db.GetDb().Transaction(func(tx *gorm.DB) error {+ return GetStorageFileStore().RequestFileDeletions(ctx, tx, references)+ })+ if err != nil {+ t.Errorf("failed to request deletion of written files: %v", err)+ return+ }++ if err := DrainStorageFileDeletions(ctx, references...); err != nil {+ t.Errorf("failed to drain deletion of written files: %v", err)+ }+}++func generateRandomContent(t *testing.T, size int) []byte {+ t.Helper()++ content := make([]byte, size)+ _, err := rand.Read(content)+ require.NoError(t, err)++ return content+}++func Test_WriteFile_WhenCredentialsChangeToWrongOnes_NextWriteFailsUntilCorrected(t *testing.T) {+ for _, target := range getRemoteStorageTargets() {+ t.Run(target.name, func(t *testing.T) {+ t.Parallel()++ storage := createTestStorageOfType(t, target.storageType, target.startContainer(t))+ writtenFiles := trackWrittenFiles(t, storage.ID)+ content := generateRandomContent(t, metadataSizedFileBytes)++ require.NoError(t, writtenFiles.writeUniqueFile(t.Context(), content))++ setTestStorageSecret(t, storage.ID, getWrongTestSecret(t, target.storageType))+ require.Error(+ t,+ writtenFiles.writeUniqueFile(t.Context(), content),+ "a write after the credentials changed must use the new, wrong ones",+ )++ setTestStorageSecret(t, storage.ID, getCorrectTestSecret(t, target.storageType))+ require.NoError(+ t,+ writtenFiles.writeUniqueFile(t.Context(), content),+ "a write after the credentials were corrected must use the corrected ones",+ )+ })+ }+}++func Test_WriteFile_AfterStorageIsDeleted_Fails(t *testing.T) {+ for _, target := range getRemoteStorageTargets() {+ t.Run(target.name, func(t *testing.T) {+ t.Parallel()++ storage := createTestStorageOfType(t, target.storageType, target.startContainer(t))+ writtenFiles := trackWrittenFiles(t, storage.ID)+ content := generateRandomContent(t, metadataSizedFileBytes)++ require.NoError(t, writtenFiles.writeUniqueFile(t.Context(), content))++ // The written file is removed while its storage still exists, since the deletion worker+ // can no longer reach a deleted storage.+ writtenFiles.requestDeletionAndDrainFunc(t)()+ RemoveTestStorage(t.Context(), storage.ID)++ // The write is not tracked: no file can reach a deleted storage, and a deletion request+ // for it would name a storage that no longer exists.+ _, err := GetStorageFileStore().WriteFile(+ t.Context(),+ storage_files.StoredFileReference{+ StorageID: storage.ID,+ FileName: "after-deletion-" + uuid.New().String(),+ },+ bytes.NewReader(content),+ )+ assert.Error(t, err)+ })+ }+}++// The deadline plus the three seconds the spec allows a write to outlive it.+const (+ silentStorageWriteDeadline = 5 * time.Second+ silentStorageReturnLimit = 8 * time.Second+)++func Test_WriteFile_WhenStorageGoesSilentMidConnection_FailsByDeadlineAndRedials(t *testing.T) {+ for _, target := range getRemoteStorageTargets() {+ if target.storageType == StorageTypeRclone {+ continue+ }++ t.Run(target.name, func(t *testing.T) {+ t.Parallel()++ container := target.startContainer(t)+ storage := createTestStorageOfType(t, target.storageType, container)+ writtenFiles := trackWrittenFiles(t, storage.ID)+ content := generateRandomContent(t, segmentSizedFileBytes)++ require.NoError(t, writtenFiles.writeUniqueFile(t.Context(), content))++ containers.SetEgressLoss(t, container)+ t.Cleanup(func() { containers.ClearEgressShaping(t, container) })++ writeCtx, cancel := context.WithTimeout(t.Context(), silentStorageWriteDeadline)+ defer cancel()++ writeResult := make(chan error, 1)+ go func() {+ writeResult <- writtenFiles.writeUniqueFile(writeCtx, content)+ }()++ select {+ case err := <-writeResult:+ require.Error(t, err, "a write to a silent storage must fail")+ case <-time.After(silentStorageReturnLimit):+ t.Fatalf("a write to a silent storage did not return within %s", silentStorageReturnLimit)+ }++ containers.ClearEgressShaping(t, container)++ require.NoError(+ t,+ writtenFiles.writeUniqueFile(t.Context(), content),+ "the next write must not reuse the connection that went silent",+ )+ })+ }+}++var rcloneSecretLine = regexp.MustCompile(`(?m)^secret_access_key = .*$`)++func setTestStorageSecret(t *testing.T, storageID uuid.UUID, secret string) {+ t.Helper()++ storage, err := storageRepository.FindByID(t.Context(), storageID)+ require.NoError(t, err)++ switch storage.Type {+ case StorageTypeS3:+ storage.S3Storage.S3SecretKey = secret+ case StorageTypeAzureBlob:+ storage.AzureBlobStorage.AccountKey = secret+ case StorageTypeSFTP:+ storage.SFTPStorage.Password = secret+ case StorageTypeFTP:+ storage.FTPStorage.Password = secret+ case StorageTypeNAS:+ storage.NASStorage.Password = secret+ case StorageTypeRclone:+ configContent, err := encryption.GetFieldEncryptor().Decrypt(storage.RcloneStorage.ConfigContent)+ require.NoError(t, err)++ storage.RcloneStorage.ConfigContent = rcloneSecretLine.ReplaceAllLiteralString(+ configContent, "secret_access_key = "+secret,+ )+ default:+ t.Fatalf("no test secret for storage type %s", storage.Type)+ }++ require.NoError(t, storage.EncryptSensitiveData(encryption.GetFieldEncryptor()))++ _, err = storageRepository.Save(storage)+ require.NoError(t, err)+}++func getCorrectTestSecret(t *testing.T, storageType StorageType) string {+ t.Helper()++ switch storageType {+ case StorageTypeS3, StorageTypeRclone:+ return containers.MinioRootPassword+ case StorageTypeAzureBlob:+ return containers.AzuriteAccountKey+ case StorageTypeSFTP:+ return containers.SftpPassword+ case StorageTypeFTP:+ return containers.FtpPassword+ case StorageTypeNAS:+ return containers.SambaPassword+ default:+ t.Fatalf("no test secret for storage type %s", storageType)+ return ""+ }+}++// An Azure account key must stay valid base64, or the client fails on decoding it and the test+// would never reach the server.+func getWrongTestSecret(t *testing.T, storageType StorageType) string {+ t.Helper()++ if storageType == StorageTypeAzureBlob {+ return base64.StdEncoding.EncodeToString(generateRandomContent(t, 64))+ }++ return "wrong-secret-" + uuid.New().String()+}+func addressOf(endpoint containers.Endpoint) string {return fmt.Sprintf("%s:%d", endpoint.Host, endpoint.Port)}@@ -344,42 +692,158 @@ func setupS3Container(ctx context.Context, address string) (*S3Container, error)}, nil}-// setupAzuriteContainer creates the two blob containers on the Azurite server at address (host:port).-func setupAzuriteContainer(ctx context.Context, address string) (*AzuriteContainer, error) {- accountName := containers.AzuriteAccountName- accountKey := containers.AzuriteAccountKey- serviceURL := fmt.Sprintf("http://%s/%s", address, accountName)- containerNameKey := "test-container-key"- containerNameStr := "test-container-connstr"+const testAzuriteContainerName = "test-container"- connectionString := fmt.Sprintf(+func getAzuriteConnectionString(address string) string {+ return fmt.Sprintf("DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=http://%s/%s",- accountName,- accountKey,+ containers.AzuriteAccountName,+ containers.AzuriteAccountKey,address,- accountName,+ containers.AzuriteAccountName,)+}- client, err := azblob.NewClientFromConnectionString(connectionString, nil)+func createAzuriteBlobContainer(ctx context.Context, address string) error {+ client, err := azblob.NewClientFromConnectionString(getAzuriteConnectionString(address), nil)if err != nil {- return nil, fmt.Errorf("failed to create azblob client: %w", err)+ return fmt.Errorf("failed to create azblob client: %w", err)}ctx, cancel := context.WithTimeout(ctx, 30*time.Second)defer cancel()- // The containers may already exist on a reused server; an AlreadyExists error is harmless here.- _, _ = client.CreateContainer(ctx, containerNameKey, nil)- _, _ = client.CreateContainer(ctx, containerNameStr, nil)-- return &AzuriteContainer{- endpoint: serviceURL,- accountName: accountName,- accountKey: accountKey,- containerNameKey: containerNameKey,- containerNameStr: containerNameStr,- connectionString: connectionString,- }, nil+ _, err = client.CreateContainer(ctx, testAzuriteContainerName, nil)+ if err != nil && !bloberror.HasCode(err, bloberror.ContainerAlreadyExists) {+ return fmt.Errorf("failed to create blob container: %w", err)+ }++ return nil+}++// buildTestStorage points a storage of the given type at the container and prepares whatever the+// server needs first, such as a bucket. Credentials stay in plain text so a caller can still edit+// them; saveTestStorage encrypts them.+func buildTestStorage(t *testing.T, storageType StorageType, container containers.ContainerHandle) *Storage {+ t.Helper()++ storage := &Storage{+ Type: storageType,+ Name: "Test " + string(storageType) + " " + uuid.New().String(),+ }++ address := addressOf(container.Endpoint)++ switch storageType {+ case StorageTypeLocal:+ storage.LocalStorage = &local_storage.LocalStorage{}+ case StorageTypeS3:+ s3Container, err := setupS3Container(t.Context(), address)+ require.NoError(t, err, "failed to set up the S3 container")++ storage.S3Storage = &s3_storage.S3Storage{+ S3Bucket: s3Container.bucketName,+ S3Region: s3Container.region,+ S3AccessKey: s3Container.accessKey,+ S3SecretKey: s3Container.secretKey,+ S3Endpoint: "http://" + address,+ }+ case StorageTypeAzureBlob:+ require.NoError(t, createAzuriteBlobContainer(t.Context(), address))++ storage.AzureBlobStorage = &azure_blob_storage.AzureBlobStorage{+ AuthMethod: azure_blob_storage.AuthMethodAccountKey,+ AccountName: containers.AzuriteAccountName,+ AccountKey: containers.AzuriteAccountKey,+ ContainerName: testAzuriteContainerName,+ Endpoint: fmt.Sprintf("http://%s/%s", address, containers.AzuriteAccountName),+ }+ case StorageTypeNAS:+ storage.NASStorage = &nas_storage.NASStorage{+ Host: container.Host,+ Port: container.Port,+ Share: containers.SambaShare,+ Username: containers.SambaUsername,+ Password: containers.SambaPassword,+ Path: "test-files",+ }+ case StorageTypeFTP:+ storage.FTPStorage = &ftp_storage.FTPStorage{+ Host: container.Host,+ Port: container.Port,+ Username: containers.FtpUsername,+ Password: containers.FtpPassword,+ Path: "test-files",+ }+ case StorageTypeSFTP:+ storage.SFTPStorage = &sftp_storage.SFTPStorage{+ Host: container.Host,+ Port: container.Port,+ Username: containers.SftpUsername,+ Password: containers.SftpPassword,+ SkipHostKeyVerify: true,+ Path: containers.SftpUploadDir,+ }+ case StorageTypeRclone:+ s3Container, err := setupS3Container(t.Context(), address)+ require.NoError(t, err, "failed to set up the S3 container behind rclone")++ storage.RcloneStorage = &rclone_storage.RcloneStorage{+ ConfigContent: getRcloneMinioConfig(s3Container, s3Container.secretKey),+ RemotePath: s3Container.bucketName,+ }+ default:+ t.Fatalf("no test setup for storage type %s", storageType)+ }++ return storage+}++func getRcloneMinioConfig(s3Container *S3Container, secretKey string) string {+ return fmt.Sprintf(`[minio]+type = s3+provider = Other+access_key_id = %s+secret_access_key = %s+endpoint = http://%s+acl = private`, s3Container.accessKey, secretKey, s3Container.endpoint)+}++func saveTestStorage(t *testing.T, storage *Storage) *Storage {+ t.Helper()++ owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)++ workspace, err := workspaces_testing.CreateTestWorkspaceDirect("Storage Test Workspace", owner.UserID)+ require.NoError(t, err)++ storage.WorkspaceID = workspace.ID+ require.NoError(t, storage.EncryptSensitiveData(encryption.GetFieldEncryptor()))++ savedStorage, err := storageRepository.Save(storage)+ require.NoError(t, err)++ t.Cleanup(func() {+ if _, err := storageRepository.FindByID(context.WithoutCancel(t.Context()), savedStorage.ID); err == nil {+ RemoveTestStorage(t.Context(), savedStorage.ID)+ }++ if err := workspaces_testing.RemoveTestWorkspaceDirect(workspace.ID); err != nil {+ t.Logf("failed to remove test workspace: %v", err)+ }+ })++ return savedStorage+}++func createTestStorageOfType(+ t *testing.T,+ storageType StorageType,+ container containers.ContainerHandle,+) *Storage {+ t.Helper()++ return saveTestStorage(t, buildTestStorage(t, storageType, container))}func Test_RcloneStorage_DeleteFile_WhenAuthFailsOnLookup_ReturnsErrorAndDoesNotDeleteObject(t *testing.T) {@@ -420,15 +884,9 @@ func Test_RcloneStorage_DeleteFile_WhenAuthFailsOnLookup_ReturnsErrorAndDoesNotD})rcloneStorage := &rclone_storage.RcloneStorage{- StorageID: uuid.New(),- ConfigContent: fmt.Sprintf(`[minio]-type = s3-provider = Other-access_key_id = %s-secret_access_key = totally-wrong-secret-key-endpoint = http://%s-acl = private`, s3Container.accessKey, s3Container.endpoint),- RemotePath: s3Container.bucketName,+ StorageID: uuid.New(),+ ConfigContent: getRcloneMinioConfig(s3Container, "totally-wrong-secret-key"),+ RemotePath: s3Container.bucketName,}encryptor := encryption.GetFieldEncryptor()diff --git a/backend/internal/features/storages/models/azure_blob/model.go b/backend/internal/features/storages/models/azure_blob/model.goindex c6e7ef6e..d41cea3d 100644--- a/backend/internal/features/storages/models/azure_blob/model.go+++ b/backend/internal/features/storages/models/azure_blob/model.go@@ -31,6 +31,10 @@ const (azureTLSHandshakeTimeout = 30 * time.SecondazureDeleteTimeout = 30 * time.Second+ // Go keeps two idle connections per host by default, which drops connections as soon as more+ // than two writes to one account run at once.+ azureMaxIdleConnsPerHost = 16+// Chunk size for block blob uploads - 16MB provides good balance between// memory usage and upload efficiency. This creates backpressure to pg_dump// by only reading one chunk at a time and waiting for Azure to confirm receipt.@@ -500,20 +504,24 @@ func (s *AzureBlobStorage) getClient(encryptor encryption.FieldEncryptor) (*azblreturn client, nil}-func (s *AzureBlobStorage) buildClientOptions() *azblob.ClientOptions {- transport := &http.Transport{- Proxy: http.ProxyFromEnvironment,- DialContext: (&net.Dialer{- Timeout: azureConnectTimeout,- }).DialContext,- TLSHandshakeTimeout: azureTLSHandshakeTimeout,- ResponseHeaderTimeout: azureResponseTimeout,- IdleConnTimeout: azureIdleConnTimeout,- }+// Clients are still built per call from freshly decrypted credentials. Only the connections are+// shared: every request is signed or carries its SAS token, so a changed credential takes effect on+// the next request even over a connection opened before the change.+var sharedHTTPClient = &http.Client{Transport: &http.Transport{+ Proxy: http.ProxyFromEnvironment,+ DialContext: (&net.Dialer{+ Timeout: azureConnectTimeout,+ }).DialContext,+ TLSHandshakeTimeout: azureTLSHandshakeTimeout,+ ResponseHeaderTimeout: azureResponseTimeout,+ IdleConnTimeout: azureIdleConnTimeout,+ MaxIdleConnsPerHost: azureMaxIdleConnsPerHost,+}}+func (s *AzureBlobStorage) buildClientOptions() *azblob.ClientOptions {return &azblob.ClientOptions{ClientOptions: azcore.ClientOptions{- Transport: &http.Client{Transport: transport},+ Transport: sharedHTTPClient,Retry: policy.RetryOptions{MaxRetries: 0,},diff --git a/backend/internal/features/storages/models/ftp/model.go b/backend/internal/features/storages/models/ftp/model.goindex 90f06ae1..c92b3ad5 100644--- a/backend/internal/features/storages/models/ftp/model.go+++ b/backend/internal/features/storages/models/ftp/model.go@@ -7,6 +7,7 @@ import ("fmt""io""log/slog"+ "net""net/textproto""strings""time"@@ -14,6 +15,7 @@ import ("github.com/google/uuid""github.com/jlaffaye/ftp"+ storage_connections "databasus-backend/internal/features/storages/connections""databasus-backend/internal/util/encryption"io_utils "databasus-backend/internal/util/io")@@ -55,46 +57,18 @@ func (f *FTPStorage) SaveFile(logger.InfoContext(ctx, "starting to save file to FTP storage", "file_name", fileName, "host", f.Host)- conn, err := f.connect(encryptor, ftpConnectTimeout)+ lease, err := f.checkoutSession(ctx, encryptor)if err != nil {logger.ErrorContext(ctx, "failed to connect to FTP", "file_name", fileName, "error", err)return fmt.Errorf("failed to connect to FTP: %w", err)}- defer func() {- if quitErr := conn.Quit(); quitErr != nil {- logger.ErrorContext(- ctx,- "failed to close FTP connection",- "file_name",- fileName,- "error",- quitErr,- )- }- }()-- if f.Path != "" {- if err := f.ensureDirectory(conn, f.Path); err != nil {- logger.ErrorContext(- ctx,- "failed to ensure directory",- "file_name",- fileName,- "path",- f.Path,- "error",- err,- )- return fmt.Errorf("failed to ensure directory: %w", err)- }- }filePath := f.getFilePath(fileName)logger.DebugContext(ctx, "uploading file to FTP", "file_name", fileName, "file_path", filePath)- ctxReader := &contextReader{ctx: ctx, reader: file}+ err = lease.GetSession().serverConn.Stor(filePath, &contextReader{ctx: ctx, reader: file})+ lease.Release(ignoreFtpReply(err))- err = conn.Stor(filePath, ctxReader)if err != nil {select {case <-ctx.Done():@@ -123,20 +97,20 @@ func (f *FTPStorage) GetFile(logger *slog.Logger,fileName string,) (io.ReadCloser, error) {- conn, err := f.connect(encryptor, ftpConnectTimeout)+ lease, err := f.checkoutSession(ctx, encryptor)if err != nil {return nil, fmt.Errorf("failed to connect to FTP: %w", err)}filePath := f.getFilePath(fileName)- totalBytes, err := conn.FileSize(filePath)+ totalBytes, err := lease.GetSession().serverConn.FileSize(filePath)if err != nil {// A server that does not implement SIZE still serves the file, so an unsupported command// costs the completeness check rather than the whole restore. Any other reply, 550 above// all, means the file itself is unreachable.if !isUnsupportedFtpCommand(err) {- _ = conn.Quit()+ lease.Release(ignoreFtpReply(err))return nil, fmt.Errorf("failed to stat file on FTP: %w", err)}@@ -146,9 +120,7 @@ func (f *FTPStorage) GetFile(totalBytes = io_utils.UnknownTotalBytes}- if err := conn.Quit(); err != nil {- logger.WarnContext(ctx, "failed to close the FTP connection used for stat", "error", err)- }+ lease.Release(nil)return io_utils.NewResumingReader(io_utils.ResumingReaderSpec{StreamCtx: ctx,@@ -172,17 +144,15 @@ func (f *FTPStorage) DeleteFile(deleteCtx, cancel := context.WithTimeout(context.Background(), ftpDeleteTimeout)defer cancel()- conn, err := f.connectWithContext(deleteCtx, encryptor, ftpDeleteTimeout)+ lease, err := f.checkoutSession(deleteCtx, encryptor)if err != nil {return fmt.Errorf("failed to connect to FTP: %w", err)}- defer func() {- _ = conn.Quit()- }()- filePath := f.getFilePath(fileName)+ err = lease.GetSession().serverConn.Delete(f.getFilePath(fileName))+ lease.Release(ignoreFtpReply(err))- if err := conn.Delete(filePath); err != nil && !isFtpFileUnavailable(err) {+ if err != nil && !isFtpFileUnavailable(err) {return fmt.Errorf("failed to delete file from FTP: %w", err)}@@ -212,16 +182,14 @@ func (f *FTPStorage) TestConnection(encryptor encryption.FieldEncryptor) error {ctx, cancel := context.WithTimeout(context.Background(), ftpTestConnectTimeout)defer cancel()- conn, err := f.connectWithContext(ctx, encryptor, ftpTestConnectTimeout)+ session, err := f.connectWithContext(ctx, encryptor, ftpTestConnectTimeout)if err != nil {return fmt.Errorf("failed to connect to FTP: %w", err)}- defer func() {- _ = conn.Quit()- }()+ defer closeSession(session)if f.Path != "" {- if err := f.ensureDirectory(conn, f.Path); err != nil {+ if err := f.ensureDirectory(session.serverConn, f.Path); err != nil {return fmt.Errorf("failed to access or create path '%s': %w", f.Path, err)}}@@ -258,18 +226,11 @@ func (f *FTPStorage) Update(incoming *FTPStorage) {}}-func (f *FTPStorage) connect(- encryptor encryption.FieldEncryptor,- timeout time.Duration,-) (*ftp.ServerConn, error) {- return f.connectWithContext(context.Background(), encryptor, timeout)-}-func (f *FTPStorage) connectWithContext(ctx context.Context,encryptor encryption.FieldEncryptor,timeout time.Duration,-) (*ftp.ServerConn, error) {+) (*ftpSession, error) {password, err := encryptor.Decrypt(f.Password)if err != nil {return nil, fmt.Errorf("failed to decrypt FTP password: %w", err)@@ -280,30 +241,48 @@ func (f *FTPStorage) connectWithContext(dialCtx, cancel := context.WithTimeout(ctx, timeout)defer cancel()- var conn *ftp.ServerConn+ connections := &trackedConnections{+ open: map[*trackedConnection]struct{}{},+ dialer: net.Dialer{Timeout: timeout},+ controlCtx: dialCtx,+ }++ dialOptions := []ftp.DialOption{ftp.DialWithDialFunc(connections.dial)}+if f.UseSSL {- tlsConfig := &tls.Config{+ connections.tlsConfig = &tls.Config{ServerName: f.Host,InsecureSkipVerify: f.SkipTLSVerify,}- conn, err = ftp.Dial(address,- ftp.DialWithContext(dialCtx),- ftp.DialWithExplicitTLS(tlsConfig),- )- } else {- conn, err = ftp.Dial(address, ftp.DialWithContext(dialCtx))+ dialOptions = append(dialOptions, ftp.DialWithExplicitTLS(connections.tlsConfig))}++ session := &ftpSession{connections: connections}++ // The greeting, the TLS upgrade and the login ignore the context, so a server that accepts the+ // TCP connection and then goes silent would hold the dial forever without this.+ stopCloseOnCancel := context.AfterFunc(dialCtx, func() {+ closeSession(session)+ })+ defer stopCloseOnCancel()++ session.serverConn, err = ftp.Dial(address, dialOptions...)if err != nil {+ closeSession(session)return nil, fmt.Errorf("failed to dial FTP server: %w", err)}- err = conn.Login(f.Username, password)- if err != nil {- _ = conn.Quit()+ if err := session.serverConn.Login(f.Username, password); err != nil {+ closeSession(session)return nil, fmt.Errorf("failed to login to FTP server: %w", err)}- return conn, nil+ if !stopCloseOnCancel() {+ closeSession(session)+ return nil, fmt.Errorf("failed to connect to FTP server: %w", dialCtx.Err())+ }++ return session, nil}func (f *FTPStorage) ensureDirectory(conn *ftp.ServerConn, path string) error {@@ -329,16 +308,16 @@ func (f *FTPStorage) ensureDirectory(conn *ftp.ServerConn, path string) error {continue}- err := conn.ChangeDir(part)- if err != nil {- err = conn.MakeDir(part)- if err != nil {- return fmt.Errorf("failed to create directory '%s': %w", part, err)- }- err = conn.ChangeDir(part)- if err != nil {- return fmt.Errorf("failed to change into directory '%s': %w", part, err)- }+ if err := conn.ChangeDir(part); err == nil {+ continue+ }++ // Concurrent sessions dial at once, and the one that loses the race to create the directory+ // finds it in place, so a failed MKD is judged by whether the directory can be entered.+ makeDirErr := conn.MakeDir(part)++ if err := conn.ChangeDir(part); err != nil {+ return fmt.Errorf("failed to create directory '%s': %w", part, errors.Join(makeDirErr, err))}}@@ -357,39 +336,29 @@ func (f *FTPStorage) getFilePath(filename string) string {}type ftpFileReader struct {- response *ftp.Response- conn *ftp.ServerConn- stopDeadlineOnCancel func() bool-}--func (r *ftpFileReader) Read(p []byte) (n int, err error) {- return r.response.Read(p)+ response *ftp.Response+ lease *storage_connections.Lease[*ftpSession]+ readErr error}-func (r *ftpFileReader) Close() error {- var errs []error-- if r.stopDeadlineOnCancel != nil {- r.stopDeadlineOnCancel()+func (r *ftpFileReader) Read(p []byte) (int, error) {+ readBytes, err := r.response.Read(p)+ if err != nil && !errors.Is(err, io.EOF) {+ r.readErr = err}- if r.response != nil {- if err := r.response.Close(); err != nil {- errs = append(errs, fmt.Errorf("failed to close response: %w", err))- }- }+ return readBytes, err+}- if r.conn != nil {- if err := r.conn.Quit(); err != nil {- errs = append(errs, fmt.Errorf("failed to close connection: %w", err))- }+func (r *ftpFileReader) Close() error {+ closeErr := r.response.Close()+ if closeErr != nil {+ closeErr = fmt.Errorf("failed to close response: %w", closeErr)}- if len(errs) > 0 {- return errs[0]- }+ r.lease.Release(errors.Join(r.readErr, closeErr))- return nil+ return closeErr}type contextReader struct {@@ -406,37 +375,38 @@ func (r *contextReader) Read(p []byte) (n int, err error) {}}-// Each attempt dials a fresh control connection: an FTP data transfer dies together with the-// control channel that opened it, so a resumed REST cannot reuse the broken one.+// Each attempt takes its own session: an FTP data transfer dies together with the control channel+// that opened it, and a session whose read failed was closed, so a resumed REST gets a live one.func (f *FTPStorage) retrieveFileFromOffset(attemptCtx context.Context,encryptor encryption.FieldEncryptor,filePath string,offsetBytes int64,) (io.ReadCloser, error) {- conn, err := f.connect(encryptor, ftpConnectTimeout)+ lease, err := f.checkoutSession(attemptCtx, encryptor)if err != nil {return nil, fmt.Errorf("failed to connect to FTP: %w", err)}- response, err := conn.RetrFrom(filePath, uint64(offsetBytes))+ response, err := lease.GetSession().serverConn.RetrFrom(filePath, uint64(offsetBytes))if err != nil {- _ = conn.Quit()+ lease.Release(ignoreFtpReply(err))return nil, fmt.Errorf("failed to retrieve file from FTP: %w", err)}- // The FTP client takes no context, so a read blocked on a silent server would ignore the- // caller's cancellation; an expired deadline is what unblocks it.- stopDeadlineOnCancel := context.AfterFunc(attemptCtx, func() {- _ = response.SetDeadline(time.Now())- })+ return &ftpFileReader{response: response, lease: lease}, nil+}++// A protocol reply such as 550 leaves the control channel in a clean state, so only transport+// failures make a session unusable.+func ignoreFtpReply(err error) error {+ var protocolError *textproto.Error+ if errors.As(err, &protocolError) {+ return nil+ }- return &ftpFileReader{- response: response,- conn: conn,- stopDeadlineOnCancel: stopDeadlineOnCancel,- }, nil+ return err}// 550 is the only reply an FTP server has for a missing file, and also its replydiff --git a/backend/internal/features/storages/models/ftp/session_pool.go b/backend/internal/features/storages/models/ftp/session_pool.gonew file mode 100644index 00000000..cbf94be7--- /dev/null+++ b/backend/internal/features/storages/models/ftp/session_pool.go@@ -0,0 +1,179 @@+package ftp_storage++import (+ "context"+ "crypto/tls"+ "errors"+ "fmt"+ "net"+ "sync"++ "github.com/jlaffaye/ftp"++ storage_connections "databasus-backend/internal/features/storages/connections"+ "databasus-backend/internal/util/encryption"+)++type ftpSession struct {+ serverConn *ftp.ServerConn+ connections *trackedConnections+}++// The locator hands out a freshly loaded model on every write, so the pool lives at package level+// and the model only names its key.+var sessionPool = storage_connections.NewSessionPool(storage_connections.SessionOperations[*ftpSession]{+ Close: closeSession,+ Probe: probeSession,+}, storage_connections.GetDefaultPoolSettings())++// QUIT is skipped on purpose: a session is also closed to unblock a call stuck on a silent server,+// where a polite goodbye would wait just as long.+func closeSession(session *ftpSession) {+ session.connections.closeAll()+}++// The FTP client takes no context, so the probe deadline is enforced by closing the session.+func probeSession(ctx context.Context, session *ftpSession) error {+ stopCloseOnDeadline := context.AfterFunc(ctx, func() {+ closeSession(session)+ })+ defer stopCloseOnDeadline()++ return session.serverConn.NoOp()+}++// Credential fields stay in their stored, encrypted form: they change whenever the user saves new+// credentials, and the fingerprint never holds a plain secret.+func (f *FTPStorage) getSessionKey() storage_connections.Key {+ return storage_connections.Key{+ StorageID: f.StorageID,+ Fingerprint: storage_connections.Fingerprint(+ f.Host, f.Port, f.Username, f.Password, f.UseSSL, f.SkipTLSVerify, f.Path,+ ),+ }+}++func (f *FTPStorage) checkoutSession(+ ctx context.Context,+ encryptor encryption.FieldEncryptor,+) (*storage_connections.Lease[*ftpSession], error) {+ return sessionPool.Checkout(ctx, f.getSessionKey(), func(ctx context.Context) (*ftpSession, error) {+ return f.dialSession(ctx, encryptor)+ })+}++// The target directory is ensured once per session rather than walked on every write.+func (f *FTPStorage) dialSession(ctx context.Context, encryptor encryption.FieldEncryptor) (*ftpSession, error) {+ session, err := f.connectWithContext(ctx, encryptor, ftpConnectTimeout)+ if err != nil {+ return nil, err+ }++ if f.Path != "" {+ if err := f.ensureDirectory(session.serverConn, f.Path); err != nil {+ closeSession(session)+ return nil, fmt.Errorf("failed to ensure directory: %w", err)+ }+ }++ return session, nil+}++var errSessionClosed = errors.New("FTP session is closed")++// The client closes a data connection only when its transfer ends, and a transfer blocked on a+// silent server never ends. Every connection a session opens is tracked, so closing the session+// unblocks the control channel and any data transfer alike.+type trackedConnections struct {+ mutex sync.Mutex+ open map[*trackedConnection]struct{}+ isClosed bool+ dialCount int+ dialer net.Dialer+ tlsConfig *tls.Config+ controlCtx context.Context+}++type trackedConnection struct {+ net.Conn+ owner *trackedConnections+}++func (c *trackedConnection) Close() error {+ c.owner.forget(c)++ return c.Conn.Close()+}++// With a custom dial function the client upgrades only the control channel to TLS by itself, so+// data channels of a TLS session are wrapped here.+func (t *trackedConnections) dial(network, address string) (net.Conn, error) {+ t.mutex.Lock()+ isControlChannel := t.dialCount == 0+ t.dialCount+++ t.mutex.Unlock()++ var (+ rawConnection net.Conn+ err error+ )++ if isControlChannel {+ rawConnection, err = t.dialer.DialContext(t.controlCtx, network, address)+ } else {+ rawConnection, err = t.dialer.Dial(network, address)+ }++ if err != nil {+ return nil, err+ }++ connection := &trackedConnection{Conn: rawConnection, owner: t}+ if !t.remember(connection) {+ _ = rawConnection.Close()+ return nil, errSessionClosed+ }++ if !isControlChannel && t.tlsConfig != nil {+ return tls.Client(connection, t.tlsConfig), nil+ }++ return connection, nil+}++func (t *trackedConnections) remember(connection *trackedConnection) bool {+ t.mutex.Lock()+ defer t.mutex.Unlock()++ if t.isClosed {+ return false+ }++ t.open[connection] = struct{}{}++ return true+}++func (t *trackedConnections) forget(connection *trackedConnection) {+ t.mutex.Lock()+ defer t.mutex.Unlock()++ delete(t.open, connection)+}++func (t *trackedConnections) closeAll() {+ t.mutex.Lock()+ t.isClosed = true+ openConnections := make([]*trackedConnection, 0, len(t.open))++ for connection := range t.open {+ openConnections = append(openConnections, connection)+ }++ t.open = map[*trackedConnection]struct{}{}+ t.mutex.Unlock()++ for _, connection := range openConnections {+ _ = connection.Conn.Close()+ }+}diff --git a/backend/internal/features/storages/models/nas/model.go b/backend/internal/features/storages/models/nas/model.goindex 7e0d5ed2..f69bfc5c 100644--- a/backend/internal/features/storages/models/nas/model.go+++ b/backend/internal/features/storages/models/nas/model.go@@ -16,12 +16,14 @@ import ("github.com/google/uuid""github.com/hirochachacha/go-smb2"+ storage_connections "databasus-backend/internal/features/storages/connections""databasus-backend/internal/util/encryption"io_utils "databasus-backend/internal/util/io")const (- nasDeleteTimeout = 30 * time.Second+ nasDeleteTimeout = 30 * time.Second+ nasTestConnectTimeout = 30 * time.Second// Chunk size for NAS uploads - 16MB provides good balance between// memory usage and upload efficiency. This creates backpressure to pg_dump@@ -60,96 +62,21 @@ func (n *NASStorage) SaveFile(logger.InfoContext(ctx, "starting to save file to NAS storage", "file_name", fileName, "host", n.Host)- session, err := n.createSessionWithContext(ctx, encryptor)+ lease, err := n.checkoutSession(ctx, encryptor)if err != nil {logger.ErrorContext(ctx, "failed to create NAS session", "file_name", fileName, "error", err)return fmt.Errorf("failed to create NAS session: %w", err)}- defer func() {- if logoffErr := session.Logoff(); logoffErr != nil {- logger.ErrorContext(- ctx,- "failed to logoff NAS session",- "file_name",- fileName,- "error",- logoffErr,- )- }- }()-- fs, err := session.Mount(n.Share)- if err != nil {- logger.ErrorContext(- ctx,- "failed to mount NAS share",- "file_name",- fileName,- "share",- n.Share,- "error",- err,- )- return fmt.Errorf("failed to mount share '%s': %w", n.Share, err)- }- defer func() {- if umountErr := fs.Umount(); umountErr != nil {- logger.ErrorContext(- ctx,- "failed to unmount NAS share",- "file_name",- fileName,- "error",- umountErr,- )- }- }()-- // Ensure the directory exists- if n.Path != "" {- if err := n.ensureDirectory(fs, n.Path); err != nil {- logger.ErrorContext(- ctx,- "failed to ensure directory",- "file_name",- fileName,- "path",- n.Path,- "error",- err,- )- return fmt.Errorf("failed to ensure directory: %w", err)- }- }filePath := n.getFilePath(fileName)logger.DebugContext(ctx, "creating file on NAS", "file_name", fileName, "file_path", filePath)- nasFile, err := fs.Create(filePath)- if err != nil {- logger.ErrorContext(- ctx,- "failed to create file on NAS",- "file_name",- fileName,- "file_path",- filePath,- "error",- err,- )- return fmt.Errorf("failed to create file on NAS: %w", err)- }- defer func() {- if closeErr := nasFile.Close(); closeErr != nil {- logger.ErrorContext(ctx, "failed to close NAS file", "file_name", fileName, "error", closeErr)- }- }()+ err = writeNasFile(ctx, lease.GetSession().getShare(ctx), filePath, file)+ lease.Release(err)- logger.DebugContext(ctx, "copying file data to NAS", "file_name", fileName)- _, err = copyWithContext(ctx, nasFile, file)if err != nil {logger.ErrorContext(ctx, "failed to write file to NAS", "file_name", fileName, "error", err)- return fmt.Errorf("failed to write file to NAS: %w", err)+ return err}logger.InfoContext(@@ -163,39 +90,46 @@ func (n *NASStorage) SaveFile(return nil}+func writeNasFile(ctx context.Context, share *smb2.Share, filePath string, source io.Reader) error {+ nasFile, err := share.Create(filePath)+ if err != nil {+ return fmt.Errorf("failed to create file on NAS: %w", err)+ }++ if _, err := copyWithContext(ctx, nasFile, source); err != nil {+ _ = nasFile.Close()+ return fmt.Errorf("failed to write file to NAS: %w", err)+ }++ if err := nasFile.Close(); err != nil {+ return fmt.Errorf("failed to close file on NAS: %w", err)+ }++ return nil+}+func (n *NASStorage) GetFile(ctx context.Context,encryptor encryption.FieldEncryptor,logger *slog.Logger,fileName string,) (io.ReadCloser, error) {- session, err := n.createSession(encryptor)+ lease, err := n.checkoutSession(ctx, encryptor)if err != nil {return nil, fmt.Errorf("failed to create NAS session: %w", err)}- share, err := session.Mount(n.Share)- if err != nil {- _ = session.Logoff()-- return nil, fmt.Errorf("failed to mount share '%s': %w", n.Share, err)- }-filePath := n.getFilePath(fileName)- fileInfo, err := share.Stat(filePath)- if err != nil {- _ = share.Umount()- _ = session.Logoff()+ fileInfo, err := lease.GetSession().getShare(ctx).Stat(filePath)+ lease.Release(ignoreNotExist(err))+ if err != nil {return nil, fmt.Errorf("file not found: %s", fileName)}totalBytes := fileInfo.Size()- _ = share.Umount()- _ = session.Logoff()-return io_utils.NewResumingReader(io_utils.ResumingReaderSpec{StreamCtx: ctx,Logger: logger,@@ -218,25 +152,15 @@ func (n *NASStorage) DeleteFile(deleteCtx, cancel := context.WithTimeout(context.Background(), nasDeleteTimeout)defer cancel()- session, err := n.createSessionWithContext(deleteCtx, encryptor)+ lease, err := n.checkoutSession(deleteCtx, encryptor)if err != nil {return fmt.Errorf("failed to create NAS session: %w", err)}- defer func() {- _ = session.Logoff()- }()- fs, err := session.Mount(n.Share)- if err != nil {- return fmt.Errorf("failed to mount share '%s': %w", n.Share, err)- }- defer func() {- _ = fs.Umount()- }()+ err = ignoreNotExist(lease.GetSession().getShare(deleteCtx).Remove(n.getFilePath(fileName)))+ lease.Release(err)- filePath := n.getFilePath(fileName)-- if err := fs.Remove(filePath); err != nil && !errors.Is(err, iofs.ErrNotExist) {+ if err != nil {return fmt.Errorf("failed to delete file from NAS: %w", err)}@@ -265,30 +189,17 @@ func (n *NASStorage) Validate(encryptor encryption.FieldEncryptor) error {return nil}+// A connection test often runs for settings that are not saved yet, so it dials its own session+// instead of placing one in the pool under a storage ID it may not have.func (n *NASStorage) TestConnection(encryptor encryption.FieldEncryptor) error {- session, err := n.createSession(encryptor)- if err != nil {- return fmt.Errorf("failed to connect to NAS: %w", err)- }- defer func() {- _ = session.Logoff()- }()+ ctx, cancel := context.WithTimeout(context.Background(), nasTestConnectTimeout)+ defer cancel()- // Try to mount the share to verify access- fs, err := session.Mount(n.Share)+ session, err := n.dialSession(ctx, encryptor)if err != nil {- return fmt.Errorf("failed to access share '%s': %w", n.Share, err)- }- defer func() {- _ = fs.Umount()- }()-- // If path is specified, check if it exists or can be created- if n.Path != "" {- if err := n.ensureDirectory(fs, n.Path); err != nil {- return fmt.Errorf("failed to access or create path '%s': %w", n.Path, err)- }+ return fmt.Errorf("failed to connect to NAS: %w", err)}+ defer closeSession(session)return nil}@@ -323,22 +234,13 @@ func (n *NASStorage) Update(incoming *NASStorage) {}}-func (n *NASStorage) createSession(encryptor encryption.FieldEncryptor) (*smb2.Session, error) {- return n.createSessionWithContext(context.Background(), encryptor)-}--func (n *NASStorage) createSessionWithContext(+func (n *NASStorage) dialSmbSession(ctx context.Context,+ conn net.Conn,encryptor encryption.FieldEncryptor,) (*smb2.Session, error) {- conn, err := n.createConnectionWithContext(ctx)- if err != nil {- return nil, err- }-password, err := encryptor.Decrypt(n.Password)if err != nil {- _ = conn.Close()return nil, fmt.Errorf("failed to decrypt NAS password: %w", err)}@@ -350,9 +252,8 @@ func (n *NASStorage) createSessionWithContext(},}- session, err := d.Dial(conn)+ session, err := d.DialContext(ctx, conn)if err != nil {- _ = conn.Close()return nil, fmt.Errorf("failed to create SMB session: %w", err)}@@ -385,41 +286,13 @@ func (n *NASStorage) createConnectionWithContext(ctx context.Context) (net.Conn,return conn, nil}-func (n *NASStorage) ensureDirectory(fs *smb2.Share, path string) error {- // Clean and normalize the path- path = filepath.Clean(path)- path = strings.ReplaceAll(path, "\\", "/")+// MkdirAll also succeeds when a concurrent session created the directory first, which happens when+// several sessions dial at once.+func (n *NASStorage) ensureDirectory(share *smb2.Share, path string) error {+ path = strings.ReplaceAll(filepath.Clean(path), "\\", "/")- // Check if directory already exists- _, err := fs.Stat(path)- if err == nil {- return nil // Directory exists- }-- // Try to create the directory (including parent directories)- parts := strings.Split(path, "/")- currentPath := ""-- for _, part := range parts {- if part == "" || part == "." {- continue- }-- if currentPath == "" {- currentPath = part- } else {- currentPath = currentPath + "/" + part- }-- // Check if this part of the path exists- _, err := fs.Stat(currentPath)- if err != nil {- // Directory doesn't exist, try to create it- err = fs.Mkdir(currentPath, 0o755)- if err != nil {- return fmt.Errorf("failed to create directory '%s': %w", currentPath, err)- }- }+ if err := share.MkdirAll(path, 0o755); err != nil {+ return fmt.Errorf("failed to create directory '%s': %w", path, err)}return nil@@ -438,46 +311,29 @@ func (n *NASStorage) getFilePath(filename string) string {}type nasFileReader struct {- file *smb2.File- share *smb2.Share- session *smb2.Session- stopCloseOnCancel func() bool+ file *smb2.File+ lease *storage_connections.Lease[*nasSession]+ readErr error}-func (r *nasFileReader) Read(p []byte) (n int, err error) {- return r.file.Read(p)-}--func (r *nasFileReader) Close() error {- var closeErrors []error-- if r.stopCloseOnCancel != nil {- r.stopCloseOnCancel()+func (r *nasFileReader) Read(p []byte) (int, error) {+ readBytes, err := r.file.Read(p)+ if err != nil && !errors.Is(err, io.EOF) {+ r.readErr = err}- if r.file != nil {- if err := r.file.Close(); err != nil {- closeErrors = append(closeErrors, fmt.Errorf("failed to close file: %w", err))- }- }-- if r.share != nil {- if err := r.share.Umount(); err != nil {- closeErrors = append(closeErrors, fmt.Errorf("failed to unmount share: %w", err))- }- }+ return readBytes, err+}- if r.session != nil {- if err := r.session.Logoff(); err != nil {- closeErrors = append(closeErrors, fmt.Errorf("failed to logoff session: %w", err))- }+func (r *nasFileReader) Close() error {+ closeErr := r.file.Close()+ if closeErr != nil {+ closeErr = fmt.Errorf("failed to close file: %w", closeErr)}- if len(closeErrors) > 0 {- return closeErrors[0]- }+ r.lease.Release(errors.Join(r.readErr, closeErr))- return nil+ return closeErr}type writeResult struct {@@ -548,30 +404,22 @@ func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (int64,return written, nil}-// Each attempt logs in again: a resumed read follows a dropped one, and the SMB session that-// carried it is no longer usable.+// Each attempt takes its own session. A resumed read follows a dropped one, and the session that+// carried it was closed when its read failed, so the retry gets a live one.func (n *NASStorage) openFileAtOffset(attemptCtx context.Context,encryptor encryption.FieldEncryptor,filePath string,offsetBytes int64,) (io.ReadCloser, error) {- session, err := n.createSessionWithContext(attemptCtx, encryptor)+ lease, err := n.checkoutSession(attemptCtx, encryptor)if err != nil {return nil, fmt.Errorf("failed to create NAS session: %w", err)}- share, err := session.Mount(n.Share)- if err != nil {- _ = session.Logoff()-- return nil, fmt.Errorf("failed to mount share '%s': %w", n.Share, err)- }-- nasFile, err := share.Open(filePath)+ nasFile, err := lease.GetSession().getShare(attemptCtx).Open(filePath)if err != nil {- _ = share.Umount()- _ = session.Logoff()+ lease.Release(ignoreNotExist(err))return nil, fmt.Errorf("failed to open file from NAS: %w", err)}@@ -579,23 +427,19 @@ func (n *NASStorage) openFileAtOffset(if offsetBytes > 0 {if _, err := nasFile.Seek(offsetBytes, io.SeekStart); err != nil {_ = nasFile.Close()- _ = share.Umount()- _ = session.Logoff()+ lease.Release(err)return nil, fmt.Errorf("failed to resume NAS read at offset %d: %w", offsetBytes, err)}}- // The SMB file takes no context, so a read blocked on a silent server would ignore the- // caller's cancellation; closing the file is what unblocks it.- stopCloseOnCancel := context.AfterFunc(attemptCtx, func() {- _ = nasFile.Close()- })-- return &nasFileReader{- file: nasFile,- share: share,- session: session,- stopCloseOnCancel: stopCloseOnCancel,- }, nil+ return &nasFileReader{file: nasFile, lease: lease}, nil+}++func ignoreNotExist(err error) error {+ if errors.Is(err, iofs.ErrNotExist) {+ return nil+ }++ return err}diff --git a/backend/internal/features/storages/models/nas/model_test.go b/backend/internal/features/storages/models/nas/model_test.goindex 202561e4..3b473ef7 100644--- a/backend/internal/features/storages/models/nas/model_test.go+++ b/backend/internal/features/storages/models/nas/model_test.go@@ -35,18 +35,11 @@ func Test_DeleteFile_WhenTheServerRefusesTheRemoval_ReturnsTheError(t *testing.Tencryptor := encryption.GetFieldEncryptor()directory := "refused-" + uuid.New().String()- session, err := storage.createSessionWithContext(t.Context(), encryptor)+ session, err := storage.dialSession(t.Context(), encryptor)require.NoError(t, err)+ t.Cleanup(func() { closeSession(session) })- share, err := session.Mount(storage.Share)- require.NoError(t, err)-- require.NoError(t, share.MkdirAll(storage.getFilePath(directory), 0o755))-- t.Cleanup(func() {- _ = share.Umount()- _ = session.Logoff()- })+ require.NoError(t, session.getShare(t.Context()).MkdirAll(storage.getFilePath(directory), 0o755))require.NoError(t, storage.SaveFile(t.Context(), encryptor, logger.GetLogger(), directory+"/artifact", strings.NewReader("payload"),diff --git a/backend/internal/features/storages/models/nas/session_pool.go b/backend/internal/features/storages/models/nas/session_pool.gonew file mode 100644index 00000000..79524fa8--- /dev/null+++ b/backend/internal/features/storages/models/nas/session_pool.go@@ -0,0 +1,95 @@+package nas_storage++import (+ "context"+ "fmt"+ "net"++ "github.com/hirochachacha/go-smb2"++ storage_connections "databasus-backend/internal/features/storages/connections"+ "databasus-backend/internal/util/encryption"+)++type nasSession struct {+ connection net.Conn+ session *smb2.Session+ share *smb2.Share+}++// Every call goes through the share bound to the caller's context, so the library itself gives up+// on a request when the caller does.+func (s *nasSession) getShare(ctx context.Context) *smb2.Share {+ return s.share.WithContext(ctx)+}++// The locator hands out a freshly loaded model on every write, so the pool lives at package level+// and the model only names its key.+var sessionPool = storage_connections.NewSessionPool(storage_connections.SessionOperations[*nasSession]{+ Close: closeSession,+ Probe: probeSession,+}, storage_connections.GetDefaultPoolSettings())++// Unmounting and logging off are skipped on purpose: a session is also closed to unblock a call+// stuck on a silent server, where each would wait just as long. The server drops the tree and the+// session together with the connection.+func closeSession(session *nasSession) {+ _ = session.connection.Close()+}++func probeSession(ctx context.Context, session *nasSession) error {+ _, err := session.getShare(ctx).Stat(".")++ return err+}++// Credential fields stay in their stored, encrypted form: they change whenever the user saves new+// credentials, and the fingerprint never holds a plain secret.+func (n *NASStorage) getSessionKey() storage_connections.Key {+ return storage_connections.Key{+ StorageID: n.StorageID,+ Fingerprint: storage_connections.Fingerprint(+ n.Host, n.Port, n.Share, n.Username, n.Password, n.Domain, n.UseSSL, n.Path,+ ),+ }+}++func (n *NASStorage) checkoutSession(+ ctx context.Context,+ encryptor encryption.FieldEncryptor,+) (*storage_connections.Lease[*nasSession], error) {+ return sessionPool.Checkout(ctx, n.getSessionKey(), func(ctx context.Context) (*nasSession, error) {+ return n.dialSession(ctx, encryptor)+ })+}++// The share is mounted and the target directory ensured once per session rather than on every write.+func (n *NASStorage) dialSession(ctx context.Context, encryptor encryption.FieldEncryptor) (*nasSession, error) {+ connection, err := n.createConnectionWithContext(ctx)+ if err != nil {+ return nil, err+ }++ smbSession, err := n.dialSmbSession(ctx, connection, encryptor)+ if err != nil {+ _ = connection.Close()+ return nil, err+ }++ share, err := smbSession.WithContext(ctx).Mount(n.Share)+ if err != nil {+ _ = connection.Close()+ return nil, fmt.Errorf("failed to mount share '%s': %w", n.Share, err)+ }++ session := &nasSession{connection: connection, session: smbSession, share: share}++ if n.Path != "" {+ if err := n.ensureDirectory(session.getShare(ctx), n.Path); err != nil {+ closeSession(session)+ return nil, fmt.Errorf("failed to ensure directory: %w", err)+ }+ }++ return session, nil+}diff --git a/backend/internal/features/storages/models/rclone/model.go b/backend/internal/features/storages/models/rclone/model.goindex dc7c9fdb..37cc4c2d 100644--- a/backend/internal/features/storages/models/rclone/model.go+++ b/backend/internal/features/storages/models/rclone/model.go@@ -17,6 +17,7 @@ import ("github.com/rclone/rclone/fs/config""github.com/rclone/rclone/fs/operations"+ storage_connections "databasus-backend/internal/features/storages/connections""databasus-backend/internal/util/encryption"io_utils "databasus-backend/internal/util/io")@@ -53,11 +54,12 @@ func (r *RcloneStorage) SaveFile(logger.InfoContext(ctx, "starting to save file to rclone storage", "file_name", fileName)- remoteFs, err := r.getFs(ctx, encryptor)+ remoteFs, releaseRemote, err := r.acquireFs(ctx, encryptor)if err != nil {logger.ErrorContext(ctx, "failed to create rclone filesystem", "file_name", fileName, "error", err)return fmt.Errorf("failed to create rclone filesystem: %w", err)}+ defer releaseRemote()filePath := r.getFilePath(fileName)logger.DebugContext(ctx, "uploading file via rclone", "file_name", fileName, "file_path", filePath)@@ -98,7 +100,7 @@ func (r *RcloneStorage) GetFile(logger *slog.Logger,fileName string,) (io.ReadCloser, error) {- remoteFs, err := r.getFs(ctx, encryptor)+ remoteFs, releaseRemote, err := r.acquireFs(ctx, encryptor)if err != nil {return nil, fmt.Errorf("failed to create rclone filesystem: %w", err)}@@ -107,10 +109,11 @@ func (r *RcloneStorage) GetFile(obj, err := remoteFs.NewObject(ctx, filePath)if err != nil {+ releaseRemote()return nil, fmt.Errorf("failed to get object from rclone: %w", err)}- return io_utils.NewResumingReader(io_utils.ResumingReaderSpec{+ fileReader := io_utils.NewResumingReader(io_utils.ResumingReaderSpec{StreamCtx: ctx,Logger: logger,FileName: fileName,@@ -123,7 +126,22 @@ func (r *RcloneStorage) GetFile(return obj.Open(attemptCtx, &fs.RangeOption{Start: offsetBytes, End: -1})},IsRetryableError: isRetryableRcloneError,- }), nil+ })++ return &remoteHoldingReader{ReadCloser: fileReader, releaseRemote: releaseRemote}, nil+}++// The stream keeps reading through the remote after GetFile returns, so the remote stays in use+// until the caller closes the stream.+type remoteHoldingReader struct {+ io.ReadCloser+ releaseRemote func()+}++func (r *remoteHoldingReader) Close() error {+ defer r.releaseRemote()++ return r.ReadCloser.Close()}func isRetryableRcloneError(err error) bool {@@ -144,10 +162,11 @@ func (r *RcloneStorage) DeleteFile(deleteCtx, cancel := context.WithTimeout(context.Background(), rcloneDeleteTimeout)defer cancel()- remoteFs, err := r.getFs(deleteCtx, encryptor)+ remoteFs, releaseRemote, err := r.acquireFs(deleteCtx, encryptor)if err != nil {return fmt.Errorf("failed to create rclone filesystem: %w", err)}+ defer releaseRemote()filePath := r.getFilePath(fileName)@@ -201,10 +220,13 @@ func (r *RcloneStorage) TestConnection(encryptor encryption.FieldEncryptor) erroctx, cancel := context.WithTimeout(context.Background(), rcloneOperationTimeout)defer cancel()- remoteFs, err := r.getFs(ctx, encryptor)+ // A connection test often runs for settings that are not saved yet, so it builds its own remote+ // instead of placing one in the cache under a storage ID it may not have.+ remoteFs, err := r.buildFs(ctx, encryptor)if err != nil {return fmt.Errorf("failed to create rclone filesystem: %w", err)}+ defer shutdownRemote(remoteFs)testFileID := uuid.New().String() + "-test"testFilePath := r.getFilePath(testFileID)@@ -259,7 +281,18 @@ func (r *RcloneStorage) Update(incoming *RcloneStorage) {}}-func (r *RcloneStorage) getFs(+func (r *RcloneStorage) acquireFs(+ ctx context.Context,+ encryptor encryption.FieldEncryptor,+) (fs.Fs, func(), error) {+ fingerprint := storage_connections.Fingerprint(r.ConfigContent, r.RemotePath)++ return remotes.acquire(ctx, r.StorageID, fingerprint, func(ctx context.Context) (fs.Fs, error) {+ return r.buildFs(ctx, encryptor)+ })+}++func (r *RcloneStorage) buildFs(ctx context.Context,encryptor encryption.FieldEncryptor,) (fs.Fs, error) {diff --git a/backend/internal/features/storages/models/rclone/model_test.go b/backend/internal/features/storages/models/rclone/model_test.goindex df2c91c7..4a8a17d9 100644--- a/backend/internal/features/storages/models/rclone/model_test.go+++ b/backend/internal/features/storages/models/rclone/model_test.go@@ -3,7 +3,9 @@ package rclone_storageimport ("os""path/filepath"+ "strings""testing"+ "time""github.com/google/uuid""github.com/stretchr/testify/assert"@@ -123,7 +125,7 @@ func Test_ParseConfigContent_WhitespaceAroundKeysAndValues_Trimmed(t *testing.T)assert.Equal(t, "us-west-2", sections["myremote"]["region"])}-// getFs strips a leading slash from RemotePath, so an absolute root would become+// buildFs strips a leading slash from RemotePath, so an absolute root would become// relative to the working directory. The test moves there instead and leaves the// remote path empty, which puts the rclone root on the temporary directory.func newLocalRcloneStorage(t *testing.T) (*RcloneStorage, string) {@@ -180,3 +182,80 @@ func Test_DeleteFile_WhenCalledTwice_Succeeds(t *testing.T) {t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), "backup-1",))}++func Test_SaveFile_WhenConfigurationIsUnchanged_BuildsRemoteOnce(t *testing.T) {+ storage, _ := newLocalRcloneStorage(t)++ saveTestFile(t, storage, "first")+ firstRemote := getCurrentCachedRemote(t, storage)++ saveTestFile(t, storage, "second")++ assert.Same(t, firstRemote, getCurrentCachedRemote(t, storage))+}++func Test_SaveFile_WhenConfigurationChanges_BuildsNewRemoteAndRetiresOld(t *testing.T) {+ storage, _ := newLocalRcloneStorage(t)++ saveTestFile(t, storage, "before-change")+ oldRemote := getCurrentCachedRemote(t, storage)++ changedConfig, err := encryption.GetFieldEncryptor().Encrypt("[testlocal]\ntype = local\nlinks = true\n")+ require.NoError(t, err)+ storage.ConfigContent = changedConfig++ saveTestFile(t, storage, "after-change")+ newRemote := getCurrentCachedRemote(t, storage)++ assert.NotSame(t, oldRemote, newRemote)+ assert.True(t, isRemoteRetired(oldRemote), "the remote built from the old configuration must be retired")+ assert.False(t, isRemoteRetired(newRemote))+}++func Test_SaveFile_WhenRemoteStaysUnusedPastIdleTimeout_DropsRemote(t *testing.T) {+ previousRemotes := remotes+ remotes = newRemoteCache(50 * time.Millisecond)+ t.Cleanup(func() { remotes = previousRemotes })++ storage, _ := newLocalRcloneStorage(t)++ saveTestFile(t, storage, "only-write")+ idleRemote := getCurrentCachedRemote(t, storage)++ require.Eventually(t, func() bool {+ remotes.mutex.Lock()+ defer remotes.mutex.Unlock()++ return remotes.remotes[storage.StorageID] == nil+ }, 5*time.Second, 10*time.Millisecond, "an idle remote must be dropped after the idle timeout")++ assert.True(t, isRemoteRetired(idleRemote))+}++func saveTestFile(t *testing.T, storage *RcloneStorage, fileName string) {+ t.Helper()++ err := storage.SaveFile(+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName, strings.NewReader("payload"),+ )+ require.NoError(t, err)+}++func getCurrentCachedRemote(t *testing.T, storage *RcloneStorage) *cachedRemote {+ t.Helper()++ remotes.mutex.Lock()+ defer remotes.mutex.Unlock()++ cached := remotes.remotes[storage.StorageID]+ require.NotNil(t, cached, "no remote is cached for the storage")++ return cached+}++func isRemoteRetired(cached *cachedRemote) bool {+ remotes.mutex.Lock()+ defer remotes.mutex.Unlock()++ return cached.isRetired+}diff --git a/backend/internal/features/storages/models/rclone/remote_cache.go b/backend/internal/features/storages/models/rclone/remote_cache.gonew file mode 100644index 00000000..3d290cfb--- /dev/null+++ b/backend/internal/features/storages/models/rclone/remote_cache.go@@ -0,0 +1,185 @@+package rclone_storage++import (+ "context"+ "sync"+ "time"++ "github.com/google/uuid"+ "github.com/rclone/rclone/fs"+)++const (+ rcloneRemoteIdleTimeout = 5 * time.Minute+ rcloneRemoteShutdownTimeout = 30 * time.Second+)++// Building a remote runs fs.NewFs under the process-wide rcloneConfigMu, which would serialize every+// write of every rclone storage. A remote is kept per storage and rebuilt only when the fingerprint+// of its stored configuration changes, so a write never reaches the remote through old settings.+type remoteCache struct {+ idleTimeout time.Duration++ mutex sync.Mutex+ remotes map[uuid.UUID]*cachedRemote+}++// A remote is shut down only once nobody uses it, so a full backup streaming for hours through a+// remote whose configuration changed meanwhile is never cut off by the change.+type cachedRemote struct {+ storageID uuid.UUID+ remoteFs fs.Fs+ fingerprint string+ activeUses int+ isRetired bool+ idleTimer *time.Timer+}++var remotes = newRemoteCache(rcloneRemoteIdleTimeout)++func newRemoteCache(idleTimeout time.Duration) *remoteCache {+ return &remoteCache{+ idleTimeout: idleTimeout,+ remotes: make(map[uuid.UUID]*cachedRemote),+ }+}++func (c *remoteCache) acquire(+ ctx context.Context,+ storageID uuid.UUID,+ fingerprint string,+ build func(ctx context.Context) (fs.Fs, error),+) (fs.Fs, func(), error) {+ if cached, isHit := c.takeCurrent(storageID, fingerprint); isHit {+ return cached.remoteFs, c.releaseFunc(cached), nil+ }++ builtFs, err := build(ctx)+ if err != nil {+ return nil, nil, err+ }++ stored := c.storeBuilt(storageID, fingerprint, builtFs)+ if stored.remoteFs != builtFs {+ shutdownRemote(builtFs)+ }++ return stored.remoteFs, c.releaseFunc(stored), nil+}++func (c *remoteCache) takeCurrent(storageID uuid.UUID, fingerprint string) (*cachedRemote, bool) {+ c.mutex.Lock()+ defer c.mutex.Unlock()++ cached := c.remotes[storageID]+ if cached == nil || cached.fingerprint != fingerprint {+ return nil, false+ }++ c.markUsed(cached)++ return cached, true+}++// Two writes can miss at once and both build. The first one stored wins, and the loser's remote is+// shut down by the caller, so no more than one remote per configuration outlives the race.+func (c *remoteCache) storeBuilt(storageID uuid.UUID, fingerprint string, builtFs fs.Fs) *cachedRemote {+ c.mutex.Lock()++ if cached := c.remotes[storageID]; cached != nil && cached.fingerprint == fingerprint {+ c.markUsed(cached)+ c.mutex.Unlock()++ return cached+ }++ replaced := c.remotes[storageID]+ stored := &cachedRemote{storageID: storageID, remoteFs: builtFs, fingerprint: fingerprint, activeUses: 1}+ c.remotes[storageID] = stored++ isReplacedIdle := replaced != nil && c.retire(replaced)+ c.mutex.Unlock()++ if isReplacedIdle {+ shutdownRemote(replaced.remoteFs)+ }++ return stored+}++func (c *remoteCache) markUsed(cached *cachedRemote) {+ cached.activeUses++++ if cached.idleTimer != nil {+ cached.idleTimer.Stop()+ cached.idleTimer = nil+ }+}++// Reports whether the retired remote is idle, so the caller shuts it down outside the mutex.+func (c *remoteCache) retire(cached *cachedRemote) bool {+ cached.isRetired = true++ if cached.idleTimer != nil {+ cached.idleTimer.Stop()+ cached.idleTimer = nil+ }++ return cached.activeUses == 0+}++func (c *remoteCache) releaseFunc(cached *cachedRemote) func() {+ return sync.OnceFunc(func() {+ c.release(cached)+ })+}++func (c *remoteCache) release(cached *cachedRemote) {+ c.mutex.Lock()++ cached.activeUses--+ if cached.activeUses > 0 {+ c.mutex.Unlock()+ return+ }++ if cached.isRetired {+ c.mutex.Unlock()+ shutdownRemote(cached.remoteFs)++ return+ }++ cached.idleTimer = time.AfterFunc(c.idleTimeout, func() {+ c.expire(cached)+ })+ c.mutex.Unlock()+}++// A use that started after the timer fired already holds the remote, so the expiry yields to it.+func (c *remoteCache) expire(cached *cachedRemote) {+ c.mutex.Lock()++ if cached.activeUses > 0 || cached.isRetired || c.remotes[cached.storageID] != cached {+ c.mutex.Unlock()+ return+ }++ delete(c.remotes, cached.storageID)+ c.retire(cached)+ c.mutex.Unlock()++ shutdownRemote(cached.remoteFs)+}++func shutdownRemote(remoteFs fs.Fs) {+ shutdowner, isShutdowner := remoteFs.(fs.Shutdowner)+ if !isShutdowner {+ return+ }++ ctx, cancel := context.WithTimeout(context.Background(), rcloneRemoteShutdownTimeout)+ defer cancel()++ _ = shutdowner.Shutdown(ctx)+}diff --git a/backend/internal/features/storages/models/s3/model.go b/backend/internal/features/storages/models/s3/model.goindex 8c0a81ef..f1436ee2 100644--- a/backend/internal/features/storages/models/s3/model.go+++ b/backend/internal/features/storages/models/s3/model.go@@ -5,7 +5,6 @@ import ("context""crypto/md5""crypto/sha256"- "crypto/tls""encoding/base64""encoding/hex""encoding/json"@@ -14,7 +13,6 @@ import ("hash""io""log/slog"- "net""net/http""strings""time"@@ -98,7 +96,7 @@ func (s *S3Storage) SaveFile(default:}- client, err := s.getClient(encryptor)+ client, err := s.getClient(ctx, encryptor)if err != nil {logger.ErrorContext(ctx, "failed to build the s3 client", "bucket", s.S3Bucket, "error", err)@@ -165,12 +163,7 @@ func (s *S3Storage) SaveFile(return nil}- coreClient, err := s.getCoreClient(encryptor)- if err != nil {- logger.ErrorContext(ctx, "failed to build the s3 core client", "bucket", s.S3Bucket, "error", err)-- return err- }+ coreClient := &minio.Core{Client: client}source := io.MultiReader(bytes.NewReader(firstPart), bytes.NewReader(peek[:peekLen]), file)@@ -209,13 +202,15 @@ func (s *S3Storage) GetFile(logger *slog.Logger,fileName string,) (io.ReadCloser, error) {- coreClient, err := s.getCoreClient(encryptor)+ client, err := s.getClient(ctx, encryptor)if err != nil {- logger.ErrorContext(ctx, "failed to build the s3 core client", "bucket", s.S3Bucket, "error", err)+ logger.ErrorContext(ctx, "failed to build the s3 client", "bucket", s.S3Bucket, "error", err)return nil, err}+ coreClient := &minio.Core{Client: client}+baseKey := s.buildObjectKey(fileName)logger.DebugContext(ctx, "reading file from s3", "file_name", fileName, "object_key", baseKey)@@ -263,18 +258,18 @@ func (s *S3Storage) DeleteFile(logger *slog.Logger,fileName string,) error {- client, err := s.getClient(encryptor)+ // Deletes run from cleanup paths whose caller context is often already cancelled, so the+ // operation carries its own deadline; the caller's ctx stays for log correlation only.+ deleteCtx, cancel := context.WithTimeout(context.Background(), s3DeleteTimeout)+ defer cancel()++ client, err := s.getClient(deleteCtx, encryptor)if err != nil {return err}baseKey := s.buildObjectKey(fileName)- // Deletes run from cleanup paths whose caller context is often already cancelled, so the- // operation carries its own deadline; the caller's ctx stays for log correlation only.- deleteCtx, cancel := context.WithTimeout(context.Background(), s3DeleteTimeout)- defer cancel()-keys, err := s.derivedObjectKeys(deleteCtx, client, baseKey)if err != nil {logger.WarnContext(ctx, "failed to discover s3 objects to delete",@@ -290,7 +285,7 @@ func (s *S3Storage) DeleteFile(return fmt.Errorf("failed to delete file from S3: %w", err)}- if err := s.abortDerivedMultipartUploads(deleteCtx, encryptor, client, baseKey); err != nil {+ if err := s.abortDerivedMultipartUploads(deleteCtx, client, baseKey); err != nil {logger.WarnContext(ctx, "failed to abort incomplete s3 uploads","file_name", fileName, "object_key", baseKey, "error", err)@@ -321,7 +316,10 @@ func (s *S3Storage) Validate(encryptor encryption.FieldEncryptor) error {// probe stays on a single key inside the prefix and never lets a refused delete fail the storage:// writing and reading back is everything a backup needs from S3.func (s *S3Storage) TestConnection(encryptor encryption.FieldEncryptor) error {- client, err := s.getClient(encryptor)+ ctx, cancel := context.WithTimeout(context.Background(), s3ConnectionTestTimeout)+ defer cancel()++ client, err := s.getClient(ctx, encryptor)if err != nil {return err}@@ -440,14 +438,10 @@ func (s *S3Storage) derivedObjectKeys(func (s *S3Storage) abortDerivedMultipartUploads(ctx context.Context,- encryptor encryption.FieldEncryptor,client *minio.Client,baseKey string,) error {- coreClient, err := s.getCoreClient(encryptor)- if err != nil {- return err- }+ coreClient := &minio.Core{Client: client}for upload := range client.ListIncompleteUploads(ctx, s.S3Bucket, baseKey, true) {if upload.Err != nil {@@ -776,21 +770,18 @@ func (s *S3Storage) buildObjectKey(fileName string) string {return prefix + fileName}-func (s *S3Storage) getClient(encryptor encryption.FieldEncryptor) (*minio.Client, error) {- endpoint, useSSL, accessKey, secretKey, bucketLookup, transport, err := s.getClientParams(- encryptor,- )+func (s *S3Storage) getClient(ctx context.Context, encryptor encryption.FieldEncryptor) (*minio.Client, error) {+ params, err := s.getClientParams(encryptor)if err != nil {return nil, err}- minioClient, err := minio.New(endpoint, &minio.Options{- Creds: credentials.NewStaticV4(accessKey, secretKey, ""),- Secure: useSSL,- Region: s.S3Region,- BucketLookup: bucketLookup,- Transport: transport,- })+ region := s.S3Region+ if region == "" {+ region = getCachedBucketRegion(ctx, params, s.S3Bucket)+ }++ minioClient, err := newMinioClient(params, region)if err != nil {return nil, fmt.Errorf("failed to initialize MinIO client: %w", err)}@@ -798,33 +789,28 @@ func (s *S3Storage) getClient(encryptor encryption.FieldEncryptor) (*minio.Clienreturn minioClient, nil}-func (s *S3Storage) getCoreClient(encryptor encryption.FieldEncryptor) (*minio.Core, error) {- endpoint, useSSL, accessKey, secretKey, bucketLookup, transport, err := s.getClientParams(- encryptor,- )- if err != nil {- return nil, err- }+type clientParams struct {+ endpoint string+ useSSL bool+ accessKey string+ secretKey string+ bucketLookup minio.BucketLookupType+ transport *http.Transport+}- coreClient, err := minio.NewCore(endpoint, &minio.Options{- Creds: credentials.NewStaticV4(accessKey, secretKey, ""),- Secure: useSSL,- Region: s.S3Region,- BucketLookup: bucketLookup,- Transport: transport,+func newMinioClient(params clientParams, region string) (*minio.Client, error) {+ return minio.New(params.endpoint, &minio.Options{+ Creds: credentials.NewStaticV4(params.accessKey, params.secretKey, ""),+ Secure: params.useSSL,+ Region: region,+ BucketLookup: params.bucketLookup,+ Transport: params.transport,})- if err != nil {- return nil, fmt.Errorf("failed to initialize MinIO Core client: %w", err)- }-- return coreClient, nil}-func (s *S3Storage) getClientParams(- encryptor encryption.FieldEncryptor,-) (endpoint string, useSSL bool, accessKey, secretKey string, bucketLookup minio.BucketLookupType, transport *http.Transport, err error) {- endpoint = s.S3Endpoint- useSSL = true+func (s *S3Storage) getClientParams(encryptor encryption.FieldEncryptor) (clientParams, error) {+ endpoint := s.S3Endpoint+ useSSL := trueif after, ok := strings.CutPrefix(endpoint, "http://"); ok {useSSL = false@@ -837,35 +823,29 @@ func (s *S3Storage) getClientParams(endpoint = fmt.Sprintf("s3.%s.amazonaws.com", s.S3Region)}- accessKey, err = encryptor.Decrypt(s.S3AccessKey)+ accessKey, err := encryptor.Decrypt(s.S3AccessKey)if err != nil {- return "", false, "", "", 0, nil, fmt.Errorf("failed to decrypt S3 access key: %w", err)+ return clientParams{}, fmt.Errorf("failed to decrypt S3 access key: %w", err)}- secretKey, err = encryptor.Decrypt(s.S3SecretKey)+ secretKey, err := encryptor.Decrypt(s.S3SecretKey)if err != nil {- return "", false, "", "", 0, nil, fmt.Errorf("failed to decrypt S3 secret key: %w", err)+ return clientParams{}, fmt.Errorf("failed to decrypt S3 secret key: %w", err)}- bucketLookup = minio.BucketLookupAuto+ bucketLookup := minio.BucketLookupAutoif s.S3UseVirtualHostedStyle {bucketLookup = minio.BucketLookupDNS}- transport = &http.Transport{- Proxy: http.ProxyFromEnvironment,- DialContext: (&net.Dialer{- Timeout: s3ConnectTimeout,- }).DialContext,- TLSHandshakeTimeout: s3TLSHandshakeTimeout,- ResponseHeaderTimeout: s3ResponseTimeout,- IdleConnTimeout: s3IdleConnTimeout,- TLSClientConfig: &tls.Config{- InsecureSkipVerify: s.SkipTLSVerify,- },- }-- return endpoint, useSSL, accessKey, secretKey, bucketLookup, transport, nil+ return clientParams{+ endpoint: endpoint,+ useSSL: useSSL,+ accessKey: accessKey,+ secretKey: secretKey,+ bucketLookup: bucketLookup,+ transport: getSharedTransport(s.SkipTLSVerify),+ }, nil}// chunkedManifest lists, in order, the objects a single logical file was split into, with eachdiff --git a/backend/internal/features/storages/models/s3/model_test.go b/backend/internal/features/storages/models/s3/model_test.goindex 8415d0e5..a406de8c 100644--- a/backend/internal/features/storages/models/s3/model_test.go+++ b/backend/internal/features/storages/models/s3/model_test.go@@ -615,8 +615,8 @@ func buildTestS3Transport(t *testing.T, isSkipTLSVerify bool) *http.Transport {SkipTLSVerify: isSkipTLSVerify,}- _, _, _, _, _, transport, err := storage.getClientParams(encryptor)+ params, err := storage.getClientParams(encryptor)require.NoError(t, err)- return transport+ return params.transport}diff --git a/backend/internal/features/storages/models/s3/transport.go b/backend/internal/features/storages/models/s3/transport.gonew file mode 100644index 00000000..2f08a531--- /dev/null+++ b/backend/internal/features/storages/models/s3/transport.go@@ -0,0 +1,83 @@+package s3_storage++import (+ "context"+ "crypto/tls"+ "net"+ "net/http"+ "sync"+)++// Go keeps two idle connections per host by default, which drops connections as soon as more than+// two writes to one bucket run at once.+const s3MaxIdleConnsPerHost = 16++// Clients are still built per call from freshly decrypted credentials. Only the connections are+// shared: S3 signs every request, so a changed key takes effect on the next request even over a+// connection opened before the change, and a changed endpoint lands in its own connection pool.+var sharedTransports = map[bool]*http.Transport{+ false: newSharedTransport(false),+ true: newSharedTransport(true),+}++func getSharedTransport(isSkipTLSVerify bool) *http.Transport {+ return sharedTransports[isSkipTLSVerify]+}++func newSharedTransport(isSkipTLSVerify bool) *http.Transport {+ return &http.Transport{+ Proxy: http.ProxyFromEnvironment,+ DialContext: (&net.Dialer{+ Timeout: s3ConnectTimeout,+ }).DialContext,+ TLSHandshakeTimeout: s3TLSHandshakeTimeout,+ ResponseHeaderTimeout: s3ResponseTimeout,+ IdleConnTimeout: s3IdleConnTimeout,+ MaxIdleConnsPerHost: s3MaxIdleConnsPerHost,+ TLSClientConfig: &tls.Config{+ InsecureSkipVerify: isSkipTLSVerify,+ },+ }+}++type bucketRegionKey struct {+ endpoint string+ bucket string+}++// A client built per call loses minio's own bucket-location cache, so without this every write to a+// storage with no region set would pay an extra location request.+var (+ bucketRegionsMutex sync.RWMutex+ bucketRegions = map[bucketRegionKey]string{}+)++// A failed lookup is not cached and yields an empty region, so the client falls back to minio's own+// lookup and reports the real error on the request it was built for.+func getCachedBucketRegion(ctx context.Context, params clientParams, bucket string) string {+ key := bucketRegionKey{endpoint: params.endpoint, bucket: bucket}++ bucketRegionsMutex.RLock()+ cachedRegion, isCached := bucketRegions[key]+ bucketRegionsMutex.RUnlock()++ if isCached {+ return cachedRegion+ }++ lookupClient, err := newMinioClient(params, "")+ if err != nil {+ return ""+ }++ region, err := lookupClient.GetBucketLocation(ctx, bucket)+ if err != nil {+ return ""+ }++ bucketRegionsMutex.Lock()+ bucketRegions[key] = region+ bucketRegionsMutex.Unlock()++ return region+}diff --git a/backend/internal/features/storages/models/sftp/model.go b/backend/internal/features/storages/models/sftp/model.goindex 328e3e81..93356c7a 100644--- a/backend/internal/features/storages/models/sftp/model.go+++ b/backend/internal/features/storages/models/sftp/model.go@@ -15,6 +15,7 @@ import ("github.com/pkg/sftp""golang.org/x/crypto/ssh"+ storage_connections "databasus-backend/internal/features/storages/connections""databasus-backend/internal/util/encryption"io_utils "databasus-backend/internal/util/io")@@ -55,65 +56,18 @@ func (s *SFTPStorage) SaveFile(logger.InfoContext(ctx, "starting to save file to SFTP storage", "file_name", fileName, "host", s.Host)- client, sshConn, err := s.connect(encryptor, sftpConnectTimeout)+ lease, err := s.checkoutSession(ctx, encryptor)if err != nil {logger.ErrorContext(ctx, "failed to connect to SFTP", "file_name", fileName, "error", err)return fmt.Errorf("failed to connect to SFTP: %w", err)}- defer func() {- if closeErr := client.Close(); closeErr != nil {- logger.ErrorContext(- ctx,- "failed to close SFTP client",- "file_name",- fileName,- "error",- closeErr,- )- }- if closeErr := sshConn.Close(); closeErr != nil {- logger.ErrorContext(- ctx,- "failed to close SSH connection",- "file_name",- fileName,- "error",- closeErr,- )- }- }()-- if s.Path != "" {- if err := s.ensureDirectory(client, s.Path); err != nil {- logger.ErrorContext(- ctx,- "failed to ensure directory",- "file_name",- fileName,- "path",- s.Path,- "error",- err,- )- return fmt.Errorf("failed to ensure directory: %w", err)- }- }filePath := s.getFilePath(fileName)logger.DebugContext(ctx, "uploading file to SFTP", "file_name", fileName, "file_path", filePath)- remoteFile, err := client.Create(filePath)- if err != nil {- logger.ErrorContext(ctx, "failed to create remote file", "file_name", fileName, "error", err)- return fmt.Errorf("failed to create remote file: %w", err)- }- defer func() {- _ = remoteFile.Close()- }()-- ctxReader := &contextReader{ctx: ctx, reader: file}+ err = writeRemoteFile(lease.GetSession().sftpClient, filePath, &contextReader{ctx: ctx, reader: file})+ lease.Release(err)- _, err = io.Copy(remoteFile, ctxReader)if err != nil {select {case <-ctx.Done():@@ -136,32 +90,51 @@ func (s *SFTPStorage) SaveFile(return nil}+// The plain copy ends in the library's ReadFrom, which for a reader of unknown size, such as the+// compression pipe, writes one 32 KB block at a time and waits for each answer. Over a distant+// server that is hundreds of round trips per WAL segment. The concurrent path keeps blocks in flight+// whatever the reader's size. A failed write may leave holes in the remote file, and the caller+// discards the attempt as it does any failed write.+func writeRemoteFile(client *sftp.Client, filePath string, source io.Reader) error {+ remoteFile, err := client.Create(filePath)+ if err != nil {+ return fmt.Errorf("failed to create remote file: %w", err)+ }++ if _, err := remoteFile.ReadFromWithConcurrency(source, 0); err != nil {+ _ = remoteFile.Close()+ return err+ }++ if err := remoteFile.Close(); err != nil {+ return fmt.Errorf("failed to close remote file: %w", err)+ }++ return nil+}+func (s *SFTPStorage) GetFile(ctx context.Context,encryptor encryption.FieldEncryptor,logger *slog.Logger,fileName string,) (io.ReadCloser, error) {- client, sshConn, err := s.connect(encryptor, sftpConnectTimeout)+ lease, err := s.checkoutSession(ctx, encryptor)if err != nil {return nil, fmt.Errorf("failed to connect to SFTP: %w", err)}filePath := s.getFilePath(fileName)- fileInfo, err := client.Stat(filePath)- if err != nil {- _ = client.Close()- _ = sshConn.Close()+ fileInfo, err := lease.GetSession().sftpClient.Stat(filePath)+ lease.Release(ignoreNotExist(err))+ if err != nil {return nil, fmt.Errorf("failed to stat file on SFTP: %w", err)}totalBytes := fileInfo.Size()- _ = client.Close()- _ = sshConn.Close()-return io_utils.NewResumingReader(io_utils.ResumingReaderSpec{StreamCtx: ctx,Logger: logger,@@ -184,18 +157,15 @@ func (s *SFTPStorage) DeleteFile(deleteCtx, cancel := context.WithTimeout(context.Background(), sftpDeleteTimeout)defer cancel()- client, sshConn, err := s.connectWithContext(deleteCtx, encryptor, sftpDeleteTimeout)+ lease, err := s.checkoutSession(deleteCtx, encryptor)if err != nil {return fmt.Errorf("failed to connect to SFTP: %w", err)}- defer func() {- _ = client.Close()- _ = sshConn.Close()- }()- filePath := s.getFilePath(fileName)+ err = ignoreNotExist(lease.GetSession().sftpClient.Remove(s.getFilePath(fileName)))+ lease.Release(err)- if err := client.Remove(filePath); err != nil && !errors.Is(err, fs.ErrNotExist) {+ if err != nil {return fmt.Errorf("failed to delete file from SFTP: %w", err)}@@ -284,13 +254,6 @@ func (s *SFTPStorage) Update(incoming *SFTPStorage) {}}-func (s *SFTPStorage) connect(- encryptor encryption.FieldEncryptor,- timeout time.Duration,-) (*sftp.Client, *ssh.Client, error) {- return s.connectWithContext(context.Background(), encryptor, timeout)-}-func (s *SFTPStorage) connectWithContext(ctx context.Context,encryptor encryption.FieldEncryptor,@@ -336,6 +299,13 @@ func (s *SFTPStorage) connectWithContext(return nil, nil, fmt.Errorf("failed to dial SFTP server: %w", err)}+ // The SSH handshake and the SFTP init ignore the context, so a server that accepts the TCP+ // connection and then goes silent would hold the dial forever without this.+ stopCloseOnCancel := context.AfterFunc(ctx, func() {+ _ = conn.Close()+ })+ defer stopCloseOnCancel()+sshConn, chans, reqs, err := ssh.NewClientConn(conn, address, config)if err != nil {_ = conn.Close()@@ -350,6 +320,11 @@ func (s *SFTPStorage) connectWithContext(return nil, nil, fmt.Errorf("failed to create SFTP client: %w", err)}+ if !stopCloseOnCancel() {+ closeSession(&sftpSession{sshClient: sshClient, sftpClient: sftpClient})+ return nil, nil, fmt.Errorf("failed to connect to SFTP server: %w", ctx.Err())+ }+return sftpClient, sshClient, nil}@@ -375,12 +350,18 @@ func (s *SFTPStorage) ensureDirectory(client *sftp.Client, path string) error {currentPath = currentPath + "/" + part}- _, err := client.Stat(currentPath)- if err != nil {- err = client.Mkdir(currentPath)- if err != nil {- return fmt.Errorf("failed to create directory '%s': %w", currentPath, err)+ if _, err := client.Stat(currentPath); err == nil {+ continue+ }++ // Concurrent sessions dial at once, and the one that loses the race to create the directory+ // finds it in place.+ if err := client.Mkdir(currentPath); err != nil {+ if info, statErr := client.Stat(currentPath); statErr == nil && info.IsDir() {+ continue}++ return fmt.Errorf("failed to create directory '%s': %w", currentPath, err)}}@@ -399,46 +380,29 @@ func (s *SFTPStorage) getFilePath(filename string) string {}type sftpFileReader struct {- file *sftp.File- client *sftp.Client- sshConn *ssh.Client- stopCloseOnCancel func() bool-}--func (r *sftpFileReader) Read(p []byte) (n int, err error) {- return r.file.Read(p)+ file *sftp.File+ lease *storage_connections.Lease[*sftpSession]+ readErr error}-func (r *sftpFileReader) Close() error {- var errs []error-- if r.stopCloseOnCancel != nil {- r.stopCloseOnCancel()+func (r *sftpFileReader) Read(p []byte) (int, error) {+ readBytes, err := r.file.Read(p)+ if err != nil && !errors.Is(err, io.EOF) {+ r.readErr = err}- if r.file != nil {- if err := r.file.Close(); err != nil {- errs = append(errs, fmt.Errorf("failed to close file: %w", err))- }- }-- if r.client != nil {- if err := r.client.Close(); err != nil {- errs = append(errs, fmt.Errorf("failed to close SFTP client: %w", err))- }- }+ return readBytes, err+}- if r.sshConn != nil {- if err := r.sshConn.Close(); err != nil {- errs = append(errs, fmt.Errorf("failed to close SSH connection: %w", err))- }+func (r *sftpFileReader) Close() error {+ closeErr := r.file.Close()+ if closeErr != nil {+ closeErr = fmt.Errorf("failed to close file: %w", closeErr)}- if len(errs) > 0 {- return errs[0]- }+ r.lease.Release(errors.Join(r.readErr, closeErr))- return nil+ return closeErr}type contextReader struct {@@ -455,47 +419,41 @@ func (r *contextReader) Read(p []byte) (n int, err error) {}}-// Each attempt dials a fresh SSH connection: a resumed read follows a dropped one, and the client-// that carried it is no longer usable.+// Each attempt takes its own session. A resumed read follows a dropped one, and the session that+// carried it was closed when its read failed, so the retry gets a live one.func (s *SFTPStorage) openFileAtOffset(attemptCtx context.Context,encryptor encryption.FieldEncryptor,filePath string,offsetBytes int64,) (io.ReadCloser, error) {- client, sshConn, err := s.connect(encryptor, sftpConnectTimeout)+ lease, err := s.checkoutSession(attemptCtx, encryptor)if err != nil {return nil, fmt.Errorf("failed to connect to SFTP: %w", err)}- remoteFile, err := client.Open(filePath)+ remoteFile, err := lease.GetSession().sftpClient.Open(filePath)if err != nil {- _ = client.Close()- _ = sshConn.Close()-+ lease.Release(ignoreNotExist(err))return nil, fmt.Errorf("failed to open file from SFTP: %w", err)}if offsetBytes > 0 {if _, err := remoteFile.Seek(offsetBytes, io.SeekStart); err != nil {_ = remoteFile.Close()- _ = client.Close()- _ = sshConn.Close()+ lease.Release(err)return nil, fmt.Errorf("failed to resume SFTP read at offset %d: %w", offsetBytes, err)}}- // The SFTP client takes no context, so a read blocked on a silent server would ignore the- // caller's cancellation; closing the connection is what unblocks it.- stopCloseOnCancel := context.AfterFunc(attemptCtx, func() {- _ = sshConn.Close()- })+ return &sftpFileReader{file: remoteFile, lease: lease}, nil+}++func ignoreNotExist(err error) error {+ if errors.Is(err, fs.ErrNotExist) {+ return nil+ }- return &sftpFileReader{- file: remoteFile,- client: client,- sshConn: sshConn,- stopCloseOnCancel: stopCloseOnCancel,- }, nil+ return err}diff --git a/backend/internal/features/storages/models/sftp/session_pool.go b/backend/internal/features/storages/models/sftp/session_pool.gonew file mode 100644index 00000000..41c190be--- /dev/null+++ b/backend/internal/features/storages/models/sftp/session_pool.go@@ -0,0 +1,80 @@+package sftp_storage++import (+ "context"+ "fmt"++ "github.com/pkg/sftp"+ "golang.org/x/crypto/ssh"++ storage_connections "databasus-backend/internal/features/storages/connections"+ "databasus-backend/internal/util/encryption"+)++type sftpSession struct {+ sshClient *ssh.Client+ sftpClient *sftp.Client+}++// The locator hands out a freshly loaded model on every write, so the pool lives at package level+// and the model only names its key.+var sessionPool = storage_connections.NewSessionPool(storage_connections.SessionOperations[*sftpSession]{+ Close: closeSession,+ Probe: probeSession,+}, storage_connections.GetDefaultPoolSettings())++// The SFTP client's Close waits for its receive loop, which only ends once the SSH connection is+// gone, so the SSH side closes first or a session stuck on a silent server never closes.+func closeSession(session *sftpSession) {+ _ = session.sshClient.Close()+ _ = session.sftpClient.Close()+}++// The SFTP client takes no context, so the probe deadline is enforced by closing the session.+func probeSession(ctx context.Context, session *sftpSession) error {+ stopCloseOnDeadline := context.AfterFunc(ctx, func() {+ closeSession(session)+ })+ defer stopCloseOnDeadline()++ _, err := session.sftpClient.Getwd()++ return err+}++// Credential fields stay in their stored, encrypted form: they change whenever the user saves new+// credentials, and the fingerprint never holds a plain secret.+func (s *SFTPStorage) getSessionKey() storage_connections.Key {+ return storage_connections.Key{+ StorageID: s.StorageID,+ Fingerprint: storage_connections.Fingerprint(s.Host, s.Port, s.Username, s.Password, s.PrivateKey, s.Path),+ }+}++func (s *SFTPStorage) checkoutSession(+ ctx context.Context,+ encryptor encryption.FieldEncryptor,+) (*storage_connections.Lease[*sftpSession], error) {+ return sessionPool.Checkout(ctx, s.getSessionKey(), func(ctx context.Context) (*sftpSession, error) {+ return s.dialSession(ctx, encryptor)+ })+}++// The target directory is ensured once per session rather than walked on every write.+func (s *SFTPStorage) dialSession(ctx context.Context, encryptor encryption.FieldEncryptor) (*sftpSession, error) {+ sftpClient, sshClient, err := s.connectWithContext(ctx, encryptor, sftpConnectTimeout)+ if err != nil {+ return nil, err+ }++ session := &sftpSession{sshClient: sshClient, sftpClient: sftpClient}++ if s.Path != "" {+ if err := s.ensureDirectory(sftpClient, s.Path); err != nil {+ closeSession(session)+ return nil, fmt.Errorf("failed to ensure directory: %w", err)+ }+ }++ return session, nil+}diff --git a/backend/internal/util/testing/containers/azurite.go b/backend/internal/util/testing/containers/azurite.goindex 66e51d30..f6e7b642 100644--- a/backend/internal/util/testing/containers/azurite.go+++ b/backend/internal/util/testing/containers/azurite.go@@ -22,6 +22,12 @@ const azuriteBlobPort = "10000/tcp"func StartAzurite(t *testing.T) Endpoint {t.Helper()+ return StartAzuriteContainer(t).Endpoint+}++func StartAzuriteContainer(t *testing.T) ContainerHandle {+ t.Helper()+req := testcontainers.ContainerRequest{Image: "mcr.microsoft.com/azure-storage/azurite",ExposedPorts: []string{azuriteBlobPort},@@ -29,5 +35,5 @@ func StartAzurite(t *testing.T) Endpoint {WaitingFor: wait.ForListeningPort(azuriteBlobPort).WithStartupTimeout(120 * time.Second),}- return start(t, req, azuriteBlobPort)+ return startContainer(t, req, azuriteBlobPort)}diff --git a/backend/internal/util/testing/containers/ftp.go b/backend/internal/util/testing/containers/ftp.goindex d9465f2d..1472aa77 100644--- a/backend/internal/util/testing/containers/ftp.go+++ b/backend/internal/util/testing/containers/ftp.go@@ -34,6 +34,12 @@ const (func StartFtp(t *testing.T) Endpoint {t.Helper()+ return StartFtpContainer(t).Endpoint+}++func StartFtpContainer(t *testing.T) ContainerHandle {+ t.Helper()+req := testcontainers.ContainerRequest{Image: "stilliard/pure-ftpd:latest",ExposedPorts: ftpExposedPorts(),@@ -50,7 +56,7 @@ func StartFtp(t *testing.T) Endpoint {WaitingFor: wait.ForListeningPort(ftpControlPort).WithStartupTimeout(120 * time.Second),}- return start(t, req, ftpControlPort)+ return startContainer(t, req, ftpControlPort)}// ftpExposedPorts lists the control port plus every passive port. testcontainers drops any hostdiff --git a/backend/internal/util/testing/containers/minio.go b/backend/internal/util/testing/containers/minio.goindex 4faba876..f8d077c0 100644--- a/backend/internal/util/testing/containers/minio.go+++ b/backend/internal/util/testing/containers/minio.go@@ -29,6 +29,12 @@ const MinioImage = "pgsty/minio:RELEASE.2026-08-04T00-00-00Z"func StartMinio(t *testing.T) Endpoint {t.Helper()+ return StartMinioContainer(t).Endpoint+}++func StartMinioContainer(t *testing.T) ContainerHandle {+ t.Helper()+req := testcontainers.ContainerRequest{Image: MinioImage,ExposedPorts: []string{minioPort},@@ -40,5 +46,5 @@ func StartMinio(t *testing.T) Endpoint {WaitingFor: wait.ForHTTP("/minio/health/live").WithPort(minioPort).WithStartupTimeout(120 * time.Second),}- return start(t, req, minioPort)+ return startContainer(t, req, minioPort)}diff --git a/backend/internal/util/testing/containers/netem.go b/backend/internal/util/testing/containers/netem.gonew file mode 100644index 00000000..e80dfb44--- /dev/null+++ b/backend/internal/util/testing/containers/netem.go@@ -0,0 +1,166 @@+package containers++import (+ "context"+ "fmt"+ "io"+ "net"+ "strconv"+ "testing"+ "time"++ "github.com/moby/moby/api/types/container"+ "github.com/testcontainers/testcontainers-go"+ "github.com/testcontainers/testcontainers-go/wait"+)++// Any image with iproute2 fits; netshoot is multi-arch and maintained. The tag is pinned for the+// same reason MinioImage is.+const NetemSidecarImage = "nicolaka/netshoot:v0.14"++const netemSidecarTimeout = 60 * time.Second++const shapedProbeDialTimeout = 2 * time.Second++// AddEgressDelay delays every packet the container sends, so each round trip a test makes to it+// grows by delay, FTP passive data connections included. A host where shaping silently does+// nothing fails the test here instead of producing a fast, meaningless run.+func AddEgressDelay(t *testing.T, handle ContainerHandle, delay time.Duration) {+ t.Helper()++ runTrafficControl(t, handle, "qdisc", "add", "dev", "eth0", "root", "netem", "delay", formatNetemDuration(delay))++ handshakeDuration, err := MeasureTcpHandshake(t, handle)+ if err != nil {+ t.Fatalf("failed to dial the container after adding egress delay: %v", err)+ }++ if handshakeDuration < delay {+ t.Fatalf("egress delay of %s is not in effect: the TCP handshake took %s", delay, handshakeDuration)+ }+}++// SetEgressLoss drops everything the container sends, so it goes silent mid-connection without+// closing anything.+func SetEgressLoss(t *testing.T, handle ContainerHandle) {+ t.Helper()++ runTrafficControl(t, handle, "qdisc", "replace", "dev", "eth0", "root", "netem", "loss", "100%")+}++// A plain FIFO replaces whatever shaping is in place, so a test may also call this from a cleanup+// that runs after it already cleared the shaping itself.+func ClearEgressShaping(t *testing.T, handle ContainerHandle) {+ t.Helper()++ runTrafficControl(t, handle, "qdisc", "replace", "dev", "eth0", "root", "pfifo")+}++// The mapped host port is served by Docker's userland proxy, which completes the handshake itself+// before it reaches the container. Only a dial to the container's own address crosses the shaped+// interface during the handshake.+func MeasureTcpHandshake(t *testing.T, handle ContainerHandle) (time.Duration, error) {+ t.Helper()++ address := getContainerAddress(t, handle)+ startedAt := time.Now()++ dialer := net.Dialer{Timeout: shapedProbeDialTimeout}++ connection, err := dialer.DialContext(t.Context(), "tcp", address)+ if err != nil {+ return 0, err+ }++ handshakeDuration := time.Since(startedAt)+ _ = connection.Close()++ return handshakeDuration, nil+}++func getContainerAddress(t *testing.T, handle ContainerHandle) string {+ t.Helper()++ ctx := context.Background()++ inspect, err := handle.Container.Inspect(ctx)+ if err != nil {+ t.Fatalf("failed to inspect container: %v", err)+ }++ containerIP, err := handle.Container.ContainerIP(ctx)+ if err != nil {+ t.Fatalf("failed to get container IP: %v", err)+ }++ mappedPort := strconv.Itoa(handle.Port)++ for containerPort, bindings := range inspect.NetworkSettings.Ports {+ for _, binding := range bindings {+ if binding.HostPort == mappedPort {+ return net.JoinHostPort(containerIP, strconv.Itoa(int(containerPort.Num())))+ }+ }+ }++ t.Fatalf("no container port is published on host port %s", mappedPort)++ return ""+}++func runTrafficControl(t *testing.T, handle ContainerHandle, args ...string) {+ t.Helper()++ ctx, cancel := context.WithTimeout(context.Background(), netemSidecarTimeout)+ defer cancel()++ sidecar, err := testcontainers.Run(+ ctx,+ NetemSidecarImage,+ testcontainers.WithCmd(append([]string{"tc"}, args...)...),+ testcontainers.WithHostConfigModifier(func(hostConfig *container.HostConfig) {+ hostConfig.NetworkMode = container.NetworkMode("container:" + handle.Container.GetContainerID())+ hostConfig.CapAdd = []string{"NET_ADMIN"}+ }),+ testcontainers.WithWaitStrategy(wait.ForExit().WithExitTimeout(netemSidecarTimeout)),+ )+ if sidecar != nil {+ defer func() {+ if err := testcontainers.TerminateContainer(sidecar); err != nil {+ t.Logf("failed to terminate netem sidecar: %v", err)+ }+ }()+ }++ if err != nil {+ t.Fatalf("failed to run tc %v in a netem sidecar: %v", args, err)+ }++ state, err := sidecar.State(ctx)+ if err != nil {+ t.Fatalf("failed to read netem sidecar state: %v", err)+ }++ if state.ExitCode != 0 {+ t.Fatalf("tc %v exited with code %d: %s", args, state.ExitCode, readContainerLog(ctx, sidecar))+ }+}++func readContainerLog(ctx context.Context, target testcontainers.Container) string {+ logReader, err := target.Logs(ctx)+ if err != nil {+ return fmt.Sprintf("<failed to read log: %v>", err)+ }+ defer func() { _ = logReader.Close() }()++ logContent, err := io.ReadAll(logReader)+ if err != nil {+ return fmt.Sprintf("<failed to read log: %v>", err)+ }++ return string(logContent)+}++func formatNetemDuration(delay time.Duration) string {+ return strconv.FormatInt(delay.Microseconds(), 10) + "us"+}diff --git a/backend/internal/util/testing/containers/netem_test.go b/backend/internal/util/testing/containers/netem_test.gonew file mode 100644index 00000000..d67f505d--- /dev/null+++ b/backend/internal/util/testing/containers/netem_test.go@@ -0,0 +1,56 @@+package containers++import (+ "testing"+ "time"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++const testEgressDelay = 50 * time.Millisecond++func Test_AddEgressDelay_WhenAppliedToMinio_TcpHandshakeTakesAtLeastTheDelay(t *testing.T) {+ minio := StartMinioContainer(t)++ handshakeBeforeDelay, err := MeasureTcpHandshake(t, minio)+ require.NoError(t, err)+ assert.Less(t, handshakeBeforeDelay, testEgressDelay)++ AddEgressDelay(t, minio, testEgressDelay)++ handshakeAfterDelay, err := MeasureTcpHandshake(t, minio)+ require.NoError(t, err)+ assert.GreaterOrEqual(t, handshakeAfterDelay, testEgressDelay)+}++func Test_SetEgressLoss_WhenApplied_DialTimesOut(t *testing.T) {+ minio := StartMinioContainer(t)++ SetEgressLoss(t, minio)++ _, err := MeasureTcpHandshake(t, minio)+ assert.Error(t, err)+}++func Test_ClearEgressShaping_AfterLoss_DialSucceedsAgain(t *testing.T) {+ minio := StartMinioContainer(t)+ SetEgressLoss(t, minio)++ ClearEgressShaping(t, minio)++ handshakeDuration, err := MeasureTcpHandshake(t, minio)+ require.NoError(t, err)+ assert.Less(t, handshakeDuration, testEgressDelay)+}++func Test_ClearEgressShaping_WhenCalledTwice_Succeeds(t *testing.T) {+ minio := StartMinioContainer(t)+ AddEgressDelay(t, minio, testEgressDelay)++ ClearEgressShaping(t, minio)+ ClearEgressShaping(t, minio)++ _, err := MeasureTcpHandshake(t, minio)+ assert.NoError(t, err)+}diff --git a/backend/internal/util/testing/containers/samba.go b/backend/internal/util/testing/containers/samba.goindex d40d27cf..bf064e56 100644--- a/backend/internal/util/testing/containers/samba.go+++ b/backend/internal/util/testing/containers/samba.go@@ -21,6 +21,12 @@ const sambaPort = "445/tcp"func StartSamba(t *testing.T) Endpoint {t.Helper()+ return StartSambaContainer(t).Endpoint+}++func StartSambaContainer(t *testing.T) ContainerHandle {+ t.Helper()+req := testcontainers.ContainerRequest{Image: "dperson/samba:latest",ExposedPorts: []string{sambaPort},@@ -33,5 +39,5 @@ func StartSamba(t *testing.T) Endpoint {WaitingFor: wait.ForListeningPort(sambaPort).WithStartupTimeout(120 * time.Second),}- return start(t, req, sambaPort)+ return startContainer(t, req, sambaPort)}diff --git a/backend/internal/util/testing/containers/sftp.go b/backend/internal/util/testing/containers/sftp.goindex fd560a13..94b35a6a 100644--- a/backend/internal/util/testing/containers/sftp.go+++ b/backend/internal/util/testing/containers/sftp.go@@ -33,6 +33,12 @@ func sftpReady() wait.Strategy {func StartSftp(t *testing.T) Endpoint {t.Helper()+ return StartSftpContainer(t).Endpoint+}++func StartSftpContainer(t *testing.T) ContainerHandle {+ t.Helper()+req := testcontainers.ContainerRequest{Image: "atmoz/sftp:latest",ExposedPorts: []string{sftpPort},@@ -40,5 +46,5 @@ func StartSftp(t *testing.T) Endpoint {WaitingFor: sftpReady(),}- return start(t, req, sftpPort)+ return startContainer(t, req, sftpPort)}diff --git a/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/.openspec.yaml b/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/.openspec.yamlnew file mode 100644index 00000000..5c881ba7--- /dev/null+++ b/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/.openspec.yaml@@ -0,0 +1,2 @@+schema: spec-driven+created: 2026-10-18diff --git a/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/design.md b/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/design.mdnew file mode 100644index 00000000..bc33b43e--- /dev/null+++ b/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/design.md@@ -0,0 +1,212 @@+## Context++See proposal.md for the motivation and the specs for the required behavior. Paths are under `backend/internal/features/` unless stated otherwise.++### How archiving works today++All paths in this part are under `backups/backups/`.++- `pg_receivewal` writes finalized segments into `wal-queue/<database_id>/`. The uploader loop wakes every second (`usecases/physical/postgresql/wal_watch_dir.go:30`). `scanAndUpload` (`wal_watch_dir.go:77`) walks the directory in name order and calls `ProcessSegment` for one file at a time. `sweepPendingUploads` (`wal_watch_dir.go:111`) then does the same for segments that `realignResumePath` moved into `pending-upload/` (`wal_resume.go:108`). On startup, `recoverLocalSegmentsOnStartup` (`wal_watch_dir.go:47`) takes over this database's own leftover claims in a single pass, one file at a time, before the receiver spawns (`wal_stream.go:277`).+- `ProcessSegment` (`wal_upload.go:82`) and `RecoverSegment` (`wal_upload.go:128`) insert the claim row (`wal_upload.go:109`, `wal_upload.go:155`), then `uploadAndCommit` (`wal_upload.go:229`) does the rest. It compresses and encrypts the segment while writing it to storage, writes the metadata sidecar, and commits the row and both write receipts in one transaction. After that it runs the gap probe and removes the local file. A failed step releases the claim and keeps the file for the next pass.+- `probeChainGap` (`wal_upload.go:324`) compares the committed segment with the newest committed segment before it, using `FindLatestCommittedBefore` (`core/physical/repositories/wal_segment_repository.go:298`). Repeat notifications are suppressed through `lastNotifiedGapEnd` (`wal_upload.go:64`), which only moves forward.+- The archive-staleness tracker (`wal_staleness.go:37`) treats any rise of the newest committed segment as progress. It reads that segment through `getLastCommittedWal` (`wal_staleness.go:77`), which calls `FindLatestCommittedSegment`. That query orders by timeline first, then by position (`wal_segment_repository.go:335`).+- Every 30 seconds the lag monitor logs one rollup line (`lag_monitor.go:173`). The counters it reads are swapped to zero on read (`wal_upload.go:184`).+- The compression encoder is created per segment with default options (`wal_upload.go:597`). Its stream encoder holds about 17 to 18 MB at the default level, mostly the 8 MB window and its 16 MB history buffer.+- Restore planning already stops at the first missing segment: `contiguousWalRun` (`core/physical/chain_view/restore_set.go:275`). The restorable point users see is therefore correct no matter in which order segments commit.++### How a storage write works today++- `Store.WriteFile` (`storages/files/store.go:45`) inserts a pending-deletion row (`store.go:57`), asks the locator for the storage (`store.go:62`), calls the model's `SaveFile` and then extends the row's window (`store.go:77`). The locator runs `FindByID` on every call (`storages/file_locator.go:18`), which is one query plus one per storage type (`storages/repository.go:135`). Against the internal database this costs a few milliseconds.+- No storage model keeps a client between calls. Each `SaveFile` builds its own:+ - S3: a new `minio.Client` with a new `http.Transport` (`storages/models/s3/model.go:101`, `model.go:855`), so every write starts a new TCP connection, and a new TLS session over HTTPS. Each S3 write also allocates a 16 MB buffer for the first part (`model.go:120`).+ - Azure Blob: a new client and transport (`storages/models/azure_blob/model.go:83`).+ - SFTP: a new TCP connection, SSH handshake and SFTP session (`storages/models/sftp/model.go:58`). The copy at `sftp/model.go:116` ends in the library's `ReadFrom`. For a reader of unknown size, such as our compression pipe, it writes one 32 KB block at a time and waits for each answer (`pkg/[email protected]/client.go:2022`). One 16 MB segment takes about 512 round trips.+ - FTP: a new connection and a full login (`storages/models/ftp/model.go:58`), then a directory walk with one command per path segment, then a new data connection.+ - NAS: a new SMB session and share mount (`storages/models/nas/model.go:63`), then a directory walk with one stat per path segment.+ - rclone: a new remote through `fs.NewFs` (`storages/models/rclone/model.go:56`), under a process-wide mutex (`rclone/model.go:29`) that serializes every rclone storage in the process.+- Requests made by the SFTP and SMB libraries ignore the caller's context. A connection whose peer vanished without closing it hangs the write until the operating system gives up on the socket.+- The storage test containers (`backend/internal/util/testing/containers/`) run on the same host as the tests. A round trip takes microseconds, so none of the costs above shows up in any test.++### Constraints++- `backend/AGENTS.md`, Stored files: every write attempt gets its own object name, and a file is kept only when a committed transaction claims it. The per-attempt object name (`wal_upload.go:500`) and the commit transaction already satisfy both rules for concurrent writes. Reused connections change how bytes travel, not which files are kept.+- `backend/AGENTS.md`, Background services, Logging and Testing: one background loop per file, `job_id`/`job_name` scoping, and unit tests only where no API exists. No HTTP surface covers WAL archiving or storage writes, so tests drive the uploader, the dispatcher and `Store.WriteFile` directly.+- ADR-0004 asks for bounded memory through streaming with backpressure. Upload concurrency and pooled connections are both bounded by fixed limits, and the local queue keeps its existing high watermark (`wal_backpressure.go`).+- ADR-0011 keeps partial segments out of storage. Nothing here changes that.+- The `single-instance-runtime` spec runs one application process per installation. In-memory state of the uploader is authoritative for its database, and an in-memory connection cache is authoritative for the process.++## Goals / Non-Goals++**Goals:**++- Remove per-write connection setup from every storage type except Google Drive and local storage.+- Make SFTP write a segment in a handful of round trips instead of hundreds.+- Hide the remaining per-segment latency by overlapping up to four uploads per database.+- Keep the gap notification and the staleness alert correct when segments commit out of order.+- Leave enough timing in the logs to tell latency-bound archiving from bandwidth-bound archiving.+- Prove the write rate against every covered storage type with a test that injects real network delay.++**Non-Goals:**++- Faster compression, sidecar removal or fewer database round trips per segment.+- Any change to receiving WAL, the local queue limit or restore.+- Google Drive's write path.++## Decisions++### D1. A continuous pool of four workers fed in WAL order++The uploader loop becomes a dispatcher. On each tick, and whenever a worker finishes, it lists the watch directory and `pending-upload/`, sorts the segment files by timeline and then by WAL position, and starts uploads for files that are not in flight while fewer than four are running. Every start goes through one path: live segments, pending-upload segments, and the startup recovery of leftover claims. Recovery keeps calling `RecoverSegment` and live files keep calling `ProcessSegment`. Timeline `.history` files stay on the dispatcher goroutine, as they are rare and need a source connection.++Startup recovery stays a single pass. The dispatcher lists the two directories once, starts every segment found there through the four slots, and returns when those uploads have finished, whatever their outcome. A failed segment waits for the live loop, as it does today. Recovery never loops until the queue is empty; otherwise a storage outage at startup would keep the receiver from ever spawning.++Starting in WAL order keeps the archive progress point (D4) moving forward and drains the oldest WAL first under backpressure. Nothing depends on claims being inserted in that order. A failed segment is retried after later segments have claimed, and goroutines started in order may reach the database out of order. The gap probe (D3) works from local files and the catalog, so it does not need that order either.++Alternatives rejected:++- *A batch per tick.* Start up to four uploads from the current listing and wait for all of them before the next tick. There is no shared in-flight state, but one slow upload holds the next batch for up to `walSegmentUploadTimeout`, two minutes (`wal_upload.go:33`). The three free workers would sit idle exactly when the queue is longest.+- *Pipelining inside one worker.* Compress segment N+1 while N uploads. At most a 2x gain, and none when the storage round trip, not compression, dominates.+- *A setting for the worker count.* Kept out of scope by the proposal. A constant can become a setting later without breaking anyone, while removing a setting would break existing configurations.++Why four: the user's source needs about 1.3 segments per second. Connection reuse (D7) and pipelined SFTP writes (D8) bring a single write against a storage 50 ms away down to a few hundred milliseconds, but one TCP stream still caps how fast a single upload moves bytes over a long path. Four workers reach the required rate with a full cycle of up to about three seconds per segment. Higher numbers raise the risk of hitting FTP or SFTP per-user connection limits and multiply encoder and buffer memory across every streaming database.++### D2. An in-memory in-flight set owned by the dispatcher++The dispatcher keeps the set of file names currently uploading. Only the dispatcher reads it; the gap probe (D3) and the progress point (D4) work from the directory listings instead. A name enters the set when its upload starts and leaves it when the worker returns, whatever the outcome. The set prevents a second start of the same segment. It also prevents the "held by another in-flight claim" warning (`wal_upload.go:219`), which would otherwise fire on every tick for a slow upload.++The set is keyed by WAL file name, not path. `realignResumePath` can move a file from the watch directory into `pending-upload/` while its upload runs, and the same segment must not start again from the new path. On Linux an open file survives a rename, so the running upload finishes normally. Its `removeLocal` of the old path then finds nothing, and the next pass sees the moved copy as already committed and drops it through `handleClaimConflict`. That is today's behavior and stays as it is.++Alternative rejected: *rely on the claim row's uniqueness alone.* It is already race-safe (`Test_WalUpload_ConcurrentClaimSameSegment_OnlyWinnerInserts`), but every loser would do a probe query and log a warning on every tick. Under the single-instance runtime the database adds no safety that the in-process set lacks.++### D3. The gap probe asks whether the missing predecessor is still coming++The uploader gets the watch directory in its dependencies. When the newest committed predecessor does not end where the committed segment starts, the probe looks up the segment directly before it, the one ending at `StartLSN`, before notifying:++1. If its full-size file exists in the watch directory, or after that in `pending-upload/`, it is still coming, and no gap is reported. A file shorter than a full segment does not count. `pg_receivewal` resumes after the highest complete segment (`GetResumeSegmentNo`, `wal_resume.go:39`), `movePendingUploadsOutOfResumePath` leaves short files where they are (`wal_resume.go:87`), and the uploader only warns about them (`wal_upload.go:88`), so a short file below the resume point is never replaced. Counting it would suppress the notification for that hole forever. An uploading segment's file stays in one of these two places until `removeLocal` runs after the commit, so the in-flight set is not consulted and the uploader keeps no dependency on dispatcher state.+2. Otherwise the probe looks it up in the catalog by chain key, whether committed or claimed. If a row exists, it was committed or claimed between the two reads, and no gap is reported.+3. Otherwise the predecessor will not arrive, and a gap is reported.++The order of the checks closes both races:++- The watch directory is checked before `pending-upload/`. `realignResumePath` only moves files from the first into the second, so a file renamed between the two checks is still found in the second.+- The local check runs before the catalog check. A predecessor that commits concurrently writes its row before it removes its local file (`wal_upload.go:229` onwards), so at least one of the two checks sees it.++The reported gap ends at `StartLSN`. It starts at the larger of two positions: the end of the newest committed predecessor, and the end of the highest full-size segment file on the same timeline that still sits in the two directories below `StartLSN`. Under concurrent uploads the newest committed predecessor can lag behind a segment that is still uploading, and the first position alone would report a hole wider than the real one. The probe already lists both directories for step 1, so the second position costs nothing extra.++Each segment checks only its direct predecessor, so only the first segment after a real hole reports it. That segment commits at most once, because the commit is guarded by `file_name IS NULL`, and re-processing a committed segment goes through `handleClaimConflict`, which runs no probe. Each hole is therefore reported at most once per session without any bookkeeping. `lastNotifiedGapEnd` and the lock around it are deleted rather than replaced. Under out-of-order commits the forward-only marker would also hide a lower gap found after a higher one.++Alternatives rejected:++- *Report a gap only when the hole outlives a grace period.* It needs a timer and per-hole state, delays real gap alerts, and still needs the "is it still coming" check to avoid alerting on a long retry.+- *Keep a set of reported gaps.* It guards against a duplicate that the commit guard already rules out, and it grows for the whole session.++### D4. Staleness follows the archive progress point++The tracker's input changes from "newest committed segment" to the archive progress point defined in the spec. Two goroutines hold the two halves, so the value is assembled in the lag monitor:++- **The dispatcher owns the lowest pending segment.** After each listing of the watch directory and `pending-upload/`, it stores the timeline and start position of the lowest segment it found, or "none", in a field on the streamer guarded by a mutex. "Lowest" compares the timeline first and the position second, the same order `FindLatestCommittedSegment` uses. Files of uploading segments are still in the listing, so in-flight segments count without a separate lookup. Files shorter than a full segment count as well: restore cannot pass that position until the segment arrives, so the point must not pass it either. This is how a torn segment surfaces: through the staleness alert and the existing incomplete-segment warning, never through a gap notification (D3). Between listings the stored value can only be lower than the truth, never higher, which errs towards reporting staleness rather than hiding it.+- **The lag monitor combines the two.** `reportArchiveStalenessIfDue` (`lag_monitor.go:194`) already reads the newest committed segment through `getLastCommittedWal`. It reads the dispatcher's value and passes both to a pure function. That function returns the lowest pending start when the lowest pending segment lies below the newest committed one in (timeline, position) order, and the newest committed end otherwise. The result goes to `recordSampleAndDetectStaleness` in place of the newest committed end. `getLastCommittedWal` returns the newest committed segment's timeline alongside its end for this comparison.++This adds no extra query. The tracker logic in `wal_staleness.go:37` stays the same; only the value it receives changes, so the new behavior is tested on the pure function. `ChainRiskReport.LastArchivedWalAt` keeps reporting the newest committed segment's time, since the alert text explains that the recovery point stopped advancing.++Alternative rejected: *compute the gap-free run from the catalog with `contiguousWalRun`.* It is exact, but it reads the whole chain's WAL every 30 seconds for every streaming database. The `background-load` spec exists to keep that kind of work out of periodic loops.++### D5. Step timing collected per upload, reported in the existing rollup++Four steps are measured. The claim insert is timed in `ProcessSegment` and `RecoverSegment`, where it happens, and handed to `uploadAndCommit` with the claim. `uploadAndCommit` times the segment write (compression and encryption run inside it through the pipe), the sidecar write, and the commit including the gap probe. Each upload adds its durations to accumulators under the uploader's mutex.++The dispatcher knows how many uploads run, so it reports the count to the uploader through one method each time it starts an upload. The uploader keeps the highest count since the last report. `TakeArchivedSinceLastReport` becomes a take of one snapshot struct with the counts, the per-step totals and maximums, and the highest concurrency, and resets all of them. `reportArchivingProgress` (`lag_monitor.go:173`) formats the snapshot into the existing INFO line. Values go into the message and IDs stay as kv pairs, per `backend/AGENTS.md` Logging.++Reading the line: if the segment write takes about 16 MB divided by the link speed, archiving is bandwidth-bound and more workers will not help. If it takes much longer, the time is round trips.++Alternatives rejected:++- *One log line per segment at INFO.* A busy source finalizes several segments a second, and the existing comment at `wal_upload.go:301` already rejects that rate.+- *The dispatcher keeps its own counter and the lag monitor reads two sources.* Two snapshots taken at different moments would not describe the same interval.++### D6. Bounded encoder memory++Each worker builds its own zstd encoder (`wal_upload.go:597`) with a concurrency of one and the library's lower-memory mode. A concurrency of one encodes on the caller's goroutine instead of a second one. The lower-memory mode shrinks the history buffer from twice the window to the window plus one block, which takes the encoder from about 18 MB to about 9.5 MB. Neither option changes the compressed format or the window, so existing segments, restore and compression ratio stay the same.++Alternatives rejected:++- *A shared encoder pool.* It adds lifecycle code for a saving that matters only with dozens of streaming databases on one instance.+- *A smaller window.* It saves more memory but changes the compression ratio of every segment.++### D7. Reused connections: shared transports for HTTP storages, a session pool for the rest++The fix differs by protocol, because what each write rebuilds differs.++**S3 and Azure Blob keep building a client per call, over a shared transport.** The only per-write cost these two types have beyond the request itself is the fresh `http.Transport` (`s3/model.go:855`, `azure_blob/model.go:504`). Each package keeps its transports at package level instead: S3 one per `SkipTLSVerify` value, Azure one. `MaxIdleConnsPerHost` is raised from Go's default of two, so four concurrent writes keep their connections. The client is still built per call from freshly decrypted credentials. TCP and TLS connections are then reused, a changed endpoint lands in a different connection pool by itself, and changed credentials take effect on the next request, because both protocols sign every request rather than authenticating a connection. The existing `IdleConnTimeout` of 90 seconds closes idle connections.++A client built per call loses minio's per-client bucket-location cache. When `S3Region` is empty, which `Validate` allows, every write would pay an extra location request (`minio-go/[email protected]/bucket-cache.go:49`). The S3 package therefore keeps a small map from endpoint and bucket to the region it looked up once with `GetBucketLocation`, and builds each client with that region.++**rclone caches the remote.** `fs.NewFs` is the expensive part and runs under a process-wide mutex. The rclone package keeps a map from storage ID to a remote plus a fingerprint, a SHA-256 over the remote's configuration text. A write whose fingerprint differs from the cached one builds a new remote, so `fs.NewFs` and the mutex run only on a miss or after a change, and retires the old remote. A remote unused for five minutes is retired and dropped. A retired remote is shut down once the last call using it returns, so a full backup streaming through it is never cut off by a settings change. The connection test builds its own remote, since it often runs for settings that are not saved under a storage ID yet.++**SFTP, FTP and SMB get a session pool with exclusive checkout.** These protocols authenticate the connection itself, so a session carries the credentials it was opened with. A new package, `storages/connections`, provides `SessionPool`. Each of the three model packages keeps one pool at package level, because the locator hands out a freshly loaded model on every write. The pool key is the storage ID plus a fingerprint: a SHA-256 over the fields the session is built from, as loaded from the database, with credential fields in their encrypted form. A changed host, user, password, key or TLS flag gives a new fingerprint, so the next write opens a new session without any invalidation hook. When the pool sees a new fingerprint for a storage ID, it closes every idle session under the old one, and a session that comes back under a stale fingerprint is closed instead of kept.++A write takes an idle session or dials a new one, and gives it back when it finishes. A session given back after an error is closed. At most four idle sessions stay per key, and extra ones are closed. A session idle for more than 30 seconds is probed before reuse with one cheap request under a five-second timeout: `Getwd` for SFTP, `NoOp` for FTP, a stat of the share root for SMB. A failed probe closes it, and the write takes the next one or dials. A session idle for five minutes is closed. Each session ensures the target directory once, when it is dialed, instead of walking the path on every write.++The five-minute idle timeout frees server-side connection slots for storages nobody writes to, including deleted ones. No write can reach a deleted storage, because the locator no longer finds it.++Alternatives rejected:++- *Cache whole S3 and Azure clients behind a settings fingerprint.* It needs fingerprint methods and invalidation for clients whose only per-call cost is the transport. Sharing the transport gives the same connection reuse with none of that.+- *Cache the storage model and invalidate it on save and delete.* It needs a hook in every path that saves a storage, including token refreshes and workspace transfers, to save a few milliseconds per write. A missed hook would keep using old credentials, which is the failure the spec forbids.+- *One shared session per SFTP or SMB storage.* Both libraries allow concurrent files on one session, but a full backup streaming over the same TCP connection would make WAL uploads wait behind it, and one dead connection would fail every write in flight.+- *rclone's own `fs/cache`.* It keys on the remote's name and root, not on its settings, so a changed password would keep the old remote until its expiry.++### D8. SFTP writes through the library's concurrent path++`SaveFile` for SFTP replaces the plain copy with `ReadFromWithConcurrency` on the remote file, with the library's default limit of 64 requests in flight. It dispatches blocks without waiting for each answer, whatever the reader's size. On an error the remote file may hold holes. The write then fails, and the caller discards the attempt through the pending-deletion path, as for any failed write.++Alternatives rejected:++- *Enable `UseConcurrentWrites` and keep the copy.* For a reader of unknown size, such as the compression pipe, the library silently falls back to sequential writes (`pkg/[email protected]/client.go:2022`).+- *Wrap the reader in an `io.LimitedReader` with a huge limit.* It reaches the same code path through a side effect of the size check, which a library update could change.++### D9. Cancelling a write closes its connection++Every checkout from the session pool registers `context.AfterFunc` on the write's context. If the context ends before the write returns, the function closes the session. The blocked library call then fails, the write returns, and the session goes back as failed and is not reused. The registration is stopped when the write returns normally. FTP data connections are dialed with the same timeout as the control connection instead of none. The FTP client closes a data connection only when its transfer ends, so an FTP session tracks every connection it opens and closing the session closes them all.++S3 and Azure need no extra handling, because their HTTP requests already honor the context and a failed request does not return its connection to the transport's idle pool.++Alternatives rejected:++- *An SSH or SMB keepalive timer.* It finds a dead connection between writes, but a write already blocked on it stays blocked.+- *A read and write deadline on the raw socket.* The libraries own the socket after the handshake, and a fixed deadline would also cut off a large full-backup upload that is progressing.++### D10. A throughput test with real network delay++Three helpers join the test container package. `AddEgressDelay` starts a one-shot sidecar inside the network namespace of a storage container, with the `NET_ADMIN` capability, and runs `tc qdisc add dev eth0 root netem delay 50ms`. The delay applies to every packet the storage sends, so every round trip from the test grows by 50 ms, including FTP passive data connections. `SetEgressLoss` replaces the root qdisc with one that drops every packet, which makes the storage go silent mid-connection. `ClearEgressShaping` replaces it with a plain queue again, and calling it twice is harmless. Each helper waits for the sidecar to exit and fails the test on a non-zero code. After adding the delay, the helper dials the storage port at the container's own address, since Docker's userland proxy completes a handshake on the mapped host port by itself, and fails the test when the TCP handshake took less than the delay, so a host where shaping silently does nothing cannot produce a passing run. The sidecar image, `nicolaka/netshoot`, carries iproute2, is pinned by tag and is added to the `pull-testcontainers` recipe. The storage helpers gain variants that return the container handle the sidecar needs.++The throughput test runs one subtest per covered storage type, in parallel. Each subtest starts its container, adds the delay (except for local storage), and registers a storage row pointing at it. It then writes 24 pairs of a 16 MB file and a 1 KB file through `Store.WriteFile`, the call WAL archiving uses, from four writers. The clock starts with the first write, so connection setup counts. The subtest fails when twelve seconds pass before all pairs are written, and the failure states how many pairs were written and how long that took. At the end it requests deletion of every written file and drains the deletion worker. Before any storage fix, the test runs once and the numbers per storage type are recorded under its task as the baseline.++Two seconds per pair is about 1.5 times the rate of the user who reported the problem. The expected rate after the fixes is several pairs per second for every type, and the current SFTP rate is far below one pair per second, so the floor has room for a loaded CI runner while still catching a regression.++Alternatives rejected:++- *toxiproxy.* FTP in passive mode tells the client to open a data connection to a second port, which goes around the proxy. Every protocol would also need its own proxy setup.+- *A delay inside the test process, wrapping each `net.Conn`.* Several libraries build their own connections and accept no custom dialer, so the test would miss exactly the paths it is meant to measure.+- *Assert the number of connections or handshakes.* It is deterministic, but it pins the mechanism rather than the rate the user needs, and it breaks under refactoring that keeps the rate.+- *A bandwidth test without delay.* On a shared CI runner it measures the runner's disk and CPU and would flake.++## Risks / Trade-offs++- [An FTP or SFTP server limits connections per user below four, so some uploads fail every pass.] → A failed upload releases its claim and retries, so throughput never drops below one upload at a time. The step-timing line and the existing upload-failure warning make the cause visible.+- [Storage providers with request-rate limits throttle four parallel writers.] → Same retry path. Four is low enough to stay under common per-user limits.+- [A server drops an idle pooled session in less than 30 seconds, and the next write fails on it without a probe.] → The write fails, the session is closed, and the segment is retried on the next pass one second later. The failure shows up as an upload warning.+- [The netem sidecar needs the `sch_netem` kernel module and permission to add `NET_ADMIN`.] → Both exist on the Fedora development host and on GitHub's Ubuntu runners with rootful Docker. Rootless Docker may refuse the capability; the helper then fails loudly instead of skipping.+- [The FTP test container binds its passive ports one to one on the host, so two FTP containers cannot run at once.] → The throughput test starts one FTP container, and no other package starts one.+- [The storage test package writes about 400 MB per storage container.] → Files are deleted at the end of each subtest. Subtests run in parallel, so the extra wall time is close to the slowest one.+- [The out-of-order window shows a transient "after a gap" mark on a WAL range row in the backup list.] → It lasts as long as one upload, since the `physical-backup-wal-ranges` spec marks gaps from committed rows. The mark clears when the predecessor commits. No notification is sent.+- [More segments pending at once make `realignResumePath` move files more often.] → The move is a rename within one directory tree. D2 keys the in-flight set by name, so a moved file is not uploaded twice.+- [Memory per streaming database on S3 grows to about 100 MB.] → It is bounded by the worker count. A shared limit across databases can be added later if many streaming databases share one instance.++## Migration Plan++No schema or API change. Deploying the new binary switches to concurrent uploads on the next streamer start and to reused storage connections on the first write. Rolling back to the previous binary restores sequential uploads and per-write connections. Claims left by either version are taken over by startup recovery as they are today.++## Open Questions++None.diff --git a/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/proposal.md b/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/proposal.mdnew file mode 100644index 00000000..9ad34a2d--- /dev/null+++ b/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/proposal.md@@ -0,0 +1,55 @@+## Why++Databasus archives a database's WAL one segment at a time. Each 16 MB segment waits for a claim row, compression, two storage writes and a commit before the next one starts. A source that writes about 1.25 GB of WAL per minute produces roughly 80 segments a minute, so every segment has about 750 ms for that whole cycle.++Each storage write also costs far more round trips than it needs. Every write opens a new connection to the storage: a new TCP and TLS session for S3 and Azure, a new SSH handshake for SFTP, a new login for FTP and a new SMB session for a NAS. SFTP then sends the data in 32 KB blocks and waits for an answer to each one, so one 16 MB segment takes about 500 round trips. With a storage 50 ms away, a single SFTP segment needs about 26 seconds, and the other storage types spend most of the 750 ms budget on handshakes.++When archiving falls behind, the local queue reaches its limit and Databasus stops the WAL receiver. The replication slot then keeps WAL on the source, and the source reports the slot as `extended`. Operators see a repeating "WAL chain at risk" notification and a source disk that keeps filling. A user running a busy production database hit this the day they enabled WAL streaming. Nothing can be tuned to fix it, because both the one-at-a-time loop and the per-write connections are fixed in code.++## What Changes++- Each streaming database uploads up to four finalized WAL segments at the same time. The limit is fixed in code and has no setting.+- Uploads start in WAL order, and the same segment is never uploaded twice at once.+- Storage connections are reused between writes for S3-compatible storage, Azure Blob, SFTP, FTP, SMB and rclone. A connection opened with settings that have since changed is never used again.+- SFTP sends a file's blocks without waiting for each block's answer.+- A storage that stops answering in the middle of a write no longer holds the upload past its timeout.+- An automated test adds 50 ms of network delay to each storage type except Google Drive and requires at least two segments per second through the same write path WAL archiving uses.+- The WAL gap notification fires only for a segment that will never arrive, and its range starts at the nearest segment that is archived or still queued. A later segment that finishes before an earlier one no longer counts as a gap.+- The "archiving fell behind" alert follows the point up to which WAL is archived without holes. A single stuck segment trips the alert even while later segments keep finishing.+- The periodic archiving log line also reports how long each upload step took and how many uploads ran at once. Operators can then tell slow storage round trips from a saturated network link.++## Capabilities++### New Capabilities++- `physical-backups/wal-archiving`: how finalized WAL segments of a streaming database reach storage. It covers how many segments upload at once and in what order, the write rate archiving sustains against a distant storage, how storage connections are reused and retired, when a WAL gap is reported, when archiving counts as fallen behind, and what the periodic archiving log reports.++### Modified Capabilities++None. Restore planning already stops at the first missing segment, so the recovery point users restore to does not change.++## Impact++- Module rules: `AGENTS.md` and `backend/AGENTS.md`.+- Code:+ - the WAL uploader, the watch-directory upload loop, the gap probe and the archive-staleness tracker under `backend/internal/features/backups/backups/usecases/physical/postgresql/`;+ - the S3, Azure Blob, SFTP, FTP, NAS and rclone storage models under `backend/internal/features/storages/models/`, plus a new session pool package next to them;+ - the test container helpers under `backend/internal/util/testing/containers/`, which gain a network-delay sidecar.+- Every storage write uses the reused connections, not only WAL archiving. Full backups, metadata files and deletions of the six storage types above share them.+- Storage servers: each streaming database keeps up to four connections open to its storage instead of opening one per write. Idle connections close after five minutes. An FTP or SFTP server with a low limit on connections per user can refuse some of them. A refused upload is retried on a later pass, as it is today.+- Memory per streaming database on S3 grows from about 34 MB to about 100 MB: four workers, each with a compression encoder of about 9.5 MB and the S3 write buffer of 16 MB. Today one worker holds an encoder of about 18 MB and the same buffer.+- Tests: the storage test package starts six more storage containers, each with a short-lived sidecar, and writes about 400 MB to each.+- No database migration. No API change. No frontend change.++## Out of scope++- A setting for upload concurrency, per database or global.+- Google Drive. No emulator exists to test it against, and its write path stays as it is.+- Caching storage settings read from the database. Each write still reads them, so a change takes effect on the next write.+- The two security defects found while planning this change: SFTP never verifies the server's host key, and the Google Drive token refresh prints token prefixes to standard output. Each gets its own fix.+- Section names that collide in the in-memory rclone configuration.+- The monitoring tab and stored metric history. The `wal-stream-monitoring-tab` change covers them.+- Changing how `pg_receivewal` receives WAL, its `--synchronous` flag or the local queue limit.+- Uploading partial segments. ADR-0011 keeps them out.+- Dropping or merging the per-segment metadata sidecar.+- Notification wording, including adding the workspace name.diff --git a/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/specs/physical-backups/wal-archiving/spec.md b/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/specs/physical-backups/wal-archiving/spec.mdnew file mode 100644index 00000000..7a6c4da2--- /dev/null+++ b/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/specs/physical-backups/wal-archiving/spec.md@@ -0,0 +1,179 @@+## Purpose++Defines how finalized WAL segments of a streaming PostgreSQL database reach storage, how fast that must work against a distant storage, and when Databasus reports archiving as broken or fallen behind.++## ADDED Requirements++### Requirement: Up to four segments of one database upload at the same time++For a database in WAL streaming mode, Databasus SHALL upload up to four finalized WAL segments at the same time. It SHALL NOT upload more than four segments of one database at once. Segments waiting in the local queue SHALL start uploading in ascending order of timeline first and WAL position second. Segments recovered after a restart and segments set aside from the receiver's resume path SHALL go through the same limit.++#### Scenario: Slow storage with a queued backlog++- **WHEN** eight finalized segments wait in the local queue and each segment's upload, from claim to commit, takes one second+- **THEN** four segments are uploading at once+- **AND** all eight are archived in about two seconds rather than eight++#### Scenario: Limit is never exceeded++- **WHEN** fifty finalized segments wait in the local queue+- **THEN** at no moment are more than four of that database's segments uploading++#### Scenario: Start order follows WAL position++- **WHEN** segments 10, 11, 12, 13 and 15 wait in the local queue, segment 14 waits among the segments set aside from the receiver's resume path, and one upload slot frees up after 10 to 13 have started+- **THEN** segment 14 starts next+- **AND** segment 15 starts only after segment 14++### Requirement: A segment is uploaded by at most one upload at a time++Databasus SHALL NOT start a second upload of a segment while an upload of that segment is still running. A segment whose upload failed SHALL stay in the local queue and SHALL be retried on a later pass.++#### Scenario: Queue is scanned while a slow upload runs++- **WHEN** a segment's upload is still running and the next scan of the local queue finds that segment's file+- **THEN** no second upload of that segment starts+- **AND** no "held by another in-flight claim" warning is logged for it++#### Scenario: Failed upload is retried++- **WHEN** a segment's storage write fails+- **THEN** the segment's file stays in the local queue+- **AND** a later pass uploads it again under a new object name++### Requirement: Archiving keeps up with a busy source when the storage is far away++For a storage of type S3-compatible, Azure Blob, SFTP, FTP, SMB, rclone or local, four concurrent writers SHALL store at least two pairs per second of a 16 MB segment-sized file and a 1 KB metadata file, when every response from the storage arrives 50 ms later than on a local network. These are the files WAL archiving writes for each segment. Google Drive is not covered by this requirement.++#### Scenario: Storage in another region++- **WHEN** a storage answers every request 50 ms late and four concurrent writers store twenty-four pairs of a 16 MB file and a 1 KB file+- **THEN** all twenty-four pairs are stored within twelve seconds++#### Scenario: Local storage++- **WHEN** four concurrent writers store twenty-four pairs of a 16 MB file and a 1 KB file in a local storage+- **THEN** all twenty-four pairs are stored within twelve seconds++### Requirement: Storage connections opened with old settings are never used again++A write SHALL use only a connection opened with the storage's current settings, and it SHALL authenticate with the storage's current credentials. After a storage's connection settings or credentials change, the next write SHALL use the new ones. A connection that has been idle for five minutes SHALL be closed. A deleted storage SHALL receive no further writes.++#### Scenario: Credentials are changed to wrong ones++- **WHEN** a write to a storage succeeds, the storage's password is then changed to a wrong one, and another write follows+- **THEN** the second write fails with an authentication error++#### Scenario: Credentials are corrected++- **WHEN** a write fails because of wrong credentials and the credentials are then corrected+- **THEN** the next write succeeds++#### Scenario: Idle connection++- **WHEN** no write reaches a storage for five minutes+- **THEN** every connection Databasus kept open to that storage is closed++#### Scenario: Storage is deleted++- **WHEN** a write to a storage succeeds and the storage is then deleted+- **THEN** a further write for that storage fails+- **AND** no connection to that storage stays open five minutes after the last write++### Requirement: A storage that stops answering does not hold an upload++For a storage of type S3-compatible, Azure Blob, SFTP, FTP or SMB, when the storage stops answering in the middle of a write, the write SHALL fail no later than three seconds after the deadline of the operation that started it, and the connection it used SHALL be closed. A later write SHALL NOT reuse that connection. rclone and Google Drive are not covered, because how they wait depends on the remote service behind them.++#### Scenario: Storage goes silent mid-write++- **WHEN** a storage stops answering while a segment is being written and the segment's upload deadline passes+- **THEN** the write fails within three seconds after the deadline+- **AND** the upload slot is free for the next segment+- **AND** the next write to that storage opens a new connection++### Requirement: A WAL gap is reported only for a segment that will not arrive++When a segment is archived and the archived segment just before it on the same timeline does not end where it starts, Databasus SHALL check the segment that ends where the archived segment starts. It SHALL report a WAL gap only if that segment is neither waiting as a full-size file in the local queue or among the segments set aside from the receiver's resume path, including while it uploads, nor recorded in the catalog as claimed or archived. A file shorter than a full segment does not count as waiting, because the receiver never streams a segment below its resume point again. When the hole spans several segments, only that one segment decides. The reported gap SHALL end at the start of the archived segment and SHALL start at the end of the nearest earlier segment on the same timeline that is archived or still waiting as a full-size file in either place. A gap SHALL be reported at most once per streaming session, even when segments around it finish out of order.++#### Scenario: Later segment finishes first++- **WHEN** segment 11 finishes uploading while segment 10 is still uploading+- **THEN** no WAL gap is reported+- **AND** no gap is reported after segment 10 finishes either++#### Scenario: Earlier segment failed and waits for a retry++- **WHEN** segment 11 is archived while segment 10's last upload attempt failed and its file still waits in the local queue+- **THEN** no WAL gap is reported++#### Scenario: Segment that was never received++- **WHEN** segment 20 is archived, segment 19 was never received, and segment 18 is the newest archived segment before it+- **THEN** one WAL gap notification is sent for the range between the end of segment 18 and the start of segment 20++#### Scenario: Segment before the hole is still uploading++- **WHEN** segment 20 is archived, segment 19 was never received, segment 18 is still uploading, and segment 17 is the newest archived segment before segment 20+- **THEN** one WAL gap notification is sent for the range between the end of segment 18 and the start of segment 20++#### Scenario: Segment before the gap left only a short file++- **WHEN** segment 21 is archived, segment 20 exists in the local queue only as a file shorter than a full segment, and segment 19 is the newest archived segment before it+- **THEN** one WAL gap notification is sent for the range between the end of segment 19 and the start of segment 21++#### Scenario: Hole spanning several segments++- **WHEN** segment 30 is archived, segment 27 is the newest archived segment before it, and neither segment 28 nor segment 29 is queued or in the catalog+- **THEN** one WAL gap notification is sent for the range between the end of segment 27 and the start of segment 30++#### Scenario: Two gaps finish out of order++- **WHEN** a gap before segment 40 is reported first and a gap before segment 30 is found afterwards+- **THEN** the gap before segment 30 is reported as well++### Requirement: Archiving counts as fallen behind when the gap-free archive stops advancing++Databasus SHALL track the archive progress point: the WAL position up to which every received segment is archived. It is the start of the oldest segment still uploading, waiting in the local queue or set aside from the receiver's resume path, or the end of the newest archived segment when no older segment is pending. A segment on an older timeline than the newest archived segment counts as older. A segment file counts as pending even when it is shorter than a full segment, because restore cannot pass that position until the segment is archived. Such a file is reported through this alert, not through a WAL gap notification. The "archiving fell behind" alert SHALL fire when the source keeps writing while the archive progress point does not advance for longer than the staleness threshold, even if newer segments keep being archived.++#### Scenario: One segment keeps failing while others succeed++- **WHEN** segment 50's upload fails on every attempt for longer than the staleness threshold, the source keeps writing, and segments 51 to 90 are archived in that time+- **THEN** the "archiving fell behind" alert fires++#### Scenario: Short segment file stays in the queue++- **WHEN** a segment file shorter than a full segment stays in the local queue for longer than the staleness threshold while the source keeps writing and later segments are archived+- **THEN** the "archiving fell behind" alert fires++#### Scenario: Out-of-order finishing within the threshold++- **WHEN** segments finish out of order but every pending segment is archived within a few seconds+- **THEN** the "archiving fell behind" alert does not fire++#### Scenario: Idle source++- **WHEN** the source writes nothing and no segment is pending+- **THEN** the "archiving fell behind" alert does not fire++### Requirement: The periodic archiving log reports upload timing++For each streaming database, the periodic archiving log entry SHALL report the following for the interval since the previous entry:++- the number of segments archived and their size;+- the average and the longest time of each upload step: claiming the segment, writing the segment, writing its metadata and committing;+- the highest number of uploads that ran at the same time.++It SHALL keep reporting how many bytes the archive is behind the source.++#### Scenario: Busy interval++- **WHEN** segments were archived since the previous entry+- **THEN** the entry states how many were archived and their size+- **AND** it states the average and the longest time of each upload step+- **AND** it states the highest number of uploads that ran at once++#### Scenario: Quiet interval++- **WHEN** no segment was archived since the previous entry+- **THEN** the entry reports no step timings+- **AND** it still states how many bytes the archive is behind the sourcediff --git a/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/tasks.md b/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/tasks.mdnew file mode 100644index 00000000..81d38fa3--- /dev/null+++ b/openspec/changes/archive/2026-10-18-parallel-wal-segment-uploads/tasks.md@@ -0,0 +1,170 @@+All commands run in `backend/` unless stated otherwise. Short names used below:++- `<containers>` stands for `./internal/util/testing/containers/...`+- `<storages>` stands for `./internal/features/storages/...`+- `<wal>` stands for `./internal/features/backups/backups/usecases/physical/postgresql/...`++The storage tasks (1 to 5) and the WAL tasks (6 to 10) touch different packages and can be done in either order. Within each group the order below is the dependency order.++## 1. Network delay rig for storage tests++- [x] 1.1 Give each storage container helper a variant that returns the container handle: `StartMinioContainer`, `StartAzuriteContainer`, `StartSambaContainer`, `StartFtpContainer` and `StartSftpContainer` in `internal/util/testing/containers/`, each returning `ContainerHandle` through `startContainer`. Make the existing `Start*` functions call the new variant and return its `Endpoint`, so no current caller changes. Verify with `just test-package <storages>`: `Test_Storage_BasicOperations` and the model tests still pass.+- [x] 1.2 Add `internal/util/testing/containers/netem.go` with three helpers, each taking `t` and a `ContainerHandle` (design D10):+ - `AddEgressDelay(t, handle, delay)` runs `tc qdisc add dev eth0 root netem delay <delay>`;+ - `SetEgressLoss(t, handle)` runs `tc qdisc replace dev eth0 root netem loss 100%`;+ - `ClearEgressShaping(t, handle)` runs `tc qdisc replace dev eth0 root pfifo`, so a cleanup may call it again after the test already cleared the shaping.++ Each helper starts a one-shot sidecar through `testcontainers.Run` with `WithHostConfigModifier` setting `NetworkMode` to `container:<id>` and `CapAdd` to `NET_ADMIN`, with no exposed ports and `wait.ForExit()`. It fails the test when the exit code is not zero and includes the sidecar's log in the message. After adding the delay, `AddEgressDelay` dials the container's own address and fails the test when the TCP handshake took less than the delay. The mapped host port does not work for this check, because Docker's userland proxy completes that handshake itself. Pin the sidecar image by tag in a constant next to the helpers and extend the `pull-testcontainers` regex in `backend/Justfile` so the image is pulled with the others. Verify with `netem_test.go`, run through `just test-package <containers>`:+ - `Test_AddEgressDelay_WhenAppliedToMinio_TcpHandshakeTakesAtLeastTheDelay`: the handshake is under 50 ms before the call and at least 50 ms after it;+ - `Test_SetEgressLoss_WhenApplied_DialTimesOut`: a dial with a two-second timeout fails after the call;+ - `Test_ClearEgressShaping_AfterLoss_DialSucceedsAgain`: the dial succeeds after shaping is cleared.+ - `Test_ClearEgressShaping_WhenCalledTwice_Succeeds`: a second clear succeeds and the dial still works.++## 2. Baseline throughput test++- [x] 2.1 Extract the per-type storage setup that `Test_Storage_BasicOperations` builds inline in `internal/features/storages/model_test.go` (the MinIO bucket, the Azurite container, the SFTP, FTP and Samba credentials, the rclone remote pointing at MinIO, the local temp directory) into one test helper. It takes a storage type and a container handle and returns a storage row saved through `storageRepository.Save` with encrypted credentials, removed through `RemoveTestStorage` in `t.Cleanup`. Make `Test_Storage_BasicOperations` use the helper instead of its inline setup. Verify with `just test-package <storages>`: `Test_Storage_BasicOperations` still passes for every type it covered.+- [x] 2.2 Add `Test_WriteFile_WhenStorageAnswers50msLate_SustainsTwoSegmentsPerSecond` to `model_test.go`, with one parallel subtest per type, labelled `S3`, `AzureBlob`, `SFTP`, `FTP`, `NAS`, `Rclone` and `Local`. Each subtest:+ - starts its container (the rclone subtest starts MinIO) and calls `AddEgressDelay` with 50 ms, except `Local`;+ - registers the storage with the helper from 2.1;+ - generates one random 16 MiB buffer and one 1 KiB buffer;+ - feeds 24 jobs to four goroutines, and each job writes the 16 MiB file and then the 1 KiB file under unique names through `GetStorageFileStore().WriteFile`;+ - starts its clock at the first write and fails when 12 seconds pass before all 24 pairs are written, stating how many pairs were written and the elapsed time;+ - requests deletion of every written file in one transaction through `RequestFileDeletions` and calls `DrainStorageFileDeletions` in `t.Cleanup`.++ Verify by running `just test-package <storages>` once before any fix from section 4. Record under this task, as an indented note, each subtest's result with pairs written and elapsed time. `Local` is expected to pass and `SFTP` to fail far below the floor; the other results are recorded as observed and are the baseline for section 4.++ Baseline recorded on 2026-10-18 on the Fedora development host:+ - `Local`: passed, 24 pairs in 1.1 s.+ - `S3`: passed, 24 pairs in 5.3 s.+ - `AzureBlob`: passed, 24 pairs in 6.3 s.+ - `Rclone`: passed, 24 pairs in 6.7 s.+ - `FTP`: failed, 4 pairs within 12 s.+ - `SFTP`: failed, 0 pairs within 12 s.+ - `NAS`: failed after 4 pairs. Concurrent writes race on creating the target directory, and the losers fail with "mkdir test-files: file already exists".++## 3. Session pool for SFTP, FTP and SMB++These tasks touch only the new package, so their verification runs only that package. The throughput subtests from 2.2 stay red until section 4.++- [x] 3.1 Create the package `internal/features/storages/connections` (package `storage_connections`) with:+ - `Key`, holding the storage ID and a fingerprint;+ - `Fingerprint(parts ...any) string`, a hex SHA-256 over the JSON encoding of its arguments;+ - `SessionPool[T]`, built with a close function and an idle cap (4 in production), whose `Checkout(ctx, key, dial)` returns a lease that hands out the session and has `Release(err error)`. The dial function comes with each checkout, because only the caller holds the decrypted settings the key was fingerprinted from.++ An idle session for the key is reused, otherwise a new one is dialed. `Release(nil)` returns the session to the idle list unless the idle cap is reached, in which case the session is closed. `Release` with an error closes the session. Verify with `session_pool_test.go` against a fake session type, run through `just test-package ./internal/features/storages/connections/...`:+ - `Fingerprint` gives the same value for equal arguments and a different one when any argument changes;+ - sequential checkouts reuse one session;+ - two concurrent checkouts get two different sessions;+ - a release with an error closes the session and the next checkout dials;+ - a fifth idle session is closed on release.+- [x] 3.2 Extend `SessionPool` with a probe function, a probe threshold (30 seconds), a probe timeout (5 seconds) and an idle timeout (5 minutes), all passed at construction so tests can use short values (design D7, D9):+ - a session idle longer than the probe threshold is probed under the probe timeout before reuse, and a failed probe closes it and moves on to the next idle session or a dial;+ - `context.AfterFunc` on the checkout's context closes the session if the context ends before `Release`, and `Release` stops it;+ - a checkout with a new fingerprint for a known storage ID closes every idle session of the old one, and `Release` of a lease under a stale fingerprint closes the session instead of keeping it;+ - an idle session is closed after the idle timeout through a timer, with no background loop.++ Verify with `session_pool_test.go`, run through `just test-package ./internal/features/storages/connections/...`:+ - a session idle past the probe threshold is probed, and when the probe fails the checkout dials a new one;+ - cancelling the context of a lease whose holder is blocked closes its session, and the blocked call returns;+ - a new fingerprint closes the old fingerprint's idle sessions, and a lease of the old fingerprint released afterwards is closed rather than kept;+ - an idle session is closed after the idle timeout, which the test waits for as a condition with a limit.++## 4. Reused connections per storage type++Each task here makes its own throughput subtest pass. Until the last task of this section is done, subtests of types whose task is still open may keep failing, and each verification below says which subtests must pass.++- [x] 4.1 S3: replace the per-call `http.Transport` built in `getClientParams` with package-level transports, one per `SkipTLSVerify` value, created once with the existing timeouts and `MaxIdleConnsPerHost` set to 16. Keep building `minio.Client` and `minio.Core` per call from freshly decrypted credentials over those transports. When `S3Region` is empty, look the region up once with `GetBucketLocation` and keep it in a package-level map keyed by endpoint and bucket, so later clients get it without a request (design D7). Verify with `just test-package <storages>`: the `S3` and `Local` throughput subtests, `Test_Storage_BasicOperations` and the S3 model tests (`internal/features/storages/models/s3`) pass. Throughput subtests of types whose task is still open may fail.+- [x] 4.2 Azure Blob: replace the per-call transport built in `buildClientOptions` with one package-level transport with the existing timeouts and `MaxIdleConnsPerHost` set to 16, and keep building the client per call. Verify with `just test-package <storages>`: the `AzureBlob` throughput subtest, `Test_Storage_BasicOperations` and the Azure model tests pass. Throughput subtests of types whose task is still open may fail.+- [x] 4.3 rclone: keep a package-level map from storage ID to the built remote and a fingerprint of the remote's configuration text (design D7). A write with a matching fingerprint reuses the remote. A write with a different fingerprint builds a new remote and retires the old one. A remote unused for the idle timeout is retired and dropped through a timer that starts when its last use ends. A retired remote is shut down once no call uses it, so a long full backup through it is never cut off. `TestConnection` builds its own remote and shuts it down afterwards, because it often runs for settings that are not saved yet. The idle timeout is five minutes in production and injectable for tests. `fs.NewFs` and `rcloneConfigMu` then run only on a miss. Route every remote-building method through the map. Verify with tests in `internal/features/storages/models/rclone/model_test.go` against the existing local-type remote, with a short idle timeout:+ - two writes with the same configuration build the remote once;+ - a write with changed configuration text builds a new remote and retires the old one;+ - a remote that is not used is retired and dropped after the idle timeout, which the test waits for as a condition with a limit.++ They must pass with `just test-package <storages>`, together with the `Rclone` throughput subtest, `Test_Storage_BasicOperations` and the existing rclone model tests. Throughput subtests of types whose task is still open may fail.+- [x] 4.4 SFTP: add a fingerprint method over the host, port, user, password, private key and path. Keep a package-level `SessionPool` whose session is the SSH client plus the SFTP client. The dial creates the target directory once, the probe calls `Getwd`, and the close function closes both clients. Make `SaveFile` check out a session under its context and write with `ReadFromWithConcurrency(reader, 0)` instead of `io.Copy` (design D8). Make each read attempt of `GetFile` hold its own lease until that attempt's reader is closed, and make the other methods check out and release around their calls. Make the dial honor its context through the SSH handshake, and make the directory creation succeed when a concurrent session created the same directory first. Verify with `just test-package <storages>`: the `SFTP` throughput subtest, `Test_Storage_BasicOperations` and the SFTP model tests pass.+- [x] 4.5 FTP: add a fingerprint method over the host, port, user, password, TLS mode and path. Keep a package-level `SessionPool` whose session is a logged-in `ServerConn`. The dial creates the target directory once, the probe calls `NoOp`, and the close function closes every connection the session opened. A custom dial function tracks the control and data connections and dials both with the existing 30-second timeout, so closing a session also unblocks a data transfer stuck on a silent server. Make the directory creation succeed when a concurrent session created the same directory first. Route every method through the pool, with `GetFile` holding its lease until its reader is closed. Verify with `just test-package <storages>`: the `FTP` throughput subtest and `Test_Storage_BasicOperations` pass.+- [x] 4.6 NAS: add a fingerprint method over the host, port, share, user, password, domain, TLS flag and path. Keep a package-level `SessionPool` whose session is the TCP connection, the SMB session and the mounted share. The dial creates the target directory once, the probe stats the share root, and the close function closes the connection, which drops the tree and the session with it. Make every call use the share through `WithContext` with the caller's context, and route every method through the pool. Make the directory creation succeed when a concurrent session created the same directory first, which the baseline in 2.2 showed failing. Verify with `just test-package <storages>`: the `NAS` throughput subtest, `Test_Storage_BasicOperations` and the NAS model tests pass.+- [x] 4.7 Local: check that the `Local` throughput subtest still passes with every other change in place. If it does not, find the cause before changing the floor. Verify with `just test-package <storages>`: every throughput subtest and every other test in the package pass.++## 5. Credential changes and silent storages++- [x] 5.1 Add two tests to `model_test.go` for the spec requirement "Storage connections opened with old settings are never used again", each with subtests for `S3`, `AzureBlob`, `SFTP`, `FTP`, `NAS` and `Rclone`:+ - `Test_WriteFile_WhenCredentialsChangeToWrongOnes_NextWriteFailsUntilCorrected` writes one file successfully through `GetStorageFileStore().WriteFile`, then saves the storage with a wrong password or secret through `storageRepository.Save` with the value encrypted, and requires the next write to fail. It then saves the correct value back and requires the next write to succeed.+ - `Test_WriteFile_AfterStorageIsDeleted_Fails` writes one file successfully, removes the storage with `RemoveTestStorage`, and requires the next write for that storage ID to fail.++ The five-minute close of idle connections is covered by the pool test in 3.2 for SFTP, FTP and SMB, by the remote-cache test in 4.3 for rclone, and by the transports' existing 90-second idle timeout for S3 and Azure. Verify with `just test-package <storages>`.+- [x] 5.2 Add `Test_WriteFile_WhenStorageGoesSilentMidConnection_FailsByDeadlineAndRedials` to `model_test.go`, with subtests for `S3`, `AzureBlob`, `SFTP`, `FTP` and `NAS` (spec: "A storage that stops answering does not hold an upload"). Each subtest:+ - writes one file successfully, so a pooled connection exists;+ - calls `SetEgressLoss`;+ - writes again with a five-second context and requires the call to return an error within eight seconds, the deadline plus the three seconds the spec allows, waiting on a channel with that limit;+ - calls `ClearEgressShaping` and requires the next write to succeed, which shows that the dead connection was not reused.++ Verify with `just test-package <storages>`.++## 6. Step timing in the archiving rollup++- [x] 6.1 Time the claim insert in `ProcessSegment` and `RecoverSegment` and pass the duration to `uploadAndCommit`. Time the segment write, the sidecar write and the commit with the gap probe inside `uploadAndCommit`. Add `RecordConcurrentUploads(count int)`, which keeps the highest count since the last take. Replace `TakeArchivedSinceLastReport` with a method returning one snapshot struct that carries the archived count and bytes, the per-step totals and maximums and the highest concurrency, and resets all of them. Verify with a test in `wal_upload_test.go` that archives two segments through the mock storage, records a concurrency of 3, and asserts the snapshot's counts, the non-zero step maximums, the concurrency, and that a second take is empty. It must pass with `just test-package <wal>`.+- [x] 6.2 Format the snapshot into the existing INFO line in `reportArchivingProgress`: segments and size, average and longest time per step, highest concurrency, and bytes behind the source. Keep the DEBUG line for a quiet interval with only the bytes behind. Verify with a test in `lag_monitor_test.go` that captures the log for a busy interval and a quiet interval, and passes with `just test-package <wal>`.++## 7. Gap probe for out-of-order commits++- [x] 7.1 Add `WatchDir` to `WalUploadDeps`, set it in `NewWalStreamSupervisor`, and make the test uploader helper in `wal_upload_test.go` use the directory its tests write segment files into. Make `probeChainGap` follow design D3:+ - check the segment that ends at the committed segment's start: its file in the watch directory, then in `pending-upload/`, then its row in the catalog by chain key, and report only when all three miss;+ - count only full-size files in both directories, so a short file never suppresses a gap;+ - start the reported range at the larger of the newest committed predecessor's end and the end of the highest full-size same-timeline segment file below the committed segment in the two directories;+ - delete `lastNotifiedGapEnd`, its lock use in the probe and the comments that describe it.++ Verify with tests in `wal_upload_test.go`, run through `just test-package <wal>`, for these spec scenarios:+ - "Later segment finishes first";+ - "Earlier segment failed and waits for a retry";+ - "Segment that was never received";+ - "Segment before the hole is still uploading": segment 17 committed, the file of 18 in the watch directory, 20 committed, and the reported range starts at the end of 18;+ - "Hole spanning several segments";+ - "Two gaps finish out of order";+ - one case where the predecessor's file sits only in `pending-upload/`;+ - "Segment before the gap left only a short file": the file of segment 20 is shorter than a full segment, 19 is committed, 21 commits, and one gap from the end of 19 to the start of 21 is reported.++ The existing `Test_WalUpload_PostUpdateGapProbe_EmitsSingleNotificationPerOutage` and `Test_WalUpload_ContiguousSegments_NoGapNotification` must still pass.++## 8. Concurrent dispatcher++- [x] 8.1 Replace the sequential `scanAndUpload` and `sweepPendingUploads` walk with a dispatcher. It lists the watch directory and `pending-upload/`, sorts segments by timeline and WAL position, keeps an in-flight set keyed by WAL file name, starts uploads while fewer than four run, reports each new count through `RecordConcurrentUploads`, and wakes on its tick and on every worker completion. Timeline history files stay on the dispatcher. Verify with tests in `wal_watch_dir_test.go`, run through `just test-package <wal>`, against a mock storage whose writes block until released:+ - eight queued segments reach four concurrent writes and never five;+ - the fifth segment starts as soon as one write is released, which the test waits for as a condition with a limit;+ - the spec scenario "Start order follows WAL position": with 10 to 13 and 15 in the watch directory and 14 in `pending-upload/`, 14 starts before 15 when the first slot frees up;+ - a segment whose write is still blocked is not started again on the next tick, and the captured log shows no "held by another in-flight claim" warning while the write stays blocked.+- [x] 8.2 Route the startup recovery of leftover segments through the same dispatcher with `RecoverSegment`, as one pass over the files listed at startup that returns when those uploads have finished (design D1). `Run` keeps calling it before the receiver spawns. Verify with tests in `wal_watch_dir_test.go`, run through `just test-package <wal>`:+ - several finalized segments with their own NULL claims all commit, and recovery used more than one concurrent write;+ - with a mock storage whose writes always fail, recovery returns after one attempt per segment and the segment files stay in place.+- [x] 8.3 Make the existing pending-upload tests pass on the dispatcher: `Test_SweepPendingUploads_WhenUploadSucceeds_RemovesSegmentFromStagingDir` and `Test_SweepPendingUploads_WhenUploadFails_RetainsSegmentForNextTick`, renamed if the function they name no longer exists. Add one test where a segment is renamed into `pending-upload/` while its upload is blocked, and assert it is written to storage once. Verify with `just test-package <wal>`.++## 9. Staleness on the archive progress point++- [x] 9.1 Make `getLastCommittedWal` also return the newest committed segment's timeline. Add the pure function that combines the lowest pending segment (timeline and start, or none) with the newest committed segment (timeline and end) into the archive progress point (design D4). Verify with tests in `wal_staleness_test.go`, run through `just test-package <wal>`:+ - "One segment keeps failing while others succeed": the point stays at segment 50's start while the newest committed end moves through 51 to 90, and the tracker reports staleness once the threshold passes;+ - "Out-of-order finishing within the threshold": the point moves to the newest committed end once nothing is pending below it;+ - "Idle source": the point equals the newest committed end when nothing is pending;+ - a pending segment on an older timeline holds the point even when its position is higher than the newest committed end on the newer timeline.+- [x] 9.2 Have the dispatcher store the lowest listed segment, short files included, under a mutex after each listing. Have `reportArchiveStalenessIfDue` pass the combined progress point to the tracker. Verify with tests in `wal_watch_dir_test.go`, run through `just test-package <wal>`:+ - with segment 50's write blocked while 51 to 53 commit, the stored lowest segment is 50;+ - with a short file for segment 60 in the watch directory, the stored lowest segment is 60.++ The existing staleness tests must still pass.++## 10. Encoder memory++- [x] 10.1 Create the WAL segment zstd encoder with `zstd.WithEncoderConcurrency(1)` and `zstd.WithLowerEncoderMem(true)` (design D6). Verify with `just test-package <wal>`: `Test_BuildWalSegmentArtifactReader_WhenLargeSegment_StreamsAndCountsArtifact` and the restore-side decoding tests in the package pass.++## 11. Wrap-up++- [x] 11.1 Run the neighbouring packages that exercise WAL streaming end to end. Verify with `just test-package ./internal/features/backups/backups/backuping/physical/... ./internal/features/backups/backups/core/physical/...` passing.+- [x] 11.2 Search `docs/`, the READMEs, `adr/` and code comments for statements that segments upload one at a time, that each storage write opens its own connection, or that describe `lastNotifiedGapEnd`, and bring each in line with the new behavior. Then run the formatter and the linters. Verify with `just format` and `just lint` at the repository root finishing clean.++## Change verification++```sh+just test-package ./internal/util/testing/containers/...+just test-package ./internal/features/storages/...+just test-package ./internal/features/backups/backups/usecases/physical/postgresql/...+just test-package ./internal/features/backups/backups/backuping/physical/... ./internal/features/backups/backups/core/physical/...+```diff --git a/openspec/specs/physical-backups/wal-archiving/spec.md b/openspec/specs/physical-backups/wal-archiving/spec.mdnew file mode 100644index 00000000..4d3d9220--- /dev/null+++ b/openspec/specs/physical-backups/wal-archiving/spec.md@@ -0,0 +1,181 @@+# WAL Archiving Specification++## Purpose++Defines how finalized WAL segments of a streaming PostgreSQL database reach storage, how fast that must work against a distant storage, and when Databasus reports archiving as broken or fallen behind.++## Requirements++### Requirement: Up to four segments of one database upload at the same time++For a database in WAL streaming mode, Databasus SHALL upload up to four finalized WAL segments at the same time. It SHALL NOT upload more than four segments of one database at once. Segments waiting in the local queue SHALL start uploading in ascending order of timeline first and WAL position second. Segments recovered after a restart and segments set aside from the receiver's resume path SHALL go through the same limit.++#### Scenario: Slow storage with a queued backlog++- **WHEN** eight finalized segments wait in the local queue and each segment's upload, from claim to commit, takes one second+- **THEN** four segments are uploading at once+- **AND** all eight are archived in about two seconds rather than eight++#### Scenario: Limit is never exceeded++- **WHEN** fifty finalized segments wait in the local queue+- **THEN** at no moment are more than four of that database's segments uploading++#### Scenario: Start order follows WAL position++- **WHEN** segments 10, 11, 12, 13 and 15 wait in the local queue, segment 14 waits among the segments set aside from the receiver's resume path, and one upload slot frees up after 10 to 13 have started+- **THEN** segment 14 starts next+- **AND** segment 15 starts only after segment 14++### Requirement: A segment is uploaded by at most one upload at a time++Databasus SHALL NOT start a second upload of a segment while an upload of that segment is still running. A segment whose upload failed SHALL stay in the local queue and SHALL be retried on a later pass.++#### Scenario: Queue is scanned while a slow upload runs++- **WHEN** a segment's upload is still running and the next scan of the local queue finds that segment's file+- **THEN** no second upload of that segment starts+- **AND** no "held by another in-flight claim" warning is logged for it++#### Scenario: Failed upload is retried++- **WHEN** a segment's storage write fails+- **THEN** the segment's file stays in the local queue+- **AND** a later pass uploads it again under a new object name++### Requirement: Archiving keeps up with a busy source when the storage is far away++For a storage of type S3-compatible, Azure Blob, SFTP, FTP, SMB, rclone or local, four concurrent writers SHALL store at least two pairs per second of a 16 MB segment-sized file and a 1 KB metadata file, when every response from the storage arrives 50 ms later than on a local network. These are the files WAL archiving writes for each segment. Google Drive is not covered by this requirement.++#### Scenario: Storage in another region++- **WHEN** a storage answers every request 50 ms late and four concurrent writers store twenty-four pairs of a 16 MB file and a 1 KB file+- **THEN** all twenty-four pairs are stored within twelve seconds++#### Scenario: Local storage++- **WHEN** four concurrent writers store twenty-four pairs of a 16 MB file and a 1 KB file in a local storage+- **THEN** all twenty-four pairs are stored within twelve seconds++### Requirement: Storage connections opened with old settings are never used again++A write SHALL use only a connection opened with the storage's current settings, and it SHALL authenticate with the storage's current credentials. After a storage's connection settings or credentials change, the next write SHALL use the new ones. A connection that has been idle for five minutes SHALL be closed. A deleted storage SHALL receive no further writes.++#### Scenario: Credentials are changed to wrong ones++- **WHEN** a write to a storage succeeds, the storage's password is then changed to a wrong one, and another write follows+- **THEN** the second write fails with an authentication error++#### Scenario: Credentials are corrected++- **WHEN** a write fails because of wrong credentials and the credentials are then corrected+- **THEN** the next write succeeds++#### Scenario: Idle connection++- **WHEN** no write reaches a storage for five minutes+- **THEN** every connection Databasus kept open to that storage is closed++#### Scenario: Storage is deleted++- **WHEN** a write to a storage succeeds and the storage is then deleted+- **THEN** a further write for that storage fails+- **AND** no connection to that storage stays open five minutes after the last write++### Requirement: A storage that stops answering does not hold an upload++For a storage of type S3-compatible, Azure Blob, SFTP, FTP or SMB, when the storage stops answering in the middle of a write, the write SHALL fail no later than three seconds after the deadline of the operation that started it, and the connection it used SHALL be closed. A later write SHALL NOT reuse that connection. rclone and Google Drive are not covered, because how they wait depends on the remote service behind them.++#### Scenario: Storage goes silent mid-write++- **WHEN** a storage stops answering while a segment is being written and the segment's upload deadline passes+- **THEN** the write fails within three seconds after the deadline+- **AND** the upload slot is free for the next segment+- **AND** the next write to that storage opens a new connection++### Requirement: A WAL gap is reported only for a segment that will not arrive++When a segment is archived and the archived segment just before it on the same timeline does not end where it starts, Databasus SHALL check the segment that ends where the archived segment starts. It SHALL report a WAL gap only if that segment is neither waiting as a full-size file in the local queue or among the segments set aside from the receiver's resume path, including while it uploads, nor recorded in the catalog as claimed or archived. A file shorter than a full segment does not count as waiting, because the receiver never streams a segment below its resume point again. When the hole spans several segments, only that one segment decides. The reported gap SHALL end at the start of the archived segment and SHALL start at the end of the nearest earlier segment on the same timeline that is archived or still waiting as a full-size file in either place. A gap SHALL be reported at most once per streaming session, even when segments around it finish out of order.++#### Scenario: Later segment finishes first++- **WHEN** segment 11 finishes uploading while segment 10 is still uploading+- **THEN** no WAL gap is reported+- **AND** no gap is reported after segment 10 finishes either++#### Scenario: Earlier segment failed and waits for a retry++- **WHEN** segment 11 is archived while segment 10's last upload attempt failed and its file still waits in the local queue+- **THEN** no WAL gap is reported++#### Scenario: Segment that was never received++- **WHEN** segment 20 is archived, segment 19 was never received, and segment 18 is the newest archived segment before it+- **THEN** one WAL gap notification is sent for the range between the end of segment 18 and the start of segment 20++#### Scenario: Segment before the hole is still uploading++- **WHEN** segment 20 is archived, segment 19 was never received, segment 18 is still uploading, and segment 17 is the newest archived segment before segment 20+- **THEN** one WAL gap notification is sent for the range between the end of segment 18 and the start of segment 20++#### Scenario: Segment before the gap left only a short file++- **WHEN** segment 21 is archived, segment 20 exists in the local queue only as a file shorter than a full segment, and segment 19 is the newest archived segment before it+- **THEN** one WAL gap notification is sent for the range between the end of segment 19 and the start of segment 21++#### Scenario: Hole spanning several segments++- **WHEN** segment 30 is archived, segment 27 is the newest archived segment before it, and neither segment 28 nor segment 29 is queued or in the catalog+- **THEN** one WAL gap notification is sent for the range between the end of segment 27 and the start of segment 30++#### Scenario: Two gaps finish out of order++- **WHEN** a gap before segment 40 is reported first and a gap before segment 30 is found afterwards+- **THEN** the gap before segment 30 is reported as well++### Requirement: Archiving counts as fallen behind when the gap-free archive stops advancing++Databasus SHALL track the archive progress point: the WAL position up to which every received segment is archived. It is the start of the oldest segment still uploading, waiting in the local queue or set aside from the receiver's resume path, or the end of the newest archived segment when no older segment is pending. A segment on an older timeline than the newest archived segment counts as older. A segment file counts as pending even when it is shorter than a full segment, because restore cannot pass that position until the segment is archived. Such a file is reported through this alert, not through a WAL gap notification. The "archiving fell behind" alert SHALL fire when the source keeps writing while the archive progress point does not advance for longer than the staleness threshold, even if newer segments keep being archived.++#### Scenario: One segment keeps failing while others succeed++- **WHEN** segment 50's upload fails on every attempt for longer than the staleness threshold, the source keeps writing, and segments 51 to 90 are archived in that time+- **THEN** the "archiving fell behind" alert fires++#### Scenario: Short segment file stays in the queue++- **WHEN** a segment file shorter than a full segment stays in the local queue for longer than the staleness threshold while the source keeps writing and later segments are archived+- **THEN** the "archiving fell behind" alert fires++#### Scenario: Out-of-order finishing within the threshold++- **WHEN** segments finish out of order but every pending segment is archived within a few seconds+- **THEN** the "archiving fell behind" alert does not fire++#### Scenario: Idle source++- **WHEN** the source writes nothing and no segment is pending+- **THEN** the "archiving fell behind" alert does not fire++### Requirement: The periodic archiving log reports upload timing++For each streaming database, the periodic archiving log entry SHALL report the following for the interval since the previous entry:++- the number of segments archived and their size;+- the average and the longest time of each upload step: claiming the segment, writing the segment, writing its metadata and committing;+- the highest number of uploads that ran at the same time.++It SHALL keep reporting how many bytes the archive is behind the source.++#### Scenario: Busy interval++- **WHEN** segments were archived since the previous entry+- **THEN** the entry states how many were archived and their size+- **AND** it states the average and the longest time of each upload step+- **AND** it states the highest number of uploads that ran at once++#### Scenario: Quiet interval++- **WHEN** no segment was archived since the previous entry+- **THEN** the entry reports no step timings+- **AND** it still states how many bytes the archive is behind the source