PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (tests): Check that storage writes reuse their connection by counting connections and bound only a single segment write by time, so the 50 ms latency test stops measuring how loaded the CI runner is
Rostislav Dugin487b39b2aa2e
diff --git a/backend/internal/features/storages/model_test.go b/backend/internal/features/storages/model_test.goindex b875a7b9..ac21267f 100644--- a/backend/internal/features/storages/model_test.go+++ b/backend/internal/features/storages/model_test.go@@ -38,6 +38,7 @@ import (db "databasus-backend/internal/storage""databasus-backend/internal/util/encryption""databasus-backend/internal/util/logger"+ test_utils "databasus-backend/internal/util/testing""databasus-backend/internal/util/testing/containers")@@ -217,20 +218,23 @@ func Test_Storage_BasicOperations(t *testing.T) {}}-// WAL archiving writes a 16 MiB segment and a 1 KiB metadata sidecar per segment, and a busy source-// needs about 1.3 segments per second. Two pairs per second against a storage 50 ms away leaves room-// for a loaded CI runner while still catching a write path that pays a round trip per block or a-// handshake per file.+// The storage answers every packet 50 ms late, so each round trip a write pays shows up in its+// duration and in nothing else the runner's load can inflate as much. The check targets the two+// mistakes that keep WAL archiving from keeping up with a distant storage: a handshake per file and+// a round trip per block. The connection count catches the first exactly, while S3, Azure and+// Rclone reconnect for only one TCP round trip, too little for any timing to tell apart.const (- throughputEgressDelay = 50 * time.Millisecond- throughputPairCount = 24- throughputWriterCount = 4- throughputTimeLimit = 12 * time.Second- segmentSizedFileBytes = 16 * 1024 * 1024- metadataSizedFileBytes = 1024- writeFailureReportLimit = 3+ roundTripEgressDelay = 50 * time.Millisecond+ reusedFileWriteCount = 10+ segmentSizedFileBytes = 16 * 1024 * 1024+ metadataSizedFileBytes = 1024)+// A round trip per 32 KiB block, the buffer io.Copy streams with, costs a segment 512 round trips,+// about 25s. A streamed segment takes 0.5-1s on a developer machine, so a quarter of the slow path+// leaves the loaded CI runner several times the room it needs and still fails the slow path.+const segmentWriteLimit = 128 * roundTripEgressDelay+type remoteStorageTarget struct {name stringstorageType StorageType@@ -248,124 +252,58 @@ func getRemoteStorageTargets() []remoteStorageTarget {}}-func Test_WriteFile_WhenStorageAnswers50msLate_SustainsTwoSegmentsPerSecond(t *testing.T) {- targets := append(getRemoteStorageTargets(), remoteStorageTarget{"Local", StorageTypeLocal, nil})-- for _, target := range targets {+func Test_WriteFile_WhenStorageAnswers50msLate_PaysNoHandshakePerFileOrRoundTripPerBlock(t *testing.T) {+ for _, target := range getRemoteStorageTargets() {t.Run(target.name, func(t *testing.T) {t.Parallel()- var container containers.ContainerHandle- if target.startContainer != nil {- container = target.startContainer(t)- containers.AddEgressDelay(t, container, throughputEgressDelay)- }+ container := target.startContainer(t)+ containers.AddEgressDelay(t, container, roundTripEgressDelay)+ t.Cleanup(func() { containers.ClearEgressShaping(t, container) })- storage := createTestStorageOfType(t, target.storageType, container)+ // FTP data connections still go straight to the container's passive ports, so only+ // control connections, one per session, cross the forwarder.+ forwarder := test_utils.StartCountingForwarder(t, addressOf(container.Endpoint))+ forwardedContainer := container+ forwardedContainer.Endpoint = containers.Endpoint{Host: forwarder.Host, Port: forwarder.Port}- segmentContent := generateRandomContent(t, segmentSizedFileBytes)+ storage := createTestStorageOfType(t, target.storageType, forwardedContainer)+ writtenFiles := trackWrittenFiles(t, storage.ID)metadataContent := generateRandomContent(t, metadataSizedFileBytes)+ segmentContent := generateRandomContent(t, segmentSizedFileBytes)- written := writeSegmentPairsConcurrently(t, storage.ID, segmentContent, metadataContent)+ // The first write opens the session, and its handshake is not what the test measures.+ require.NoError(t, writtenFiles.writeUniqueFile(t.Context(), metadataContent))+ connectionsAfterFirstWrite := forwarder.GetAcceptedCount()- if target.startContainer != nil {- t.Cleanup(func() { containers.ClearEgressShaping(t, container) })+ for range reusedFileWriteCount {+ require.NoError(t, writtenFiles.writeUniqueFile(t.Context(), metadataContent))}- require.Empty(t, written.failures, "writes failed after %d pairs in %s", written.pairCount, written.elapsed)- require.Equal(+ assert.Equal(t,- throughputPairCount,- written.pairCount,- "only %d of %d pairs were written within %s, the writers stopped after %s",- written.pairCount,- throughputPairCount,- throughputTimeLimit,- written.elapsed,+ connectionsAfterFirstWrite,+ forwarder.GetAcceptedCount(),+ "%d files written one after another must reuse the connection the first write opened",+ reusedFileWriteCount,)- t.Logf("%d pairs written in %s", written.pairCount, written.elapsed)- })- }-}--type writtenPairs struct {- pairCount int- elapsed time.Duration- failures []error-}+ startedAt := time.Now()+ require.NoError(t, writtenFiles.writeUniqueFile(t.Context(), segmentContent))+ segmentWriteDuration := time.Since(startedAt)-// Writers stop taking jobs once the time limit passes, and the call waits for the writes already in-// flight, so every file that reached the storage is handed to cleanup.-func writeSegmentPairsConcurrently(- t *testing.T,- storageID uuid.UUID,- segmentContent []byte,- metadataContent []byte,-) writtenPairs {- t.Helper()-- var (- resultMutex sync.Mutex- result writtenPairs- writersWaitGroup sync.WaitGroup- )-- writtenFiles := trackWrittenFiles(t, storageID)-- pairIndexes := make(chan int, throughputPairCount)- for pairIndex := range throughputPairCount {- pairIndexes <- pairIndex- }- close(pairIndexes)-- startedAt := time.Now()-- writePair := func(pairIndex int) error {- pairName := fmt.Sprintf("throughput-%s-%d", uuid.New(), pairIndex)-- if err := writtenFiles.write(t.Context(), pairName, segmentContent); err != nil {- return err- }-- return writtenFiles.write(t.Context(), pairName+".metadata", metadataContent)- }-- for range throughputWriterCount {- writersWaitGroup.Go(func() {- for pairIndex := range pairIndexes {- if time.Since(startedAt) > throughputTimeLimit {- return- }-- err := writePair(pairIndex)- isWithinLimit := time.Since(startedAt) <= throughputTimeLimit-- resultMutex.Lock()- if err != nil && len(result.failures) < writeFailureReportLimit {- result.failures = append(result.failures, err)- }-- if err == nil && isWithinLimit {- result.pairCount++- result.elapsed = time.Since(startedAt)- }- resultMutex.Unlock()+ assert.Less(+ t,+ segmentWriteDuration,+ segmentWriteLimit,+ "a %d MiB segment took %s, as long as paying a round trip for every four 32 KiB blocks",+ segmentSizedFileBytes/1024/1024,+ segmentWriteDuration,+ )- if err != nil {- return- }- }+ t.Logf("segment written in %s", segmentWriteDuration)})}-- writersWaitGroup.Wait()-- if result.pairCount < throughputPairCount {- result.elapsed = time.Since(startedAt)- }-- return result}// Every file a test writes, the failed attempts included, is handed to the deletion worker at thediff --git a/backend/internal/util/testing/counting_forwarder.go b/backend/internal/util/testing/counting_forwarder.gonew file mode 100644index 00000000..0f979f49--- /dev/null+++ b/backend/internal/util/testing/counting_forwarder.go@@ -0,0 +1,120 @@+package testing++import (+ "context"+ "net"+ "sync"+ "testing"+ "time"+)++// CountingForwarder relays TCP connections to a target address and counts them, so a test can see+// how many connections a client opened instead of inferring it from how long the calls took.+type CountingForwarder struct {+ Host string+ Port int++ listener net.Listener+ targetAddress string++ mutex sync.Mutex+ acceptedCount int+ isClosed bool+ connections []net.Conn+}++func StartCountingForwarder(t *testing.T, targetAddress string) *CountingForwarder {+ t.Helper()++ listener, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("listen for counting forwarder: %v", err)+ }++ listenerAddress, isTCPAddress := listener.Addr().(*net.TCPAddr)+ if !isTCPAddress {+ t.Fatalf("counting forwarder listens on a non-TCP address %s", listener.Addr())+ }++ forwarder := &CountingForwarder{+ Host: listenerAddress.IP.String(),+ Port: listenerAddress.Port,+ listener: listener,+ targetAddress: targetAddress,+ }++ go forwarder.acceptConnections()+ t.Cleanup(forwarder.close)++ return forwarder+}++func (f *CountingForwarder) GetAcceptedCount() int {+ f.mutex.Lock()+ defer f.mutex.Unlock()++ return f.acceptedCount+}++func (f *CountingForwarder) acceptConnections() {+ for {+ clientConnection, err := f.listener.Accept()+ if err != nil {+ return+ }++ f.mutex.Lock()+ f.acceptedCount+++ f.mutex.Unlock()++ go f.forward(clientConnection)+ }+}++func (f *CountingForwarder) forward(clientConnection net.Conn) {+ targetConnection, err := (&net.Dialer{Timeout: 5 * time.Second}).Dial("tcp", f.targetAddress)+ if err != nil {+ _ = clientConnection.Close()+ return+ }++ // A connection accepted while the forwarder closes would otherwise outlive the test.+ if !f.track(clientConnection, targetConnection) {+ _ = clientConnection.Close()+ _ = targetConnection.Close()++ return+ }++ go pipeAndClose(targetConnection, clientConnection)+ go pipeAndClose(clientConnection, targetConnection)+}++func (f *CountingForwarder) track(connections ...net.Conn) bool {+ f.mutex.Lock()+ defer f.mutex.Unlock()++ if f.isClosed {+ return false+ }++ f.connections = append(f.connections, connections...)++ return true+}++// Clients such as the storage session pools keep idle connections open past the test, so the+// forwarder closes them itself rather than leave its relay goroutines running.+func (f *CountingForwarder) close() {+ _ = f.listener.Close()++ f.mutex.Lock()+ f.isClosed = true+ connections := f.connections+ f.connections = nil+ f.mutex.Unlock()++ for _, connection := range connections {+ _ = connection.Close()+ }+}