PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (tests): Retry test container boots and wait for real SSH and log-cleanup readiness, so transient Mongo exits, bastion handshake resets and lumberjack cleanup lag stop failing the backend CI job
Rostislav Dugin43e0a098ce19
diff --git a/backend/internal/features/tests/physical/postgresql/shared/sshtunnel.go b/backend/internal/features/tests/physical/postgresql/shared/sshtunnel.goindex b41ea199..6eca69a1 100644--- a/backend/internal/features/tests/physical/postgresql/shared/sshtunnel.go+++ b/backend/internal/features/tests/physical/postgresql/shared/sshtunnel.go@@ -10,6 +10,7 @@ import ("github.com/gin-gonic/gin""github.com/jackc/pgx/v5""github.com/stretchr/testify/require"+ "golang.org/x/crypto/ssh"physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"@@ -89,32 +90,61 @@ func startBastion(t *testing.T, topology containers.BastionedDatabase) {t.Helper()require.NoError(t, topology.BastionContainer.Start(context.Background()))- waitForBastionAccepting(t, topology.Bastion, 90*time.Second)+ waitForBastionAuthenticating(t, topology.Bastion, 90*time.Second)}-// Dialling rather than checking the container state: a running container whose sshd has not finished-// binding still refuses connections. The address is the one from before the outage on purpose — the-// backend's forwarder has been retrying it all along, and if the restart moved it the premise of the-// test is gone rather than merely slow.-func waitForBastionAccepting(t *testing.T, bastion containers.Endpoint, timeout time.Duration) {+// A completed SSH handshake rather than an accepted TCP connection: after a restart the published+// port and even sshd's own listener can be up while the first handshake is still reset, and the+// test's next step is exactly such a handshake. The address is the one from before the outage on+// purpose — the backend's forwarder has been retrying it all along, and if the restart moved it the+// premise of the test is gone rather than merely slow.+func waitForBastionAuthenticating(t *testing.T, bastion containers.Endpoint, timeout time.Duration) {t.Helper()address := net.JoinHostPort(bastion.Host, strconv.Itoa(bastion.Port))- dialer := net.Dialer{Timeout: time.Second}++ var lastErr errordeadline := time.Now().UTC().Add(timeout)for time.Now().UTC().Before(deadline) {- conn, err := dialer.DialContext(context.Background(), "tcp", address)- if err == nil {- _ = conn.Close()-+ lastErr = probeBastionHandshake(address)+ if lastErr == nil {return}time.Sleep(500 * time.Millisecond)}- t.Fatalf("the bastion never accepted connections at %s again within %s", address, timeout)+ t.Fatalf("the bastion at %s never completed an ssh handshake again within %s: %v", address, timeout, lastErr)+}++func probeBastionHandshake(address string) error {+ const probeTimeout = 5 * time.Second++ dialer := net.Dialer{Timeout: probeTimeout}++ conn, err := dialer.DialContext(context.Background(), "tcp", address)+ if err != nil {+ return err+ }+ defer func() { _ = conn.Close() }()++ // The deadline bounds the handshake too, which ssh.NewClientConn would otherwise wait on forever.+ if err := conn.SetDeadline(time.Now().UTC().Add(probeTimeout)); err != nil {+ return err+ }++ clientConn, channels, requests, err := ssh.NewClientConn(conn, address, &ssh.ClientConfig{+ User: containers.SshBastionUsername,+ Auth: []ssh.AuthMethod{ssh.Password(containers.SshBastionPassword)},+ // The throwaway test bastion regenerates its host key per image build; nothing to pin.+ HostKeyCallback: ssh.InsecureIgnoreHostKey(),+ })+ if err != nil {+ return err+ }++ return ssh.NewClient(clientConn, channels, requests).Close()}// A FAILED row is how the supervisor hands a streamer back for reclaim, so it is exactly what andiff --git a/backend/internal/util/logger/file_sink_test.go b/backend/internal/util/logger/file_sink_test.goindex fee6bf80..61a0f3d9 100644--- a/backend/internal/util/logger/file_sink_test.go+++ b/backend/internal/util/logger/file_sink_test.go@@ -6,6 +6,7 @@ import ("path/filepath""strings""testing"+ "time""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"@@ -43,12 +44,23 @@ func Test_RotatingFileWriter_WhenSizeExceeded_KeepsAtMostMaxBackupsPlusActive(trequire.NoError(t, writer.Shutdown(t.Context()))- logFiles, err := filepath.Glob(filepath.Join(logDirectory, "databasus*.log"))- require.NoError(t, err)+ globLogFiles := func() []string {+ logFiles, globErr := filepath.Glob(filepath.Join(logDirectory, "databasus*.log"))+ require.NoError(t, globErr)++ return logFiles+ }++ // lumberjack deletes surplus backups in a background goroutine that Close does not wait for,+ // so the cap holds only once that cleanup has caught up with the last rotation.+ require.Eventually(t, func() bool {+ return len(globLogFiles()) <= 4+ }, 5*time.Second, 50*time.Millisecond, "active file plus at most 3 backups")++ logFiles := globLogFiles()// Both bounds matter: >1 proves rotation happened at all, <=4 proves it was capped. Asserting// only the upper bound would pass if rotation never ran.assert.Greater(t, len(logFiles), 1, "5.6 MB through a 1 MB limit must rotate")- assert.LessOrEqual(t, len(logFiles), 4, "active file plus at most 3 backups")for _, logFile := range logFiles {info, statErr := os.Stat(logFile)diff --git a/backend/internal/util/testing/containers/containers.go b/backend/internal/util/testing/containers/containers.goindex 4ec7248b..3934f3bd 100644--- a/backend/internal/util/testing/containers/containers.go+++ b/backend/internal/util/testing/containers/containers.go@@ -22,6 +22,10 @@ const dataDirTmpfsOptions = "rw,size=512m"const loopbackIP = "127.0.0.1"+// One retry covers a transient boot failure, a second covers a back-to-back repeat under the same+// load spike. More would only delay the report of a genuinely broken image.+const containerBootAttempts = 3+// Endpoint is the reachable address of a started container's primary port.type Endpoint struct {Host string@@ -87,19 +91,7 @@ func startUnpublished(t *testing.T, req testcontainers.ContainerRequest, placemereq.Networks = []string{placement.NetworkName}req.NetworkAliases = map[string][]string{placement.NetworkName: {placement.Alias}}- container, err := testcontainers.GenericContainer(context.Background(), testcontainers.GenericContainerRequest{- ContainerRequest: req,- Started: true,- })- if err != nil {- t.Fatalf("failed to start %s container: %v", req.Image, err)- }-- t.Cleanup(func() {- if err := container.Terminate(context.Background()); err != nil {- t.Logf("failed to terminate %s container: %v", req.Image, err)- }- })+ bootContainer(t, req)}func start(t *testing.T, req testcontainers.ContainerRequest, mappedPort string) Endpoint {@@ -111,21 +103,46 @@ func start(t *testing.T, req testcontainers.ContainerRequest, mappedPort string)func startContainer(t *testing.T, req testcontainers.ContainerRequest, mappedPort string) ContainerHandle {t.Helper()- container, err := testcontainers.GenericContainer(context.Background(), testcontainers.GenericContainerRequest{- ContainerRequest: req,- Started: true,- })- if err != nil {- t.Fatalf("failed to start %s container: %v", req.Image, err)- }+ container := bootContainer(t, req)- t.Cleanup(func() {- if err := container.Terminate(context.Background()); err != nil {- t.Logf("failed to terminate %s container: %v", req.Image, err)+ return ContainerHandle{Endpoint: endpointOf(t, container, mappedPort), Container: container}+}++// A boot can fail for reasons outside the test under CPU contention: the mongo entrypoint, for one,+// starts mongod twice and the second start sometimes cannot bind its port and exits with code 48.+// Each failed attempt is logged, so an image that keeps needing a retry still shows up in the output.+func bootContainer(t *testing.T, req testcontainers.ContainerRequest) testcontainers.Container {+ t.Helper()++ var lastErr error++ for attempt := 1; attempt <= containerBootAttempts; attempt++ {+ container, err := testcontainers.GenericContainer(context.Background(), testcontainers.GenericContainerRequest{+ ContainerRequest: req,+ Started: true,+ })+ if err == nil {+ t.Cleanup(func() {+ if err := container.Terminate(context.Background()); err != nil {+ t.Logf("failed to terminate %s container: %v", req.Image, err)+ }+ })++ return container}- })- return ContainerHandle{Endpoint: endpointOf(t, container, mappedPort), Container: container}+ lastErr = err+ t.Logf("attempt %d/%d to start %s container failed: %v", attempt, containerBootAttempts, req.Image, err)++ // GenericContainer hands back the half-started container alongside the error.+ if err := testcontainers.TerminateContainer(container); err != nil {+ t.Logf("failed to terminate %s container after a failed start: %v", req.Image, err)+ }+ }++ t.Fatalf("failed to start %s container after %d attempts: %v", req.Image, containerBootAttempts, lastErr)++ return nil}// The MySQL client resolves the name "localhost" to a unix socket rather than TCP, and the serverdiff --git a/openspec/changes/archive/2026-10-10-harden-flaky-backend-tests/.openspec.yaml b/openspec/changes/archive/2026-10-10-harden-flaky-backend-tests/.openspec.yamlnew file mode 100644index 00000000..683a56f6--- /dev/null+++ b/openspec/changes/archive/2026-10-10-harden-flaky-backend-tests/.openspec.yaml@@ -0,0 +1,3 @@+schema: spec-driven+created: 2026-10-10+skip_specs: truediff --git a/openspec/changes/archive/2026-10-10-harden-flaky-backend-tests/design.md b/openspec/changes/archive/2026-10-10-harden-flaky-backend-tests/design.mdnew file mode 100644index 00000000..9e1d2e31--- /dev/null+++ b/openspec/changes/archive/2026-10-10-harden-flaky-backend-tests/design.md@@ -0,0 +1,65 @@+## Context++See proposal.md - Why for the three observed failures. The current state behind each:++- **Container start.** Every test container goes through one of two helpers, `startContainer` (`backend/internal/util/testing/containers/containers.go:110`) and `startUnpublished` (`containers.go:83`). Both call `testcontainers.GenericContainer` once and `t.Fatalf` on any error. When creation or start fails, `GenericContainer` still returns the half-started container alongside the error (`[email protected]/generic.go:79-97`), and today nobody terminates it. The MongoDB image's entrypoint boots `mongod` twice, first on loopback to create the root user and then on all interfaces (`containers/mongodb.go:24-35`). Under `go test -p=8` on a 4-vCPU runner the second boot can fail to bind and exit with code 48, which surfaces as `wait until ready: container exited with code 48`.+- **Bastion restart.** `startBastion` (`backend/internal/features/tests/physical/postgresql/shared/sshtunnel.go:88-93`) restarts the container and then `waitForBastionAccepting` (`sshtunnel.go:99-118`) polls a plain TCP dial. `Container.Start` already reran the container's own wait strategy, which is a listening-port check inside the container and on the published port (`containers/ssh_bastion.go:210`; `[email protected]/docker.go:243-268`). Both TCP-level checks passed, and the very next SSH handshake from `OpenAdminConn` was reset (`bodies.go:521`). The logs show the supervisor's own tunnel opening successfully in the same second, so the bastion recovered moments later. The exact reason sshd reset that one handshake is not proven. What is proven is that TCP readiness did not imply SSH readiness.+- **Log rotation.** The rotating sink writes through `lumberjack` v2.2.1. After a rotation, `lumberjack` deletes surplus backups in a background goroutine (`mill`/`millRun`), and `Close` does not wait for it. The test pushes about 5.6 MB through a 1 MB limit with three backups allowed, shuts the writer down and immediately counts files (`backend/internal/util/logger/file_sink_test.go:31-56`). If the last cleanup has not run yet, it sees five files.++Constraints: `backend/AGENTS.md` Testing section (run the touched packages via `just test-package`, never the full suite), the principle that nothing observes cleanup by sleeping (stated for stored-file cleanup in `backend/AGENTS.md:269` and applied here to log-file cleanup), and ADR-0013, which keeps one container per engine version alive and makes container boots comparatively rare and expensive.++## Goals / Non-Goals++**Goals:**++- A transient container boot failure no longer fails a test, for every container the shared helpers start.+- A restarted bastion is used only after it completes an SSH handshake.+- The rotation test asserts the cap the sink actually guarantees: eventually at most four files.++**Non-Goals:**++- Retrying anything after a container has booted. Test logic, queries and tunnels opened by the code under test keep failing on the first error.+- Diagnosing why sshd reset the handshake. The fix makes the test wait for the condition it needs, whatever the cause.++## Decisions++### 1. Retry the whole boot in the shared start helpers++Both helpers delegate to one function that runs up to three attempts. A failed attempt terminates the container `GenericContainer` returned, if any, logs the attempt number, image and error with `t.Logf`, and moves on. After the last attempt the test fails with the last error and the attempt count. The loop lives in one shared function in `containers.go` and calls `GenericContainer` directly. It gets no injectable start call and no unit test of its own: `backend/AGENTS.md` reserves unit tests for complex model logic, and a three-iteration loop in a test helper does not qualify. Every test that boots a container exercises the success path. There is no pause between attempts: the failures seen are load spikes, and an attempt already spends seconds pulling state and booting.++Three attempts: one retry covers the observed failure, and a second keeps a back-to-back repeat under the same load spike from failing the test. More attempts only delay the report of a genuinely broken image.++Alternatives rejected:++- *Retry only MongoDB.* The failure mode, an entrypoint that boots twice or a container that dies under CPU contention, is not specific to MongoDB, and the shared helpers are the one place every engine passes through. Per-engine retries would duplicate the same loop.+- *Change MongoDB's readiness instead.* Exit 48 happens inside the container before any readiness check can help. Waiting longer cannot revive an exited process.+- *Suite-wide rerun of failed tests.* Rejected in proposal.md - Out of scope.+- *Retry only for specific errors (exit code 48, "wait until ready").* Matching testcontainers error strings is brittle across library upgrades. A deterministic failure such as a missing image costs at most two extra fast failures.++### 2. Bastion readiness is a full SSH handshake with the test credentials++`waitForBastionAccepting` becomes a wait that dials the bastion's published address with an SSH client, authenticates with `SshBastionUsername` and `SshBastionPassword`, and closes the client on success. Each try has a short timeout, tries repeat every 500 ms, and the existing 90-second deadline stays. The function is renamed to match what it now proves. It keeps dialling the address from before the outage, for the reason its current comment gives. Host key handling matches the production forwarder (`backend/internal/features/sshtunnel/forwarder.go:107-110`).++Alternatives rejected:++- *Read the SSH version banner only.* The banner arrives before key exchange, and the observed reset happened during the handshake. A banner check could pass and still leave the next handshake to fail.+- *Retry inside `OpenAdminConn`.* Every physical test uses that helper. A retry there would also hide a tunnel regression in tests that are not restarting anything.+- *Open a real tunnel through the production `sshtunnel` package as the probe.* It needs a target and more setup, and adds nothing over a successful authenticated handshake, which is the step that failed.++### 3. The rotation test polls the cap++The upper-bound assertion becomes `assert.Eventually` over the file count with a deadline of a few seconds, checked every 50 ms or so. The lower bound (more than one file, so rotation happened) and the per-file size check stay as they are, run once after the eventual condition holds.++Alternatives rejected:++- *Make `Shutdown` wait for `lumberjack`'s cleanup.* `lumberjack` exposes no way to wait for its background goroutine, and production does not need it: the cap holds within moments of every rotation.+- *Sleep before counting.* It breaks the no-sleeping principle that `backend/AGENTS.md:269` states for stored-file cleanup, and a fixed sleep is either too short on a slow runner or wasted on a fast one.+- *Relax the bound to five.* That would accept a sink that keeps one backup too many forever.++## Risks / Trade-offs++- [A broken image or misconfigured request now fails three times before the test reports it] → Fast failures (pull errors, bad commands) add seconds. A readiness timeout triples, up to 12 minutes for MongoDB's 240-second startup timeout. That only happens for a container that never becomes ready, which is already a red build. Each attempt's error is in the test log.+- [Retries hide a container that has become chronically flaky] → Every failed attempt is logged with its image, so a recurring retry shows up in CI output instead of disappearing.+- [The bastion's pinned host port could be taken between attempts] → The failed container is terminated first, which releases the binding. The same small window exists today between the port reservation and the first start.+- [The handshake probe passes but a later handshake still resets] → That would point at a real problem in sshd or the tunnel under reconnect load rather than readiness, which is worth seeing as a failure.+- [The root cause of the bastion reset stays unknown] → Accepted. The test needs SSH readiness and now waits for exactly that.diff --git a/openspec/changes/archive/2026-10-10-harden-flaky-backend-tests/proposal.md b/openspec/changes/archive/2026-10-10-harden-flaky-backend-tests/proposal.mdnew file mode 100644index 00000000..b7c23fab--- /dev/null+++ b/openspec/changes/archive/2026-10-10-harden-flaky-backend-tests/proposal.md@@ -0,0 +1,43 @@+## Why++The `test-backend` CI job takes about 16 minutes, and one flaky test anywhere in it turns the job red and costs a full rerun. Two attempts of the same `main` run (`38072212295`, PR #824) failed on three unrelated flakes, none of them a product bug:++- A `mongo:7.0` test container exited with code 48 while booting. `mongod` failed to bind its port during the image entrypoint's second start, so the test never got a database.+- `Test_PhysicalWalStream_WhenTheBastionDiesMidStream_KeepsTheSlotAndResumes` restarted its SSH bastion, waited until the port accepted TCP, and then had its first SSH handshake reset.+- `Test_RotatingFileWriter_WhenSizeExceeded_KeepsAtMostMaxBackupsPlusActive` counted log files while the rotation library was still deleting old backups in the background, and saw five files instead of at most four.++Each failure has a known cause in test code or test infrastructure, so each is fixed where it starts. A blanket rerun of failed tests would hide the next real race in the WAL-stream resilience tests, which exist to catch exactly that.++## What Changes++- Starting a test container retries a failed boot up to three attempts in total. The failed container is removed before the next attempt, and every failed attempt is logged in the test output so a recurring boot problem stays visible. This covers every engine and auxiliary container the shared container helpers start, not just MongoDB.+- After the bastion restart, the WAL-stream outage test treats the bastion as ready only once a full SSH handshake with the test credentials succeeds through the published port. An accepted TCP connection no longer counts.+- The log rotation test waits, with a bounded deadline, for the file count to drop to the cap instead of checking it once right after shutdown. The check that rotation happened at all stays as it is.++No production code changes. Nothing is **BREAKING**.++## Capabilities++### New Capabilities++None.++### Modified Capabilities++None. The change touches only test code and test helpers, so no spec-level behavior changes and the change sets `skip_specs: true`.++## Impact++- `backend/internal/util/testing/containers/containers.go`: both container start paths gain the retry.+- `backend/internal/features/tests/physical/postgresql/shared/sshtunnel.go`: the bastion readiness wait after a restart.+- `backend/internal/util/logger/file_sink_test.go`: the rotation cap assertion.+- Governing rules: root `AGENTS.md` and `backend/AGENTS.md`, notably its Testing section and the no-sleeping principle its Stored files section states for cleanup.++## Out of scope++- Rerunning failed tests suite-wide (`gotestsum --rerun-fails` or similar). It would mask real races, and the observed flakes all have fixable causes.+- Splitting `test-backend` into several CI jobs to make a rerun cheaper. Every shard would repeat the ~22 GB image pull and setup.+- Changing `-failfast`, `-p`, timeouts or any other CI workflow setting.+- Reworking readiness checks of containers that have not flaked (Mailpit, FTP, Samba, Azurite and others that wait only for a listening port).+- Retrying inside shared connection helpers such as the physical fixture's admin connection opener. A retry there would also hide tunnel regressions in every other test that uses it.+- Any change to the production SSH tunnel, WAL streamer or file log sink.diff --git a/openspec/changes/archive/2026-10-10-harden-flaky-backend-tests/tasks.md b/openspec/changes/archive/2026-10-10-harden-flaky-backend-tests/tasks.mdnew file mode 100644index 00000000..fdde5062--- /dev/null+++ b/openspec/changes/archive/2026-10-10-harden-flaky-backend-tests/tasks.md@@ -0,0 +1,15 @@+## 1. Log rotation test++- [x] 1.1 In `backend/internal/util/logger/file_sink_test.go`, turn the "at most four files" assertion of `Test_RotatingFileWriter_WhenSizeExceeded_KeepsAtMostMaxBackupsPlusActive` into an `assert.Eventually` poll with a deadline of a few seconds (design.md - Decision 3). Keep the "more than one file" and per-file size checks, run after the poll. Verify with `just test-package -count=50 -run Test_RotatingFileWriter ./internal/util/logger/` from `backend/`.++## 2. Container boot retry++- [x] 2.1 In `backend/internal/util/testing/containers/containers.go`, move the `GenericContainer` call into one shared function that both `startContainer` and `startUnpublished` call. It makes up to three attempts, terminates the container a failed attempt returned, logs each failed attempt with `t.Logf` (attempt number, image, error), and fails the test with the last error and the attempt count (design.md - Decision 1). Verify that real containers boot through the new path with `just test-package ./internal/features/databases/databases/mongodb/... ./internal/features/databases/databases/mysql/...` from `backend/`, and check the terminate-and-log branch by reading the diff.++## 3. Bastion readiness after restart++- [x] 3.1 In `backend/internal/features/tests/physical/postgresql/shared/sshtunnel.go`, replace the TCP dial in `waitForBastionAccepting` with an authenticated SSH handshake using `SshBastionUsername` and `SshBastionPassword`, retried every 500 ms under the existing 90-second deadline. Rename the function after what it now proves and update its comment (design.md - Decision 2). Verify with `just test-package -run Test_PhysicalWalStream_WhenTheBastionDiesMidStream ./internal/features/tests/physical/postgresql/...` from `backend/`.++## 4. Lint++- [x] 4.1 Run `just lint` in `backend/` and fix every finding in the touched files.