PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FEATURE (timescaledb): Support hypertable backup, restore and verification
Rostislav Dugin3c390d73060c
diff --git a/agent/verification/e2e/fixture-gen/entrypoint.sh b/agent/verification/e2e/fixture-gen/entrypoint.shindex fbfb1aa9..3878f318 100644--- a/agent/verification/e2e/fixture-gen/entrypoint.sh+++ b/agent/verification/e2e/fixture-gen/entrypoint.sh@@ -1,10 +1,11 @@#!/bin/bash-# Produces /artifacts/good-pgN.dump for each supported Postgres major and a-# single /artifacts/broken.dump sentinel. For each major: spawn a sibling-# postgres:N container against the host daemon, wait for pg_isready, seed the-# t_a + t_b schema, pg_dump -Fc to the artifacts dir, and remove the-# container. Producer and consumer (pg_restore in the agent's spawned target)-# share the major so the custom-format archive header matches.+# Produces /artifacts/good-pgN.dump for each supported Postgres major, one+# /artifacts/good-timescale-pg17.dump (a hypertable dump), and a single+# /artifacts/broken.dump sentinel. For each fixture: spawn a sibling container+# against the host daemon, wait for pg_isready, seed the schema, pg_dump -Fc to+# the artifacts dir, and remove the container. Producer and consumer (pg_restore+# in the agent's spawned target) share the image so the archive header and the+# extension catalog version match.set -euo pipefailVERSIONS=(12 13 14 15 16 17 18)@@ -60,6 +61,51 @@ SQLecho "fixture: good-pg${V}.dump=$(stat -c%s "$OUT")B"done+# TimescaleDB fixture: dumped from the exact image the agent restores into+# (timescale/timescaledb:2.17.0-pg17), so the archive and the extension catalog+# version match. It carries a hypertable spanning many chunks; restoring it+# needs the agent's timescaledb_pre_restore / timescaledb_post_restore wrapping+# and single-threaded -j, or the _timescaledb_catalog restore fails.+TS_IMAGE="timescale/timescaledb:2.17.0-pg17"+TS_OUT="$ARTIFACTS/good-timescale-pg17.dump"++CID="$(docker run -d -e POSTGRES_PASSWORD=test "$TS_IMAGE")"++READY=0+for _ in $(seq 1 30); do+ if docker exec "$CID" pg_isready -U postgres -h 127.0.0.1 >/dev/null 2>&1; then+ READY=1+ break+ fi+ sleep 1+done++if [ "$READY" -ne 1 ]; then+ echo "FAIL: timescaledb never became ready within 30s"+ docker logs "$CID" 2>&1 | tail -30+ exit 1+fi++docker exec -i "$CID" psql -U postgres -d postgres -v ON_ERROR_STOP=1 <<'SQL'+CREATE EXTENSION IF NOT EXISTS timescaledb;+CREATE TABLE public.sensor_data (+ time TIMESTAMPTZ NOT NULL,+ sensor_id int NOT NULL,+ temperature double precision NOT NULL+);+SELECT create_hypertable('public.sensor_data', 'time');+INSERT INTO public.sensor_data (time, sensor_id, temperature)+SELECT ts, (random() * 10)::int, random() * 100+FROM generate_series('2024-01-01'::timestamptz, '2024-03-01'::timestamptz, interval '1 hour') AS ts;+SQL++docker exec "$CID" pg_dump -Fc -U postgres postgres > "$TS_OUT"++docker rm -f "$CID" >/dev/null+CID=""++echo "fixture: good-timescale-pg17.dump=$(stat -c%s "$TS_OUT")B"+printf 'not-a-valid-pg-dump-custom-format-archive\n' > "$BROKEN"-chmod 644 "$ARTIFACTS"/good-pg*.dump "$BROKEN"+chmod 644 "$ARTIFACTS"/good-*.dump "$BROKEN"echo "fixture: broken.dump=$(stat -c%s "$BROKEN")B"diff --git a/agent/verification/e2e/scripts/run-all.sh b/agent/verification/e2e/scripts/run-all.shindex d7825c0b..d200aefb 100644--- a/agent/verification/e2e/scripts/run-all.sh+++ b/agent/verification/e2e/scripts/run-all.sh@@ -9,6 +9,7 @@ TESTS=("test-startup-purge.sh""test-restore-broken.sh""test-restore-disk-budget.sh"+ "test-restore-timescale.sh")PG_VERSIONS=(12 13 14 15 16 17 18)diff --git a/agent/verification/e2e/scripts/test-restore-timescale.sh b/agent/verification/e2e/scripts/test-restore-timescale.shnew file mode 100644index 00000000..d4c0f5ba--- /dev/null+++ b/agent/verification/e2e/scripts/test-restore-timescale.sh@@ -0,0 +1,61 @@+#!/bin/bash+# TimescaleDB restore e2e: the mock serves a hypertable -Fc dump and a claim+# whose timescaledbVersion is set, so the agent must spawn a version-matched+# timescale/timescaledb:<ver>-pg<major> container, run timescaledb_pre_restore /+# timescaledb_post_restore around a single-threaded pg_restore, then the+# verifier must collect stats and the agent must POST COMPLETED with exit 0.+# Without the hooks (or with parallel -j) the _timescaledb_catalog restore fails+# and the agent would report FAILED instead.+set -euo pipefail++source "$(dirname "$0")/lib.sh"++PG_VERSION="17"+TS_VERSION="2.17.0"++WORK="/tmp/agent-work-restore-timescale"+AGENT_ID="55555555-5555-5555-5555-555555555555"+VERIFICATION_ID="55555555-aaaa-aaaa-aaaa-555555555555"+BACKUP_ID="55555555-bbbb-bbbb-bbbb-555555555555"++rm -rf "$WORK"+mkdir -p "$WORK"+cd "$WORK"++reset_mock_state+reset_mock_version++cp "$ARTIFACTS/agent-v1" ./databasus-verification-agent+chmod +x ./databasus-verification-agent++start_agent "$AGENT_ID"++curl -sf -X POST "$MOCK/mock/set-backup-fixture" \+ -H 'Content-Type: application/json' \+ -d '{"path":"/artifacts/good-timescale-pg17.dump"}'++curl -sf -X POST "$MOCK/mock/set-claim" \+ -H 'Content-Type: application/json' \+ -d "{\"verificationId\":\"$VERIFICATION_ID\",\"backupId\":\"$BACKUP_ID\",\"backupSizeMb\":1,\"maxContainerDiskMb\":4096,\"timescaledbVersion\":\"${TS_VERSION}\",\"database\":{\"type\":\"POSTGRES_LOGICAL\",\"postgresql\":{\"version\":\"${PG_VERSION}\"}}}"++wait_for_report '"status":"COMPLETED"' 300 '"status":"FAILED"'++assert_report "$VERIFICATION_ID" '.pgRestoreExitCode == 0'+assert_report "$VERIFICATION_ID" '.dbSizeBytesAfterRestore > 0'+# The hypertable's parent table is restored into public...+assert_report "$VERIFICATION_ID" '(.tableStats | map(.name) | any(. == "sensor_data"))'+# ...and its data lands in _timescaledb_internal chunks, proving the catalog ++# chunk restore (the part that fails without the pre/post hooks) succeeded.+assert_report "$VERIFICATION_ID" '(.tableStats | map(select(.name | startswith("_hyper_")) | .rowCount) | add) > 0'++echo "TimescaleDB restore report OK: COMPLETED with sensor_data hypertable + chunk rows restored"++stop_agent++if ! leak_check "$AGENT_ID"; then+ echo "---- agent.out ----"+ cat agent.out+ exit 1+fi++echo "Verification agent restore-timescale e2e passed"diff --git a/agent/verification/internal/features/api/dto.go b/agent/verification/internal/features/api/dto.goindex 2fd85c17..810aa8c4 100644--- a/agent/verification/internal/features/api/dto.go+++ b/agent/verification/internal/features/api/dto.go@@ -51,6 +51,7 @@ type JobAssignment struct {BackupSizeMb float64 `json:"backupSizeMb"`MaxContainerDiskMb float64 `json:"maxContainerDiskMb"`Database AssignedDatabase `json:"database"`+ TimescaledbVersion string `json:"timescaledbVersion"`}type ReportTableStat struct {diff --git a/agent/verification/internal/features/container/manager.go b/agent/verification/internal/features/container/manager.goindex 91f16e81..830eed4c 100644--- a/agent/verification/internal/features/container/manager.go+++ b/agent/verification/internal/features/container/manager.go@@ -21,6 +21,11 @@ const (// engineImageRepo is the official Postgres image; the job's major version// (from the backend assignment) is appended as the tag.engineImageRepo = "postgres"++ // timescaleImageRepo carries the timescaledb extension; the tag is+ // "<extversion>-pg<major>" so the engine matches the dump exactly+ // (pg_restore cannot cross extension versions).+ timescaleImageRepo = "timescale/timescaledb")// minimalCaps are the only capabilities added back after CapDrop ALL — exactly@@ -94,7 +99,7 @@ func (m *Manager) PurgeContainers(ctx context.Context) {// Spawn's every failure is pre-pg_restore — the runner reports it with no exit// code (retryable AgentSetupFailed), never BackupRejected.func (m *Manager) Spawn(jobCtx context.Context, req SpawnRequest) (*PostgresContainer, error) {- image := imageForMajor(req.PgMajor)+ image := imageForJob(req)if err := m.engine.EnsureImage(jobCtx, image); err != nil {return nil, fmt.Errorf("ensure image: %w", err)@@ -239,9 +244,14 @@ func pingOnce(ctx context.Context, conn dbconn.Conn) bool {return pgConn.Ping(pingCtx) == nil}-// imageForMajor returns the Docker image tag the agent spawns for a given-// Postgres major. Extracted so the major-to-image contract is unit-testable-// without standing up a Docker engine.+func imageForJob(req SpawnRequest) string {+ if req.TimescaledbVersion != "" {+ return fmt.Sprintf("%s:%s-pg%s", timescaleImageRepo, req.TimescaledbVersion, req.PgMajor)+ }++ return imageForMajor(req.PgMajor)+}+func imageForMajor(pgMajor string) string {return engineImageRepo + ":" + pgMajor}diff --git a/agent/verification/internal/features/container/postgres_test.go b/agent/verification/internal/features/container/postgres_test.goindex dfadf151..38330ac8 100644--- a/agent/verification/internal/features/container/postgres_test.go+++ b/agent/verification/internal/features/container/postgres_test.go@@ -95,3 +95,15 @@ func Test_ImageForMajor_ReturnsPostgresOfficialTagPerMajor(t *testing.T) {})}}++func Test_ImageForJob_WhenTimescaleVersionSet_ReturnsVersionMatchedTimescaleImage(t *testing.T) {+ image := imageForJob(SpawnRequest{PgMajor: "17", TimescaledbVersion: "2.17.0"})++ assert.Equal(t, "timescale/timescaledb:2.17.0-pg17", image)+}++func Test_ImageForJob_WhenNoTimescaleVersion_ReturnsStockPostgresImage(t *testing.T) {+ image := imageForJob(SpawnRequest{PgMajor: "17"})++ assert.Equal(t, "postgres:17", image)+}diff --git a/agent/verification/internal/features/container/spawnspec.go b/agent/verification/internal/features/container/spawnspec.goindex 9b5e55b6..f5ba02f7 100644--- a/agent/verification/internal/features/container/spawnspec.go+++ b/agent/verification/internal/features/container/spawnspec.go@@ -3,10 +3,11 @@ package containerimport "github.com/google/uuid"type SpawnRequest struct {- PgMajor string- CPUPerJob int- RAMMbPerJob int- VerificationID uuid.UUID+ PgMajor string+ CPUPerJob int+ RAMMbPerJob int+ VerificationID uuid.UUID+ TimescaledbVersion string}type spawnPlan struct {diff --git a/agent/verification/internal/features/restore/timescaledb.go b/agent/verification/internal/features/restore/timescaledb.gonew file mode 100644index 00000000..4a670bb8--- /dev/null+++ b/agent/verification/internal/features/restore/timescaledb.go@@ -0,0 +1,49 @@+package restore++import (+ "context"+ "fmt"++ "github.com/jackc/pgx/v5"++ "databasus-verification-agent/internal/features/dbconn"+)++// TimescaleDB restore procedure run against the throwaway restore target.+//+// timescaledb_pre_restore() sets a database-level GUC (ALTER DATABASE ... SET+// timescaledb.restoring='on') that the separate in-container pg_restore connection inherits, and+// stops background workers. timescaledb_post_restore() resets it and restarts the workers, returning+// the database to normal mode — it must run before the verifier collects stats. The connection used+// here is the agent-host verifier conn (published port); pg_restore uses the in-container conn, but+// both reach the same database, so the GUC carries across.++// RunTimescalePreRestore installs the extension (a freshly initialised target may lack it) and+// enters restoring mode.+func (r *Restorer) RunTimescalePreRestore(ctx context.Context, conn dbconn.Conn) error {+ return execStatements(ctx, conn,+ "CREATE EXTENSION IF NOT EXISTS timescaledb",+ "SELECT timescaledb_pre_restore()",+ )+}++// RunTimescalePostRestore leaves restoring mode and restarts background workers.+func (r *Restorer) RunTimescalePostRestore(ctx context.Context, conn dbconn.Conn) error {+ return execStatements(ctx, conn, "SELECT timescaledb_post_restore()")+}++func execStatements(ctx context.Context, conn dbconn.Conn, statements ...string) error {+ pgConn, err := pgx.Connect(ctx, conn.DSN())+ if err != nil {+ return fmt.Errorf("connect restore target: %w", err)+ }+ defer func() { _ = pgConn.Close(ctx) }()++ for _, statement := range statements {+ if _, err := pgConn.Exec(ctx, statement); err != nil {+ return fmt.Errorf("run %q: %w", statement, err)+ }+ }++ return nil+}diff --git a/agent/verification/internal/features/runner/concurrent_restores_integration_test.go b/agent/verification/internal/features/runner/concurrent_restores_integration_test.goindex f6b014b9..771edfb2 100644--- a/agent/verification/internal/features/runner/concurrent_restores_integration_test.go+++ b/agent/verification/internal/features/runner/concurrent_restores_integration_test.go@@ -192,6 +192,9 @@ func (r *barrierRestorer) RunPgRestore(return restore.Result{PgRestoreExitCode: 0, DurationMs: 1}, nil}+func (r *barrierRestorer) RunTimescalePreRestore(context.Context, dbconn.Conn) error { return nil }+func (r *barrierRestorer) RunTimescalePostRestore(context.Context, dbconn.Conn) error { return nil }+func makeConcurrentAssignment(id uuid.UUID) *api.JobAssignment {return &api.JobAssignment{VerificationID: id,diff --git a/agent/verification/internal/features/runner/fakes_test.go b/agent/verification/internal/features/runner/fakes_test.goindex 562cf25a..95b67c7c 100644--- a/agent/verification/internal/features/runner/fakes_test.go+++ b/agent/verification/internal/features/runner/fakes_test.go@@ -136,6 +136,11 @@ type fakeRestorer struct {runEntered atomic.BoolrunCtxCancelled atomic.Bool++ // calls records the restore-related call order ("pre", "restore", "post") for a single job.+ calls []string+ // runParallelJobs records the -j value RunPgRestore was invoked with.+ runParallelJobs int}func (r *fakeRestorer) StageBackupViaExec(@@ -149,8 +154,11 @@ func (r *fakeRestorer) StageBackupViaExec(}func (r *fakeRestorer) RunPgRestore(- ctx context.Context, _ restore.ExecRunner, _ string, _ dbconn.Conn, _ int,+ ctx context.Context, _ restore.ExecRunner, _ string, _ dbconn.Conn, parallelJobs int,) (restore.Result, error) {+ r.calls = append(r.calls, "restore")+ r.runParallelJobs = parallelJobs+if r.runBlocks {r.runEntered.Store(true)<-ctx.Done()@@ -162,6 +170,18 @@ func (r *fakeRestorer) RunPgRestore(return r.runResult, r.runErr}+func (r *fakeRestorer) RunTimescalePreRestore(context.Context, dbconn.Conn) error {+ r.calls = append(r.calls, "pre")++ return nil+}++func (r *fakeRestorer) RunTimescalePostRestore(context.Context, dbconn.Conn) error {+ r.calls = append(r.calls, "post")++ return nil+}+type fakeStats struct {stats verifier.Statserr errordiff --git a/agent/verification/internal/features/runner/interfaces.go b/agent/verification/internal/features/runner/interfaces.goindex 17cf7fee..f78de084 100644--- a/agent/verification/internal/features/runner/interfaces.go+++ b/agent/verification/internal/features/runner/interfaces.go@@ -39,6 +39,8 @@ type Restorer interface {conn dbconn.Conn,parallelJobs int,) (restore.Result, error)+ RunTimescalePreRestore(ctx context.Context, conn dbconn.Conn) error+ RunTimescalePostRestore(ctx context.Context, conn dbconn.Conn) error}type StatsCollector interface {diff --git a/agent/verification/internal/features/runner/runner.go b/agent/verification/internal/features/runner/runner.goindex c840415c..5670d190 100644--- a/agent/verification/internal/features/runner/runner.go+++ b/agent/verification/internal/features/runner/runner.go@@ -43,10 +43,11 @@ var errAbortNoReport = errors.New("verification aborted; no report")var backupDownloadBackoffFn = backupDownloadBackofftype SpawnRequest struct {- PgMajor string- CPUPerJob int- RAMMbPerJob int- VerificationID uuid.UUID+ PgMajor string+ CPUPerJob int+ RAMMbPerJob int+ VerificationID uuid.UUID+ TimescaledbVersion string}type Runner struct {@@ -162,10 +163,11 @@ func (r *Runner) executeJob(ctx context.Context, job *api.JobAssignment) {}jobContainer, err := r.spawner.Spawn(jobCtx, SpawnRequest{- PgMajor: pgMajor,- CPUPerJob: r.capacity.CPUPerJob,- RAMMbPerJob: r.capacity.RAMMbPerJob,- VerificationID: job.VerificationID,+ PgMajor: pgMajor,+ CPUPerJob: r.capacity.CPUPerJob,+ RAMMbPerJob: r.capacity.RAMMbPerJob,+ VerificationID: job.VerificationID,+ TimescaledbVersion: job.TimescaledbVersion,})if err != nil {if jobCtx.Err() != nil {@@ -233,6 +235,27 @@ func (r *Runner) executeJob(ctx context.Context, job *api.JobAssignment) {parallelJobs := min(maxParallelRestoreJobs, r.capacity.CPUPerJob)+ // TimescaleDB restores single-threaded and in restoring mode. Single-threaded because parallel+ // pg_restore loads dependent _timescaledb_catalog rows before the rows they reference, tripping+ // the catalog's own foreign keys. post_restore runs on the success path below, before stats; on+ // failure the throwaway container is discarded, so nothing is left in restoring mode to repair+ // (no defer, unlike the backend's user-facing restore). The hooks use the verifier (host) conn+ // while pg_restore uses the in-container conn, but the database-level restoring GUC carries across.+ if job.TimescaledbVersion != "" {+ parallelJobs = 1++ if err := r.restorer.RunTimescalePreRestore(jobCtx, jobContainer.GetVerifierConn()); err != nil {+ if jobCtx.Err() != nil {+ return+ }++ r.reportFailure(ctx, job.VerificationID, nil,+ fmt.Sprintf("timescaledb_pre_restore: %v", err), runLogger)++ return+ }+ }+restoreResult, err := r.restorer.RunPgRestore(jobCtx, jobContainer, archivePath, jobContainer.GetInContainerConn(), parallelJobs)if err != nil {@@ -274,6 +297,19 @@ func (r *Runner) executeJob(ctx context.Context, job *api.JobAssignment) {return}+ if job.TimescaledbVersion != "" {+ if err := r.restorer.RunTimescalePostRestore(jobCtx, jobContainer.GetVerifierConn()); err != nil {+ if jobCtx.Err() != nil {+ return+ }++ r.reportFailure(ctx, job.VerificationID, &restoreResult,+ fmt.Sprintf("timescaledb_post_restore: %v", err), runLogger)++ return+ }+ }+verifierConn := jobContainer.GetVerifierConn()stats, err := r.verifier.CollectStats(jobCtx, verifierConn)diff --git a/agent/verification/internal/features/runner/runner_test.go b/agent/verification/internal/features/runner/runner_test.goindex 570a14f9..9cff5723 100644--- a/agent/verification/internal/features/runner/runner_test.go+++ b/agent/verification/internal/features/runner/runner_test.go@@ -269,6 +269,46 @@ func Test_ExecuteJob_WhenAllSucceeds_ReportsCompletedWithStats(t *testing.T) {assert.Equal(t, "public", report.TableStats[0].SchemaName)}+func timescaleJob() *api.JobAssignment {+ job := postgresJob()+ job.TimescaledbVersion = "2.17.0"++ return job+}++func Test_ExecuteJob_WhenTimescaleJobSucceeds_RunsPreRestoreThenRestoreThenPostRestore(t *testing.T) {+ apiClient := &fakeAPI{downloadBody: []byte("ARCHIVE")}+ restorer := &fakeRestorer{runResult: restore.Result{PgRestoreExitCode: 0}}+ stats := &fakeStats{stats: verifier.Stats{TableCount: 1}}+ r := newTestRunner(apiClient, okSpawner(), restorer, stats, newFakeRegistrar())+ r.connAlive = func(context.Context, dbconn.Conn) bool { return true }++ r.executeJob(t.Context(), timescaleJob())++ report, ok := apiClient.lastReport()+ require.True(t, ok)+ assert.Equal(t, api.VerificationStatusCompleted, report.Status)+ assert.Equal(t, []string{"pre", "restore", "post"}, restorer.calls,+ "timescaledb restore must enter restoring mode, restore, then leave it")+ assert.Equal(t, 1, restorer.runParallelJobs,+ "timescaledb restore must be single-threaded to keep _timescaledb_catalog FK order")+}++func Test_ExecuteJob_WhenNonTimescaleJob_DoesNotRunTimescaleHooks(t *testing.T) {+ apiClient := &fakeAPI{downloadBody: []byte("ARCHIVE")}+ restorer := &fakeRestorer{runResult: restore.Result{PgRestoreExitCode: 0}}+ stats := &fakeStats{stats: verifier.Stats{TableCount: 1}}+ r := newTestRunner(apiClient, okSpawner(), restorer, stats, newFakeRegistrar())+ r.connAlive = func(context.Context, dbconn.Conn) bool { return true }++ r.executeJob(t.Context(), postgresJob())++ assert.Equal(t, []string{"restore"}, restorer.calls,+ "a non-timescaledb restore must not run the timescaledb hooks")+ assert.Equal(t, 2, restorer.runParallelJobs,+ "a non-timescaledb restore keeps parallel jobs (CPUPerJob)")+}+func Test_ExecuteJob_WhenAbortedBeforeFailedReport_DoesNotReport(t *testing.T) {apiClient := &fakeAPI{}spawner := &fakeSpawner{err: errors.New("spawn failed")}diff --git a/agent/verification/internal/features/start/spawner.go b/agent/verification/internal/features/start/spawner.goindex 12082875..8a076974 100644--- a/agent/verification/internal/features/start/spawner.go+++ b/agent/verification/internal/features/start/spawner.go@@ -15,10 +15,11 @@ func (s containerManagerSpawner) Spawn(ctx context.Context, req runner.SpawnRequest,) (runner.JobContainer, error) {jobContainer, err := s.containerManager.Spawn(ctx, container.SpawnRequest{- PgMajor: req.PgMajor,- CPUPerJob: req.CPUPerJob,- RAMMbPerJob: req.RAMMbPerJob,- VerificationID: req.VerificationID,+ PgMajor: req.PgMajor,+ CPUPerJob: req.CPUPerJob,+ RAMMbPerJob: req.RAMMbPerJob,+ VerificationID: req.VerificationID,+ TimescaledbVersion: req.TimescaledbVersion,})if err != nil {return nil, errdiff --git a/backend/internal/features/backups/backups/core/logical/model.go b/backend/internal/features/backups/backups/core/logical/model.goindex 4536de0b..67bd1852 100644--- a/backend/internal/features/backups/backups/core/logical/model.go+++ b/backend/internal/features/backups/backups/core/logical/model.go@@ -32,6 +32,12 @@ type LogicalBackup struct {EncryptionIV *string `json:"-" gorm:"column:encryption_iv"`Encryption backups_core_enums.BackupEncryption `json:"encryption" gorm:"column:encryption;type:text;not null;default:'NONE'"`+ // TimescaledbVersion is the source's timescaledb extension version captured at backup+ // time, or "" when the source has no timescaledb. Non-empty marks the backup as needing+ // the TimescaleDB restore procedure, and the verification agent uses the exact version to+ // pull a matching engine image (pg_restore cannot cross extension versions).+ TimescaledbVersion string `json:"timescaledbVersion" gorm:"column:timescaledb_version;type:text;not null;default:''"`+CreatedAt time.Time `json:"createdAt" gorm:"column:created_at"`}diff --git a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.goindex 7502e47e..cfe4bd7e 100644--- a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go@@ -97,6 +97,15 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(backup.BackupRawDbSizeMb = rawSizeMB}+ timescaledbVersion, err := pg.GetTimescaleDBVersion(ctx, uc.fieldEncryptor)+ if err != nil {+ uc.logger.Warn("failed to detect timescaledb extension before backup",+ "database_id", db.ID,+ "error", err)+ } else {+ backup.TimescaledbVersion = timescaledbVersion+ }+return uc.streamToStorage(ctx,backup,diff --git a/backend/internal/features/databases/databases/postgresql/logical/timescaledb.go b/backend/internal/features/databases/databases/postgresql/logical/timescaledb.gonew file mode 100644index 00000000..01ab3b8c--- /dev/null+++ b/backend/internal/features/databases/databases/postgresql/logical/timescaledb.go@@ -0,0 +1,90 @@+package postgresql_logical++import (+ "context"+ "errors"+ "fmt"++ "github.com/jackc/pgx/v5"++ "databasus-backend/internal/util/encryption"+)++// GetTimescaleDBVersion returns the installed timescaledb extension version on this database,+// or "" when the extension is absent. Captured at backup time so restore knows whether the+// TimescaleDB restore procedure is required, and the verification agent knows which engine+// image version to pull.+func (p *PostgresqlLogicalDatabase) GetTimescaleDBVersion(+ ctx context.Context,+ encryptor encryption.FieldEncryptor,+) (string, error) {+ conn, err := openPgConn(ctx, p, *p.Database, encryptor)+ if err != nil {+ return "", fmt.Errorf("failed to connect to database '%s': %w", *p.Database, err)+ }+ defer func() { _ = conn.Close(ctx) }()++ var version string++ err = conn.QueryRow(+ ctx, "SELECT extversion FROM pg_extension WHERE extname = 'timescaledb'",+ ).Scan(&version)+ if errors.Is(err, pgx.ErrNoRows) {+ return "", nil+ }++ if err != nil {+ return "", fmt.Errorf("failed to query timescaledb extension version: %w", err)+ }++ return version, nil+}++// TimescaleDB restore procedure, executed against the restore target.+//+// timescaledb_pre_restore() runs `ALTER DATABASE <db> SET timescaledb.restoring='on'` — a+// database-level GUC that persists across new connections — and stops background workers. Because+// the GUC lives on the database (not the session), the separate pg_restore process that runs next+// inherits restoring mode; that is the whole reason these hooks can run on connections distinct+// from pg_restore's. timescaledb_post_restore() resets the GUC and restarts the workers, so it MUST+// run even when pg_restore fails — otherwise the database is left stuck in restoring mode with its+// workers stopped. Callers run RunTimescaleDBPostRestore via defer.++// RunTimescaleDBPreRestore installs the extension (a freshly created restore target has none, and+// pre_restore needs its functions present) and enters restoring mode.+func (p *PostgresqlLogicalDatabase) RunTimescaleDBPreRestore(+ ctx context.Context,+ encryptor encryption.FieldEncryptor,+) error {+ return p.execOnTarget(ctx, encryptor,+ "CREATE EXTENSION IF NOT EXISTS timescaledb",+ "SELECT timescaledb_pre_restore()",+ )+}++func (p *PostgresqlLogicalDatabase) RunTimescaleDBPostRestore(+ ctx context.Context,+ encryptor encryption.FieldEncryptor,+) error {+ return p.execOnTarget(ctx, encryptor, "SELECT timescaledb_post_restore()")+}++func (p *PostgresqlLogicalDatabase) execOnTarget(+ ctx context.Context,+ encryptor encryption.FieldEncryptor,+ statements ...string,+) error {+ conn, err := openPgConn(ctx, p, *p.Database, encryptor)+ if err != nil {+ return fmt.Errorf("failed to connect to database '%s': %w", *p.Database, err)+ }+ defer func() { _ = conn.Close(ctx) }()++ for _, statement := range statements {+ if _, err := conn.Exec(ctx, statement); err != nil {+ return fmt.Errorf("failed to run %q: %w", statement, err)+ }+ }++ return nil+}diff --git a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.goindex b384889c..9f24daf9 100644--- a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go@@ -99,22 +99,55 @@ func (uc *RestorePostgresqlBackupUsecase) restoreCustomType(pg.CpuCount,)- // If excluding extensions, we must use file-based restore (requires TOC file generation)- // Also use file-based restore for parallel jobs (multiple CPUs)- if isExcludeExtensions || pg.CpuCount > 1 {- return uc.restoreViaFile(- parentCtx,- originalDB,- pgBin,- backup,- storage,- pg,- isExcludeExtensions,- )+ // File-based restore for parallel jobs (multiple CPUs) or extension exclusion (needs a TOC file);+ // otherwise stream directly via stdin. A timescaledb backup uses whichever path applies — the+ // pre/post hooks wrap it the same way (they only set a database-level GUC, so streaming is kept).+ runRestore := func() error {+ if isExcludeExtensions || pg.CpuCount > 1 {+ return uc.restoreViaFile(parentCtx, originalDB, pgBin, backup, storage, pg, isExcludeExtensions)+ }++ return uc.restoreViaStdin(parentCtx, originalDB, pgBin, backup, storage, pg)}- // Single CPU without extension exclusion: stream directly via stdin- return uc.restoreViaStdin(parentCtx, originalDB, pgBin, backup, storage, pg)+ if backup.TimescaledbVersion != "" {+ return uc.withTimescaleHooks(parentCtx, pg, runRestore)+ }++ return runRestore()+}++// withTimescaleHooks wraps a restore in the TimescaleDB procedure: ensure the extension exists and+// enter restoring mode before pg_restore, then leave restoring mode after — unconditionally, so a+// failed restore never leaves the target stuck in restoring mode. See timescaledb.go for why the+// database-level GUC set here is inherited by the separate pg_restore process.+func (uc *RestorePostgresqlBackupUsecase) withTimescaleHooks(+ ctx context.Context,+ pg *pgtypes.PostgresqlLogicalDatabase,+ runRestore func() error,+) (err error) {+ encryptor := util_encryption.GetFieldEncryptor()++ if err := pg.RunTimescaleDBPreRestore(ctx, encryptor); err != nil {+ return fmt.Errorf("timescaledb_pre_restore failed: %w", err)+ }++ uc.logger.Info("entered timescaledb restoring mode")++ defer func() {+ if postErr := pg.RunTimescaleDBPostRestore(ctx, encryptor); postErr != nil {+ uc.logger.Error(+ "timescaledb_post_restore failed; target database may be left in restoring mode",+ "error", postErr,+ )++ if err == nil {+ err = fmt.Errorf("timescaledb_post_restore failed: %w", postErr)+ }+ }+ }()++ return runRestore()}// restoreViaStdin streams backup via stdin for single CPU restore@@ -136,8 +169,11 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin("-U", pg.Username,"-d", *pg.Database,"--verbose",- "--clean",- "--if-exists",+ }+ // --clean would DROP EXTENSION timescaledb, taking the catalog tables that pre_restore needs+ // with it; TimescaleDB restores into a clean target without it. Non-timescaledb keeps --clean.+ if backup.TimescaledbVersion == "" {+ args = append(args, "--clean", "--if-exists")}if !pg.IsRestoreOwnership {args = append(args, "--no-owner")@@ -352,9 +388,15 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaFile(pg.CpuCount,)- // Use parallel jobs based on CPU count- // Cap between 1 and 8 to avoid overwhelming the server+ isTimescale := backup.TimescaledbVersion != ""++ // TimescaleDB restores single-threaded: with -j the parallel loop loads dependent+ // _timescaledb_catalog rows (chunk_constraint, chunk_index) before the chunk rows they reference,+ // tripping the catalog's own foreign keys. Otherwise cap between 1 and 8.parallelJobs := max(1, min(pg.CpuCount, 8))+ if isTimescale {+ parallelJobs = 1+ }args := []string{"-Fc", // expect custom type@@ -365,8 +407,11 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaFile("-U", pg.Username,"-d", *pg.Database,"--verbose",- "--clean",- "--if-exists",+ }+ // --clean would DROP EXTENSION timescaledb, taking the catalog tables that pre_restore needs+ // with it; TimescaleDB restores into a clean target without it. Non-timescaledb keeps --clean.+ if !isTimescale {+ args = append(args, "--clean", "--if-exists")}if !pg.IsRestoreOwnership {args = append(args, "--no-owner")diff --git a/backend/internal/features/tests/logical/postgresql/timescaledb_test.go b/backend/internal/features/tests/logical/postgresql/timescaledb_test.gonew file mode 100644index 00000000..7bc747f3--- /dev/null+++ b/backend/internal/features/tests/logical/postgresql/timescaledb_test.go@@ -0,0 +1,171 @@+package postgresql_logical++import (+ "fmt"+ "net/http"+ "os"+ "path/filepath"+ "testing"+ "time"++ "github.com/google/uuid"+ "github.com/jmoiron/sqlx"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ "databasus-backend/internal/config"+ backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"+ backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"+ restores_core "databasus-backend/internal/features/restores/core"+ "databasus-backend/internal/features/storages"+ logicaltesting "databasus-backend/internal/features/tests/logical/shared"+ users_enums "databasus-backend/internal/features/users/enums"+ users_testing "databasus-backend/internal/features/users/testing"+ workspaces_testing "databasus-backend/internal/features/workspaces/testing"+ test_utils "databasus-backend/internal/util/testing"+ "databasus-backend/internal/util/testing/containers"+)++const timescaleImage = "timescale/timescaledb:2.17.0-pg17"++// Test_PostgresqlBackupRestore_TimescaleDB boots one timescaledb container and runs the full+// backup→restore cycle of a hypertable through it, on both the CPU=1 streaming path and the CPU=4+// parallel path (the path that previously failed with _timescaledb_catalog FK violations). See+// ADR-0013 for the one-container-per-version pattern.+func Test_PostgresqlBackupRestore_TimescaleDB(t *testing.T) {+ endpoint := containers.StartTimescaleDB(t, timescaleImage)++ for _, cpuCount := range []int{1, 4} {+ t.Run(fmt.Sprintf("CPU=%d", cpuCount), func(t *testing.T) {+ testTimescaleBackupRestoreForCpuCount(t, endpoint, cpuCount)+ })+ }+}++func seedHypertableQuery(tableName string) string {+ return fmt.Sprintf(`+CREATE EXTENSION IF NOT EXISTS timescaledb;++DROP TABLE IF EXISTS %s;++CREATE TABLE %s (+ time TIMESTAMPTZ NOT NULL,+ sensor_id INTEGER NOT NULL,+ temperature DOUBLE PRECISION NOT NULL+);++SELECT create_hypertable('%s', 'time');++INSERT INTO %s (time, sensor_id, temperature)+SELECT ts, (random() * 10)::int, random() * 100+FROM generate_series('2024-01-01'::timestamptz, '2024-03-01'::timestamptz, interval '1 hour') AS ts;+`, tableName, tableName, tableName, tableName)+}++func testTimescaleBackupRestoreForCpuCount(t *testing.T, endpoint containers.Endpoint, cpuCount int) {+ container, err := connectToPostgresEndpoint(t, endpoint)+ require.NoError(t, err)+ defer func() {+ if container.DB != nil {+ container.DB.Close()+ }+ }()++ tableName := fmt.Sprintf("sensor_data_%s", uuid.New().String()[:8])+ _, err = container.DB.Exec(seedHypertableQuery(tableName))+ require.NoError(t, err)++ defer func() {+ _, _ = container.DB.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s;", tableName))+ }()++ var sourceRowCount int+ require.NoError(t, container.DB.Get(&sourceRowCount, fmt.Sprintf("SELECT count(*) FROM %s", tableName)))+ require.Positive(t, sourceRowCount)++ router := logicaltesting.CreateTestRouter()+ user := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", user, router)+ storage := storages.CreateTestStorage(workspace.ID)++ database := createDatabaseWithCpuCountViaAPI(+ t, router, "Timescale Database", workspace.ID,+ container.Host, container.Port,+ container.Username, container.Password, container.Database,+ cpuCount,+ user.Token,+ )++ logicaltesting.EnableBackupsViaAPI(+ t, router, database.ID, storage.ID,+ backups_core_enums.BackupEncryptionNone, user.Token,+ )++ logicaltesting.CreateBackupViaAPI(t, router, database.ID, user.Token)++ backup := logicaltesting.WaitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute)+ assert.Equal(t, backups_core_logical.BackupStatusCompleted, backup.Status)+ assert.NotEmpty(t, backup.TimescaledbVersion, "backup should record the source timescaledb version")++ newDBName := fmt.Sprintf("restored_ts_cpu%d_%s", cpuCount, uuid.New().String()[:8])+ _, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName))+ require.NoError(t, err)++ _, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName))+ require.NoError(t, err)++ defer func() {+ _, _ = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName))+ }()++ newDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",+ container.Host, container.Port, container.Username, container.Password, newDBName)+ restoredDB, err := sqlx.Connect("postgres", newDSN)+ require.NoError(t, err)+ defer restoredDB.Close()++ createRestoreWithCpuCountViaAPI(+ t, router, backup.ID,+ container.Host, container.Port,+ container.Username, container.Password, newDBName,+ cpuCount,+ user.Token,+ )++ restore := waitForRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute)+ assert.Equal(t, restores_core.RestoreStatusCompleted, restore.Status)++ assertHypertableRestored(t, restoredDB, tableName, sourceRowCount)++ err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))+ if err != nil {+ t.Logf("Warning: Failed to delete backup file: %v", err)+ }++ test_utils.MakeDeleteRequest(+ t, router,+ "/api/v1/databases/"+database.ID.String(),+ "Bearer "+user.Token,+ http.StatusNoContent,+ )+ storages.RemoveTestStorage(storage.ID)+ workspaces_testing.RemoveTestWorkspace(workspace, router)+}++func assertHypertableRestored(t *testing.T, restoredDB *sqlx.DB, tableName string, expectedRows int) {+ t.Helper()++ var restoredRowCount int+ require.NoError(t, restoredDB.Get(&restoredRowCount, fmt.Sprintf("SELECT count(*) FROM %s", tableName)))+ assert.Equal(t, expectedRows, restoredRowCount, "restored hypertable should keep every row")++ var hypertableCount int+ require.NoError(t, restoredDB.Get(&hypertableCount,+ "SELECT count(*) FROM timescaledb_information.hypertables WHERE hypertable_name = $1", tableName))+ assert.Equal(t, 1, hypertableCount, "restored table should be a hypertable")++ var chunkCount int+ require.NoError(t, restoredDB.Get(&chunkCount,+ "SELECT count(*) FROM timescaledb_information.chunks WHERE hypertable_name = $1", tableName))+ assert.Greater(t, chunkCount, 1, "restored hypertable should keep its chunks")+}diff --git a/backend/internal/features/verification/runs/agent_controller_test.go b/backend/internal/features/verification/runs/agent_controller_test.goindex ccc9d0d7..933ef2a9 100644--- a/backend/internal/features/verification/runs/agent_controller_test.go+++ b/backend/internal/features/verification/runs/agent_controller_test.go@@ -73,6 +73,38 @@ func Test_ClaimVerification_WhenPendingFitsBudget_ReturnsJobAssignment(t *testinassert.Equal(t, agent.Agent.ID, *updated.AgentID)}+func Test_ClaimVerification_WhenBackupUsesTimescaleDB_ForwardsTimescaledbVersion(t *testing.T) {+ router := createTestRouter()+ owner := users_testing.CreateTestUser(users_enums.UserRoleAdmin)+ workspace := workspaces_testing.CreateTestWorkspace("ws "+uuid.New().String(), owner, router)+ defer workspaces_testing.RemoveTestWorkspace(workspace, router)++ testStorage := storages.CreateTestStorage(workspace.ID)+ defer storages.RemoveTestStorage(testStorage.ID)++ notifier := notifiers.CreateTestNotifier(workspace.ID)+ defer notifiers.RemoveTestNotifier(notifier)++ database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier)+ defer databases.RemoveTestDatabase(database)++ backup := backuping_logical.SeedTestBackup(t, database.ID, testStorage.ID, 100)+ backup.TimescaledbVersion = "2.17.0"+ require.NoError(t, backups_core_logical.GetBackupRepository().Save(backup))++ EnqueueManualVerificationViaAPI(t, router, owner.Token, backup.ID)++ agent := verification_agents.CreateTestVerificationAgent(t, router, owner.Token, "ts-claim-"+uuid.New().String())+ defer verification_agents.RemoveTestVerificationAgent(t, router, owner.Token, agent.Agent.ID)++ assignment := ClaimVerificationViaAPI(+ t, router, agent.Agent.ID, agent.Token,+ AgentCapacity{MaxCPU: 4, MaxRAMMb: 4096, MaxDiskGb: 50, MaxConcurrentJobs: 2},+ )++ assert.Equal(t, "2.17.0", assignment.TimescaledbVersion)+}+func Test_ClaimVerification_WhenBackupTooBigForBudget_DoNotAssignJobWithoutError(t *testing.T) {router := createTestRouter()owner := users_testing.CreateTestUser(users_enums.UserRoleAdmin)diff --git a/backend/internal/features/verification/runs/dto.go b/backend/internal/features/verification/runs/dto.goindex 397cbf95..964ef62f 100644--- a/backend/internal/features/verification/runs/dto.go+++ b/backend/internal/features/verification/runs/dto.go@@ -39,6 +39,7 @@ type JobAssignment struct {BackupSizeMb float64 `json:"backupSizeMb"`MaxContainerDiskMb float64 `json:"maxContainerDiskMb"`Database *databases.Database `json:"database"`+ TimescaledbVersion string `json:"timescaledbVersion"`}type ReportTableStat struct {diff --git a/backend/internal/features/verification/runs/service.go b/backend/internal/features/verification/runs/service.goindex 0f528d94..01353452 100644--- a/backend/internal/features/verification/runs/service.go+++ b/backend/internal/features/verification/runs/service.go@@ -298,6 +298,7 @@ func (s *VerificationService) ClaimVerification(BackupSizeMb: candidate.Backup.BackupSizeMb,MaxContainerDiskMb: float64(EstimateRequiredForRestoreDiskMb(candidate.Backup)),Database: sanitizeDatabaseForAgent(database),+ TimescaledbVersion: candidate.Backup.TimescaledbVersion,}return nildiff --git a/backend/internal/util/testing/containers/timescaledb.go b/backend/internal/util/testing/containers/timescaledb.gonew file mode 100644index 00000000..42054ab2--- /dev/null+++ b/backend/internal/util/testing/containers/timescaledb.go@@ -0,0 +1,30 @@+package containers++import (+ "testing"++ "github.com/testcontainers/testcontainers-go"+)++// StartTimescaleDB boots a PostgreSQL server with the timescaledb extension from a+// timescale/timescaledb image (e.g. "timescale/timescaledb:2.17.0-pg17").+func StartTimescaleDB(t *testing.T, image string) Endpoint {+ t.Helper()++ return start(t, timescaleDBRequest(image), postgresPort)+}++// timescaleDBRequest mirrors postgresRequest but pins the data dir to the pre-18 layout: every+// published timescaledb image is pre-18, and postgresDataDir's tag parser would misread a+// "<ts>-pg<major>" tag. The timescaledb shared library is loaded by the image's own tuning, so no+// shared_preload_libraries override is needed here.+func timescaleDBRequest(image string) testcontainers.ContainerRequest {+ return testcontainers.ContainerRequest{+ Image: image,+ ExposedPorts: []string{postgresPort},+ Env: postgresEnv(),+ Cmd: []string{"-c", "fsync=off", "-c", "full_page_writes=off", "-c", "synchronous_commit=off"},+ Tmpfs: map[string]string{"/var/lib/postgresql/data": dataDirTmpfsOptions},+ WaitingFor: postgresReady(),+ }+}diff --git a/backend/migrations/20260617120000_add_timescaledb_version_to_backups.sql b/backend/migrations/20260617120000_add_timescaledb_version_to_backups.sqlnew file mode 100644index 00000000..c7eb36fd--- /dev/null+++ b/backend/migrations/20260617120000_add_timescaledb_version_to_backups.sql@@ -0,0 +1,7 @@+-- +goose Up+ALTER TABLE logical_backups+ ADD COLUMN timescaledb_version TEXT NOT NULL DEFAULT '';++-- +goose Down+ALTER TABLE logical_backups+ DROP COLUMN timescaledb_version;