Light, fluffy, and always free - The AWS Local Emulator alternative
feat(rds): add the snapshot export task API (#5787)
Simon Claessens772ec8d0c911
diff --git a/docs/services/index.md b/docs/services/index.mdindex 879ee3e9c1..7830075d71 100644--- a/docs/services/index.md+++ b/docs/services/index.md@@ -63,7 +63,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac| [Amazon Data Lifecycle Manager](dlm.md) | `/policies`, `/policies/{policyId}`, `/tags/{resourceArn}` | REST JSON | 8 || [ElastiCache](elasticache.md) | `POST /` with `Action=` param + TCP proxy | Query + RESP | 30 || [MemoryDB](memorydb.md) | `POST /` + `X-Amz-Target: AmazonMemoryDB.*` + TCP proxy | JSON 1.1 + RESP | 13 |-| [RDS](rds.md) | `POST /` with `Action=` param + TCP proxy | Query + wire | 102 |+| [RDS](rds.md) | `POST /` with `Action=` param + TCP proxy | Query + wire | 105 || [RDS Data API](rds-data.md) | `/Execute`, `/BeginTransaction`, `/CommitTransaction`, `/RollbackTransaction` | REST JSON | 4 || [Timestream for InfluxDB](timestream-influxdb.md) | `POST /` + `X-Amz-Target: AmazonTimestreamInfluxDB.*` + InfluxDB container | JSON 1.0 + InfluxDB HTTP | 24 || [MSK](msk.md) | `/v1/clusters/...`, `/api/v2/clusters/...` + Redpanda broker | REST JSON + Kafka | 8 |diff --git a/docs/services/rds.md b/docs/services/rds.mdindex 40cbc361e1..bd9d3e7fc2 100644--- a/docs/services/rds.md+++ b/docs/services/rds.md@@ -134,6 +134,9 @@ RDS Data API (`rds-data`) is documented separately because it uses REST JSON rou| `RestoreDBClusterToPointInTime` | Create a cluster from a source cluster at a time in its restorable window; see [Point in time restore](#point-in-time-restore) || `DescribeDBClusterSnapshotAttributes` | Return the `restore` attribute of a cluster snapshot || `ModifyDBClusterSnapshotAttribute` | Add or remove `restore` values (account ids or `all`) on a cluster snapshot |+| `StartExportTask` | Start an export task for a DB snapshot or DB cluster snapshot; see [Exporting snapshots to Amazon S3](#exporting-snapshots-to-amazon-s3) |+| `DescribeExportTasks` | List export tasks, by `ExportTaskIdentifier`, `SourceArn`, `SourceType` and the `export-task-identifier`, `s3-bucket`, `source-arn` and `status` filters |+| `CancelExportTask` | Cancel a running export task; what it already wrote stays in the bucket || `DescribeGlobalClusters` | List the account's global clusters with their primary and secondary members; see [Global clusters](#global-clusters) || `CreateGlobalCluster` | Create an Aurora global database, empty or with an existing Aurora cluster as its primary || `ModifyGlobalCluster` | Rename a global cluster, set deletion protection, or upgrade its engine version (members follow) |@@ -721,6 +724,40 @@ replicas use, so the restored data is the source's latest state whatever time inasked for; this is also why `LatestRestorableTime` is always the current time. As with`CreateDBSnapshot`, only PostgreSQL sources are accepted.+## Exporting snapshots to Amazon S3++`StartExportTask`, `DescribeExportTasks` and `CancelExportTask` manage RDS snapshot export tasks.+`ExportTaskIdentifier`, `SourceArn`, `S3BucketName`, `IamRoleArn` and `KmsKeyId` are required,+`S3Prefix` and `ExportOnly` are optional.++A task reports `STARTING`, then `IN_PROGRESS` with a `PercentProgress` while its tables are written,+then `COMPLETE` or `FAILED` with a `FailureCause`. `CancelExportTask` moves a running task to+`CANCELING`; it stops before its next table and becomes `CANCELED`, and the data it already wrote+stays in the bucket, as on RDS. A table the export skips is named in `WarningMessage`. A task still+running when Floci stops is `FAILED` as soon as Floci starts again, or `CANCELED` if it was being+canceled. A `StartExportTask` that arrives while Floci is resetting waits for the reset to finish,+and is refused with `Throttling` if it lasts more than 30 seconds.++Floci does not write snapshot data to S3 yet. In [mock mode](#mock-mode-ci-tests) a task completes+without writing anything, unless its bucket is in another region (see below). Outside mock mode a task is accepted and then ends `FAILED` with the+`FailureCause` `Floci does not write snapshot data to S3 yet: export tasks complete only in mock+mode (FLOCI_SERVICES_RDS_MOCK=true)`.++`ExportOnly` takes `database`, `database.schema` or `database.schema.table` entries made of+letters, digits, `$` and `_`, and refuses anything else with `InvalidExportOnly`, since RDS supports+a partial export only for such names.++What differs from RDS:++- The IAM role is not assumed and the KMS key encrypts nothing: both are recorded and reported back+ as given. The bucket must exist, or the request fails with `InvalidS3BucketFault`. A bucket in+ another region than the snapshot's is accepted and the task then ends `FAILED`, as on RDS.+- Only PostgreSQL snapshots can be exported, since Floci's snapshots hold PostgreSQL data only.+ Any other source is refused with `InvalidExportSourceState`, and so is a snapshot that is not+ `available`.+- `SourceArn` must name a snapshot: exporting a live DB cluster (`SourceType` `CLUSTER`) is not+ supported.+## Global clusters`CreateGlobalCluster` creates an Aurora global database (`aurora-mysql` or `aurora-postgresql`),diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsQueryController.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsQueryController.javaindex e951a6af0d..6d37b8be8d 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/AwsQueryController.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsQueryController.java@@ -578,7 +578,8 @@ public class AwsQueryController {"ModifyDBClusterParameterGroup", "DeleteDBClusterParameterGroup","DescribeDBClusterParameters","CreateOptionGroup", "DescribeOptionGroups", "ModifyOptionGroup", "DeleteOptionGroup",- "DescribeDBSnapshots", "DescribeDBClusterSnapshots"+ "DescribeDBSnapshots", "DescribeDBClusterSnapshots",+ "StartExportTask", "DescribeExportTasks", "CancelExportTask");private static final Set<String> CLOUDFORMATION_ACTIONS = Set.of(diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.javaindex a857be2cb2..340e2ca554 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.AwsQueryResponse;import io.github.hectorvent.floci.core.common.XmlBuilder;import io.github.hectorvent.floci.services.docdb.DocDbQueryHandler;import io.github.hectorvent.floci.services.neptune.NeptuneQueryHandler;+import io.github.hectorvent.floci.services.rds.export.RdsExportTaskService;import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;import io.github.hectorvent.floci.services.rds.model.DbCluster;import io.github.hectorvent.floci.services.rds.model.DbClusterEndpoint;@@ -27,6 +28,7 @@ import io.github.hectorvent.floci.services.rds.model.DbRoleAssociation;import io.github.hectorvent.floci.services.rds.model.DbSnapshot;import io.github.hectorvent.floci.services.rds.model.DbSubnetGroup;import io.github.hectorvent.floci.services.rds.model.EventSubscription;+import io.github.hectorvent.floci.services.rds.model.ExportTask;import io.github.hectorvent.floci.services.rds.model.GlobalCluster;import io.github.hectorvent.floci.services.rds.model.GlobalClusterMember;import io.github.hectorvent.floci.services.rds.model.LogExportChanges;@@ -62,13 +64,15 @@ public class RdsQueryHandler {private static final Logger LOG = Logger.getLogger(RdsQueryHandler.class);private final RdsService service;+ private final RdsExportTaskService exportTaskService;private final DocDbQueryHandler docDbQueryHandler;private final NeptuneQueryHandler neptuneQueryHandler;private final EmulatorConfig config;@Inject- public RdsQueryHandler(RdsService service, EmulatorConfig config, DocDbQueryHandler docDbQueryHandler,- NeptuneQueryHandler neptuneQueryHandler) {+ public RdsQueryHandler(RdsService service, RdsExportTaskService exportTaskService, EmulatorConfig config,+ DocDbQueryHandler docDbQueryHandler, NeptuneQueryHandler neptuneQueryHandler) {+ this.exportTaskService = exportTaskService;this.docDbQueryHandler = docDbQueryHandler;this.neptuneQueryHandler = neptuneQueryHandler;this.service = service;@@ -178,6 +182,9 @@ public class RdsQueryHandler {case "RestoreDBClusterToPointInTime" -> handleRestoreDbClusterToPointInTime(params, region);case "DescribeDBClusterSnapshotAttributes" -> handleDescribeDbClusterSnapshotAttributes(params, region);case "ModifyDBClusterSnapshotAttribute" -> handleModifyDbClusterSnapshotAttribute(params, region);+ case "StartExportTask" -> handleStartExportTask(params, region);+ case "DescribeExportTasks" -> handleDescribeExportTasks(params, region);+ case "CancelExportTask" -> handleCancelExportTask(params, region);case "DescribeGlobalClusters" -> handleDescribeGlobalClusters(params);case "CreateGlobalCluster" -> handleCreateGlobalCluster(params, region);case "ModifyGlobalCluster" -> handleModifyGlobalCluster(params);@@ -1983,6 +1990,94 @@ public class RdsQueryHandler {}}+ private Response handleStartExportTask(MultivaluedMap<String, String> params, String region) {+ String exportTaskIdentifier = params.getFirst("ExportTaskIdentifier");+ String sourceArn = params.getFirst("SourceArn");+ String s3BucketName = params.getFirst("S3BucketName");+ String iamRoleArn = params.getFirst("IamRoleArn");+ String kmsKeyId = params.getFirst("KmsKeyId");+ Response missing = firstMissingParam(+ "ExportTaskIdentifier", exportTaskIdentifier,+ "SourceArn", sourceArn,+ "S3BucketName", s3BucketName,+ "IamRoleArn", iamRoleArn,+ "KmsKeyId", kmsKeyId);+ if (missing != null) {+ return missing;+ }+ ExportTask task = exportTaskService.startExportTask(exportTaskIdentifier, sourceArn, s3BucketName,+ params.getFirst("S3Prefix"), iamRoleArn, kmsKeyId, memberList(params, "ExportOnly"), region);+ return Response.ok(AwsQueryResponse.envelope("StartExportTask", AwsNamespaces.RDS, exportTaskXml(task)))+ .build();+ }++ private Response handleDescribeExportTasks(MultivaluedMap<String, String> params, String region) {+ String sourceType = params.getFirst("SourceType");+ if (sourceType != null && !List.of("SNAPSHOT", "CLUSTER").contains(sourceType)) {+ throw new AwsException("InvalidParameterValue", "SourceType must be SNAPSHOT or CLUSTER.", 400);+ }+ Integer maxRecords = parseOptionalInt(params.getFirst("MaxRecords"));+ if (maxRecords != null && (maxRecords < 20 || maxRecords > 100)) {+ throw new AwsException("InvalidParameterValue", "MaxRecords must be between 20 and 100.", 400);+ }+ Map<String, List<String>> filters = new LinkedHashMap<>();+ for (int i = 1; params.getFirst("Filters.Filter." + i + ".Name") != null; i++) {+ String name = params.getFirst("Filters.Filter." + i + ".Name");+ filters.put(name, extractRdsFilterValues(params, name));+ }+ List<ExportTask> all = exportTaskService.describeExportTasks(params.getFirst("ExportTaskIdentifier"),+ params.getFirst("SourceArn"), sourceType, filters, region);+ int from = Math.min(parseMarker(params.getFirst("Marker")), all.size());+ int to = Math.min(from + (maxRecords != null ? maxRecords : 100), all.size());+ XmlBuilder xml = new XmlBuilder().start("ExportTasks");+ for (ExportTask task : all.subList(from, to)) {+ xml.start("ExportTask").raw(exportTaskXml(task)).end("ExportTask");+ }+ xml.end("ExportTasks");+ if (to < all.size()) {+ xml.elem("Marker", String.valueOf(to));+ }+ return Response.ok(AwsQueryResponse.envelope("DescribeExportTasks", AwsNamespaces.RDS, xml.build())).build();+ }++ private Response handleCancelExportTask(MultivaluedMap<String, String> params, String region) {+ String exportTaskIdentifier = params.getFirst("ExportTaskIdentifier");+ Response missing = firstMissingParam("ExportTaskIdentifier", exportTaskIdentifier);+ if (missing != null) {+ return missing;+ }+ ExportTask task = exportTaskService.cancelExportTask(exportTaskIdentifier, region);+ return Response.ok(AwsQueryResponse.envelope("CancelExportTask", AwsNamespaces.RDS, exportTaskXml(task)))+ .build();+ }++ static String exportTaskXml(ExportTask task) {+ XmlBuilder xml = new XmlBuilder()+ .elem("ExportTaskIdentifier", task.getExportTaskIdentifier())+ .elem("SourceArn", task.getSourceArn())+ .elem("SourceType", task.getSourceType());+ if (!task.getExportOnly().isEmpty()) {+ xml.start("ExportOnly");+ for (String entry : task.getExportOnly()) {+ xml.elem("member", entry);+ }+ xml.end("ExportOnly");+ }+ xml.elem("SnapshotTime", task.getSnapshotTime() != null ? task.getSnapshotTime().toString() : null)+ .elem("TaskStartTime", task.getTaskStartTime() != null ? task.getTaskStartTime().toString() : null)+ .elem("TaskEndTime", task.getTaskEndTime() != null ? task.getTaskEndTime().toString() : null)+ .elem("S3Bucket", task.getS3Bucket())+ .elem("S3Prefix", task.getS3Prefix())+ .elem("IamRoleArn", task.getIamRoleArn())+ .elem("KmsKeyId", task.getKmsKeyId())+ .elem("Status", task.getStatus())+ .elem("PercentProgress", task.getPercentProgress())+ .elem("TotalExtractedDataInGB", task.getTotalExtractedDataInGb())+ .elem("FailureCause", task.getFailureCause())+ .elem("WarningMessage", task.getWarningMessage());+ return xml.build();+ }+private Response handleDescribeDbSnapshots(MultivaluedMap<String, String> params, String region) {String snapshotId = params.getFirst("DBSnapshotIdentifier");String instanceId = params.getFirst("DBInstanceIdentifier");diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.javaindex 843d7063c8..fcb3043ebe 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java@@ -34,6 +34,7 @@ import io.github.hectorvent.floci.services.kms.model.KmsKey;import io.github.hectorvent.floci.services.rds.container.AutoPauseListener;import io.github.hectorvent.floci.services.rds.container.RdsContainerHandle;import io.github.hectorvent.floci.services.rds.container.RdsContainerManager;+import io.github.hectorvent.floci.services.rds.export.SnapshotExportSource;import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;import io.github.hectorvent.floci.services.rds.model.DbCluster;import io.github.hectorvent.floci.services.rds.model.DbClusterEndpoint;@@ -2265,6 +2266,69 @@ public class RdsService implements Resettable, ResourceProvider {return resource.substring("cluster-snapshot:".length());}+ /**+ * The DB snapshot or DB cluster snapshot {@code sourceArn} names, as an export task reads it.+ * Floci keeps PostgreSQL data in its snapshots, so a snapshot of any other engine cannot be+ * exported, and an ARN naming a live cluster rather than a snapshot is refused.+ */+ public SnapshotExportSource snapshotExportSource(String sourceArn, String region) {+ String effectiveRegion = effectiveRegion(region);+ String accountId = currentAccountId();+ AwsArnUtils.Arn arn;+ try {+ arn = AwsArnUtils.parse(sourceArn);+ } catch (IllegalArgumentException malformed) {+ throw new AwsException("InvalidParameterValue", "Invalid SourceArn: " + sourceArn, 400);+ }+ String resource = arn.resource();+ boolean clusterSnapshot = resource.startsWith("cluster-snapshot:");+ if (!"rds".equals(arn.service()) || !(clusterSnapshot || resource.startsWith("snapshot:"))) {+ throw new AwsException("InvalidParameterValue",+ "SourceArn must name a DB snapshot or a DB cluster snapshot: " + sourceArn, 400);+ }+ boolean ours = AwsRegions.partitionFor(effectiveRegion).equals(arn.partition())+ && effectiveRegion.equals(arn.region()) && accountId.equals(arn.accountId());+ String snapshotId = resource.substring(resource.indexOf(':') + 1);+ if (clusterSnapshot) {+ if (!ours) {+ throw new AwsException("DBClusterSnapshotNotFoundFault",+ "DBClusterSnapshot " + snapshotId + " not found.", 404);+ }+ DbClusterSnapshot snapshot = requireClusterSnapshot(accountId, effectiveRegion, snapshotId);+ requireExportable(snapshot.getStatus(), snapshot.getEngine(), sourceArn);+ String sqlDump = snapshotData.get(clusterSnapshotDataKey(effectiveRegion, snapshotId))+ .orElseThrow(() -> new AwsException("DBClusterSnapshotNotFoundFault",+ "DB cluster snapshot data for " + snapshotId + " not found.", 404));+ return new SnapshotExportSource(snapshot.getDbClusterSnapshotArn(),+ imageForEngine(snapshot.getEngine(), snapshot.getEngineVersion()),+ snapshot.getMasterUsername(), snapshot.getMasterPassword(),+ snapshot.getSnapshotCreateTime(), sqlDump);+ }+ DbSnapshot snapshot = ours ? findSnapshotForScope(accountId, effectiveRegion, snapshotId) : null;+ if (snapshot == null) {+ throw new AwsException("DBSnapshotNotFound", "DBSnapshot " + snapshotId + " not found.", 404);+ }+ requireExportable(snapshot.getStatus(), snapshot.getEngine(), sourceArn);+ String sqlDump = getSnapshotDataForScope(accountId, effectiveRegion, snapshotId)+ .orElseThrow(() -> new AwsException("DBSnapshotNotFound",+ "DBSnapshot data for " + snapshotId + " not found.", 404));+ return new SnapshotExportSource(snapshot.getDbSnapshotArn(),+ imageForEngine(snapshot.getEngine(), snapshot.getEngineVersion()),+ snapshot.getMasterUsername(), snapshot.getMasterPassword(),+ snapshot.getSnapshotCreateTime(), sqlDump);+ }++ private static void requireExportable(String status, DatabaseEngine engine, String sourceArn) {+ if (!"available".equals(status)) {+ throw new AwsException("InvalidExportSourceState",+ "The snapshot " + sourceArn + " is not available for export.", 400);+ }+ if (engine != DatabaseEngine.POSTGRES) {+ throw new AwsException("InvalidExportSourceState",+ "Floci exports PostgreSQL snapshots only: " + sourceArn, 400);+ }+ }+/** Cluster snapshot dumps share the snapshot data store with instance snapshots under their own prefix. */private String clusterSnapshotDataKey(String region, String snapshotId) {return dbResourceKey(region, "cluster-snapshot:" + snapshotId);diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/export/ExportOnlySelection.java b/src/main/java/io/github/hectorvent/floci/services/rds/export/ExportOnlySelection.javanew file mode 100644index 0000000000..be95691772--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/rds/export/ExportOnlySelection.java@@ -0,0 +1,71 @@+package io.github.hectorvent.floci.services.rds.export;++import io.github.hectorvent.floci.core.common.AwsException;++import java.util.List;+import java.util.regex.Pattern;++/**+ * The {@code ExportOnly} list of a PostgreSQL export: each entry names a {@code database}, a+ * {@code database.schema} or a {@code database.schema.table}, and an empty list exports+ * everything. RDS supports a partial export only for names made of letters, digits, {@code $} and+ * {@code _}, so any other entry is refused.+ */+final class ExportOnlySelection {++ private static final Pattern IDENTIFIER = Pattern.compile("[A-Za-z0-9$_]+");++ private final List<String[]> entries;++ private ExportOnlySelection(List<String[]> entries) {+ this.entries = entries;+ }++ static ExportOnlySelection of(List<String> exportOnly) {+ if (exportOnly == null || exportOnly.isEmpty()) {+ return new ExportOnlySelection(List.of());+ }+ List<String[]> parsed = exportOnly.stream().map(ExportOnlySelection::parse).toList();+ return new ExportOnlySelection(parsed);+ }++ private static String[] parse(String entry) {+ String[] parts = entry.split("\\.", -1);+ if (parts.length > 3) {+ throw invalid(entry);+ }+ for (String part : parts) {+ if (!IDENTIFIER.matcher(part).matches()) {+ throw invalid(entry);+ }+ }+ return parts;+ }++ private static AwsException invalid(String entry) {+ return new AwsException("InvalidExportOnly",+ "The ExportOnly value " + entry + " isn't valid for a PostgreSQL export: use database, "+ + "database.schema or database.schema.table.", 400);+ }++ boolean includesDatabase(String database) {+ return entries.isEmpty() || entries.stream().anyMatch(entry -> entry[0].equals(database));+ }++ boolean includes(String database, String schema, String table) {+ if (entries.isEmpty()) {+ return true;+ }+ for (String[] entry : entries) {+ if (!entry[0].equals(database)) {+ continue;+ }+ if (entry.length == 1+ || (entry.length == 2 && entry[1].equals(schema))+ || (entry.length == 3 && entry[1].equals(schema) && entry[2].equals(table))) {+ return true;+ }+ }+ return false;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/export/RdsExportTaskService.java b/src/main/java/io/github/hectorvent/floci/services/rds/export/RdsExportTaskService.javanew file mode 100644index 0000000000..0ea3bd7e2d--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/rds/export/RdsExportTaskService.java@@ -0,0 +1,460 @@+package io.github.hectorvent.floci.services.rds.export;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.core.common.Resettable;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.rds.RdsService;+import io.github.hectorvent.floci.services.rds.model.ExportTask;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.quarkus.runtime.StartupEvent;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.event.Observes;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.time.Clock;+import java.time.Instant;+import java.util.Comparator;+import java.util.List;+import java.util.Locale;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.UUID;+import java.util.concurrent.Executor;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.regex.Pattern;++/**+ * RDS snapshot exports to Amazon S3: {@code StartExportTask}, {@code DescribeExportTasks} and+ * {@code CancelExportTask}. A task is {@code STARTING} while its snapshot is restored,+ * {@code IN_PROGRESS} while its tables are written, and then {@code COMPLETE} or {@code FAILED}.+ * A cancel moves it to {@code CANCELING} until the worker stops before its next table, then to+ * {@code CANCELED}, and leaves what was already written in the bucket, as RDS does.+ *+ * <p>Deviations, documented in {@code docs/services/rds.md}: the IAM role is not assumed and the+ * KMS key does not encrypt anything, both are recorded as given; only PostgreSQL snapshots can be+ * exported, since Floci's snapshots hold PostgreSQL data only; and a live cluster cannot be the+ * source.+ */+@ApplicationScoped+public class RdsExportTaskService implements Resettable {++ private static final Logger LOG = Logger.getLogger(RdsExportTaskService.class);++ static final String RESTART_FAILURE_CAUSE = "The emulator restarted before the export finished";+ static final long RESET_DRAIN_TIMEOUT_MILLIS = 5_000L;+ private static final long RESET_ADMISSION_WAIT_MILLIS = 30_000L;+ static final long GIB = 1_073_741_824L;+ private static final String KEY_PREFIX = "export-task::";++ // Letters, digits and single hyphens, starting with a letter and not ending with a hyphen.+ private static final Pattern IDENTIFIER = Pattern.compile("[A-Za-z](?:-?[A-Za-z0-9])*");+ private static final int IDENTIFIER_MAX_LENGTH = 60;+ private static final Set<String> FILTER_NAMES =+ Set.of("export-task-identifier", "s3-bucket", "source-arn", "status");++ private final StorageBackend<String, ExportTask> tasks;+ private final RdsService rdsService;+ private final S3Service s3Service;+ private final SnapshotExporter exporter;+ private final RegionResolver regionResolver;+ private final boolean mock;+ private final Clock clock;+ private final Executor workers;+ private final Object lock = new Object();+ // Reset support, following SesExportJobService: beforeReset bumps the generation so a running+ // worker stops writing, then waits for the workers already running, for a bounded time. A+ // StartExportTask arriving meanwhile waits for afterReset rather than being wiped by it.+ private final AtomicInteger generation = new AtomicInteger();+ private boolean resetting;+ private final Object workerMonitor = new Object();+ private int activeWorkers;++ @Inject+ public RdsExportTaskService(StorageFactory storageFactory, RdsService rdsService, S3Service s3Service,+ UnsupportedSnapshotExporter exporter, RegionResolver regionResolver,+ EmulatorConfig config, Clock clock) {+ this(storageFactory.create("rds", "rds-export-tasks.json",+ new TypeReference<Map<String, ExportTask>>() {}),+ rdsService, s3Service, exporter, regionResolver, config.services().rds().mock(), clock,+ task -> Thread.ofVirtual().name("rds-export").start(task));+ }++ RdsExportTaskService(StorageBackend<String, ExportTask> tasks, RdsService rdsService, S3Service s3Service,+ SnapshotExporter exporter, RegionResolver regionResolver, boolean mock, Clock clock,+ Executor workers) {+ this.tasks = tasks;+ this.rdsService = rdsService;+ this.s3Service = s3Service;+ this.exporter = exporter;+ this.regionResolver = regionResolver;+ this.mock = mock;+ this.clock = clock;+ this.workers = workers;+ recoverInterruptedTasks();+ }++ public ExportTask startExportTask(String exportTaskIdentifier, String sourceArn, String s3BucketName,+ String s3Prefix, String iamRoleArn, String kmsKeyId,+ List<String> exportOnly, String region) {+ String effectiveRegion = effectiveRegion(region);+ String accountId = regionResolver.getAccountId();+ if (exportTaskIdentifier.length() > IDENTIFIER_MAX_LENGTH+ || !IDENTIFIER.matcher(exportTaskIdentifier).matches()) {+ throw new AwsException("InvalidParameterValue", "The export task identifier " + exportTaskIdentifier+ + " is not valid: use 1 to 60 letters, digits or single hyphens, starting with a letter"+ + " and not ending with a hyphen.", 400);+ }+ ExportOnlySelection.of(exportOnly);+ SnapshotExportSource source = rdsService.snapshotExportSource(sourceArn, effectiveRegion);+ if (!s3Service.bucketExists(s3BucketName)) {+ throw new AwsException("InvalidS3BucketFault",+ "The S3 bucket " + s3BucketName + " doesn't exist.", 400);+ }+ String bucketRegion = s3Service.getBucketRegion(s3BucketName);+ if (bucketRegion == null || bucketRegion.isBlank()) {+ bucketRegion = regionResolver.getDefaultRegion();+ }+ boolean bucketInAnotherRegion = !bucketRegion.equalsIgnoreCase(effectiveRegion);++ ExportTask task = new ExportTask();+ task.setExportTaskIdentifier(exportTaskIdentifier);+ task.setSourceArn(source.sourceArn());+ task.setSourceType(ExportTask.SOURCE_TYPE_SNAPSHOT);+ task.setExportOnly(exportOnly);+ task.setSnapshotTime(source.snapshotTime());+ task.setTaskStartTime(Instant.now(clock));+ task.setS3Bucket(s3BucketName);+ task.setS3Prefix(s3Prefix);+ task.setIamRoleArn(iamRoleArn);+ task.setKmsKeyId(kmsKeyId);+ task.setStatus(ExportTask.STATUS_STARTING);+ task.setRegion(effectiveRegion);+ task.setAccountId(accountId);+ task.setRuntimeArn(regionResolver.buildArn("rds", effectiveRegion, accountId,+ "export-task:" + exportTaskIdentifier));+ task.setRuntimeVolumeId("export-" + UUID.randomUUID().toString().replace("-", "").substring(0, 16));++ synchronized (lock) {+ awaitResetFinished();+ if (tasks.get(key(effectiveRegion, exportTaskIdentifier)).isPresent()) {+ throw new AwsException("ExportTaskAlreadyExists",+ "The export task with the ID " + exportTaskIdentifier + " already exists.", 400);+ }+ tasks.put(key(effectiveRegion, exportTaskIdentifier), task);+ ExportTask started = new ExportTask(task);+ int startedIn = generation.get();+ // The worker runs on a fresh thread, where account-aware storage would otherwise fall+ // back to the default account and never find this task again.+ workers.execute(() -> RequestScopes.runAs(accountId, effectiveRegion,+ () -> run(effectiveRegion, exportTaskIdentifier, source, bucketInAnotherRegion, startedIn)));+ return started;+ }+ }++ public List<ExportTask> describeExportTasks(String exportTaskIdentifier, String sourceArn, String sourceType,+ Map<String, List<String>> filters, String region) {+ String effectiveRegion = effectiveRegion(region);+ for (String name : filters.keySet()) {+ if (!FILTER_NAMES.contains(name)) {+ throw new AwsException("InvalidParameterValue", "Unrecognized filter name: " + name, 400);+ }+ }+ if (exportTaskIdentifier != null && !exportTaskIdentifier.isBlank()) {+ requireTask(effectiveRegion, exportTaskIdentifier);+ }+ List<ExportTask> found;+ synchronized (lock) {+ found = tasks.scan(key -> key.startsWith(KEY_PREFIX + effectiveRegion + "::")).stream()+ .map(ExportTask::new)+ .toList();+ }+ return found.stream()+ .filter(task -> matches(exportTaskIdentifier, task.getExportTaskIdentifier()))+ .filter(task -> matches(sourceArn, task.getSourceArn()))+ .filter(task -> matches(sourceType, task.getSourceType()))+ .filter(task -> matchesFilters(task, filters))+ .sorted(Comparator.comparing(ExportTask::getTaskStartTime))+ .toList();+ }++ public ExportTask cancelExportTask(String exportTaskIdentifier, String region) {+ String effectiveRegion = effectiveRegion(region);+ synchronized (lock) {+ ExportTask task = requireTask(effectiveRegion, exportTaskIdentifier);+ if (task.isTerminal() || ExportTask.STATUS_CANCELING.equals(task.getStatus())) {+ throw new AwsException("InvalidExportTaskStateFault",+ "The export task with the ID " + exportTaskIdentifier + " can't be canceled because its "+ + "status is " + task.getStatus() + ".", 400);+ }+ task.setStatus(ExportTask.STATUS_CANCELING);+ tasks.put(key(effectiveRegion, exportTaskIdentifier), task);+ return new ExportTask(task);+ }+ }++ // Holds lock. A start arriving while a reset drains would capture the already-bumped generation,+ // so its task would be wiped by clear() after it was admitted. It waits for afterReset.+ private void awaitResetFinished() {+ long deadline = System.currentTimeMillis() + RESET_ADMISSION_WAIT_MILLIS;+ while (resetting) {+ long remaining = deadline - System.currentTimeMillis();+ if (remaining <= 0) {+ throw resettingNow();+ }+ try {+ lock.wait(remaining);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw resettingNow();+ }+ }+ }++ private static AwsException resettingNow() {+ return new AwsException("Throttling", "Export tasks are not accepted while the emulator is resetting.", 429);+ }++ private void run(String region, String taskId, SnapshotExportSource source, boolean bucketInAnotherRegion,+ int startedIn) {+ synchronized (workerMonitor) {+ activeWorkers++;+ }+ try {+ runTask(region, taskId, source, bucketInAnotherRegion, startedIn);+ } finally {+ synchronized (workerMonitor) {+ activeWorkers--;+ workerMonitor.notifyAll();+ }+ }+ }++ private void runTask(String region, String taskId, SnapshotExportSource source, boolean bucketInAnotherRegion,+ int startedIn) {+ Optional<ExportTask> current = tasks.get(key(region, taskId));+ if (current.isEmpty() || current.get().isTerminal() || generation.get() != startedIn) {+ return;+ }+ if (ExportTask.STATUS_CANCELING.equals(current.get().getStatus())) {+ // Canceled before its worker started: nothing to restore, nothing to stop.+ finish(region, taskId, startedIn, ExportTask.STATUS_CANCELED, null, null);+ return;+ }+ if (bucketInAnotherRegion) {+ finish(region, taskId, startedIn, ExportTask.STATUS_FAILED, "The S3 bucket "+ + current.get().getS3Bucket() + " isn't located in the current AWS Region. Review your S3 bucket"+ + " name and retry the export.", null);+ return;+ }+ if (mock) {+ finish(region, taskId, startedIn, ExportTask.STATUS_COMPLETE, null,+ new SnapshotExporter.Outcome(0, List.of(), Instant.now(clock)));+ return;+ }+ SnapshotExporter.Progress progress = new SnapshotExporter.Progress() {+ @Override+ public void extracting(int percent) {+ synchronized (lock) {+ if (abandonedByReset()) {+ return;+ }+ tasks.get(key(region, taskId))+ .filter(task -> !task.isTerminal() && !ExportTask.STATUS_CANCELING.equals(task.getStatus()))+ .ifPresent(task -> {+ task.setStatus(ExportTask.STATUS_IN_PROGRESS);+ task.setPercentProgress(percent);+ tasks.put(key(region, taskId), task);+ });+ }+ }++ @Override+ public boolean canceled() {+ return abandonedByReset() || tasks.get(key(region, taskId))+ .map(task -> ExportTask.STATUS_CANCELING.equals(task.getStatus()) || task.isTerminal())+ .orElse(true);+ }++ @Override+ public boolean abandonedByReset() {+ return generation.get() != startedIn;+ }+ };+ try {+ SnapshotExporter.Outcome outcome = exporter.export(current.get(), source, progress);+ finish(region, taskId, startedIn, ExportTask.STATUS_COMPLETE, null, outcome);+ } catch (RuntimeException e) {+ LOG.warnv(e, "RDS export task {0} failed", taskId);+ finish(region, taskId, startedIn, ExportTask.STATUS_FAILED,+ e.getMessage() != null ? e.getMessage() : e.toString(), null);+ }+ }++ /** A task canceled while it ran ends {@code CANCELED} whatever its worker reached. */+ private void finish(String region, String taskId, int startedIn, String status, String failureCause,+ SnapshotExporter.Outcome outcome) {+ synchronized (lock) {+ if (generation.get() != startedIn) {+ LOG.infov("RDS export task {0} abandoned by an emulator reset", taskId);+ return;+ }+ Optional<ExportTask> current = tasks.get(key(region, taskId));+ if (current.isEmpty() || current.get().isTerminal()) {+ return;+ }+ ExportTask task = current.get();+ if (ExportTask.STATUS_CANCELING.equals(task.getStatus())) {+ task.setStatus(ExportTask.STATUS_CANCELED);+ } else {+ task.setStatus(status);+ task.setFailureCause(failureCause);+ if (ExportTask.STATUS_COMPLETE.equals(status)) {+ task.setPercentProgress(100);+ }+ }+ if (outcome != null) {+ task.setTotalExtractedDataInGb((int) (outcome.bytesExtracted() / GIB));+ if (!outcome.warnings().isEmpty()) {+ task.setWarningMessage(String.join(" ", outcome.warnings()));+ }+ }+ task.setTaskEndTime(outcome != null && outcome.finishedAt() != null+ ? outcome.finishedAt() : Instant.now(clock));+ tasks.put(key(region, taskId), task);+ }+ }++ private ExportTask requireTask(String region, String taskId) {+ return tasks.get(key(region, taskId))+ .orElseThrow(() -> new AwsException("ExportTaskNotFound",+ "The export task with the ID " + taskId + " doesn't exist.", 404));+ }++ private static boolean matches(String wanted, String actual) {+ return wanted == null || wanted.isBlank() || wanted.equals(actual);+ }++ private static boolean matchesFilters(ExportTask task, Map<String, List<String>> filters) {+ for (Map.Entry<String, List<String>> filter : filters.entrySet()) {+ List<String> values = filter.getValue();+ if (values.isEmpty()) {+ continue;+ }+ boolean matched = switch (filter.getKey()) {+ case "export-task-identifier" -> values.contains(task.getExportTaskIdentifier());+ case "s3-bucket" -> values.contains(task.getS3Bucket());+ case "source-arn" -> values.contains(task.getSourceArn());+ case "status" -> values.stream()+ .anyMatch(value -> value.toUpperCase(Locale.ROOT).equals(task.getStatus()));+ default -> throw new IllegalStateException("Unvalidated filter name: " + filter.getKey());+ };+ if (!matched) {+ return false;+ }+ }+ return true;+ }++ private String effectiveRegion(String region) {+ return region != null && !region.isBlank() ? region : regionResolver.getDefaultRegion();+ }++ private static String key(String region, String taskId) {+ return KEY_PREFIX + region + "::" + taskId;+ }++ /**+ * Runs from the constructor, where no request scope is active, so the store has to be read+ * across accounts: a plain scan would only ever sweep the default account's tasks.+ */+ private void recoverInterruptedTasks() {+ if (!(tasks instanceof AccountAwareStorageBackend<ExportTask> accountTasks)) {+ return;+ }+ for (AccountAwareStorageBackend.AccountEntry<ExportTask> entry+ : accountTasks.scanAllAccountEntries(k -> k.startsWith(KEY_PREFIX))) {+ ExportTask task = entry.value();+ if (task.isTerminal()) {+ continue;+ }+ if (ExportTask.STATUS_CANCELING.equals(task.getStatus())) {+ task.setStatus(ExportTask.STATUS_CANCELED);+ } else {+ task.setStatus(ExportTask.STATUS_FAILED);+ task.setFailureCause(RESTART_FAILURE_CAUSE);+ }+ task.setTaskEndTime(Instant.now(clock));+ accountTasks.putForAccount(entry.accountId(), entry.key(), task);+ LOG.warnv("RDS export task {0} was interrupted by a restart and is now {1}",+ task.getExportTaskIdentifier(), task.getStatus());+ if (!mock) {+ try {+ exporter.discard(task);+ } catch (RuntimeException e) {+ LOG.warnv(e, "Could not remove what RDS export task {0} left behind",+ task.getExportTaskIdentifier());+ }+ }+ }+ }++ /**+ * Forces the bean, which is otherwise created on first use, at startup, so a task a restart+ * interrupted is failed and its restore container and volume removed straight away rather than+ * when a client first calls an export action.+ */+ void onStart(@Observes StartupEvent ignored) {+ LOG.debugv("RDS export task service ready");+ }++ @Override+ public void beforeReset() {+ synchronized (lock) {+ resetting = true;+ generation.incrementAndGet();+ }+ boolean interrupted = false;+ long deadline = System.currentTimeMillis() + RESET_DRAIN_TIMEOUT_MILLIS;+ synchronized (workerMonitor) {+ while (activeWorkers > 0) {+ long remaining = deadline - System.currentTimeMillis();+ if (remaining <= 0) {+ LOG.warnv("{0} RDS export worker(s) still running after {1}ms",+ activeWorkers, RESET_DRAIN_TIMEOUT_MILLIS);+ break;+ }+ try {+ workerMonitor.wait(remaining);+ } catch (InterruptedException e) {+ // Keep draining to the deadline; the interrupt is re-asserted on the way out.+ interrupted = true;+ }+ }+ }+ if (interrupted) {+ Thread.currentThread().interrupt();+ }+ }++ @Override+ public void afterReset() {+ synchronized (lock) {+ resetting = false;+ lock.notifyAll();+ }+ }++ @Override+ public void clear() {+ tasks.clear();+ LOG.info("Cleared all RDS export tasks");+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/export/SnapshotExportSource.java b/src/main/java/io/github/hectorvent/floci/services/rds/export/SnapshotExportSource.javanew file mode 100644index 0000000000..7ced829e40--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/rds/export/SnapshotExportSource.java@@ -0,0 +1,10 @@+package io.github.hectorvent.floci.services.rds.export;++import java.time.Instant;++/**+ * What an export task reads from a DB snapshot or DB cluster snapshot: the image its data+ * restores into, the master login, the time it was taken and its dump.+ */+public record SnapshotExportSource(String sourceArn, String image, String masterUsername,+ String masterPassword, Instant snapshotTime, String sqlDump) {}diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/export/SnapshotExporter.java b/src/main/java/io/github/hectorvent/floci/services/rds/export/SnapshotExporter.javanew file mode 100644index 0000000000..fa6b50a757--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/rds/export/SnapshotExporter.java@@ -0,0 +1,33 @@+package io.github.hectorvent.floci.services.rds.export;++import io.github.hectorvent.floci.services.rds.model.ExportTask;++import java.time.Instant;+import java.util.List;++/** Writes one export task's data to its bucket. */+public interface SnapshotExporter {++ /** How the exporter reports back to the task it runs for. */+ interface Progress {++ /** The snapshot is restored and tables are being written; {@code percent} of them are done. */+ void extracting(int percent);++ /** True once the task was canceled, which stops the export before its next table. */+ boolean canceled();++ boolean abandonedByReset();+ }++ /**+ * What an export wrote: its size, a warning for each table it had to leave out, and when it+ * finished, which is also the end time its {@code export_info} file records.+ */+ record Outcome(long bytesExtracted, List<String> warnings, Instant finishedAt) {}++ Outcome export(ExportTask task, SnapshotExportSource source, Progress progress);++ /** Removes whatever an export that never finished left behind, after a restart. */+ default void discard(ExportTask task) {}+}diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/export/UnsupportedSnapshotExporter.java b/src/main/java/io/github/hectorvent/floci/services/rds/export/UnsupportedSnapshotExporter.javanew file mode 100644index 0000000000..7fbb0004b5--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/rds/export/UnsupportedSnapshotExporter.java@@ -0,0 +1,16 @@+package io.github.hectorvent.floci.services.rds.export;++import io.github.hectorvent.floci.services.rds.model.ExportTask;+import jakarta.enterprise.context.ApplicationScoped;++@ApplicationScoped+public class UnsupportedSnapshotExporter implements SnapshotExporter {++ static final String FAILURE_CAUSE = "Floci does not write snapshot data to S3 yet: "+ + "export tasks complete only in mock mode (FLOCI_SERVICES_RDS_MOCK=true)";++ @Override+ public Outcome export(ExportTask task, SnapshotExportSource source, Progress progress) {+ throw new UnsupportedOperationException(FAILURE_CAUSE);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/model/ExportTask.java b/src/main/java/io/github/hectorvent/floci/services/rds/model/ExportTask.javanew file mode 100644index 0000000000..0fd4719324--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/rds/model/ExportTask.java@@ -0,0 +1,187 @@+package io.github.hectorvent.floci.services.rds.model;++import com.fasterxml.jackson.annotation.JsonIgnore;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;+import java.util.ArrayList;+import java.util.List;++/**+ * A snapshot export to Amazon S3, as {@code StartExportTask} and {@code DescribeExportTasks}+ * describe it. {@code IamRoleArn} and {@code KmsKeyId} are kept as given: Floci writes to its own+ * S3 without assuming the role or encrypting with the key.+ */+@RegisterForReflection+public class ExportTask {++ public static final String STATUS_STARTING = "STARTING";+ public static final String STATUS_IN_PROGRESS = "IN_PROGRESS";+ public static final String STATUS_COMPLETE = "COMPLETE";+ public static final String STATUS_FAILED = "FAILED";+ public static final String STATUS_CANCELING = "CANCELING";+ public static final String STATUS_CANCELED = "CANCELED";++ public static final String SOURCE_TYPE_SNAPSHOT = "SNAPSHOT";++ @JsonProperty("ExportTaskIdentifier")+ private String exportTaskIdentifier;++ @JsonProperty("SourceArn")+ private String sourceArn;++ @JsonProperty("SourceType")+ private String sourceType;++ @JsonProperty("ExportOnly")+ private List<String> exportOnly = new ArrayList<>();++ @JsonProperty("SnapshotTime")+ private Instant snapshotTime;++ @JsonProperty("TaskStartTime")+ private Instant taskStartTime;++ @JsonProperty("TaskEndTime")+ private Instant taskEndTime;++ @JsonProperty("S3Bucket")+ private String s3Bucket;++ @JsonProperty("S3Prefix")+ private String s3Prefix;++ @JsonProperty("IamRoleArn")+ private String iamRoleArn;++ @JsonProperty("KmsKeyId")+ private String kmsKeyId;++ @JsonProperty("Status")+ private String status;++ @JsonProperty("PercentProgress")+ private int percentProgress;++ @JsonProperty("TotalExtractedDataInGB")+ private int totalExtractedDataInGb;++ @JsonProperty("FailureCause")+ private String failureCause;++ @JsonProperty("WarningMessage")+ private String warningMessage;++ @JsonProperty("Region")+ private String region;++ /** The account that started the task, which its worker runs as. */+ @JsonProperty("AccountId")+ private String accountId;++ /**+ * The task's ARN as Floci names its restore container's runtime, so the container carries the+ * task's account and region. RDS gives export tasks no ARN, so this is never reported.+ */+ @JsonProperty("RuntimeArn")+ private String runtimeArn;++ /** The name the restore container and its volume are created under, unique to this task. */+ @JsonProperty("RuntimeVolumeId")+ private String runtimeVolumeId;++ public ExportTask() {}++ public ExportTask(ExportTask other) {+ this.exportTaskIdentifier = other.exportTaskIdentifier;+ this.sourceArn = other.sourceArn;+ this.sourceType = other.sourceType;+ this.exportOnly = new ArrayList<>(other.exportOnly);+ this.snapshotTime = other.snapshotTime;+ this.taskStartTime = other.taskStartTime;+ this.taskEndTime = other.taskEndTime;+ this.s3Bucket = other.s3Bucket;+ this.s3Prefix = other.s3Prefix;+ this.iamRoleArn = other.iamRoleArn;+ this.kmsKeyId = other.kmsKeyId;+ this.status = other.status;+ this.percentProgress = other.percentProgress;+ this.totalExtractedDataInGb = other.totalExtractedDataInGb;+ this.failureCause = other.failureCause;+ this.warningMessage = other.warningMessage;+ this.region = other.region;+ this.accountId = other.accountId;+ this.runtimeArn = other.runtimeArn;+ this.runtimeVolumeId = other.runtimeVolumeId;+ }++ @JsonIgnore+ public boolean isTerminal() {+ return STATUS_COMPLETE.equals(status)+ || STATUS_FAILED.equals(status)+ || STATUS_CANCELED.equals(status);+ }++ public String getExportTaskIdentifier() { return exportTaskIdentifier; }+ public void setExportTaskIdentifier(String exportTaskIdentifier) { this.exportTaskIdentifier = exportTaskIdentifier; }++ public String getSourceArn() { return sourceArn; }+ public void setSourceArn(String sourceArn) { this.sourceArn = sourceArn; }++ public String getSourceType() { return sourceType; }+ public void setSourceType(String sourceType) { this.sourceType = sourceType; }++ public List<String> getExportOnly() { return exportOnly; }+ public void setExportOnly(List<String> exportOnly) {+ this.exportOnly = exportOnly != null ? new ArrayList<>(exportOnly) : new ArrayList<>();+ }++ public Instant getSnapshotTime() { return snapshotTime; }+ public void setSnapshotTime(Instant snapshotTime) { this.snapshotTime = snapshotTime; }++ public Instant getTaskStartTime() { return taskStartTime; }+ public void setTaskStartTime(Instant taskStartTime) { this.taskStartTime = taskStartTime; }++ public Instant getTaskEndTime() { return taskEndTime; }+ public void setTaskEndTime(Instant taskEndTime) { this.taskEndTime = taskEndTime; }++ public String getS3Bucket() { return s3Bucket; }+ public void setS3Bucket(String s3Bucket) { this.s3Bucket = s3Bucket; }++ public String getS3Prefix() { return s3Prefix; }+ public void setS3Prefix(String s3Prefix) { this.s3Prefix = s3Prefix; }++ public String getIamRoleArn() { return iamRoleArn; }+ public void setIamRoleArn(String iamRoleArn) { this.iamRoleArn = iamRoleArn; }++ public String getKmsKeyId() { return kmsKeyId; }+ public void setKmsKeyId(String kmsKeyId) { this.kmsKeyId = kmsKeyId; }++ public String getStatus() { return status; }+ public void setStatus(String status) { this.status = status; }++ public int getPercentProgress() { return percentProgress; }+ public void setPercentProgress(int percentProgress) { this.percentProgress = percentProgress; }++ public int getTotalExtractedDataInGb() { return totalExtractedDataInGb; }+ public void setTotalExtractedDataInGb(int totalExtractedDataInGb) { this.totalExtractedDataInGb = totalExtractedDataInGb; }++ public String getFailureCause() { return failureCause; }+ public void setFailureCause(String failureCause) { this.failureCause = failureCause; }++ public String getWarningMessage() { return warningMessage; }+ public void setWarningMessage(String warningMessage) { this.warningMessage = warningMessage; }++ public String getRegion() { return region; }+ public void setRegion(String region) { this.region = region; }++ public String getAccountId() { return accountId; }+ public void setAccountId(String accountId) { this.accountId = accountId; }++ public String getRuntimeArn() { return runtimeArn; }+ public void setRuntimeArn(String runtimeArn) { this.runtimeArn = runtimeArn; }++ public String getRuntimeVolumeId() { return runtimeVolumeId; }+ public void setRuntimeVolumeId(String runtimeVolumeId) { this.runtimeVolumeId = runtimeVolumeId; }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsQueryHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsQueryHandlerTest.javaindex be55f9424f..c4f493816f 100644--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsQueryHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsQueryHandlerTest.java@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.docdb.DocDbQueryHandler;import io.github.hectorvent.floci.services.neptune.NeptuneQueryHandler;+import io.github.hectorvent.floci.services.rds.export.RdsExportTaskService;import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;import io.github.hectorvent.floci.services.rds.model.DbCluster;import io.github.hectorvent.floci.services.rds.model.DbClusterParameterGroup;@@ -20,6 +21,7 @@ import io.github.hectorvent.floci.services.rds.model.DbProxyTargetGroup;import io.github.hectorvent.floci.services.rds.model.DbSnapshot;import io.github.hectorvent.floci.services.rds.model.DbSubnetGroup;import io.github.hectorvent.floci.services.rds.model.EventSubscription;+import io.github.hectorvent.floci.services.rds.model.ExportTask;import io.github.hectorvent.floci.services.rds.model.OptionGroup;import io.github.hectorvent.floci.services.rds.model.OptionGroupOption;import jakarta.ws.rs.core.MultivaluedHashMap;@@ -33,6 +35,7 @@ import org.junit.jupiter.params.provider.NullSource;import org.junit.jupiter.params.provider.ValueSource;import org.mockito.ArgumentCaptor;+import java.time.Instant;import java.util.List;import java.util.Map;@@ -46,6 +49,7 @@ import static org.mockito.Mockito.*;class RdsQueryHandlerTest {private RdsService service;+ private RdsExportTaskService exportTaskService;private DocDbQueryHandler docDbHandler;private NeptuneQueryHandler neptuneHandler;private RdsQueryHandler handler;@@ -61,7 +65,8 @@ class RdsQueryHandlerTest {when(config.defaultAvailabilityZone()).thenReturn("us-east-1a");docDbHandler = mock(DocDbQueryHandler.class);neptuneHandler = mock(NeptuneQueryHandler.class);- handler = new RdsQueryHandler(service, config, docDbHandler, neptuneHandler);+ exportTaskService = mock(RdsExportTaskService.class);+ handler = new RdsQueryHandler(service, exportTaskService, config, docDbHandler, neptuneHandler);}@ParameterizedTest@@ -3035,6 +3040,107 @@ class RdsQueryHandlerTest {return group;}+ @Test+ void startExportTask_passesTheRequestAndAnswersWithTheTask() {+ ExportTask task = exportTask("appdb-export", ExportTask.STATUS_STARTING);+ when(exportTaskService.startExportTask(eq("appdb-export"), eq(task.getSourceArn()), eq("exports"),+ eq("items/rds"), eq("arn:aws:iam::000000000000:role/export"), eq("alias/export"),+ eq(List.of("appdb.public.items", "appdb.audit")), eq("us-east-1"))).thenReturn(task);+ MultivaluedMap<String, String> params = params();+ params.add("ExportTaskIdentifier", "appdb-export");+ params.add("SourceArn", task.getSourceArn());+ params.add("S3BucketName", "exports");+ params.add("S3Prefix", "items/rds");+ params.add("IamRoleArn", "arn:aws:iam::000000000000:role/export");+ params.add("KmsKeyId", "alias/export");+ params.add("ExportOnly.member.2", "appdb.audit");+ params.add("ExportOnly.member.1", "appdb.public.items");++ String body = (String) handler.handle("StartExportTask", params, "us-east-1").getEntity();++ assertTrue(body.contains("<StartExportTaskResult><ExportTaskIdentifier>appdb-export</ExportTaskIdentifier>"), body);+ assertTrue(body.contains("<ExportOnly><member>appdb.public.items</member></ExportOnly>"), body);+ assertTrue(body.contains("<SourceType>SNAPSHOT</SourceType>"), body);+ assertTrue(body.contains("<SnapshotTime>2026-10-13T08:00:00Z</SnapshotTime>"), body);+ assertTrue(body.contains("<Status>STARTING</Status>"), body);+ assertTrue(body.contains("<PercentProgress>0</PercentProgress>"), body);+ assertTrue(body.contains("<TotalExtractedDataInGB>0</TotalExtractedDataInGB>"), body);+ assertFalse(body.contains("<TaskEndTime>"), body);+ }++ @Test+ void startExportTask_requiresTheRoleAndTheKey() {+ MultivaluedMap<String, String> params = params();+ params.add("ExportTaskIdentifier", "appdb-export");+ params.add("SourceArn", "arn:aws:rds:us-east-1:000000000000:snapshot:snap");+ params.add("S3BucketName", "exports");++ String body = (String) handler.handle("StartExportTask", params, "us-east-1").getEntity();++ assertTrue(body.contains("IamRoleArn"), body);+ verifyNoInteractions(exportTaskService);+ }++ @Test+ void describeExportTasks_listsEachTaskAndPassesFilters() {+ when(exportTaskService.describeExportTasks(eq(null), eq(null), eq("SNAPSHOT"), any(), eq("us-east-1")))+ .thenReturn(List.of(exportTask("one", ExportTask.STATUS_COMPLETE),+ exportTask("two", ExportTask.STATUS_IN_PROGRESS)));+ MultivaluedMap<String, String> params = params();+ params.add("SourceType", "SNAPSHOT");+ params.add("Filters.Filter.1.Name", "status");+ params.add("Filters.Filter.1.Values.Value.1", "complete");++ String body = (String) handler.handle("DescribeExportTasks", params, "us-east-1").getEntity();++ assertTrue(body.contains("<ExportTasks><ExportTask><ExportTaskIdentifier>one</ExportTaskIdentifier>"), body);+ assertTrue(body.contains("<ExportTask><ExportTaskIdentifier>two</ExportTaskIdentifier>"), body);+ @SuppressWarnings("unchecked")+ ArgumentCaptor<Map<String, List<String>>> filters = ArgumentCaptor.forClass(Map.class);+ verify(exportTaskService).describeExportTasks(eq(null), eq(null), eq("SNAPSHOT"), filters.capture(),+ eq("us-east-1"));+ assertEquals(Map.of("status", List.of("complete")), filters.getValue());+ }++ @Test+ void cancelExportTask_answersCanceling() {+ when(exportTaskService.cancelExportTask("appdb-export", "us-east-1"))+ .thenReturn(exportTask("appdb-export", ExportTask.STATUS_CANCELING));+ MultivaluedMap<String, String> params = params();+ params.add("ExportTaskIdentifier", "appdb-export");++ String body = (String) handler.handle("CancelExportTask", params, "us-east-1").getEntity();++ assertTrue(body.contains("<CancelExportTaskResult>"), body);+ assertTrue(body.contains("<Status>CANCELING</Status>"), body);+ }++ @Test+ void exportTaskErrorsKeepTheirRdsCode() {+ when(exportTaskService.cancelExportTask("missing", "us-east-1")).thenThrow(new AwsException(+ "ExportTaskNotFound", "The export task with the ID missing doesn't exist.", 404));+ MultivaluedMap<String, String> params = params();+ params.add("ExportTaskIdentifier", "missing");++ Response response = handler.handle("CancelExportTask", params, "us-east-1");++ assertEquals(404, response.getStatus());+ assertTrue(((String) response.getEntity()).contains("<Code>ExportTaskNotFound</Code>"));+ }++ private static ExportTask exportTask(String id, String status) {+ ExportTask task = new ExportTask();+ task.setExportTaskIdentifier(id);+ task.setSourceArn("arn:aws:rds:us-east-1:000000000000:snapshot:appdb-snap");+ task.setSourceType(ExportTask.SOURCE_TYPE_SNAPSHOT);+ task.setExportOnly(List.of("appdb.public.items"));+ task.setSnapshotTime(Instant.parse("2026-10-13T08:00:00Z"));+ task.setTaskStartTime(Instant.parse("2026-10-13T09:00:00Z"));+ task.setS3Bucket("exports");+ task.setStatus(status);+ return task;+ }+private static MultivaluedMap<String, String> params() {return new MultivaluedHashMap<>();}diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.javaindex a8ec8c1017..70d7ff78f6 100644--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java@@ -21,6 +21,7 @@ import io.github.hectorvent.floci.services.rds.container.AutoPauseListener;import io.github.hectorvent.floci.services.rds.container.RdsBackendGate;import io.github.hectorvent.floci.services.rds.container.RdsContainerHandle;import io.github.hectorvent.floci.services.rds.container.RdsContainerManager;+import io.github.hectorvent.floci.services.rds.export.SnapshotExportSource;import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;import io.github.hectorvent.floci.services.rds.model.DbCluster;import io.github.hectorvent.floci.services.rds.model.DbClusterEndpoint;@@ -65,6 +66,7 @@ import java.net.ServerSocket;import java.time.Duration;import java.time.Instant;import java.util.Collection;+import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Optional;@@ -5340,6 +5342,75 @@ class RdsServiceTest {rdsService.listTagsForResource(snapshot.getDbSnapshotArn()));}+ @Test+ void snapshotExportSourceCarriesTheSnapshotsDump() {+ rdsService.createDbInstance("mydb", "postgres", "16.3",+ "admin", "password", "dbname", "db.t3.micro", 20, false, null, null, null, null, false);+ when(containerManager.createPostgresSnapshot(any(), eq("admin"))).thenReturn("MOCK_DUMP_DATA");+ DbSnapshot snapshot = rdsService.createDbSnapshot("mysnap", "mydb");++ SnapshotExportSource source = rdsService.snapshotExportSource(snapshot.getDbSnapshotArn(), "us-east-1");++ assertEquals("MOCK_DUMP_DATA", source.sqlDump());+ assertEquals(snapshot.getDbSnapshotArn(), source.sourceArn());+ assertEquals("admin", source.masterUsername());+ }++ @Test+ void snapshotExportSourceRefusesAnArnFromAnotherPartition() {+ rdsService.createDbInstance("mydb", "postgres", "16.3",+ "admin", "password", "dbname", "db.t3.micro", 20, false, null, null, null, null, false);+ when(containerManager.createPostgresSnapshot(any(), eq("admin"))).thenReturn("MOCK_DUMP_DATA");+ DbSnapshot snapshot = rdsService.createDbSnapshot("mysnap", "mydb");+ String otherPartition = snapshot.getDbSnapshotArn().replaceFirst("^arn:aws:", "arn:aws-cn:");++ AwsException error = assertThrows(AwsException.class,+ () -> rdsService.snapshotExportSource(otherPartition, "us-east-1"));++ assertEquals("DBSnapshotNotFound", error.getErrorCode());+ }++ @Test+ void snapshotExportSourceRefusesASnapshotWhoseDumpIsMissing() {+ Map<String, StorageBackend<String, ?>> stores = new HashMap<>();+ StorageFactory factory = mock(StorageFactory.class);+ when(factory.create(eq("rds"), any(), any())).thenAnswer(invocation -> stores.computeIfAbsent(+ invocation.getArgument(1), name -> AccountAwareStorageBackend.inMemory("123456789012")));+ RdsService service = new RdsService(containerManager, proxyManager, ec2Service, regionResolver, config,+ factory, null, null, null, null, kmsService, null);+ service.createDbInstance("mydb", "postgres", "16.3",+ "admin", "password", "dbname", "db.t3.micro", 20, false, null, null, null, null, false);+ when(containerManager.createPostgresSnapshot(any(), eq("admin"))).thenReturn("MOCK_DUMP_DATA");+ DbSnapshot snapshot = service.createDbSnapshot("mysnap", "mydb");+ stores.get("rds-snapshot-data.json").clear();++ AwsException error = assertThrows(AwsException.class,+ () -> service.snapshotExportSource(snapshot.getDbSnapshotArn(), "us-east-1"));++ assertEquals("DBSnapshotNotFound", error.getErrorCode());+ assertEquals(404, error.getHttpStatus());+ }++ @Test+ void snapshotExportSourceRefusesAClusterSnapshotWhoseDumpIsMissing() {+ Map<String, StorageBackend<String, ?>> stores = new HashMap<>();+ StorageFactory factory = mock(StorageFactory.class);+ when(factory.create(eq("rds"), any(), any())).thenAnswer(invocation -> stores.computeIfAbsent(+ invocation.getArgument(1), name -> AccountAwareStorageBackend.inMemory("123456789012")));+ RdsService service = new RdsService(containerManager, proxyManager, ec2Service, regionResolver, config,+ factory, null, null, null, null, kmsService, null);+ service.createDbCluster("aurora-cluster", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", false, null);+ when(containerManager.createPostgresSnapshot(any(), eq("admin"))).thenReturn("CLUSTER_DUMP");+ DbClusterSnapshot snapshot = service.createDbClusterSnapshot("csnap", "aurora-cluster", Map.of());+ stores.get("rds-snapshot-data.json").clear();++ AwsException error = assertThrows(AwsException.class,+ () -> service.snapshotExportSource(snapshot.getDbClusterSnapshotArn(), "us-east-1"));++ assertEquals("DBClusterSnapshotNotFoundFault", error.getErrorCode());+ }+private DbClusterSnapshot clusterSnapshotOfNewCluster(String snapshotId) {rdsService.createDbCluster("aurora-cluster", "aurora-postgresql", "16.3","admin", "password", "dbname", false, null);diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/export/ExportPlanTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/export/ExportPlanTest.javanew file mode 100644index 0000000000..a6e03e09f2--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/rds/export/ExportPlanTest.java@@ -0,0 +1,46 @@+package io.github.hectorvent.floci.services.rds.export;++import io.github.hectorvent.floci.core.common.AwsException;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.util.List;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.equalTo;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++/** Which tables an export's {@code ExportOnly} selects, and the entries it refuses. */+class ExportPlanTest {++ @Test+ void emptyExportOnlySelectsEverything() {+ ExportOnlySelection selection = ExportOnlySelection.of(List.of());++ assertTrue(selection.includesDatabase("appdb"));+ assertTrue(selection.includes("appdb", "public", "items"));+ }++ @Test+ void exportOnlySelectsByDatabaseSchemaOrTable() {+ ExportOnlySelection selection = ExportOnlySelection.of(+ List.of("whole", "appdb.audit", "appdb.public.items"));++ assertTrue(selection.includes("whole", "any", "table"));+ assertTrue(selection.includes("appdb", "audit", "events"));+ assertTrue(selection.includes("appdb", "public", "items"));+ assertFalse(selection.includes("appdb", "public", "orders"));+ assertFalse(selection.includesDatabase("other"));+ }++ @ParameterizedTest+ @ValueSource(strings = {"appdb.public.items.extra", "app-db", "appdb..items", "appdb.public items"})+ void exportOnlyRefusesNamesAPartialExportCannotTake(String entry) {+ AwsException error = assertThrows(AwsException.class, () -> ExportOnlySelection.of(List.of(entry)));++ assertThat(error.getErrorCode(), equalTo("InvalidExportOnly"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/export/RdsExportTaskServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/export/RdsExportTaskServiceTest.javanew file mode 100644index 0000000000..b89d998c10--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/rds/export/RdsExportTaskServiceTest.java@@ -0,0 +1,522 @@+package io.github.hectorvent.floci.services.rds.export;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.rds.RdsService;+import io.github.hectorvent.floci.services.rds.model.ExportTask;+import io.github.hectorvent.floci.services.s3.S3Service;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.time.Clock;+import java.time.Instant;+import java.time.ZoneOffset;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CopyOnWriteArrayList;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.contains;+import static org.hamcrest.Matchers.empty;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasSize;+import static org.hamcrest.Matchers.not;+import static org.hamcrest.Matchers.notNullValue;+import static org.hamcrest.Matchers.nullValue;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.argThat;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * The export task lifecycle, with the export itself replaced: the statuses a task passes+ * through, cancellation, the request checks and the recovery of tasks a restart interrupted.+ */+class RdsExportTaskServiceTest {++ private static final String ACCOUNT = "000000000000";+ private static final String REGION = "us-east-1";+ private static final String SNAPSHOT_ARN = "arn:aws:rds:us-east-1:000000000000:snapshot:appdb-snap";+ private static final Instant SNAPSHOT_TIME = Instant.parse("2026-10-13T08:00:00Z");+ private static final Clock CLOCK = Clock.fixed(Instant.parse("2026-10-13T09:00:00Z"), ZoneOffset.UTC);++ private RdsService rdsService;+ private S3Service s3Service;+ private RegionResolver regionResolver;+ private final List<Runnable> workers = new ArrayList<>();++ @BeforeEach+ void setUp() {+ rdsService = mock(RdsService.class);+ s3Service = mock(S3Service.class);+ regionResolver = new RegionResolver(REGION, ACCOUNT);+ when(rdsService.snapshotExportSource(SNAPSHOT_ARN, REGION)).thenReturn(new SnapshotExportSource(+ SNAPSHOT_ARN, "postgres:16-alpine", "master", "secret", SNAPSHOT_TIME, "-- dump"));+ when(s3Service.bucketExists("exports")).thenReturn(true);+ }++ private RdsExportTaskService service(SnapshotExporter exporter) {+ return new RdsExportTaskService(new InMemoryStorage<>(), rdsService, s3Service, exporter,+ regionResolver, false, CLOCK, workers::add);+ }++ private static SnapshotExporter.Outcome outcome(long bytes, List<String> warnings) {+ return new SnapshotExporter.Outcome(bytes, warnings, CLOCK.instant());+ }++ private ExportTask start(RdsExportTaskService service, String id, List<String> exportOnly) {+ return service.startExportTask(id, SNAPSHOT_ARN, "exports", "prefix", "arn:aws:iam::000000000000:role/export",+ "alias/export", exportOnly, REGION);+ }++ private ExportTask describe(RdsExportTaskService service, String id) {+ return service.describeExportTasks(id, null, null, Map.of(), REGION).getFirst();+ }++ private void runWorkers() {+ List<Runnable> pending = List.copyOf(workers);+ workers.clear();+ pending.forEach(Runnable::run);+ }++ @Test+ void startExportTask_isStartingWithTheSnapshotAndRequestRecorded() {+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));++ ExportTask task = start(service, "appdb-export", List.of("appdb.public.items"));++ assertThat(task.getStatus(), equalTo("STARTING"));+ assertThat(task.getSourceArn(), equalTo(SNAPSHOT_ARN));+ assertThat(task.getSourceType(), equalTo("SNAPSHOT"));+ assertThat(task.getSnapshotTime(), equalTo(SNAPSHOT_TIME));+ assertThat(task.getTaskStartTime(), equalTo(CLOCK.instant()));+ assertThat(task.getExportOnly(), contains("appdb.public.items"));+ assertThat(task.getIamRoleArn(), equalTo("arn:aws:iam::000000000000:role/export"));+ assertThat(task.getKmsKeyId(), equalTo("alias/export"));+ assertThat(task.getPercentProgress(), equalTo(0));+ }++ @Test+ void aTaskMovesThroughInProgressToComplete() {+ List<String> seen = new ArrayList<>();+ RdsExportTaskService[] holder = new RdsExportTaskService[1];+ holder[0] = service((task, source, progress) -> {+ progress.extracting(50);+ seen.add(describe(holder[0], task.getExportTaskIdentifier()).getStatus()+ + "@" + describe(holder[0], task.getExportTaskIdentifier()).getPercentProgress());+ return outcome(2_147_483_648L, List.of());+ });+ start(holder[0], "appdb-export", List.of());++ runWorkers();++ ExportTask done = describe(holder[0], "appdb-export");+ assertThat(seen, contains("IN_PROGRESS@50"));+ assertThat(done.getStatus(), equalTo("COMPLETE"));+ assertThat(done.getPercentProgress(), equalTo(100));+ assertThat(done.getTotalExtractedDataInGb(), equalTo(2));+ assertThat(done.getTaskEndTime(), equalTo(CLOCK.instant()));+ assertThat(done.getFailureCause(), nullValue());+ }++ @Test+ void aFailedExportReportsItsCause() {+ RdsExportTaskService service = service((task, source, progress) -> {+ throw new IllegalStateException("psql failed with exit code 1: relation does not exist");+ });+ start(service, "appdb-export", List.of());++ runWorkers();++ ExportTask failed = describe(service, "appdb-export");+ assertThat(failed.getStatus(), equalTo("FAILED"));+ assertThat(failed.getFailureCause(), equalTo("psql failed with exit code 1: relation does not exist"));+ assertThat(failed.getTaskEndTime(), notNullValue());+ }++ @Test+ void outsideMockModeATaskFailsUntilSnapshotDataCanBeWritten() {+ RdsExportTaskService service = service(new UnsupportedSnapshotExporter());+ start(service, "appdb-export", List.of());++ runWorkers();++ ExportTask failed = describe(service, "appdb-export");+ assertThat(failed.getStatus(), equalTo("FAILED"));+ assertThat(failed.getFailureCause(), equalTo(UnsupportedSnapshotExporter.FAILURE_CAUSE));+ assertThat(failed.getTaskEndTime(), equalTo(CLOCK.instant()));+ }++ @Test+ void aFailureWithoutAMessageStillReportsACause() {+ RdsExportTaskService service = service((task, source, progress) -> {+ throw new NullPointerException();+ });+ start(service, "appdb-export", List.of());++ runWorkers();++ assertThat(describe(service, "appdb-export").getFailureCause(), equalTo("java.lang.NullPointerException"));+ }++ @Test+ void skippedTablesBecomeTheWarningMessage() {+ RdsExportTaskService service = service((task, source, progress) ->+ outcome(10, List.of("The table appdb.public.odd was skipped.")));+ start(service, "appdb-export", List.of());++ runWorkers();++ assertThat(describe(service, "appdb-export").getWarningMessage(),+ equalTo("The table appdb.public.odd was skipped."));+ }++ @Test+ void cancelingStopsTheExportAndEndsCanceled() {+ List<Boolean> canceledSeen = new ArrayList<>();+ RdsExportTaskService[] holder = new RdsExportTaskService[1];+ holder[0] = service((task, source, progress) -> {+ progress.extracting(10);+ holder[0].cancelExportTask(task.getExportTaskIdentifier(), REGION);+ assertThat(describe(holder[0], task.getExportTaskIdentifier()).getStatus(), equalTo("CANCELING"));+ progress.extracting(60);+ canceledSeen.add(progress.canceled());+ return outcome(0, List.of());+ });+ start(holder[0], "appdb-export", List.of());++ runWorkers();++ ExportTask canceled = describe(holder[0], "appdb-export");+ assertThat(canceledSeen, contains(true));+ assertThat(canceled.getStatus(), equalTo("CANCELED"));+ assertThat(canceled.getPercentProgress(), equalTo(10));+ }++ @Test+ void aFinishedTaskCannotBeCanceled() {+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));+ start(service, "appdb-export", List.of());+ runWorkers();++ AwsException error = assertThrows(AwsException.class,+ () -> service.cancelExportTask("appdb-export", REGION));++ assertThat(error.getErrorCode(), equalTo("InvalidExportTaskStateFault"));+ }++ @Test+ void anIdentifierCanOnlyBeUsedOnce() {+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));+ start(service, "appdb-export", List.of());++ AwsException error = assertThrows(AwsException.class, () -> start(service, "appdb-export", List.of()));++ assertThat(error.getErrorCode(), equalTo("ExportTaskAlreadyExists"));+ }++ @Test+ void aMissingBucketIsRefusedBeforeAnythingRuns() {+ when(s3Service.bucketExists("exports")).thenReturn(false);+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));++ AwsException error = assertThrows(AwsException.class, () -> start(service, "appdb-export", List.of()));++ assertThat(error.getErrorCode(), equalTo("InvalidS3BucketFault"));+ assertThat(workers, empty());+ }++ @Test+ void aBucketInAnotherRegionFailsTheTaskWithoutExporting() {+ when(s3Service.getBucketRegion("exports")).thenReturn("eu-west-1");+ SnapshotExporter exporter = mock(SnapshotExporter.class);+ RdsExportTaskService service = service(exporter);++ assertThat(start(service, "appdb-export", List.of()).getStatus(), equalTo("STARTING"));+ runWorkers();++ ExportTask failed = describe(service, "appdb-export");+ assertThat(failed.getStatus(), equalTo("FAILED"));+ assertThat(failed.getFailureCause(), equalTo("The S3 bucket exports isn't located in the current AWS Region."+ + " Review your S3 bucket name and retry the export."));+ verify(exporter, never()).export(any(), any(), any());+ }++ @Test+ void aBucketWithoutALocationConstraintIsInTheDefaultRegion() {+ when(s3Service.getBucketRegion("exports")).thenReturn(null);+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));++ assertThat(start(service, "appdb-export", List.of()).getStatus(), equalTo("STARTING"));+ }++ @Test+ void identifiersFollowTheRdsRules() {+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));++ for (String invalid : List.of("1export", "export--task", "export-", "export_task", "e".repeat(61))) {+ AwsException error = assertThrows(AwsException.class, () -> start(service, invalid, List.of()));+ assertThat(invalid, error.getErrorCode(), equalTo("InvalidParameterValue"));+ }+ verify(rdsService, never()).snapshotExportSource(anyString(), any());+ }++ @Test+ void anUnknownTaskIsNotFound() {+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));++ AwsException describe = assertThrows(AwsException.class,+ () -> service.describeExportTasks("missing", null, null, Map.of(), REGION));+ AwsException cancel = assertThrows(AwsException.class, () -> service.cancelExportTask("missing", REGION));++ assertThat(describe.getErrorCode(), equalTo("ExportTaskNotFound"));+ assertThat(describe.getHttpStatus(), equalTo(404));+ assertThat(cancel.getErrorCode(), equalTo("ExportTaskNotFound"));+ }++ @Test+ void describeFiltersByStatusBucketAndSource() {+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));+ start(service, "first", List.of());+ runWorkers();+ start(service, "second", List.of());++ assertThat(service.describeExportTasks(null, null, null, Map.of("status", List.of("complete")), REGION)+ .stream().map(ExportTask::getExportTaskIdentifier).toList(), contains("first"));+ assertThat(service.describeExportTasks(null, null, null, Map.of("status", List.of("starting")), REGION)+ .stream().map(ExportTask::getExportTaskIdentifier).toList(), contains("second"));+ assertThat(service.describeExportTasks(null, SNAPSHOT_ARN, "SNAPSHOT",+ Map.of("s3-bucket", List.of("exports")), REGION), hasSize(2));+ assertThat(service.describeExportTasks(null, null, "CLUSTER", Map.of(), REGION), empty());+ AwsException unknown = assertThrows(AwsException.class, () ->+ service.describeExportTasks(null, null, null, Map.of("engine", List.of("postgres")), REGION));+ assertThat(unknown.getErrorCode(), equalTo("InvalidParameterValue"));+ }++ @Test+ void anUnknownFilterNameIsRefusedEvenWithNoTasks() {+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));++ AwsException unknown = assertThrows(AwsException.class, () ->+ service.describeExportTasks(null, null, null, Map.of("staus", List.of("COMPLETE")), REGION));++ assertThat(unknown.getErrorCode(), equalTo("InvalidParameterValue"));+ }++ @Test+ void mockModeCompletesWithoutExporting() {+ SnapshotExporter exporter = mock(SnapshotExporter.class);+ RdsExportTaskService service = new RdsExportTaskService(new InMemoryStorage<>(), rdsService, s3Service,+ exporter, regionResolver, true, CLOCK, workers::add);+ start(service, "appdb-export", List.of());++ runWorkers();++ assertThat(describe(service, "appdb-export").getStatus(), equalTo("COMPLETE"));+ verify(exporter, never()).export(any(), any(), any());+ }++ @Test+ void theResponsesKeepTheStatusTheirCallSetWhenTheWorkerFinishesFirst() {+ RdsExportTaskService service = new RdsExportTaskService(new InMemoryStorage<>(), rdsService, s3Service,+ (task, source, progress) -> outcome(0, List.of()), regionResolver, true, CLOCK, Runnable::run);++ ExportTask started = start(service, "appdb-export", List.of());+ ExportTask described = describe(service, "appdb-export");++ assertThat(started.getStatus(), equalTo("STARTING"));+ assertThat(described.getStatus(), equalTo("COMPLETE"));+ }++ @Test+ void aDescribedTaskDoesNotChangeWhenItsWorkerDoes() {+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));+ start(service, "appdb-export", List.of());+ ExportTask described = describe(service, "appdb-export");++ runWorkers();++ assertThat(described.getStatus(), equalTo("STARTING"));+ assertThat(describe(service, "appdb-export").getStatus(), equalTo("COMPLETE"));+ }++ @Test+ void theCancelResponseIsCancelingEvenOnceTheTaskIsCanceled() {+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));+ start(service, "appdb-export", List.of());++ ExportTask canceling = service.cancelExportTask("appdb-export", REGION);+ runWorkers();++ assertThat(canceling.getStatus(), equalTo("CANCELING"));+ assertThat(describe(service, "appdb-export").getStatus(), equalTo("CANCELED"));+ }++ @Test+ void aTaskARestartInterruptedIsFailed() {+ AccountAwareStorageBackend<ExportTask> persisted =+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, ACCOUNT);+ ExportTask running = new ExportTask();+ running.setExportTaskIdentifier("appdb-export");+ running.setStatus(ExportTask.STATUS_IN_PROGRESS);+ running.setTaskStartTime(SNAPSHOT_TIME);+ persisted.putForAccount(ACCOUNT, "export-task::us-east-1::appdb-export", running);++ SnapshotExporter exporter = mock(SnapshotExporter.class);++ RdsExportTaskService service = new RdsExportTaskService(persisted, rdsService, s3Service, exporter,+ regionResolver, false, CLOCK, workers::add);++ ExportTask recovered = describe(service, "appdb-export");+ assertThat(recovered.getStatus(), equalTo("FAILED"));+ assertThat(recovered.getFailureCause(), equalTo(RdsExportTaskService.RESTART_FAILURE_CAUSE));+ verify(exporter).discard(argThat(task -> "appdb-export".equals(task.getExportTaskIdentifier())));+ }++ @Test+ void aTaskBeingCanceledWhenARestartInterruptedItIsCanceled() {+ AccountAwareStorageBackend<ExportTask> persisted =+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, ACCOUNT);+ ExportTask canceling = new ExportTask();+ canceling.setExportTaskIdentifier("appdb-export");+ canceling.setStatus(ExportTask.STATUS_CANCELING);+ canceling.setTaskStartTime(SNAPSHOT_TIME);+ persisted.putForAccount(ACCOUNT, "export-task::us-east-1::appdb-export", canceling);+ SnapshotExporter exporter = mock(SnapshotExporter.class);++ RdsExportTaskService service = new RdsExportTaskService(persisted, rdsService, s3Service, exporter,+ regionResolver, false, CLOCK, workers::add);++ ExportTask recovered = describe(service, "appdb-export");+ assertThat(recovered.getStatus(), equalTo("CANCELED"));+ assertThat(recovered.getFailureCause(), nullValue());+ verify(exporter).discard(argThat(task -> "appdb-export".equals(task.getExportTaskIdentifier())));+ }++ @Test+ void aTaskCanceledBeforeItsWorkerStartsNeverExports() {+ SnapshotExporter exporter = mock(SnapshotExporter.class);+ RdsExportTaskService service = service(exporter);+ start(service, "appdb-export", List.of());++ String answered = service.cancelExportTask("appdb-export", REGION).getStatus();+ runWorkers();++ assertThat(answered, equalTo("CANCELING"));+ assertThat(describe(service, "appdb-export").getStatus(), equalTo("CANCELED"));+ verify(exporter, never()).export(any(), any(), any());+ }++ @Test+ void theSameIdentifierInTwoRegionsGetsItsOwnRuntime() {+ String otherArn = "arn:aws:rds:eu-west-1:000000000000:snapshot:appdb-snap";+ when(rdsService.snapshotExportSource(otherArn, "eu-west-1")).thenReturn(new SnapshotExportSource(+ otherArn, "postgres:16-alpine", "master", "secret", SNAPSHOT_TIME, "-- dump"));+ when(s3Service.bucketExists("exports-west")).thenReturn(true);+ when(s3Service.getBucketRegion("exports-west")).thenReturn("eu-west-1");+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));++ ExportTask east = start(service, "appdb-export", List.of());+ ExportTask west = service.startExportTask("appdb-export", otherArn, "exports-west", null,+ "arn:aws:iam::000000000000:role/export", "alias/export", List.of(), "eu-west-1");++ assertThat(east.getRuntimeArn(), equalTo("arn:aws:rds:us-east-1:000000000000:export-task:appdb-export"));+ assertThat(west.getRuntimeArn(), equalTo("arn:aws:rds:eu-west-1:000000000000:export-task:appdb-export"));+ assertThat(east.getRuntimeVolumeId(), not(equalTo(west.getRuntimeVolumeId())));+ assertThat(service.describeExportTasks(null, null, null, Map.of(), "eu-west-1"), hasSize(1));+ }++ @Test+ void aStartDuringAResetWaitsForItToFinish() throws Exception {+ RdsExportTaskService service = service((task, source, progress) -> outcome(0, List.of()));+ service.beforeReset();+ List<ExportTask> admitted = new CopyOnWriteArrayList<>();+ Thread caller = Thread.ofVirtual().start(() -> admitted.add(start(service, "appdb-export", List.of())));++ caller.join(300);+ assertThat("the start waits while the reset runs", admitted, empty());+ service.clear();+ service.afterReset();+ caller.join(5_000);++ assertThat(admitted, hasSize(1));+ assertThat(describe(service, "appdb-export").getStatus(), equalTo("STARTING"));+ }++ @Test+ void aWorkerQueuedBeforeAResetDoesNotExport() {+ SnapshotExporter exporter = mock(SnapshotExporter.class);+ RdsExportTaskService service = service(exporter);+ start(service, "appdb-export", List.of());++ service.beforeReset();+ service.afterReset();+ runWorkers();++ verify(exporter, never()).export(any(), any(), any());+ }++ @Test+ void aWorkerThatOutlivesAResetLeavesTheNewTaskWithItsIdentifierAlone() {+ RdsExportTaskService[] holder = new RdsExportTaskService[1];+ holder[0] = service((task, source, progress) -> {+ holder[0].beforeReset();+ holder[0].clear();+ holder[0].afterReset();+ start(holder[0], task.getExportTaskIdentifier(), List.of());+ progress.extracting(50);+ throw new IllegalStateException("the restore container was removed by the reset");+ });+ start(holder[0], "appdb-export", List.of());++ runWorkers();++ ExportTask restarted = describe(holder[0], "appdb-export");+ assertThat(restarted.getStatus(), equalTo("STARTING"));+ assertThat(restarted.getPercentProgress(), equalTo(0));+ assertThat(restarted.getFailureCause(), nullValue());+ }++ @Test+ void theTaskEndsAtTheTimeItsExportRecorded() {+ Instant finished = Instant.parse("2026-10-13T09:05:00Z");+ RdsExportTaskService service = service((task, source, progress) ->+ new SnapshotExporter.Outcome(0, List.of(), finished));+ start(service, "appdb-export", List.of());++ runWorkers();++ assertThat(describe(service, "appdb-export").getTaskEndTime(), equalTo(finished));+ }++ @Test+ void aFailingCleanupAtStartupDoesNotStopRecovery() {+ AccountAwareStorageBackend<ExportTask> persisted =+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, ACCOUNT);+ for (String id : List.of("first", "second")) {+ ExportTask running = new ExportTask();+ running.setExportTaskIdentifier(id);+ running.setStatus(ExportTask.STATUS_STARTING);+ running.setTaskStartTime(SNAPSHOT_TIME);+ persisted.putForAccount(ACCOUNT, "export-task::us-east-1::" + id, running);+ }+ SnapshotExporter exporter = mock(SnapshotExporter.class);+ doThrow(new IllegalStateException("Docker is not reachable")).when(exporter)+ .discard(argThat(task -> "first".equals(task.getExportTaskIdentifier())));++ RdsExportTaskService service = new RdsExportTaskService(persisted, rdsService, s3Service, exporter,+ regionResolver, false, CLOCK, workers::add);++ assertThat(describe(service, "first").getStatus(), equalTo("FAILED"));+ assertThat(describe(service, "second").getStatus(), equalTo("FAILED"));+ verify(exporter).discard(argThat(task -> "second".equals(task.getExportTaskIdentifier())));+ }+}