Light, fluffy, and always free - The AWS Local Emulator alternative
feat(redshift): create final snapshots before deleting clusters (#6346)
Vũ Thế Huy1b8548a0afbd
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseRedshiftTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseRedshiftTest.javaindex 0b94e49aec..0cc23ebf98 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseRedshiftTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseRedshiftTest.java@@ -151,7 +151,8 @@ class FirehoseRedshiftTest {// The stream does not exist when creation failed.}try {- redshift.deleteCluster(DeleteClusterRequest.builder().clusterIdentifier(clusterId).build());+ redshift.deleteCluster(DeleteClusterRequest.builder().clusterIdentifier(clusterId)+ .skipFinalClusterSnapshot(true).build());} catch (ClusterNotFoundException ignored) {// The cluster does not exist when creation failed.}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.javaindex 8be93c78eb..6e35a3d2e1 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java@@ -60,7 +60,8 @@ class RedshiftDataOperationsTest {static void cleanup() {if (redshift != null && clusterId != null) {try {- redshift.deleteCluster(DeleteClusterRequest.builder().clusterIdentifier(clusterId).build());+ redshift.deleteCluster(DeleteClusterRequest.builder().clusterIdentifier(clusterId)+ .skipFinalClusterSnapshot(true).build());} catch (Exception e) {LOG.warnf(e, "Failed to clean up Redshift cluster %s", clusterId);}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftOperationsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftOperationsTest.javaindex c16b570d9e..309677db71 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftOperationsTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftOperationsTest.java@@ -104,6 +104,7 @@ class RedshiftOperationsTest {try {client.deleteCluster(DeleteClusterRequest.builder().clusterIdentifier(clusterId)+ .skipFinalClusterSnapshot(true).build());} catch (Exception e) {LOG.log(Level.WARNING, "Failed to clean up Redshift cluster " + clusterId, e);@@ -151,15 +152,17 @@ class RedshiftOperationsTest {@Test@Order(1)- @DisplayName("Create cluster, insert data, snapshot, restore to new cluster, verify data")+ @DisplayName("Create cluster, insert data, take final snapshot on deletion, restore and verify data")void testSnapshotAndRestoreLifecycle() throws Exception {String sourceClusterId = TestFixtures.uniqueName("rs-source");String restoredClusterId = TestFixtures.uniqueName("rs-restored");String snapshotId = TestFixtures.uniqueName("rs-snap");+ String finalSnapshotId = TestFixtures.uniqueName("rs-final");clustersToCleanup.add(sourceClusterId);clustersToCleanup.add(restoredClusterId);snapshotsToCleanup.add(snapshotId);+ snapshotsToCleanup.add(finalSnapshotId);// 1. Create source clusterCreateClusterResponse createRes = client.createCluster(CreateClusterRequest.builder()@@ -205,17 +208,23 @@ class RedshiftOperationsTest {assertThat(descSnapRes.snapshots()).isNotEmpty();assertThat(descSnapRes.snapshots().get(0).snapshotIdentifier()).isEqualTo(snapshotId);- // 4. Delete source cluster+ // 4. Delete source cluster with a separate final snapshot.client.deleteCluster(DeleteClusterRequest.builder().clusterIdentifier(sourceClusterId)+ .finalClusterSnapshotIdentifier(finalSnapshotId)+ .finalClusterSnapshotRetentionPeriod(7).build());clustersToCleanup.remove(sourceClusterId);+ Snapshot finalSnapshot = client.describeClusterSnapshots(request -> request.snapshotIdentifier(finalSnapshotId))+ .snapshots().get(0);+ assertThat(finalSnapshot.manualSnapshotRetentionPeriod()).isEqualTo(7);+ assertThat(finalSnapshot.clusterIdentifier()).isEqualTo(sourceClusterId);// 5. Restore from snapshot to a new cluster identifierRestoreFromClusterSnapshotResponse restoreRes = client.restoreFromClusterSnapshot(RestoreFromClusterSnapshotRequest.builder().clusterIdentifier(restoredClusterId)- .snapshotIdentifier(snapshotId)+ .snapshotIdentifier(finalSnapshotId).nodeType("dc2.large").build());@@ -238,6 +247,7 @@ class RedshiftOperationsTest {// Cleanup restored cluster and snapshotclient.deleteCluster(DeleteClusterRequest.builder().clusterIdentifier(restoredClusterId)+ .skipFinalClusterSnapshot(true).build());clustersToCleanup.remove(restoredClusterId);diff --git a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftSpectrumTest.java b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftSpectrumTest.javaindex d57b0f09b8..146713b300 100644--- a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftSpectrumTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftSpectrumTest.java@@ -96,7 +96,8 @@ class RedshiftSpectrumTest {if (!clusterCreated) {return;}- redshift.deleteCluster(DeleteClusterRequest.builder().clusterIdentifier(clusterId).build());+ redshift.deleteCluster(DeleteClusterRequest.builder().clusterIdentifier(clusterId)+ .skipFinalClusterSnapshot(true).build());} finally {if (bucketCreated) {s3.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key("events/part-1.csv").build());diff --git a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.javaindex e160807ed2..ba755c4241 100644--- a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java@@ -208,6 +208,7 @@ public class RedshiftTest {RedshiftClient client = getClient();DeleteClusterResponse res = withRetry(() -> client.deleteCluster(DeleteClusterRequest.builder().clusterIdentifier("test-cluster")+ .skipFinalClusterSnapshot(true).build()));assertEquals("test-cluster", res.cluster().clusterIdentifier());assertEquals("deleting", res.cluster().clusterStatus());diff --git a/docs/services/redshift.md b/docs/services/redshift.mdindex 5e45347ff1..f4195ae2e2 100644--- a/docs/services/redshift.md+++ b/docs/services/redshift.md@@ -21,7 +21,7 @@ For running SQL without a PostgreSQL wire connection (the way Lambda and Step Fu| --- | --- || `CreateCluster` | Create a cluster and start a PostgreSQL container for it || `DescribeClusters` | List clusters and their connection details |-| `DeleteCluster` | Stop and remove a cluster and its container |+| `DeleteCluster` | Create the requested final snapshot, then remove the cluster; accepts `SkipFinalClusterSnapshot` to delete without a backup || `CreateClusterSnapshot` | Back up a cluster to a SQL dump via `pg_dump` || `DescribeClusterSnapshots` | List snapshots, optionally filtered by snapshot or cluster identifier || `DeleteClusterSnapshot` | Remove a snapshot and its stored dump |@@ -65,6 +65,51 @@ For running SQL without a PostgreSQL wire connection (the way Lambda and Step Fu| `GetClusterCredentialsWithIAM` | Issue short-lived credentials with the DbUser derived from the caller's IAM identity |<!-- floci:actions:end -->+### Final snapshots on cluster deletion++`DeleteCluster` defaults `SkipFinalClusterSnapshot` to `false`, so a+`FinalClusterSnapshotIdentifier` is required unless the request explicitly skips the final+snapshot. Supplying a final snapshot name together with `SkipFinalClusterSnapshot=true` is+rejected. Final snapshot names use the same validation and uniqueness rules as+`CreateClusterSnapshot`.++Floci completes the database dump and commits the snapshot metadata before stopping the proxy+or removing the cluster. Dump failures and reported snapshot storage errors leave the cluster+available. The final snapshot remains visible through `DescribeClusterSnapshots` after the+source cluster is deleted, and `RestoreFromClusterSnapshot` can restore its data to a new+cluster. Deletion is synchronous: Floci returns `deleting` after teardown rather than exposing+AWS's intermediate `final-snapshot` state.++Database dumps run under account-scoped cluster and snapshot locks, so a slow dump does not+hold the service-wide mutation monitor. Operations on the same cluster and snapshot identifier+remain coordinated, including snapshot deletion and restore. A dump whose metadata was not+committed is removed on failure; if the commit succeeded and a later flush reports an error,+the dump is retained for the committed snapshot.++`FinalClusterSnapshotRetentionPeriod` defaults to `-1` and accepts `-1` or 1 to 3,653 days.+It is persisted and returned as the snapshot's `ManualSnapshotRetentionPeriod`; automatic+expiration is not implemented. Manual snapshots created by `CreateClusterSnapshot`, and older+snapshots without stored retention metadata, also return the default `-1`. Snapshot metadata+uses the configured storage mode. Persistent and WAL metadata commit failures are reported+before teardown; hybrid retains its existing+write-behind durability and logged flush failures.++If teardown fails after the final snapshot was created, the snapshot is retained. A retry+must use a different snapshot name, or explicitly skip the final snapshot; an existing backup+is never overwritten. Internal provisioner cleanup keeps its existing deletion behavior and+explicitly skips the final snapshot.++```bash+aws redshift delete-cluster --cluster-identifier warehouse \+ --final-cluster-snapshot-identifier warehouse-final \+ --final-cluster-snapshot-retention-period 7++aws redshift delete-cluster --cluster-identifier disposable \+ --skip-final-cluster-snapshot+```++See the AWS [DeleteCluster API reference](https://docs.aws.amazon.com/redshift/latest/APIReference/API_DeleteCluster.html).+### Resetting parameter groups`ResetClusterParameterGroup` resets all parameters with `ResetAllParameters=true`, or the nameddiff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.javaindex 94d7956a50..468aecac0e 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java@@ -139,7 +139,12 @@ public class RedshiftQueryHandler {}case "DeleteCluster" -> {String identifier = params.getFirst("ClusterIdentifier");- Cluster cluster = service.deleteCluster(identifier);+ String rawSkip = params.getFirst("SkipFinalClusterSnapshot");+ if (rawSkip != null && !"true".equalsIgnoreCase(rawSkip) && !"false".equalsIgnoreCase(rawSkip)) {+ throw new AwsException("InvalidParameterValue", "SkipFinalClusterSnapshot must be a boolean.", 400);+ }+ Cluster cluster = service.deleteCluster(identifier, rawSkip == null ? null : Boolean.valueOf(rawSkip),+ params.getFirst("FinalClusterSnapshotIdentifier"), intParam(params, "FinalClusterSnapshotRetentionPeriod"));String xml = new XmlBuilder().start("DeleteClusterResponse").start("DeleteClusterResult")@@ -1058,6 +1063,7 @@ public class RedshiftQueryHandler {.elem("ClusterIdentifier", snapshot.getClusterIdentifier()).elem("SnapshotArn", snapshot.getSnapshotArn()).elem("SnapshotCreateTime", createTime)+ .elem("ManualSnapshotRetentionPeriod", snapshot.getManualSnapshotRetentionPeriod().toString()).elem("Status", snapshot.getStatus()).elem("Port", String.valueOf(snapshot.getPort())).elem("MasterUsername", snapshot.getMasterUsername());diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.javaindex 6c67db74b5..d7116fccf2 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java@@ -52,7 +52,9 @@ import java.util.Optional;import java.util.Set;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.locks.ReentrantLock;import java.util.function.Consumer;+import java.util.function.Supplier;import java.util.regex.Matcher;import java.util.regex.Pattern;@@ -91,6 +93,39 @@ public class RedshiftService {private final DynamoDbStreamReader streamReader;// Proxy ports currently handed out, so allocateProxyPort never double-assigns within this JVM.private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();+ private final ConcurrentHashMap<ResourceKey, ResourceLock> resourceLocks = new ConcurrentHashMap<>();++ private record ResourceKey(String accountId, String kind, String identifier) { }++ private static final class ResourceLock {+ private final ReentrantLock lock = new ReentrantLock();+ private int users;+ }++ private <T> T withResourceLock(String kind, String identifier, Supplier<T> work) {+ ResourceKey key = new ResourceKey(clusters.accountId(), kind, identifier);+ ResourceLock scoped = resourceLocks.compute(key, (ignored, existing) -> {+ ResourceLock selected = existing == null ? new ResourceLock() : existing;+ selected.users++;+ return selected;+ });+ scoped.lock.lock();+ try {+ return work.get();+ } finally {+ scoped.lock.unlock();+ resourceLocks.compute(key, (ignored, existing) -> --scoped.users == 0 ? null : scoped);+ }+ }++ // Acquire resource locks before the service monitor; snapshot I/O never takes the monitor.+ private <T> T mutateCluster(String identifier, Supplier<T> mutation) {+ return withResourceLock("cluster", identifier, () -> mutate(mutation));+ }++ private synchronized <T> T mutate(Supplier<T> mutation) {+ return mutation.get();+ }@Injectpublic RedshiftService(StorageFactory storageFactory, RedshiftContainerManager containerManager,@@ -183,22 +218,28 @@ public class RedshiftService {return createCluster(identifier, nodeType, username, password, null, List.of(), List.of());}- // synchronized like modify/reboot: the container + proxy + port steps must not- // interleave with another admin call on the same cluster.- public synchronized Cluster createCluster(String identifier, String nodeType, String username, String password,+ // Cluster lifecycle mutations take the scoped resource lock before the service monitor.+ public Cluster createCluster(String identifier, String nodeType, String username, String password,String clusterSubnetGroupName, List<String> vpcSecurityGroupIds) {return createCluster(identifier, nodeType, username, password, clusterSubnetGroupName,vpcSecurityGroupIds, List.of());}- public synchronized Cluster createCluster(String identifier, String nodeType, String username, String password,+ public Cluster createCluster(String identifier, String nodeType, String username, String password,String clusterSubnetGroupName, List<String> vpcSecurityGroupIds,List<String> iamRoleArns) {return createCluster(identifier, nodeType, username, password, clusterSubnetGroupName,vpcSecurityGroupIds, iamRoleArns, null);}- public synchronized Cluster createCluster(String identifier, String nodeType, String username, String password,+ public Cluster createCluster(String identifier, String nodeType, String username, String password,+ String clusterSubnetGroupName, List<String> vpcSecurityGroupIds,+ List<String> iamRoleArns, String defaultIamRoleArn) {+ return mutateCluster(identifier, () -> createClusterLocked(identifier, nodeType, username, password,+ clusterSubnetGroupName, vpcSecurityGroupIds, iamRoleArns, defaultIamRoleArn));+ }++ private Cluster createClusterLocked(String identifier, String nodeType, String username, String password,String clusterSubnetGroupName, List<String> vpcSecurityGroupIds,List<String> iamRoleArns, String defaultIamRoleArn) {if (clusters.get(identifier).isPresent()) {@@ -274,7 +315,7 @@ public class RedshiftService {return cluster;}- public synchronized Cluster createClusterWithManagedMasterPassword(+ public Cluster createClusterWithManagedMasterPassword(String identifier, String nodeType, String username, String clusterSubnetGroupName,List<String> vpcSecurityGroupIds, List<String> iamRoleArns,String kmsKeyId, String region) {@@ -282,7 +323,15 @@ public class RedshiftService {vpcSecurityGroupIds, iamRoleArns, kmsKeyId, region, null);}- public synchronized Cluster createClusterWithManagedMasterPassword(+ public Cluster createClusterWithManagedMasterPassword(+ String identifier, String nodeType, String username, String clusterSubnetGroupName,+ List<String> vpcSecurityGroupIds, List<String> iamRoleArns,+ String kmsKeyId, String region, String defaultIamRoleArn) {+ return mutateCluster(identifier, () -> createClusterWithManagedMasterPasswordLocked(identifier,+ nodeType, username, clusterSubnetGroupName, vpcSecurityGroupIds, iamRoleArns, kmsKeyId, region, defaultIamRoleArn));+ }++ private Cluster createClusterWithManagedMasterPasswordLocked(String identifier, String nodeType, String username, String clusterSubnetGroupName,List<String> vpcSecurityGroupIds, List<String> iamRoleArns,String kmsKeyId, String region, String defaultIamRoleArn) {@@ -612,13 +661,54 @@ public class RedshiftService {return clusters.scanForAccount(accountId, k -> true);}- public synchronized Cluster deleteCluster(String identifier) {+ /** Internal deletion used by provisioner cleanup, which explicitly skips the final snapshot. */+ public Cluster deleteCluster(String identifier) {+ return deleteCluster(identifier, true, null, null);+ }++ public Cluster deleteCluster(String identifier, Boolean skipFinalClusterSnapshot,+ String finalSnapshotIdentifier, Integer finalSnapshotRetentionPeriod) {+ return withResourceLock("cluster", identifier, () -> {+ Supplier<Cluster> deletion = () -> deleteClusterWithFinalSnapshot(identifier, skipFinalClusterSnapshot,+ finalSnapshotIdentifier, finalSnapshotRetentionPeriod);+ return Boolean.TRUE.equals(skipFinalClusterSnapshot) ? deletion.get()+ : withResourceLock("snapshot", finalSnapshotIdentifier, deletion);+ });+ }++ private Cluster deleteClusterWithFinalSnapshot(String identifier, Boolean skipFinalClusterSnapshot,+ String finalSnapshotIdentifier, Integer finalSnapshotRetentionPeriod) {+ boolean skipSnapshot = Boolean.TRUE.equals(skipFinalClusterSnapshot);+ if (skipSnapshot && finalSnapshotIdentifier != null) {+ throw new AwsException("InvalidParameterCombination",+ "FinalClusterSnapshotIdentifier cannot be supplied when SkipFinalClusterSnapshot is true", 400);+ }+ if (!skipSnapshot && (finalSnapshotIdentifier == null || finalSnapshotIdentifier.isBlank())) {+ throw new AwsException("InvalidParameterCombination",+ "FinalClusterSnapshotIdentifier is required unless SkipFinalClusterSnapshot is true", 400);+ }+ int retentionPeriod = finalSnapshotRetentionPeriod == null ? -1 : finalSnapshotRetentionPeriod;+ if (retentionPeriod != -1 && (retentionPeriod < 1 || retentionPeriod > 3653)) {+ throw new AwsException("InvalidRetentionPeriodFault",+ "FinalClusterSnapshotRetentionPeriod must be -1 or between 1 and 3653", 400);+ }Optional<Cluster> clusterOpt = clusters.get(identifier);if (clusterOpt.isEmpty()) {throw new AwsException("ClusterNotFound", "Cluster " + identifier + " not found", 404);}Cluster cluster = clusterOpt.get();+ if (!skipSnapshot) {+ if (!"available".equals(cluster.getClusterStatus())) {+ throw new AwsException("InvalidClusterState", "Cluster must be available to create a final snapshot", 400);+ }+ createSnapshot(finalSnapshotIdentifier, identifier, retentionPeriod);+ }++ return mutate(() -> removeCluster(identifier, cluster));+ }++ private Cluster removeCluster(String identifier, Cluster cluster) {// Tear down the auth proxy and return its port before stopping the container.// If it fails, abort deletion so the metadata remains and the user can retry.if (!stopProxyAndReleasePortSafely(identifier, cluster.getProxyPort())) {@@ -654,7 +744,14 @@ public class RedshiftService {clusterParameterGroupName, vpcSecurityGroupIds, null);}- public synchronized Cluster modifyCluster(String clusterIdentifier, String nodeType, Integer numberOfNodes,+ public Cluster modifyCluster(String clusterIdentifier, String nodeType, Integer numberOfNodes,+ String masterUserPassword, String clusterParameterGroupName,+ List<String> vpcSecurityGroupIds, Boolean multiAZ) {+ return mutateCluster(clusterIdentifier, () -> modifyClusterLocked(clusterIdentifier, nodeType,+ numberOfNodes, masterUserPassword, clusterParameterGroupName, vpcSecurityGroupIds, multiAZ));+ }++ private Cluster modifyClusterLocked(String clusterIdentifier, String nodeType, Integer numberOfNodes,String masterUserPassword, String clusterParameterGroupName,List<String> vpcSecurityGroupIds, Boolean multiAZ) {Cluster cluster = clusters.get(clusterIdentifier)@@ -693,12 +790,18 @@ public class RedshiftService {return cluster;}- public synchronized Cluster modifyClusterIamRoles(String clusterIdentifier, List<String> addIamRoles,+ public Cluster modifyClusterIamRoles(String clusterIdentifier, List<String> addIamRoles,List<String> removeIamRoles) {return modifyClusterIamRoles(clusterIdentifier, addIamRoles, removeIamRoles, null);}- public synchronized Cluster modifyClusterIamRoles(String clusterIdentifier, List<String> addIamRoles,+ public Cluster modifyClusterIamRoles(String clusterIdentifier, List<String> addIamRoles,+ List<String> removeIamRoles, String defaultIamRoleArn) {+ return mutateCluster(clusterIdentifier, () -> modifyClusterIamRolesLocked(clusterIdentifier,+ addIamRoles, removeIamRoles, defaultIamRoleArn));+ }++ private Cluster modifyClusterIamRolesLocked(String clusterIdentifier, List<String> addIamRoles,List<String> removeIamRoles, String defaultIamRoleArn) {Cluster cluster = clusters.get(clusterIdentifier).orElseThrow(() -> new AwsException("ClusterNotFound", "Cluster " + clusterIdentifier + " not found", 404));@@ -777,12 +880,19 @@ public class RedshiftService {.orElseThrow(() -> new AwsException("ClusterNotFound", "Cluster " + clusterIdentifier + " not found", 404));}- public synchronized Cluster enableLogging(String clusterIdentifier, String bucketName, String s3KeyPrefix,+ public Cluster enableLogging(String clusterIdentifier, String bucketName, String s3KeyPrefix,String logDestinationType, List<String> logExports) {return enableLogging(clusterIdentifier, bucketName, s3KeyPrefix, logDestinationType, logExports, null, null);}- public synchronized Cluster enableLogging(String clusterIdentifier, String bucketName, String s3KeyPrefix,+ public Cluster enableLogging(String clusterIdentifier, String bucketName, String s3KeyPrefix,+ String logDestinationType, List<String> logExports,+ String s3TableKmsKeyId, String s3TableGranularity) {+ return mutateCluster(clusterIdentifier, () -> enableLoggingLocked(clusterIdentifier, bucketName,+ s3KeyPrefix, logDestinationType, logExports, s3TableKmsKeyId, s3TableGranularity));+ }++ private Cluster enableLoggingLocked(String clusterIdentifier, String bucketName, String s3KeyPrefix,String logDestinationType, List<String> logExports,String s3TableKmsKeyId, String s3TableGranularity) {Cluster cluster = clusters.get(clusterIdentifier)@@ -813,7 +923,11 @@ public class RedshiftService {return cluster;}- public synchronized Cluster disableLogging(String clusterIdentifier) {+ public Cluster disableLogging(String clusterIdentifier) {+ return mutateCluster(clusterIdentifier, () -> disableLoggingLocked(clusterIdentifier));+ }++ private Cluster disableLoggingLocked(String clusterIdentifier) {Cluster cluster = clusters.get(clusterIdentifier).orElseThrow(() -> new AwsException("ClusterNotFound", "Cluster " + clusterIdentifier + " not found", 404));cluster.setLoggingEnabled(false);@@ -828,7 +942,11 @@ public class RedshiftService {return cluster;}- public synchronized Cluster rebootCluster(String clusterIdentifier) {+ public Cluster rebootCluster(String clusterIdentifier) {+ return mutateCluster(clusterIdentifier, () -> rebootClusterLocked(clusterIdentifier));+ }++ private Cluster rebootClusterLocked(String clusterIdentifier) {Cluster cluster = clusters.get(clusterIdentifier).orElseThrow(() -> new AwsException("ClusterNotFound", "Cluster " + clusterIdentifier + " not found", 404));@@ -958,6 +1076,11 @@ public class RedshiftService {}public Snapshot createSnapshot(String snapshotIdentifier, String clusterIdentifier) {+ return withResourceLock("cluster", clusterIdentifier, () -> withResourceLock("snapshot", snapshotIdentifier,+ () -> createSnapshot(snapshotIdentifier, clusterIdentifier, null)));+ }++ private Snapshot createSnapshot(String snapshotIdentifier, String clusterIdentifier, Integer retentionPeriod) {validateSnapshotIdentifier(snapshotIdentifier);Optional<Cluster> clusterOpt = clusters.get(clusterIdentifier);if (clusterOpt.isEmpty()) {@@ -976,6 +1099,7 @@ public class RedshiftService {snapshot.setMasterPassword(cluster.getMasterPassword());snapshot.setSnapshotArn(snapshotArn(clusterIdentifier, snapshotIdentifier));snapshot.setSnapshotCreateTime(Instant.now());+ snapshot.setManualSnapshotRetentionPeriod(retentionPeriod);if (cluster.getEndpoint() != null) {snapshot.setPort(cluster.getEndpoint().getPort());} else {@@ -989,18 +1113,31 @@ public class RedshiftService {if (!dumpFile.startsWith(dumpDir)) {throw new AwsException("InvalidParameterValue", "Invalid snapshot identifier", 400);}+ boolean committed = false;try {Files.createDirectories(dumpDir);containerManager.takeSnapshot(clusters.accountId(), clusterIdentifier, cluster.getMasterUsername(), dumpFile);snapshot.setSqlDump(dumpFile.toString());- } catch (AwsException e) {- throw e;+ snapshots.putAllAtomically(Map.of(snapshotIdentifier, snapshot));+ committed = true;+ snapshots.flush();} catch (Exception e) {- throw new AwsException("InternalFailure", "Failed to take snapshot for cluster " + clusterIdentifier + ": " + e.getMessage(), 500);+ if (!committed) {+ try {+ Files.deleteIfExists(dumpFile);+ } catch (IOException cleanupFailure) {+ e.addSuppressed(cleanupFailure);+ }+ }+ if (e instanceof AwsException awsFailure) {+ throw awsFailure;+ }+ AwsException failure = new AwsException("InternalFailure",+ "Failed to take snapshot for cluster " + clusterIdentifier + ": " + e.getMessage(), 500);+ failure.initCause(e);+ throw failure;}- snapshots.put(snapshotIdentifier, snapshot);- snapshots.flush();return snapshot;}@@ -1029,6 +1166,10 @@ public class RedshiftService {}public Snapshot deleteSnapshot(String snapshotIdentifier) {+ return withResourceLock("snapshot", snapshotIdentifier, () -> mutate(() -> removeSnapshot(snapshotIdentifier)));+ }++ private Snapshot removeSnapshot(String snapshotIdentifier) {Optional<Snapshot> snapshotOpt = snapshots.get(snapshotIdentifier);if (snapshotOpt.isEmpty()) {throw new AwsException("ClusterSnapshotNotFound", "Snapshot " + snapshotIdentifier + " not found", 404);@@ -1051,7 +1192,12 @@ public class RedshiftService {return restoreFromClusterSnapshot(clusterIdentifier, snapshotIdentifier, null);}- public synchronized Cluster restoreFromClusterSnapshot(String clusterIdentifier, String snapshotIdentifier, String nodeType) {+ public Cluster restoreFromClusterSnapshot(String clusterIdentifier, String snapshotIdentifier, String nodeType) {+ return withResourceLock("cluster", clusterIdentifier, () -> withResourceLock("snapshot", snapshotIdentifier,+ () -> mutate(() -> restoreSnapshot(clusterIdentifier, snapshotIdentifier, nodeType))));+ }++ private Cluster restoreSnapshot(String clusterIdentifier, String snapshotIdentifier, String nodeType) {if (clusters.get(clusterIdentifier).isPresent()) {throw new AwsException("ClusterAlreadyExists", "Cluster " + clusterIdentifier + " already exists", 400);}@@ -1480,7 +1626,14 @@ public class RedshiftService {}/** Copy is recorded on the cluster only; no snapshot is actually copied to another region. */- public synchronized Cluster enableSnapshotCopy(String clusterIdentifier, String destinationRegion,+ public Cluster enableSnapshotCopy(String clusterIdentifier, String destinationRegion,+ Integer retentionPeriod, Integer manualRetentionPeriod,+ String snapshotCopyGrantName) {+ return mutateCluster(clusterIdentifier, () -> enableSnapshotCopyLocked(clusterIdentifier,+ destinationRegion, retentionPeriod, manualRetentionPeriod, snapshotCopyGrantName));+ }++ private Cluster enableSnapshotCopyLocked(String clusterIdentifier, String destinationRegion,Integer retentionPeriod, Integer manualRetentionPeriod,String snapshotCopyGrantName) {Cluster cluster = requireCluster(clusterIdentifier);@@ -1511,7 +1664,11 @@ public class RedshiftService {return cluster;}- public synchronized Cluster disableSnapshotCopy(String clusterIdentifier) {+ public Cluster disableSnapshotCopy(String clusterIdentifier) {+ return mutateCluster(clusterIdentifier, () -> disableSnapshotCopyLocked(clusterIdentifier));+ }++ private Cluster disableSnapshotCopyLocked(String clusterIdentifier) {Cluster cluster = requireCluster(clusterIdentifier);if (cluster.getSnapshotCopyDestinationRegion() == null) {throw new AwsException("SnapshotCopyAlreadyDisabledFault",@@ -1526,7 +1683,13 @@ public class RedshiftService {return cluster;}- public synchronized Cluster modifySnapshotCopyRetentionPeriod(String clusterIdentifier, int retentionPeriod,+ public Cluster modifySnapshotCopyRetentionPeriod(String clusterIdentifier, int retentionPeriod,+ boolean manual) {+ return mutateCluster(clusterIdentifier,+ () -> modifySnapshotCopyRetentionPeriodLocked(clusterIdentifier, retentionPeriod, manual));+ }++ private Cluster modifySnapshotCopyRetentionPeriodLocked(String clusterIdentifier, int retentionPeriod,boolean manual) {Cluster cluster = requireCluster(clusterIdentifier);// SnapshotCopyDisabledFault, not ...AlreadyDisabledFault: the model lists only this one here.@@ -1670,7 +1833,13 @@ public class RedshiftService {snapshotSchedules.flush();}- public synchronized Cluster modifyClusterSnapshotSchedule(String clusterIdentifier, String scheduleIdentifier,+ public Cluster modifyClusterSnapshotSchedule(String clusterIdentifier, String scheduleIdentifier,+ boolean disassociate) {+ return mutateCluster(clusterIdentifier, () -> modifyClusterSnapshotScheduleLocked(clusterIdentifier,+ scheduleIdentifier, disassociate));+ }++ private Cluster modifyClusterSnapshotScheduleLocked(String clusterIdentifier, String scheduleIdentifier,boolean disassociate) {Cluster cluster = requireCluster(clusterIdentifier);if (disassociate) {diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/model/Snapshot.java b/src/main/java/io/github/hectorvent/floci/services/redshift/model/Snapshot.javaindex 62622d1ff8..ec95d52197 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/model/Snapshot.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/model/Snapshot.java@@ -17,6 +17,7 @@ public class Snapshot {private String sqlDump;private String snapshotArn;private Instant snapshotCreateTime;+ private Integer manualSnapshotRetentionPeriod;public Snapshot() {}@@ -109,6 +110,14 @@ public class Snapshot {this.snapshotCreateTime = snapshotCreateTime;}+ public Integer getManualSnapshotRetentionPeriod() {+ return manualSnapshotRetentionPeriod == null ? -1 : manualSnapshotRetentionPeriod;+ }++ public void setManualSnapshotRetentionPeriod(Integer manualSnapshotRetentionPeriod) {+ this.manualSnapshotRetentionPeriod = manualSnapshotRetentionPeriod;+ }+private Map<String, String> tags = new LinkedHashMap<>();public Map<String, String> getTags() { return tags; }diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotIntegrationTest.javanew file mode 100644index 0000000000..36f0998617--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotIntegrationTest.java@@ -0,0 +1,188 @@+package io.github.hectorvent.floci.services.redshift;++import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerHandle;+import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerManager;+import io.github.hectorvent.floci.services.redshift.proxy.RedshiftProxyManager;+import io.quarkus.test.InjectMock;+import io.quarkus.test.common.http.TestHTTPResource;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;+import software.amazon.awssdk.regions.Region;+import software.amazon.awssdk.services.redshift.RedshiftClient;+import software.amazon.awssdk.services.redshift.model.RedshiftException;+import software.amazon.awssdk.services.redshift.model.Snapshot;++import java.net.URI;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.*;+import static org.mockito.Mockito.*;++@QuarkusTest+class RedshiftFinalSnapshotIntegrationTest {+ private static final String ACCOUNT = "111111111111";+ private static final String DUMP = "CREATE TABLE preserved (id integer); INSERT INTO preserved VALUES (42);";++ @InjectMock+ RedshiftContainerManager containerManager;++ @InjectMock+ RedshiftProxyManager proxyManager;++ @TestHTTPResource+ URI endpoint;++ @BeforeEach+ void prepareRuntime() {+ when(containerManager.start(anyString(), anyString(), anyString(), anyString()))+ .thenReturn(new RedshiftContainerHandle("mock-container", "cluster", "localhost", 5432));+ doAnswer(invocation -> {+ Files.writeString(invocation.getArgument(3, Path.class), DUMP);+ return null;+ }).when(containerManager).takeSnapshot(anyString(), anyString(), anyString(), any(Path.class));+ }++ @ParameterizedTest+ @ValueSource(ints = {-1, 1, 3653})+ void sdkFinalSnapshotSurvivesDeletionAndFeedsRestore(int retention) {+ String source = "sdk-final-source";+ String backup = "sdk-final-backup";+ String restored = "sdk-final-restored";+ AtomicReference<String> restoredDump = new AtomicReference<>();+ doAnswer(invocation -> {+ restoredDump.set(Files.readString(invocation.getArgument(3, Path.class)));+ return null;+ }).when(containerManager).restoreSnapshot(eq(ACCOUNT), eq(restored), eq("admin"), any(Path.class));+ try (RedshiftClient client = client(ACCOUNT); RedshiftClient other = client("222222222222")) {+ create(client, source);+ try {+ assertEquals("deleting", client.deleteCluster(request -> request.clusterIdentifier(source)+ .finalClusterSnapshotIdentifier(backup).finalClusterSnapshotRetentionPeriod(retention))+ .cluster().clusterStatus());+ RedshiftException gone = assertThrows(RedshiftException.class,+ () -> client.describeClusters(request -> request.clusterIdentifier(source)));+ assertEquals("ClusterNotFound", gone.awsErrorDetails().errorCode());+ Snapshot snapshot = client.describeClusterSnapshots(request -> request.snapshotIdentifier(backup))+ .snapshots().getFirst();+ assertEquals(retention, snapshot.manualSnapshotRetentionPeriod());+ assertEquals(source, snapshot.clusterIdentifier());+ assertEquals("available", snapshot.status());+ assertNotNull(snapshot.snapshotCreateTime());+ RedshiftException hidden = assertThrows(RedshiftException.class,+ () -> other.describeClusterSnapshots(request -> request.snapshotIdentifier(backup)));+ assertEquals("ClusterSnapshotNotFound", hidden.awsErrorDetails().errorCode());+ client.restoreFromClusterSnapshot(request -> request.clusterIdentifier(restored).snapshotIdentifier(backup));+ assertEquals(DUMP, restoredDump.get());+ } finally {+ deleteIfPresent(client, source);+ deleteIfPresent(client, restored);+ try {+ client.deleteClusterSnapshot(request -> request.snapshotIdentifier(backup));+ } catch (RedshiftException expected) {+ if (!"ClusterSnapshotNotFound".equals(expected.awsErrorDetails().errorCode())) {+ throw expected;+ }+ }+ }+ }+ }++ @Test+ void sdkDefaultNeedsSnapshotNameAndExplicitSkipLeavesNoSnapshot() {+ String source = "sdk-final-validation";+ try (RedshiftClient client = client(ACCOUNT)) {+ create(client, source);+ try {+ RedshiftException missing = assertThrows(RedshiftException.class,+ () -> client.deleteCluster(request -> request.clusterIdentifier(source)));+ assertEquals("InvalidParameterCombination", missing.awsErrorDetails().errorCode());+ assertEquals("available", client.describeClusters(request -> request.clusterIdentifier(source))+ .clusters().getFirst().clusterStatus());+ client.deleteCluster(request -> request.clusterIdentifier(source).skipFinalClusterSnapshot(true));+ assertTrue(client.describeClusterSnapshots(request -> request.clusterIdentifier(source)).snapshots().isEmpty());+ verify(containerManager, never()).takeSnapshot(eq(ACCOUNT), eq(source), anyString(), any(Path.class));+ } finally {+ deleteIfPresent(client, source);+ }+ }+ }++ @Test+ void slowSnapshotDoesNotBlockTheSameNamesInAnotherAccount() throws Exception {+ String source = "sdk-account-source";+ String backup = "sdk-account-backup";+ CountDownLatch dumping = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ doAnswer(invocation -> {+ dumping.countDown();+ assertTrue(release.await(5, TimeUnit.SECONDS));+ Files.writeString(invocation.getArgument(3, Path.class), DUMP);+ return null;+ }).when(containerManager).takeSnapshot(eq(ACCOUNT), eq(source), anyString(), any(Path.class));+ try (RedshiftClient first = client(ACCOUNT); RedshiftClient other = client("222222222222");+ ExecutorService callers = Executors.newFixedThreadPool(2)) {+ create(first, source);+ Future<?> snapshot = callers.submit(() -> first.createClusterSnapshot(request -> request+ .clusterIdentifier(source).snapshotIdentifier(backup)));+ try {+ assertTrue(dumping.await(2, TimeUnit.SECONDS));+ Future<Snapshot> independent = callers.submit(() -> {+ create(other, source);+ return other.createClusterSnapshot(request -> request.clusterIdentifier(source)+ .snapshotIdentifier(backup)).snapshot();+ });+ Snapshot created = independent.get(2, TimeUnit.SECONDS);+ assertTrue(created.snapshotArn().contains(":222222222222:"));+ assertEquals(-1, created.manualSnapshotRetentionPeriod());+ assertEquals(-1, other.describeClusterSnapshots(request -> request.snapshotIdentifier(backup))+ .snapshots().getFirst().manualSnapshotRetentionPeriod());+ } finally {+ release.countDown();+ snapshot.get(3, TimeUnit.SECONDS);+ for (RedshiftClient client : new RedshiftClient[]{first, other}) {+ deleteIfPresent(client, source);+ try {+ client.deleteClusterSnapshot(request -> request.snapshotIdentifier(backup));+ } catch (RedshiftException expected) {+ if (!"ClusterSnapshotNotFound".equals(expected.awsErrorDetails().errorCode())) {+ throw expected;+ }+ }+ }+ }+ }+ }++ private RedshiftClient client(String account) {+ return RedshiftClient.builder().endpointOverride(endpoint).region(Region.US_EAST_1)+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(account, "test"))).build();+ }++ private static void create(RedshiftClient client, String name) {+ client.createCluster(request -> request.clusterIdentifier(name).nodeType("dc2.large")+ .masterUsername("admin").masterUserPassword("Secret123"));+ }++ private static void deleteIfPresent(RedshiftClient client, String name) {+ try {+ client.deleteCluster(request -> request.clusterIdentifier(name).skipFinalClusterSnapshot(true));+ } catch (RedshiftException expected) {+ if (!"ClusterNotFound".equals(expected.awsErrorDetails().errorCode())) {+ throw expected;+ }+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotStorageTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotStorageTest.javanew file mode 100644index 0000000000..1d87060be0--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotStorageTest.java@@ -0,0 +1,150 @@+package io.github.hectorvent.floci.services.redshift;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.ServiceConfigAccess;+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerHandle;+import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerManager;+import io.github.hectorvent.floci.services.redshift.model.Cluster;+import io.github.hectorvent.floci.services.redshift.model.Snapshot;+import io.github.hectorvent.floci.services.redshift.proxy.RedshiftProxyManager;+import jakarta.ws.rs.core.MultivaluedHashMap;+import jakarta.ws.rs.core.MultivaluedMap;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.nio.file.Files;+import java.nio.file.Path;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.Mockito.*;++class RedshiftFinalSnapshotStorageTest {+ private static final String ACCOUNT = "111111111111";++ @TempDir+ Path directory;++ @Test+ void persistentFinalSnapshotAndRetentionSurviveRestartAndRestore() throws Exception {+ Fixture first = fixture();+ seedCluster(first);+ delete(first, "FinalClusterSnapshotRetentionPeriod", "7");++ Fixture restarted = fixture();+ Snapshot snapshot = restarted.service().describeSnapshots("final-backup").getFirst();+ assertEquals(7, snapshot.getManualSnapshotRetentionPeriod());+ assertEquals("backup contents", Files.readString(Path.of(snapshot.getSqlDump())));+ assertTrue(restarted.service().describeClusters(null).isEmpty());+ restarted.service().restoreFromClusterSnapshot("restored", "final-backup");+ verify(restarted.manager()).restoreSnapshot(ACCOUNT, "restored", "admin", Path.of(snapshot.getSqlDump()));+ }++ @Test+ void actualPersistentCommitFailurePreservesClusterAndRejectsSnapshot() throws Exception {+ Fixture fixture = fixture();+ seedCluster(fixture);+ Path snapshotFile = directory.resolve("redshift-snapshots.json");+ Files.createDirectory(snapshotFile);+ Files.writeString(snapshotFile.resolve("block-replacement"), "keep");++ assertThrows(AwsException.class, () -> delete(fixture));++ assertEquals("available", fixture.service().describeClusters("final-cluster").getFirst().getClusterStatus());+ assertTrue(fixture.service().getSnapshot("final-backup").isEmpty());+ assertFalse(Files.exists(directory.resolve("redshift-dumps").resolve(ACCOUNT).resolve("final-backup.sql")));+ verifyNoInteractions(fixture.proxy());+ verify(fixture.manager(), never()).stop(anyString(), anyString());+ assertTrue(Files.readString(directory.resolve("redshift-clusters.json")).contains("final-cluster"));+ }++ @Test+ void failedDumpRemovesPartialOutputAndPreservesOriginalFailure() {+ Fixture fixture = fixture();+ seedCluster(fixture);+ AwsException original = new AwsException("InternalFailure", "dump failed", 500);+ doAnswer(invocation -> {+ Files.writeString(invocation.getArgument(3, Path.class), "partial dump");+ throw original;+ }).when(fixture.manager()).takeSnapshot(anyString(), anyString(), anyString(), any(Path.class));++ assertSame(original, assertThrows(AwsException.class, () -> delete(fixture)));+ assertFalse(Files.exists(directory.resolve("redshift-dumps").resolve(ACCOUNT).resolve("final-backup.sql")));+ assertTrue(fixture.service().getSnapshot("final-backup").isEmpty());+ verifyNoInteractions(fixture.proxy());+ }++ @Test+ void failedCleanupIsSuppressedOnTheOriginalDumpFailure() {+ Fixture fixture = fixture();+ seedCluster(fixture);+ AwsException original = new AwsException("InternalFailure", "dump failed", 500);+ doAnswer(invocation -> {+ Path output = invocation.getArgument(3, Path.class);+ Files.createDirectory(output);+ Files.writeString(output.resolve("block-cleanup"), "partial output");+ throw original;+ }).when(fixture.manager()).takeSnapshot(anyString(), anyString(), anyString(), any(Path.class));++ assertSame(original, assertThrows(AwsException.class, () -> delete(fixture)));+ assertEquals(1, original.getSuppressed().length);+ assertTrue(fixture.service().getSnapshot("final-backup").isEmpty());+ verifyNoInteractions(fixture.proxy());+ }++ private Fixture fixture() {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.storage().persistentPath()).thenReturn(directory.toString());+ when(config.defaultAccountId()).thenReturn(ACCOUNT);+ when(config.services().redshift().proxyBasePort()).thenReturn(7100);+ when(config.services().redshift().proxyMaxPort()).thenReturn(7199);+ ServiceConfigAccess access = mock(ServiceConfigAccess.class);+ when(access.storageMode("redshift")).thenReturn("persistent");+ StorageFactory factory = new StorageFactory(config, access);+ RedshiftContainerManager manager = mock(RedshiftContainerManager.class);+ when(manager.start(anyString(), anyString(), anyString(), anyString()))+ .thenReturn(new RedshiftContainerHandle("container", "restored", "localhost", 5432));+ doAnswer(invocation -> {+ Files.writeString(invocation.getArgument(3, Path.class), "backup contents");+ return null;+ }).when(manager).takeSnapshot(anyString(), anyString(), anyString(), any(Path.class));+ RedshiftProxyManager proxy = mock(RedshiftProxyManager.class);+ DockerHostResolver dockerHost = mock(DockerHostResolver.class);+ when(dockerHost.resolve()).thenReturn("localhost");+ RedshiftService service = new RedshiftService(factory, manager, config,+ new RegionResolver("us-east-1", ACCOUNT), proxy, dockerHost, new RedshiftCredentialBroker());+ return new Fixture(service, manager, proxy, config, factory);+ }++ private static void seedCluster(Fixture fixture) {+ Cluster cluster = new Cluster();+ cluster.setClusterIdentifier("final-cluster");+ cluster.setClusterStatus("available");+ cluster.setMasterUsername("admin");+ cluster.setMasterPassword("secret-pw");+ AccountAwareStorageBackend<Cluster> clusters = fixture.factory().create("redshift", "redshift-clusters.json",+ new TypeReference<Map<String, Cluster>>() { });+ clusters.put("final-cluster", cluster);+ }++ private static void delete(Fixture fixture, String... options) {+ MultivaluedMap<String, String> params = new MultivaluedHashMap<>();+ params.putSingle("ClusterIdentifier", "final-cluster");+ params.putSingle("FinalClusterSnapshotIdentifier", "final-backup");+ for (int index = 0; index < options.length; index += 2) {+ params.putSingle(options[index], options[index + 1]);+ }+ new RedshiftQueryHandler(fixture.service(), new RedshiftCredentialBroker(), fixture.config(),+ mock(RedshiftIamDbUserResolver.class), new RegionResolver("us-east-1", ACCOUNT))+ .handle("DeleteCluster", params);+ }++ private record Fixture(RedshiftService service, RedshiftContainerManager manager, RedshiftProxyManager proxy,+ EmulatorConfig config, StorageFactory factory) { }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.javaindex 7364635a1b..cbb4a3bcb4 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java@@ -453,6 +453,7 @@ public class RedshiftOperationsTest {.contentType("application/x-www-form-urlencoded").header("Authorization", AUTH_HEADER).formParam("Action", "DeleteCluster")+ .formParam("SkipFinalClusterSnapshot", "true").formParam("ClusterIdentifier", "cluster-src").when().post("/")@@ -512,6 +513,7 @@ public class RedshiftOperationsTest {.contentType("application/x-www-form-urlencoded").header("Authorization", AUTH_HEADER).formParam("Action", "DeleteCluster")+ .formParam("SkipFinalClusterSnapshot", "true").formParam("ClusterIdentifier", "cluster-restored").when().post("/")@@ -898,6 +900,7 @@ public class RedshiftOperationsTest {.contentType("application/x-www-form-urlencoded").header("Authorization", AUTH_HEADER).formParam("Action", "DeleteCluster")+ .formParam("SkipFinalClusterSnapshot", "true").formParam("ClusterIdentifier", "cluster-tags").when().post("/")@@ -1083,7 +1086,8 @@ public class RedshiftOperationsTest {400, "InvalidSnapshotCopyGrantStateFault");assertEquals(200, redshift("DisableSnapshotCopy", "ClusterIdentifier", "cluster-copy").statusCode());- assertEquals(200, redshift("DeleteCluster", "ClusterIdentifier", "cluster-copy").statusCode());+ assertEquals(200, redshift("DeleteCluster", "ClusterIdentifier", "cluster-copy",+ "SkipFinalClusterSnapshot", "true").statusCode());assertEquals(200, redshift("DeleteSnapshotCopyGrant", "SnapshotCopyGrantName", "copy-lifecycle-grant").statusCode());}@@ -1186,7 +1190,8 @@ public class RedshiftOperationsTest {assertEquals(0, describeSchedules("ScheduleIdentifier", "sched-1").getList("SnapshotSchedule").size());assertFault(redshift("DeleteSnapshotSchedule", "ScheduleIdentifier", "sched-1"), 400, "SnapshotScheduleNotFound");- assertEquals(200, redshift("DeleteCluster", "ClusterIdentifier", "cluster-sched").statusCode());+ assertEquals(200, redshift("DeleteCluster", "ClusterIdentifier", "cluster-sched",+ "SkipFinalClusterSnapshot", "true").statusCode());}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.javaindex 43df55d056..311482f885 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.redshift;+import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.PaginatedResult;@@ -462,7 +463,7 @@ class RedshiftQueryHandlerTest {Cluster cluster = new Cluster();cluster.setClusterIdentifier("test-cluster");cluster.setClusterStatus("deleting");- when(service.deleteCluster(any())).thenReturn(cluster);+ when(service.deleteCluster("test-cluster", null, null, null)).thenReturn(cluster);Response response = handler.handle("DeleteCluster", params);assertEquals(200, response.getStatus());@@ -489,12 +490,29 @@ class RedshiftQueryHandlerTest {assertTrue(xml.contains("<Port>5439</Port>"));assertTrue(xml.contains("<MasterUsername>admin</MasterUsername>"));assertTrue(xml.contains("<RequestId>test-req-id</RequestId>"));+ assertTrue(xml.contains("<ManualSnapshotRetentionPeriod>-1</ManualSnapshotRetentionPeriod>"));// A Snapshot built via this constructor carries no ARN/CreateTime; the elements// must be omitted rather than serialized as empty or "null".assertFalse(xml.contains("<SnapshotArn>"));assertFalse(xml.contains("<SnapshotCreateTime>"));}+ @Test+ void describeLegacySnapshotsDefaultsMissingAndNullRetention() throws Exception {+ ObjectMapper mapper = new ObjectMapper();+ for (String json : List.of("{\"snapshotIdentifier\":\"legacy\"}",+ "{\"snapshotIdentifier\":\"legacy\",\"manualSnapshotRetentionPeriod\":null}")) {+ Snapshot snapshot = mapper.readValue(json, Snapshot.class);+ when(service.describeSnapshots("legacy", null)).thenReturn(List.of(snapshot));+ MultivaluedMap<String, String> params = new MultivaluedHashMap<>();+ params.putSingle("SnapshotIdentifier", "legacy");++ String xml = handler.handle("DescribeClusterSnapshots", params).getEntity().toString();++ assertTrue(xml.contains("<ManualSnapshotRetentionPeriod>-1</ManualSnapshotRetentionPeriod>"));+ }+ }+@Testvoid describeClusterSnapshots() {MultivaluedMap<String, String> params = new MultivaluedHashMap<>();diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.javaindex e8da3973c1..6cf66c71e9 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java@@ -26,11 +26,14 @@ import io.github.hectorvent.floci.services.redshift.model.SnapshotSchedule;import io.github.hectorvent.floci.services.redshift.proxy.RedshiftProxyManager;import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;import io.github.hectorvent.floci.services.secretsmanager.model.Secret;+import jakarta.ws.rs.core.MultivaluedHashMap;+import jakarta.ws.rs.core.MultivaluedMap;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.ValueSource;import org.mockito.ArgumentCaptor;+import org.mockito.InOrder;import java.nio.file.Files;import java.nio.file.Path;@@ -44,6 +47,14 @@ import java.util.List;import java.util.Map;import java.util.Optional;import java.util.Set;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;+import java.util.concurrent.atomic.AtomicReference;import static io.github.hectorvent.floci.testing.InlineMocks.inlineMock;import static org.junit.jupiter.api.Assertions.assertEquals;@@ -56,6 +67,277 @@ import static org.mockito.Mockito.*;class RedshiftServiceTest {+ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void slowSnapshotDoesNotBlockOtherClusterMutation(boolean finalSnapshot) throws Exception {+ finalSnapshotCluster();+ Cluster other = new Cluster();+ other.setClusterIdentifier("other-cluster");+ when(clusterBackend.get("other-cluster")).thenReturn(Optional.of(other));+ CountDownLatch dumping = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ doAnswer(invocation -> {+ dumping.countDown();+ assertTrue(release.await(5, TimeUnit.SECONDS));+ return null;+ }).when(cm).takeSnapshot(eq("111111111111"), eq("final-cluster"), eq("admin"), any(Path.class));+ try (ExecutorService callers = Executors.newFixedThreadPool(2)) {+ Future<?> snapshot = callers.submit(() -> {+ if (finalSnapshot) {+ finalSnapshotDelete("FinalClusterSnapshotIdentifier", "final-backup");+ } else {+ service.createSnapshot("final-backup", "final-cluster");+ }+ });+ try {+ assertTrue(dumping.await(2, TimeUnit.SECONDS));+ Future<Cluster> mutation = callers.submit(() -> service.modifyCluster("other-cluster", "ra3.large",+ null, null, null, null));+ assertEquals("ra3.large", mutation.get(1, TimeUnit.SECONDS).getNodeType());+ } finally {+ release.countDown();+ snapshot.get(3, TimeUnit.SECONDS);+ }+ }+ }++ @Test+ void snapshotFlushFailureKeepsCommittedDump() throws Exception {+ finalSnapshotCluster();+ AtomicReference<Path> dump = new AtomicReference<>();+ doAnswer(invocation -> {+ Path path = invocation.getArgument(3, Path.class);+ Files.writeString(path, "committed backup");+ dump.set(path);+ return null;+ }).when(cm).takeSnapshot(eq("111111111111"), eq("final-cluster"), eq("admin"), any(Path.class));+ doThrow(new IllegalStateException("flush failed")).when(snapshotBackend).flush();+ try {+ assertThrows(AwsException.class,+ () -> finalSnapshotDelete("FinalClusterSnapshotIdentifier", "final-backup"));+ verify(snapshotBackend).putAllAtomically(anyMap());+ assertEquals("committed backup", Files.readString(dump.get()));+ verify(cm, never()).stop(anyString(), anyString());+ } finally {+ if (dump.get() != null) {+ Files.deleteIfExists(dump.get());+ }+ }+ }++ @Test+ void snapshotKeepsSameClusterMutationSerializedWithoutHoldingServiceMonitor() throws Exception {+ finalSnapshotCluster();+ CountDownLatch dumping = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ CountDownLatch attemptingMutation = new CountDownLatch(1);+ doAnswer(invocation -> {+ dumping.countDown();+ assertTrue(release.await(5, TimeUnit.SECONDS));+ return null;+ }).when(cm).takeSnapshot(anyString(), anyString(), anyString(), any(Path.class));+ try (ExecutorService callers = Executors.newFixedThreadPool(2)) {+ Future<Snapshot> snapshot = callers.submit(() -> service.createSnapshot("backup", "final-cluster"));+ try {+ assertTrue(dumping.await(2, TimeUnit.SECONDS));+ Future<Cluster> mutation = callers.submit(() -> {+ attemptingMutation.countDown();+ return service.modifyCluster("final-cluster", "ra3.large", null, null, null, null);+ });+ assertTrue(attemptingMutation.await(2, TimeUnit.SECONDS));+ assertThrows(TimeoutException.class, () -> mutation.get(150, TimeUnit.MILLISECONDS));+ release.countDown();+ assertNotNull(snapshot.get(2, TimeUnit.SECONDS));+ assertEquals("ra3.large", mutation.get(2, TimeUnit.SECONDS).getNodeType());+ } finally {+ release.countDown();+ }+ }+ }++ @Test+ void concurrentSnapshotsOnDifferentClustersCannotOverwriteTheSameBackup() throws Exception {+ Cluster source = finalSnapshotCluster();+ when(clusterBackend.get("other-cluster")).thenReturn(Optional.of(source));+ Map<String, Snapshot> committed = new ConcurrentHashMap<>();+ when(snapshotBackend.get(anyString())).thenAnswer(invocation -> Optional.ofNullable(committed.get(invocation.getArgument(0))));+ doAnswer(invocation -> {+ committed.putAll(invocation.getArgument(0));+ return null;+ }).when(snapshotBackend).putAllAtomically(anyMap());+ CountDownLatch dumping = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ CountDownLatch attempting = new CountDownLatch(1);+ doAnswer(invocation -> {+ dumping.countDown();+ assertTrue(release.await(5, TimeUnit.SECONDS));+ return null;+ }).when(cm).takeSnapshot(anyString(), anyString(), anyString(), any(Path.class));+ try (ExecutorService callers = Executors.newFixedThreadPool(2)) {+ Future<Snapshot> first = callers.submit(() -> service.createSnapshot("backup", "final-cluster"));+ try {+ assertTrue(dumping.await(2, TimeUnit.SECONDS));+ Future<AwsException> second = callers.submit(() -> {+ attempting.countDown();+ return assertThrows(AwsException.class, () -> service.createSnapshot("backup", "other-cluster"));+ });+ assertTrue(attempting.await(2, TimeUnit.SECONDS));+ assertThrows(TimeoutException.class, () -> second.get(150, TimeUnit.MILLISECONDS));+ release.countDown();+ assertEquals("final-cluster", first.get(2, TimeUnit.SECONDS).getClusterIdentifier());+ assertEquals("ClusterSnapshotAlreadyExists", second.get(2, TimeUnit.SECONDS).getErrorCode());+ verify(cm, times(1)).takeSnapshot(anyString(), anyString(), anyString(), any(Path.class));+ } finally {+ release.countDown();+ }+ }+ }++ @Test+ void deleteWithFinalSnapshotCommitsBackupBeforeStoppingCluster() {+ Cluster cluster = finalSnapshotCluster();+ finalSnapshotDelete("FinalClusterSnapshotIdentifier", "final-backup",+ "FinalClusterSnapshotRetentionPeriod", "7");++ ArgumentCaptor<Map<String, Snapshot>> committed = ArgumentCaptor.captor();+ InOrder order = inOrder(cm, snapshotBackend, proxyManager, clusterBackend);+ order.verify(cm).takeSnapshot(eq("111111111111"), eq("final-cluster"), eq("admin"), any(Path.class));+ order.verify(snapshotBackend).putAllAtomically(committed.capture());+ order.verify(snapshotBackend).flush();+ order.verify(proxyManager).stopProxy("111111111111:final-cluster");+ order.verify(cm).stop("111111111111", "final-cluster");+ order.verify(clusterBackend).delete("final-cluster");+ Snapshot snapshot = committed.getValue().get("final-backup");+ assertEquals("final-cluster", snapshot.getClusterIdentifier());+ assertEquals("secret-pw", snapshot.getMasterPassword());+ assertEquals(7, snapshot.getManualSnapshotRetentionPeriod());+ assertEquals("deleting", cluster.getClusterStatus());+ }++ @Test+ void deleteDefaultsToRequiringFinalSnapshotName() {+ finalSnapshotCluster();+ AwsException failure = assertThrows(AwsException.class, () -> finalSnapshotDelete());+ assertEquals("InvalidParameterCombination", failure.getErrorCode());+ verifyNoInteractions(cm, proxyManager);+ verify(clusterBackend, never()).delete(anyString());+ }++ @Test+ void finalSnapshotDefaultsToIndefiniteRetentionAndSurvivesTeardownFailure() {+ Cluster cluster = finalSnapshotCluster();+ doThrow(new IllegalStateException("proxy busy")).when(proxyManager).stopProxy(anyString());+ assertThrows(AwsException.class,+ () -> finalSnapshotDelete("FinalClusterSnapshotIdentifier", "final-backup"));+ ArgumentCaptor<Map<String, Snapshot>> committed = ArgumentCaptor.captor();+ verify(snapshotBackend).putAllAtomically(committed.capture());+ assertEquals(-1, committed.getValue().get("final-backup").getManualSnapshotRetentionPeriod());+ verify(snapshotBackend, never()).delete(anyString());+ verify(cm, never()).stop(anyString(), anyString());+ assertEquals("available", cluster.getClusterStatus());+ }++ @ParameterizedTest+ @ValueSource(strings = {"../escape", "bad--name", "bad-", "1bad", " "})+ void deleteRejectsMalformedFinalSnapshotName(String name) {+ finalSnapshotCluster();+ assertThrows(AwsException.class, () -> finalSnapshotDelete("FinalClusterSnapshotIdentifier", name));+ verifyNoInteractions(cm, proxyManager);+ verify(clusterBackend, never()).delete(anyString());+ }++ @ParameterizedTest+ @ValueSource(strings = {"0", "-2", "3654"})+ void deleteRejectsInvalidRetentionBeforeDumpOrTeardown(String retention) {+ finalSnapshotCluster();+ AwsException failure = assertThrows(AwsException.class, () -> finalSnapshotDelete(+ "FinalClusterSnapshotIdentifier", "final-backup", "FinalClusterSnapshotRetentionPeriod", retention));+ assertEquals("InvalidRetentionPeriodFault", failure.getErrorCode());+ verifyNoInteractions(cm, proxyManager);+ verify(clusterBackend, never()).delete(anyString());+ }++ @Test+ void deleteRejectsSkipWithFinalSnapshotName() {+ finalSnapshotCluster();+ AwsException failure = assertThrows(AwsException.class, () -> finalSnapshotDelete(+ "SkipFinalClusterSnapshot", "true", "FinalClusterSnapshotIdentifier", "final-backup"));+ assertEquals("InvalidParameterCombination", failure.getErrorCode());+ verifyNoInteractions(cm, proxyManager);+ }++ @Test+ void deleteRejectsMalformedSkipAndRetention() {+ finalSnapshotCluster();+ for (String[] option : List.of(new String[]{"SkipFinalClusterSnapshot", "yes"},+ new String[]{"FinalClusterSnapshotRetentionPeriod", "abc"},+ new String[]{"FinalClusterSnapshotRetentionPeriod", "2147483648"})) {+ AwsException failure = assertThrows(AwsException.class, () -> finalSnapshotDelete(option));+ assertEquals("InvalidParameterValue", failure.getErrorCode());+ }+ verifyNoInteractions(cm, proxyManager);+ }++ @Test+ void deleteKeepsClusterWhenDumpFails() {+ Cluster cluster = finalSnapshotCluster();+ doThrow(new AwsException("InternalFailure", "dump failed", 500)).when(cm)+ .takeSnapshot(eq("111111111111"), eq("final-cluster"), eq("admin"), any(Path.class));+ assertThrows(AwsException.class,+ () -> finalSnapshotDelete("FinalClusterSnapshotIdentifier", "final-backup"));+ verifyNoInteractions(proxyManager);+ verify(cm, never()).stop(anyString(), anyString());+ verify(clusterBackend, never()).delete(anyString());+ assertEquals("available", cluster.getClusterStatus());+ }++ @Test+ void deleteKeepsClusterWhenSnapshotCommitFails() {+ Cluster cluster = finalSnapshotCluster();+ doThrow(new IllegalStateException("storage failed")).when(snapshotBackend).putAllAtomically(anyMap());+ assertThrows(RuntimeException.class,+ () -> finalSnapshotDelete("FinalClusterSnapshotIdentifier", "final-backup"));+ verifyNoInteractions(proxyManager);+ verify(cm, never()).stop(anyString(), anyString());+ verify(clusterBackend, never()).delete(anyString());+ assertEquals("available", cluster.getClusterStatus());+ }++ @Test+ void deleteRejectsDuplicateBackupAndUnavailableCluster() {+ Cluster cluster = finalSnapshotCluster();+ when(snapshotBackend.get("existing-backup")).thenReturn(Optional.of(new Snapshot()));+ AwsException duplicate = assertThrows(AwsException.class,+ () -> finalSnapshotDelete("FinalClusterSnapshotIdentifier", "existing-backup"));+ assertEquals("ClusterSnapshotAlreadyExists", duplicate.getErrorCode());+ cluster.setClusterStatus("unavailable");+ AwsException unavailable = assertThrows(AwsException.class,+ () -> finalSnapshotDelete("FinalClusterSnapshotIdentifier", "final-backup"));+ assertEquals("InvalidClusterState", unavailable.getErrorCode());+ verifyNoInteractions(cm, proxyManager);+ }++ private Cluster finalSnapshotCluster() {+ Cluster cluster = new Cluster();+ cluster.setClusterIdentifier("final-cluster");+ cluster.setClusterStatus("available");+ cluster.setMasterUsername("admin");+ cluster.setMasterPassword("secret-pw");+ when(clusterBackend.get("final-cluster")).thenReturn(Optional.of(cluster));+ return cluster;+ }++ private void finalSnapshotDelete(String... options) {+ MultivaluedMap<String, String> params = new MultivaluedHashMap<>();+ params.putSingle("ClusterIdentifier", "final-cluster");+ for (int index = 0; index < options.length; index += 2) {+ params.putSingle(options[index], options[index + 1]);+ }+ RedshiftQueryHandler query = new RedshiftQueryHandler(service, credentialBroker,+ mock(EmulatorConfig.class), mock(RedshiftIamDbUserResolver.class), regionResolver);+ query.handle("DeleteCluster", params);+ }+@Testvoid blankDefaultRoleCreatesWithoutDefaultAndClearsExistingDefault() {String role = "arn:aws:iam::111111111111:role/copy";@@ -871,7 +1153,7 @@ class RedshiftServiceTest {// sqlDump is now an absolute path scoped by account to avoid collisions across accountsassertTrue(snapshot.getSqlDump().contains("111111111111"));assertTrue(snapshot.getSqlDump().endsWith("my-snapshot.sql"));- verify(snapshotBackend).put(eq("my-snapshot"), any(Snapshot.class));+ verify(snapshotBackend).putAllAtomically(Map.of("my-snapshot", snapshot));verify(snapshotBackend).flush();verify(cm).takeSnapshot(eq("111111111111"), eq("my-cluster"), eq("admin"), any(Path.class));}