Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(rds): persist metadata across storage-backed restarts (#945)

Albert Einshutoin0a8732b34951
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
index 095e26ca80..ec3b691a3b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
@@ -1,10 +1,13 @@
package io.github.hectorvent.floci.services.rds;
+import com.fasterxml.jackson.core.type.TypeReference;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
-import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.rds.container.RdsContainerHandle;
import io.github.hectorvent.floci.services.rds.container.RdsContainerManager;
import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;
@@ -15,6 +18,7 @@ import io.github.hectorvent.floci.services.rds.model.DbInstance;
import io.github.hectorvent.floci.services.rds.model.DbInstanceStatus;
import io.github.hectorvent.floci.services.rds.model.DbParameterGroup;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager;
+import jakarta.annotation.PostConstruct;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
@@ -24,6 +28,7 @@ import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -50,15 +55,44 @@ public class RdsService {
public RdsService(RdsContainerManager containerManager,
RdsProxyManager proxyManager,
RegionResolver regionResolver,
- EmulatorConfig config) {
+ EmulatorConfig config,
+ StorageFactory storageFactory) {
this.containerManager = containerManager;
this.proxyManager = proxyManager;
this.regionResolver = regionResolver;
this.config = config;
- this.instances = new InMemoryStorage<>();
- this.clusters = new InMemoryStorage<>();
- this.parameterGroups = new InMemoryStorage<>();
- this.clusterParameterGroups = new InMemoryStorage<>();
+ this.instances = storageFactory.create("rds", "rds-instances.json",
+ new TypeReference<Map<String, DbInstance>>() {});
+ this.clusters = storageFactory.create("rds", "rds-clusters.json",
+ new TypeReference<Map<String, DbCluster>>() {});
+ this.parameterGroups = storageFactory.create("rds", "rds-parameter-groups.json",
+ new TypeReference<Map<String, DbParameterGroup>>() {});
+ this.clusterParameterGroups = storageFactory.create("rds", "rds-cluster-parameter-groups.json",
+ new TypeReference<Map<String, DbClusterParameterGroup>>() {});
+ }
+
+ RdsService(RdsContainerManager containerManager,
+ RdsProxyManager proxyManager,
+ RegionResolver regionResolver,
+ EmulatorConfig config,
+ StorageBackend<String, DbInstance> instances,
+ StorageBackend<String, DbCluster> clusters,
+ StorageBackend<String, DbParameterGroup> parameterGroups,
+ StorageBackend<String, DbClusterParameterGroup> clusterParameterGroups) {
+ this.containerManager = containerManager;
+ this.proxyManager = proxyManager;
+ this.regionResolver = regionResolver;
+ this.config = config;
+ this.instances = instances;
+ this.clusters = clusters;
+ this.parameterGroups = parameterGroups;
+ this.clusterParameterGroups = clusterParameterGroups;
+ }
+
+ @PostConstruct
+ void restorePersistedRuntime() {
+ restoreClusters();
+ restoreInstances();
}
// ── DB Instances ──────────────────────────────────────────────────────────
@@ -82,6 +116,7 @@ public class RdsService {
String containerHost = null;
int containerPort = 0;
String instanceVolumeId = null;
+ String instanceDockerVolumeName = null;
if (dbClusterIdentifier != null && !dbClusterIdentifier.isBlank()) {
// Cluster member — share the cluster's container
@@ -93,6 +128,9 @@ public class RdsService {
containerId = cluster.getContainerId();
containerHost = cluster.getContainerHost();
containerPort = cluster.getContainerPort();
+ instanceDockerVolumeName = cluster.getDockerVolumeName() != null
+ ? cluster.getDockerVolumeName()
+ : volumeName(cluster.getVolumeId(), cluster.getDbClusterIdentifier());
} else {
// Standalone instance — start its own container
String image = imageForEngine(engine);
@@ -103,6 +141,7 @@ public class RdsService {
containerId = handle.getContainerId();
containerHost = handle.getHost();
containerPort = handle.getPort();
+ instanceDockerVolumeName = volumeName(instanceVolumeId, id);
}
DbEndpoint endpoint = new DbEndpoint("localhost", proxyPort);
@@ -113,6 +152,7 @@ public class RdsService {
instance.setContainerHost(containerHost);
instance.setContainerPort(containerPort);
instance.setVolumeId(instanceVolumeId);
+ instance.setDockerVolumeName(instanceDockerVolumeName);
String region = regionResolver.getDefaultRegion();
instance.setDbiResourceId("db-" + java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 24).toUpperCase());
@@ -263,6 +303,7 @@ public class RdsService {
cluster.setContainerHost(handle.getHost());
cluster.setContainerPort(handle.getPort());
cluster.setVolumeId(clusterVolumeId);
+ cluster.setDockerVolumeName(volumeName(clusterVolumeId, id));
String region = regionResolver.getDefaultRegion();
cluster.setDbClusterResourceId("cluster-" + java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 24).toUpperCase());
@@ -478,6 +519,127 @@ public class RdsService {
usedPorts.remove(port);
}
+ private void restoreClusters() {
+ for (DbCluster cluster : allClusters()) {
+ if (cluster.getStatus() == DbInstanceStatus.DELETING) {
+ continue;
+ }
+ int proxyPort = reserveOrAllocateProxyPort(cluster.getProxyPort());
+ cluster.setProxyPort(proxyPort);
+ cluster.setEndpoint(new DbEndpoint("localhost", proxyPort));
+ cluster.setReaderEndpoint(new DbEndpoint("localhost", proxyPort));
+ if (cluster.getDockerVolumeName() == null) {
+ cluster.setDockerVolumeName(volumeName(cluster.getVolumeId(), cluster.getDbClusterIdentifier()));
+ }
+ try {
+ String image = imageForEngine(cluster.getEngine());
+ RdsContainerHandle handle = containerManager.start(cluster.getDbClusterIdentifier(),
+ cluster.getVolumeId(), cluster.getEngine(), image,
+ cluster.getMasterUsername(), cluster.getMasterPassword(), cluster.getDatabaseName());
+ cluster.setContainerId(handle.getContainerId());
+ cluster.setContainerHost(handle.getHost());
+ cluster.setContainerPort(handle.getPort());
+
+ String effectiveMasterUser = cluster.getMasterUsername() != null
+ ? cluster.getMasterUsername() : "root";
+ proxyManager.startProxy(cluster.getDbClusterIdentifier(), cluster.getEngine(),
+ cluster.isIamDatabaseAuthenticationEnabled(), proxyPort,
+ handle.getHost(), handle.getPort(), effectiveMasterUser,
+ cluster.getMasterPassword(), cluster.getDatabaseName(),
+ (user, pw) -> validateDbClusterPassword(cluster.getDbClusterIdentifier(), user, pw));
+ cluster.setStatus(DbInstanceStatus.AVAILABLE);
+ } catch (Exception e) {
+ releaseProxyPort(proxyPort);
+ LOG.warnv(e, "Failed to restore RDS cluster {0}", cluster.getDbClusterIdentifier());
+ }
+ }
+ }
+
+ private void restoreInstances() {
+ for (DbInstance instance : allInstances()) {
+ if (instance.getStatus() == DbInstanceStatus.DELETING) {
+ continue;
+ }
+ int proxyPort = reserveOrAllocateProxyPort(instance.getProxyPort());
+ instance.setProxyPort(proxyPort);
+ instance.setEndpoint(new DbEndpoint("localhost", proxyPort));
+ try {
+ String backendHost;
+ int backendPort;
+ String clusterId = instance.getDbClusterIdentifier();
+ if (clusterId != null && !clusterId.isBlank()) {
+ DbCluster cluster = clusters.get(clusterId).orElseThrow(() ->
+ new AwsException("DBClusterNotFoundFault",
+ "DB cluster " + clusterId + " not found.", 404));
+ backendHost = cluster.getContainerHost();
+ backendPort = cluster.getContainerPort();
+ if (backendHost == null || backendPort <= 0) {
+ throw new AwsException("InvalidDBClusterStateFault",
+ "DB cluster " + clusterId + " runtime is not available.", 400);
+ }
+ instance.setContainerId(cluster.getContainerId());
+ instance.setContainerHost(cluster.getContainerHost());
+ instance.setContainerPort(cluster.getContainerPort());
+ if (instance.getDockerVolumeName() == null) {
+ instance.setDockerVolumeName(cluster.getDockerVolumeName() != null
+ ? cluster.getDockerVolumeName()
+ : volumeName(cluster.getVolumeId(), cluster.getDbClusterIdentifier()));
+ }
+ } else {
+ if (instance.getDockerVolumeName() == null) {
+ instance.setDockerVolumeName(volumeName(instance.getVolumeId(), instance.getDbInstanceIdentifier()));
+ }
+ String image = imageForEngine(instance.getEngine());
+ RdsContainerHandle handle = containerManager.start(instance.getDbInstanceIdentifier(),
+ instance.getVolumeId(), instance.getEngine(), image,
+ instance.getMasterUsername(), instance.getMasterPassword(), instance.getDbName());
+ backendHost = handle.getHost();
+ backendPort = handle.getPort();
+ instance.setContainerId(handle.getContainerId());
+ instance.setContainerHost(handle.getHost());
+ instance.setContainerPort(handle.getPort());
+ }
+
+ String effectiveMasterUser = instance.getMasterUsername() != null
+ ? instance.getMasterUsername() : "root";
+ proxyManager.startProxy(instance.getDbInstanceIdentifier(), instance.getEngine(),
+ instance.isIamDatabaseAuthenticationEnabled(), proxyPort,
+ backendHost, backendPort, effectiveMasterUser,
+ instance.getMasterPassword(), instance.getDbName(),
+ (user, pw) -> validateDbPassword(instance.getDbInstanceIdentifier(), user, pw));
+ instance.setStatus(DbInstanceStatus.AVAILABLE);
+ } catch (Exception e) {
+ releaseProxyPort(proxyPort);
+ LOG.warnv(e, "Failed to restore RDS instance {0}", instance.getDbInstanceIdentifier());
+ }
+ }
+ }
+
+ private Collection<DbCluster> allClusters() {
+ if (clusters instanceof AccountAwareStorageBackend<DbCluster> aware) {
+ return aware.scanAllAccounts();
+ }
+ return clusters.scan(k -> true);
+ }
+
+ private Collection<DbInstance> allInstances() {
+ if (instances instanceof AccountAwareStorageBackend<DbInstance> aware) {
+ return aware.scanAllAccounts();
+ }
+ return instances.scan(k -> true);
+ }
+
+ private int reserveOrAllocateProxyPort(int persistedPort) {
+ if (persistedPort > 0 && usedPorts.add(persistedPort)) {
+ return persistedPort;
+ }
+ return allocateProxyPort();
+ }
+
+ private String volumeName(String volumeId, String fallbackId) {
+ return ContainerStorageHelper.resourceName("rds", volumeId, fallbackId);
+ }
+
private RdsContainerHandle buildHandle(DbInstance instance) {
return new RdsContainerHandle(instance.getContainerId(), instance.getDbInstanceIdentifier(),
instance.getContainerHost(), instance.getContainerPort());
diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
index 5a1eef097f..a295aae1b2 100644
--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
@@ -3,22 +3,26 @@ package io.github.hectorvent.floci.services.rds;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;
import io.github.hectorvent.floci.services.rds.model.DbCluster;
import io.github.hectorvent.floci.services.rds.model.DbClusterParameterGroup;
import io.github.hectorvent.floci.services.rds.container.RdsContainerHandle;
import io.github.hectorvent.floci.services.rds.container.RdsContainerManager;
import io.github.hectorvent.floci.services.rds.model.DbInstance;
+import io.github.hectorvent.floci.services.rds.model.DbParameterGroup;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
import java.util.Collection;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class RdsServiceTest {
@@ -43,7 +47,9 @@ class RdsServiceTest {
when(rdsConfig.proxyBasePort()).thenReturn(7000);
when(rdsConfig.proxyMaxPort()).thenReturn(7099);
- rdsService = new RdsService(containerManager, proxyManager, regionResolver, config);
+ rdsService = newService(containerManager, proxyManager,
+ new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new InMemoryStorage<>(), new InMemoryStorage<>());
when(containerManager.start(any(), any(), any(), any(), any(), any(), any()))
.thenReturn(new RdsContainerHandle("cont-id", "id", "localhost", 5432));
@@ -169,4 +175,106 @@ class RdsServiceTest {
assertEquals("DBParameterGroupNotFound", exception.getErrorCode());
}
+
+ @Test
+ void restorePersistedRuntimeRestartsStandaloneInstanceWithSameVolumeAndProxyPort() {
+ StorageBackend<String, DbInstance> instances = new InMemoryStorage<>();
+ StorageBackend<String, DbCluster> clusters = new InMemoryStorage<>();
+ StorageBackend<String, DbParameterGroup> parameterGroups = new InMemoryStorage<>();
+ StorageBackend<String, DbClusterParameterGroup> clusterParameterGroups = new InMemoryStorage<>();
+
+ when(containerManager.start(any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(new RdsContainerHandle("initial-container", "mydb", "localhost", 5432));
+
+ RdsService initialService = newService(containerManager, proxyManager,
+ instances, clusters, parameterGroups, clusterParameterGroups);
+ DbInstance created = initialService.createDbInstance("mydb", "postgres", "16.3",
+ "admin", "secret", "app", "db.t3.micro",
+ 20, false, null, null);
+
+ String persistedVolumeId = created.getVolumeId();
+ int persistedProxyPort = created.getProxyPort();
+
+ RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);
+ RdsProxyManager restoredProxyManager = mock(RdsProxyManager.class);
+ when(restoredContainerManager.start(any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(new RdsContainerHandle("restored-container", "mydb", "127.0.0.1", 15432));
+
+ RdsService restoredService = newService(restoredContainerManager, restoredProxyManager,
+ instances, clusters, parameterGroups, clusterParameterGroups);
+ restoredService.restorePersistedRuntime();
+
+ DbInstance restored = restoredService.getDbInstance("mydb");
+ assertEquals(persistedVolumeId, restored.getVolumeId());
+ assertEquals("floci-rds-" + persistedVolumeId, restored.getDockerVolumeName());
+ assertEquals(persistedProxyPort, restored.getProxyPort());
+ assertEquals(persistedProxyPort, restored.getEndpoint().port());
+ assertEquals("restored-container", restored.getContainerId());
+ assertEquals("127.0.0.1", restored.getContainerHost());
+ assertEquals(15432, restored.getContainerPort());
+
+ verify(restoredContainerManager).start(eq("mydb"), eq(persistedVolumeId),
+ eq(DatabaseEngine.POSTGRES), any(), eq("admin"), eq("secret"), eq("app"));
+ verify(restoredProxyManager).startProxy(eq("mydb"), eq(DatabaseEngine.POSTGRES),
+ eq(false), eq(persistedProxyPort), eq("127.0.0.1"), eq(15432),
+ eq("admin"), eq("secret"), eq("app"), any());
+ }
+
+ @Test
+ void restorePersistedRuntimeRestoresClusterAndMemberInstance() {
+ StorageBackend<String, DbInstance> instances = new InMemoryStorage<>();
+ StorageBackend<String, DbCluster> clusters = new InMemoryStorage<>();
+ StorageBackend<String, DbParameterGroup> parameterGroups = new InMemoryStorage<>();
+ StorageBackend<String, DbClusterParameterGroup> clusterParameterGroups = new InMemoryStorage<>();
+
+ when(containerManager.start(any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(new RdsContainerHandle("initial-cluster-container", "cluster1", "localhost", 5432));
+
+ RdsService initialService = newService(containerManager, proxyManager,
+ instances, clusters, parameterGroups, clusterParameterGroups);
+ DbCluster cluster = initialService.createDbCluster("cluster1", "aurora-postgresql", "16.3",
+ "admin", "secret", "app", false, null);
+ DbInstance member = initialService.createDbInstance("member1", "aurora-postgresql", "16.3",
+ "admin", "secret", "app", "db.t3.medium",
+ 20, false, null, "cluster1");
+
+ RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);
+ RdsProxyManager restoredProxyManager = mock(RdsProxyManager.class);
+ when(restoredContainerManager.start(any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(new RdsContainerHandle("restored-cluster-container", "cluster1", "127.0.0.1", 15432));
+
+ RdsService restoredService = newService(restoredContainerManager, restoredProxyManager,
+ instances, clusters, parameterGroups, clusterParameterGroups);
+ restoredService.restorePersistedRuntime();
+
+ DbCluster restoredCluster = restoredService.getDbCluster("cluster1");
+ DbInstance restoredMember = restoredService.getDbInstance("member1");
+
+ assertEquals(cluster.getVolumeId(), restoredCluster.getVolumeId());
+ assertEquals(cluster.getProxyPort(), restoredCluster.getProxyPort());
+ assertEquals(member.getProxyPort(), restoredMember.getProxyPort());
+ assertEquals("restored-cluster-container", restoredCluster.getContainerId());
+ assertEquals("restored-cluster-container", restoredMember.getContainerId());
+ assertEquals("127.0.0.1", restoredMember.getContainerHost());
+ assertEquals(15432, restoredMember.getContainerPort());
+
+ verify(restoredContainerManager).start(eq("cluster1"), eq(cluster.getVolumeId()),
+ eq(DatabaseEngine.POSTGRES), any(), eq("admin"), eq("secret"), eq("app"));
+ verify(restoredProxyManager).startProxy(eq("cluster1"), eq(DatabaseEngine.POSTGRES),
+ eq(false), eq(cluster.getProxyPort()), eq("127.0.0.1"), eq(15432),
+ eq("admin"), eq("secret"), eq("app"), any());
+ verify(restoredProxyManager).startProxy(eq("member1"), eq(DatabaseEngine.POSTGRES),
+ eq(false), eq(member.getProxyPort()), eq("127.0.0.1"), eq(15432),
+ eq("admin"), eq("secret"), eq("app"), any());
+ }
+
+ private RdsService newService(RdsContainerManager containerManager,
+ RdsProxyManager proxyManager,
+ StorageBackend<String, DbInstance> instances,
+ StorageBackend<String, DbCluster> clusters,
+ StorageBackend<String, DbParameterGroup> parameterGroups,
+ StorageBackend<String, DbClusterParameterGroup> clusterParameterGroups) {
+ return new RdsService(containerManager, proxyManager, regionResolver, config,
+ instances, clusters, parameterGroups, clusterParameterGroups);
+ }
}