Light, fluffy, and always free - The AWS Local Emulator alternative
fix(rds): persist metadata across storage-backed restarts (#945)
Albert Einshutoin0a8732b34951
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.javaindex 095e26ca80..ec3b691a3b 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java@@ -1,10 +1,13 @@package io.github.hectorvent.floci.services.rds;+import com.fasterxml.jackson.core.type.TypeReference;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;-import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.rds.container.RdsContainerHandle;import io.github.hectorvent.floci.services.rds.container.RdsContainerManager;import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;@@ -15,6 +18,7 @@ import io.github.hectorvent.floci.services.rds.model.DbInstance;import io.github.hectorvent.floci.services.rds.model.DbInstanceStatus;import io.github.hectorvent.floci.services.rds.model.DbParameterGroup;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager;+import jakarta.annotation.PostConstruct;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -24,6 +28,7 @@ import java.time.Instant;import java.util.ArrayList;import java.util.Collection;import java.util.List;+import java.util.Map;import java.util.Set;import java.util.concurrent.ConcurrentHashMap;@@ -50,15 +55,44 @@ public class RdsService {public RdsService(RdsContainerManager containerManager,RdsProxyManager proxyManager,RegionResolver regionResolver,- EmulatorConfig config) {+ EmulatorConfig config,+ StorageFactory storageFactory) {this.containerManager = containerManager;this.proxyManager = proxyManager;this.regionResolver = regionResolver;this.config = config;- this.instances = new InMemoryStorage<>();- this.clusters = new InMemoryStorage<>();- this.parameterGroups = new InMemoryStorage<>();- this.clusterParameterGroups = new InMemoryStorage<>();+ this.instances = storageFactory.create("rds", "rds-instances.json",+ new TypeReference<Map<String, DbInstance>>() {});+ this.clusters = storageFactory.create("rds", "rds-clusters.json",+ new TypeReference<Map<String, DbCluster>>() {});+ this.parameterGroups = storageFactory.create("rds", "rds-parameter-groups.json",+ new TypeReference<Map<String, DbParameterGroup>>() {});+ this.clusterParameterGroups = storageFactory.create("rds", "rds-cluster-parameter-groups.json",+ new TypeReference<Map<String, DbClusterParameterGroup>>() {});+ }++ RdsService(RdsContainerManager containerManager,+ RdsProxyManager proxyManager,+ RegionResolver regionResolver,+ EmulatorConfig config,+ StorageBackend<String, DbInstance> instances,+ StorageBackend<String, DbCluster> clusters,+ StorageBackend<String, DbParameterGroup> parameterGroups,+ StorageBackend<String, DbClusterParameterGroup> clusterParameterGroups) {+ this.containerManager = containerManager;+ this.proxyManager = proxyManager;+ this.regionResolver = regionResolver;+ this.config = config;+ this.instances = instances;+ this.clusters = clusters;+ this.parameterGroups = parameterGroups;+ this.clusterParameterGroups = clusterParameterGroups;+ }++ @PostConstruct+ void restorePersistedRuntime() {+ restoreClusters();+ restoreInstances();}// ── DB Instances ──────────────────────────────────────────────────────────@@ -82,6 +116,7 @@ public class RdsService {String containerHost = null;int containerPort = 0;String instanceVolumeId = null;+ String instanceDockerVolumeName = null;if (dbClusterIdentifier != null && !dbClusterIdentifier.isBlank()) {// Cluster member — share the cluster's container@@ -93,6 +128,9 @@ public class RdsService {containerId = cluster.getContainerId();containerHost = cluster.getContainerHost();containerPort = cluster.getContainerPort();+ instanceDockerVolumeName = cluster.getDockerVolumeName() != null+ ? cluster.getDockerVolumeName()+ : volumeName(cluster.getVolumeId(), cluster.getDbClusterIdentifier());} else {// Standalone instance — start its own containerString image = imageForEngine(engine);@@ -103,6 +141,7 @@ public class RdsService {containerId = handle.getContainerId();containerHost = handle.getHost();containerPort = handle.getPort();+ instanceDockerVolumeName = volumeName(instanceVolumeId, id);}DbEndpoint endpoint = new DbEndpoint("localhost", proxyPort);@@ -113,6 +152,7 @@ public class RdsService {instance.setContainerHost(containerHost);instance.setContainerPort(containerPort);instance.setVolumeId(instanceVolumeId);+ instance.setDockerVolumeName(instanceDockerVolumeName);String region = regionResolver.getDefaultRegion();instance.setDbiResourceId("db-" + java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 24).toUpperCase());@@ -263,6 +303,7 @@ public class RdsService {cluster.setContainerHost(handle.getHost());cluster.setContainerPort(handle.getPort());cluster.setVolumeId(clusterVolumeId);+ cluster.setDockerVolumeName(volumeName(clusterVolumeId, id));String region = regionResolver.getDefaultRegion();cluster.setDbClusterResourceId("cluster-" + java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 24).toUpperCase());@@ -478,6 +519,127 @@ public class RdsService {usedPorts.remove(port);}+ private void restoreClusters() {+ for (DbCluster cluster : allClusters()) {+ if (cluster.getStatus() == DbInstanceStatus.DELETING) {+ continue;+ }+ int proxyPort = reserveOrAllocateProxyPort(cluster.getProxyPort());+ cluster.setProxyPort(proxyPort);+ cluster.setEndpoint(new DbEndpoint("localhost", proxyPort));+ cluster.setReaderEndpoint(new DbEndpoint("localhost", proxyPort));+ if (cluster.getDockerVolumeName() == null) {+ cluster.setDockerVolumeName(volumeName(cluster.getVolumeId(), cluster.getDbClusterIdentifier()));+ }+ try {+ String image = imageForEngine(cluster.getEngine());+ RdsContainerHandle handle = containerManager.start(cluster.getDbClusterIdentifier(),+ cluster.getVolumeId(), cluster.getEngine(), image,+ cluster.getMasterUsername(), cluster.getMasterPassword(), cluster.getDatabaseName());+ cluster.setContainerId(handle.getContainerId());+ cluster.setContainerHost(handle.getHost());+ cluster.setContainerPort(handle.getPort());++ String effectiveMasterUser = cluster.getMasterUsername() != null+ ? cluster.getMasterUsername() : "root";+ proxyManager.startProxy(cluster.getDbClusterIdentifier(), cluster.getEngine(),+ cluster.isIamDatabaseAuthenticationEnabled(), proxyPort,+ handle.getHost(), handle.getPort(), effectiveMasterUser,+ cluster.getMasterPassword(), cluster.getDatabaseName(),+ (user, pw) -> validateDbClusterPassword(cluster.getDbClusterIdentifier(), user, pw));+ cluster.setStatus(DbInstanceStatus.AVAILABLE);+ } catch (Exception e) {+ releaseProxyPort(proxyPort);+ LOG.warnv(e, "Failed to restore RDS cluster {0}", cluster.getDbClusterIdentifier());+ }+ }+ }++ private void restoreInstances() {+ for (DbInstance instance : allInstances()) {+ if (instance.getStatus() == DbInstanceStatus.DELETING) {+ continue;+ }+ int proxyPort = reserveOrAllocateProxyPort(instance.getProxyPort());+ instance.setProxyPort(proxyPort);+ instance.setEndpoint(new DbEndpoint("localhost", proxyPort));+ try {+ String backendHost;+ int backendPort;+ String clusterId = instance.getDbClusterIdentifier();+ if (clusterId != null && !clusterId.isBlank()) {+ DbCluster cluster = clusters.get(clusterId).orElseThrow(() ->+ new AwsException("DBClusterNotFoundFault",+ "DB cluster " + clusterId + " not found.", 404));+ backendHost = cluster.getContainerHost();+ backendPort = cluster.getContainerPort();+ if (backendHost == null || backendPort <= 0) {+ throw new AwsException("InvalidDBClusterStateFault",+ "DB cluster " + clusterId + " runtime is not available.", 400);+ }+ instance.setContainerId(cluster.getContainerId());+ instance.setContainerHost(cluster.getContainerHost());+ instance.setContainerPort(cluster.getContainerPort());+ if (instance.getDockerVolumeName() == null) {+ instance.setDockerVolumeName(cluster.getDockerVolumeName() != null+ ? cluster.getDockerVolumeName()+ : volumeName(cluster.getVolumeId(), cluster.getDbClusterIdentifier()));+ }+ } else {+ if (instance.getDockerVolumeName() == null) {+ instance.setDockerVolumeName(volumeName(instance.getVolumeId(), instance.getDbInstanceIdentifier()));+ }+ String image = imageForEngine(instance.getEngine());+ RdsContainerHandle handle = containerManager.start(instance.getDbInstanceIdentifier(),+ instance.getVolumeId(), instance.getEngine(), image,+ instance.getMasterUsername(), instance.getMasterPassword(), instance.getDbName());+ backendHost = handle.getHost();+ backendPort = handle.getPort();+ instance.setContainerId(handle.getContainerId());+ instance.setContainerHost(handle.getHost());+ instance.setContainerPort(handle.getPort());+ }++ String effectiveMasterUser = instance.getMasterUsername() != null+ ? instance.getMasterUsername() : "root";+ proxyManager.startProxy(instance.getDbInstanceIdentifier(), instance.getEngine(),+ instance.isIamDatabaseAuthenticationEnabled(), proxyPort,+ backendHost, backendPort, effectiveMasterUser,+ instance.getMasterPassword(), instance.getDbName(),+ (user, pw) -> validateDbPassword(instance.getDbInstanceIdentifier(), user, pw));+ instance.setStatus(DbInstanceStatus.AVAILABLE);+ } catch (Exception e) {+ releaseProxyPort(proxyPort);+ LOG.warnv(e, "Failed to restore RDS instance {0}", instance.getDbInstanceIdentifier());+ }+ }+ }++ private Collection<DbCluster> allClusters() {+ if (clusters instanceof AccountAwareStorageBackend<DbCluster> aware) {+ return aware.scanAllAccounts();+ }+ return clusters.scan(k -> true);+ }++ private Collection<DbInstance> allInstances() {+ if (instances instanceof AccountAwareStorageBackend<DbInstance> aware) {+ return aware.scanAllAccounts();+ }+ return instances.scan(k -> true);+ }++ private int reserveOrAllocateProxyPort(int persistedPort) {+ if (persistedPort > 0 && usedPorts.add(persistedPort)) {+ return persistedPort;+ }+ return allocateProxyPort();+ }++ private String volumeName(String volumeId, String fallbackId) {+ return ContainerStorageHelper.resourceName("rds", volumeId, fallbackId);+ }+private RdsContainerHandle buildHandle(DbInstance instance) {return new RdsContainerHandle(instance.getContainerId(), instance.getDbInstanceIdentifier(),instance.getContainerHost(), instance.getContainerPort());diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.javaindex 5a1eef097f..a295aae1b2 100644--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java@@ -3,22 +3,26 @@ package io.github.hectorvent.floci.services.rds;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;import io.github.hectorvent.floci.services.rds.model.DbCluster;import io.github.hectorvent.floci.services.rds.model.DbClusterParameterGroup;import io.github.hectorvent.floci.services.rds.container.RdsContainerHandle;import io.github.hectorvent.floci.services.rds.container.RdsContainerManager;import io.github.hectorvent.floci.services.rds.model.DbInstance;+import io.github.hectorvent.floci.services.rds.model.DbParameterGroup;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;-import org.mockito.Mockito;import java.util.Collection;import static org.junit.jupiter.api.Assertions.*;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;class RdsServiceTest {@@ -43,7 +47,9 @@ class RdsServiceTest {when(rdsConfig.proxyBasePort()).thenReturn(7000);when(rdsConfig.proxyMaxPort()).thenReturn(7099);- rdsService = new RdsService(containerManager, proxyManager, regionResolver, config);+ rdsService = newService(containerManager, proxyManager,+ new InMemoryStorage<>(), new InMemoryStorage<>(),+ new InMemoryStorage<>(), new InMemoryStorage<>());when(containerManager.start(any(), any(), any(), any(), any(), any(), any())).thenReturn(new RdsContainerHandle("cont-id", "id", "localhost", 5432));@@ -169,4 +175,106 @@ class RdsServiceTest {assertEquals("DBParameterGroupNotFound", exception.getErrorCode());}++ @Test+ void restorePersistedRuntimeRestartsStandaloneInstanceWithSameVolumeAndProxyPort() {+ StorageBackend<String, DbInstance> instances = new InMemoryStorage<>();+ StorageBackend<String, DbCluster> clusters = new InMemoryStorage<>();+ StorageBackend<String, DbParameterGroup> parameterGroups = new InMemoryStorage<>();+ StorageBackend<String, DbClusterParameterGroup> clusterParameterGroups = new InMemoryStorage<>();++ when(containerManager.start(any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(new RdsContainerHandle("initial-container", "mydb", "localhost", 5432));++ RdsService initialService = newService(containerManager, proxyManager,+ instances, clusters, parameterGroups, clusterParameterGroups);+ DbInstance created = initialService.createDbInstance("mydb", "postgres", "16.3",+ "admin", "secret", "app", "db.t3.micro",+ 20, false, null, null);++ String persistedVolumeId = created.getVolumeId();+ int persistedProxyPort = created.getProxyPort();++ RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);+ RdsProxyManager restoredProxyManager = mock(RdsProxyManager.class);+ when(restoredContainerManager.start(any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(new RdsContainerHandle("restored-container", "mydb", "127.0.0.1", 15432));++ RdsService restoredService = newService(restoredContainerManager, restoredProxyManager,+ instances, clusters, parameterGroups, clusterParameterGroups);+ restoredService.restorePersistedRuntime();++ DbInstance restored = restoredService.getDbInstance("mydb");+ assertEquals(persistedVolumeId, restored.getVolumeId());+ assertEquals("floci-rds-" + persistedVolumeId, restored.getDockerVolumeName());+ assertEquals(persistedProxyPort, restored.getProxyPort());+ assertEquals(persistedProxyPort, restored.getEndpoint().port());+ assertEquals("restored-container", restored.getContainerId());+ assertEquals("127.0.0.1", restored.getContainerHost());+ assertEquals(15432, restored.getContainerPort());++ verify(restoredContainerManager).start(eq("mydb"), eq(persistedVolumeId),+ eq(DatabaseEngine.POSTGRES), any(), eq("admin"), eq("secret"), eq("app"));+ verify(restoredProxyManager).startProxy(eq("mydb"), eq(DatabaseEngine.POSTGRES),+ eq(false), eq(persistedProxyPort), eq("127.0.0.1"), eq(15432),+ eq("admin"), eq("secret"), eq("app"), any());+ }++ @Test+ void restorePersistedRuntimeRestoresClusterAndMemberInstance() {+ StorageBackend<String, DbInstance> instances = new InMemoryStorage<>();+ StorageBackend<String, DbCluster> clusters = new InMemoryStorage<>();+ StorageBackend<String, DbParameterGroup> parameterGroups = new InMemoryStorage<>();+ StorageBackend<String, DbClusterParameterGroup> clusterParameterGroups = new InMemoryStorage<>();++ when(containerManager.start(any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(new RdsContainerHandle("initial-cluster-container", "cluster1", "localhost", 5432));++ RdsService initialService = newService(containerManager, proxyManager,+ instances, clusters, parameterGroups, clusterParameterGroups);+ DbCluster cluster = initialService.createDbCluster("cluster1", "aurora-postgresql", "16.3",+ "admin", "secret", "app", false, null);+ DbInstance member = initialService.createDbInstance("member1", "aurora-postgresql", "16.3",+ "admin", "secret", "app", "db.t3.medium",+ 20, false, null, "cluster1");++ RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);+ RdsProxyManager restoredProxyManager = mock(RdsProxyManager.class);+ when(restoredContainerManager.start(any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(new RdsContainerHandle("restored-cluster-container", "cluster1", "127.0.0.1", 15432));++ RdsService restoredService = newService(restoredContainerManager, restoredProxyManager,+ instances, clusters, parameterGroups, clusterParameterGroups);+ restoredService.restorePersistedRuntime();++ DbCluster restoredCluster = restoredService.getDbCluster("cluster1");+ DbInstance restoredMember = restoredService.getDbInstance("member1");++ assertEquals(cluster.getVolumeId(), restoredCluster.getVolumeId());+ assertEquals(cluster.getProxyPort(), restoredCluster.getProxyPort());+ assertEquals(member.getProxyPort(), restoredMember.getProxyPort());+ assertEquals("restored-cluster-container", restoredCluster.getContainerId());+ assertEquals("restored-cluster-container", restoredMember.getContainerId());+ assertEquals("127.0.0.1", restoredMember.getContainerHost());+ assertEquals(15432, restoredMember.getContainerPort());++ verify(restoredContainerManager).start(eq("cluster1"), eq(cluster.getVolumeId()),+ eq(DatabaseEngine.POSTGRES), any(), eq("admin"), eq("secret"), eq("app"));+ verify(restoredProxyManager).startProxy(eq("cluster1"), eq(DatabaseEngine.POSTGRES),+ eq(false), eq(cluster.getProxyPort()), eq("127.0.0.1"), eq(15432),+ eq("admin"), eq("secret"), eq("app"), any());+ verify(restoredProxyManager).startProxy(eq("member1"), eq(DatabaseEngine.POSTGRES),+ eq(false), eq(member.getProxyPort()), eq("127.0.0.1"), eq(15432),+ eq("admin"), eq("secret"), eq("app"), any());+ }++ private RdsService newService(RdsContainerManager containerManager,+ RdsProxyManager proxyManager,+ StorageBackend<String, DbInstance> instances,+ StorageBackend<String, DbCluster> clusters,+ StorageBackend<String, DbParameterGroup> parameterGroups,+ StorageBackend<String, DbClusterParameterGroup> clusterParameterGroups) {+ return new RdsService(containerManager, proxyManager, regionResolver, config,+ instances, clusters, parameterGroups, clusterParameterGroups);+ }}