Light, fluffy, and always free - The AWS Local Emulator alternative
fix(rds): report backing container failures (#3455)
Franklin G. Mendoza68c783b576cc
diff --git a/docs/services/rds.md b/docs/services/rds.mdindex 6a30cd725d..4f0dc3a3d7 100644--- a/docs/services/rds.md+++ b/docs/services/rds.md@@ -19,6 +19,7 @@ RDS Data API (`rds-data`) is documented separately because it uses REST JSON rou| `ModifyDBInstance` | Update instance settings || `RebootDBInstance` | Restart a database instance || `DescribeOrderableDBInstanceOptions` | List deterministic instance class options |+| `DescribeEvents` | - || `CreateDBSubnetGroup` | Create a DB subnet group; tags given here are readable through `ListTagsForResource` || `DescribeDBSubnetGroups` | List DB subnet groups || `ModifyDBSubnetGroup` | Update DB subnet group description and subnet list |diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.javaindex 33cc5aa8e2..653cbedb96 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.java@@ -22,6 +22,7 @@ import io.github.hectorvent.floci.services.rds.model.DbProxyTargetGroup;import io.github.hectorvent.floci.services.rds.model.DbSubnetGroup;import io.github.hectorvent.floci.services.rds.model.OptionGroup;import io.github.hectorvent.floci.services.rds.model.OptionGroupOption;+import io.github.hectorvent.floci.services.rds.model.RdsEvent;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import jakarta.ws.rs.core.MultivaluedMap;@@ -70,6 +71,7 @@ public class RdsQueryHandler {case "ModifyDBInstance" -> handleModifyDbInstance(params, region);case "RebootDBInstance" -> handleRebootDbInstance(params, region);case "DescribeOrderableDBInstanceOptions" -> handleDescribeOrderableDbInstanceOptions(params);+ case "DescribeEvents" -> handleDescribeEvents(params, region);case "CreateDBSubnetGroup" -> handleCreateDbSubnetGroup(params, region);case "DescribeDBSubnetGroups" -> handleDescribeDbSubnetGroups(params, region);case "ModifyDBSubnetGroup" -> handleModifyDbSubnetGroup(params, region);@@ -201,8 +203,12 @@ public class RdsQueryHandler {List<String> engines = engineFilter(params);XmlBuilder xml = new XmlBuilder().start("DBInstances");for (DbInstance i : result) {- if (engines.isEmpty() || engines.contains(instanceEngine(i))) {- xml.start("DBInstance").raw(dbInstanceInnerXml(i)).end("DBInstance");+ DbInstance reconciled = service.refreshDbInstanceRuntimeHealth(i);+ if (reconciled == null) {+ reconciled = i;+ }+ if (engines.isEmpty() || engines.contains(instanceEngine(reconciled))) {+ xml.start("DBInstance").raw(dbInstanceInnerXml(reconciled)).end("DBInstance");}}boolean listForm = (identifier == null || identifier.isBlank())@@ -224,6 +230,99 @@ public class RdsQueryHandler {}}+ private Response handleDescribeEvents(MultivaluedMap<String, String> params, String region) {+ String sourceIdentifier = params.getFirst("SourceIdentifier");+ String sourceType = params.getFirst("SourceType");+ if (sourceIdentifier != null && (sourceType == null || sourceType.isBlank())) {+ throw new AwsException("InvalidParameterCombination",+ "SourceType must be provided when SourceIdentifier is specified.", 400);+ }+ if (sourceType != null && !List.of("db-instance", "db-parameter-group", "db-security-group",+ "db-snapshot", "db-cluster", "db-cluster-snapshot", "custom-engine-version",+ "db-proxy", "blue-green-deployment", "db-shard-group", "zero-etl").contains(sourceType)) {+ throw new AwsException("InvalidParameterValue", "SourceType is invalid.", 400);+ }+ Integer duration = parseOptionalInt(params.getFirst("Duration"));+ if (duration != null && (duration < 1 || duration > 20_160)) {+ throw new AwsException("InvalidParameterValue", "Duration must be between 1 and 20160.", 400);+ }+ Integer maxRecords = parseOptionalInt(params.getFirst("MaxRecords"));+ if (maxRecords != null && (maxRecords < 20 || maxRecords > 100)) {+ throw new AwsException("InvalidParameterValue", "MaxRecords must be between 20 and 100.", 400);+ }+ java.time.Instant start = parseOptionalInstant(params.getFirst("StartTime"));+ java.time.Instant end = parseOptionalInstant(params.getFirst("EndTime"));+ if (start != null && end != null && start.isAfter(end)) {+ throw new AwsException("InvalidParameterCombination", "StartTime must be before EndTime.", 400);+ }++ // Reconcile current instance health before returning the event history. This mirrors the+ // same control-plane observation used by DescribeDBInstances and records the transition once.+ for (DbInstance instance : service.listDbInstances(null, region)) {+ service.refreshDbInstanceRuntimeHealth(instance);+ }+ List<RdsEvent> all = service.describeEvents(sourceIdentifier, sourceType, start, end, duration);+ int offset = parseMarker(params.getFirst("Marker"));+ int limit = maxRecords != null ? maxRecords : 100;+ int from = Math.min(offset, all.size());+ int to = Math.min(from + limit, all.size());+ XmlBuilder xml = new XmlBuilder().start("Events");+ for (RdsEvent event : all.subList(from, to)) {+ xml.start("Event")+ .elem("SourceIdentifier", event.sourceIdentifier())+ .elem("SourceType", event.sourceType())+ .elem("Message", event.message())+ .start("EventCategories");+ for (String category : event.eventCategories()) {+ xml.elem("EventCategory", category);+ }+ xml.end("EventCategories")+ .elem("Date", event.date().toString())+ .elem("SourceArn", event.sourceArn())+ .end("Event");+ }+ xml.end("Events");+ if (to < all.size()) {+ xml.elem("Marker", String.valueOf(to));+ }+ return Response.ok(AwsQueryResponse.envelope("DescribeEvents", AwsNamespaces.RDS, xml.build())).build();+ }++ private static Integer parseOptionalInt(String value) {+ if (value == null || value.isBlank()) {+ return null;+ }+ try {+ return Integer.parseInt(value);+ } catch (NumberFormatException e) {+ throw new AwsException("InvalidParameterValue", "The parameter must be an integer.", 400);+ }+ }++ private static java.time.Instant parseOptionalInstant(String value) {+ if (value == null || value.isBlank()) {+ return null;+ }+ try {+ return java.time.Instant.parse(value);+ } catch (java.time.format.DateTimeParseException e) {+ throw new AwsException("InvalidParameterValue", "Invalid timestamp.", 400);+ }+ }++ private static int parseMarker(String marker) {+ if (marker == null || marker.isBlank()) {+ return 0;+ }+ try {+ int value = Integer.parseInt(marker);+ if (value < 0) throw new NumberFormatException();+ return value;+ } catch (NumberFormatException e) {+ throw new AwsException("InvalidParameterValue", "Marker is invalid.", 400);+ }+ }+private Response handleDeleteDbInstance(MultivaluedMap<String, String> params, String region) {String id = params.getFirst("DBInstanceIdentifier");diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.javaindex 8a8cae22d1..5633251271 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java@@ -35,6 +35,7 @@ import io.github.hectorvent.floci.services.rds.model.DbProxy;import io.github.hectorvent.floci.services.rds.model.DbProxyAuth;import io.github.hectorvent.floci.services.rds.model.DbProxyTarget;import io.github.hectorvent.floci.services.rds.model.DbProxyTargetGroup;+import io.github.hectorvent.floci.services.rds.model.RdsEvent;import io.github.hectorvent.floci.services.rds.model.DbSnapshot;import io.github.hectorvent.floci.services.rds.model.DbSubnetGroup;import io.github.hectorvent.floci.services.rds.model.OptionGroup;@@ -140,6 +141,7 @@ public class RdsService implements Resettable, ResourceProvider {private final StorageBackend<String, DbProxyTargetGroup> proxyTargetGroups;private final StorageBackend<String, DbSnapshot> snapshots;private final StorageBackend<String, String> snapshotData;+ private StorageBackend<String, RdsEvent> events = new InMemoryStorage<>();private final RdsContainerManager containerManager;private final RdsProxyManager proxyManager;// CreateDBCluster/CreateDBInstance register the new resource only after the@@ -221,6 +223,8 @@ public class RdsService implements Resettable, ResourceProvider {new TypeReference<Map<String, DbSnapshot>>() {});this.snapshotData = storageFactory.create("rds", "rds-snapshot-data.json",new TypeReference<Map<String, String>>() {});+ this.events = storageFactory.create("rds", "rds-events.json",+ new TypeReference<Map<String, RdsEvent>>() {});}RdsService(RdsContainerManager containerManager,@@ -1232,6 +1236,65 @@ public class RdsService implements Resettable, ResourceProvider {return unique.values();}+ /**+ * Reconciles the control-plane status with the Docker runtime before a describe response.+ * AWS does not keep reporting an instance as available after its database process is gone.+ */+ public synchronized DbInstance refreshDbInstanceRuntimeHealth(DbInstance instance) {+ if (instance == null+ || config.services().rds().mock()+ || instance.getStatus() != DbInstanceStatus.AVAILABLE+ || instance.getContainerId() == null+ || instance.getContainerId().isBlank()) {+ return instance;+ }+ if (containerManager.isContainerRunning(instance.getContainerId())) {+ return instance;+ }++ instance.setStatus(DbInstanceStatus.FAILED);+ recordRuntimeFailureEvent(instance);+ String accountId = accountIdFromArn(instance.getDbInstanceArn());+ String region = regionFromArn(instance.getDbInstanceArn());+ putInstanceForScope(accountId, region, instance.getDbInstanceIdentifier(), instance);+ try {+ proxyManager.stopProxy(rdsResourceRelayKey(+ instance.getDbInstanceArn(), instance.getDbInstanceIdentifier()));+ } catch (RuntimeException e) {+ // Health reconciliation must still return the failed control-plane state even when+ // closing the local relay encounters a stale or already-closed listener.+ LOG.warnv(e, "Failed to stop RDS proxy for unhealthy instance {0}",+ instance.getDbInstanceIdentifier());+ }+ LOG.warnv("RDS instance {0} backing container is no longer running; marking it failed",+ instance.getDbInstanceIdentifier());+ return instance;+ }++ private void recordRuntimeFailureEvent(DbInstance instance) {+ String eventId = "runtime-failure:" + instance.getDbInstanceArn();+ if (events.get(eventId).isPresent()) {+ return;+ }+ events.put(eventId, new RdsEvent(+ eventId, instance.getDbInstanceIdentifier(), "db-instance",+ "DB instance backing database process is unavailable.",+ List.of("availability"), Instant.now(), instance.getDbInstanceArn()));+ }++ public List<RdsEvent> describeEvents(String sourceIdentifier, String sourceType,+ Instant startTime, Instant endTime, Integer durationMinutes) {+ Instant effectiveEnd = endTime != null ? endTime : Instant.now();+ Instant effectiveStart = startTime != null ? startTime+ : effectiveEnd.minusSeconds((durationMinutes != null ? durationMinutes : 60) * 60L);+ return events.scan(_ -> true).stream()+ .filter(event -> sourceIdentifier == null || sourceIdentifier.equals(event.sourceIdentifier()))+ .filter(event -> sourceType == null || sourceType.equals(event.sourceType()))+ .filter(event -> !event.date().isBefore(effectiveStart) && !event.date().isAfter(effectiveEnd))+ .sorted(java.util.Comparator.comparing(RdsEvent::date))+ .toList();+ }+public Collection<DbInstance> listDbInstancesByDbiResourceIds(Collection<String> resourceIds) {return listDbInstancesByDbiResourceIds(resourceIds, regionResolver.getDefaultRegion());}diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.javaindex 225fd25aac..88215f63f2 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java@@ -381,6 +381,13 @@ public class RdsContainerManager {}}+ /** Returns whether a backing RDS container still exists and is running. */+ public boolean isContainerRunning(String containerId) {+ return containerId != null+ && !containerId.isBlank()+ && lifecycleManager.isContainerRunning(containerId);+ }+/** Returns the retained runtime handle used to persist cleanup identity after a failed start. */public RdsContainerHandle getActiveHandle(String runtimeId) {if (runtimeId == null || runtimeId.isBlank()) {diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/model/RdsEvent.java b/src/main/java/io/github/hectorvent/floci/services/rds/model/RdsEvent.javanew file mode 100644index 0000000000..8ca5c42b02--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/rds/model/RdsEvent.java@@ -0,0 +1,17 @@+package io.github.hectorvent.floci.services.rds.model;++import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;+import java.util.List;++@RegisterForReflection+public record RdsEvent(+ String id,+ String sourceIdentifier,+ String sourceType,+ String message,+ List<String> eventCategories,+ Instant date,+ String sourceArn) {+}diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.javaindex 899d9b9241..098dd3a51a 100644--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java@@ -5484,6 +5484,29 @@ class RdsServiceTest {assertEquals("og1", modified.getOptionGroupName());}+ @Test+ void refreshRuntimeHealthMarksDeadContainerFailedAndStopsProxy() {+ when(rdsConfig.mock()).thenReturn(false);+ when(containerManager.isContainerRunning("cont-id")).thenReturn(false);+ DbInstance instance = rdsService.createDbInstance(+ "dead-db", "postgres", "16", "admin", "password", "dbname",+ "db.t3.micro", 20, false, null, null, null, null, false, false,+ null, Map.of(), List.of(), null, null, true);++ DbInstance refreshed = rdsService.refreshDbInstanceRuntimeHealth(instance);++ assertEquals(DbInstanceStatus.FAILED, refreshed.getStatus());+ verify(proxyManager).stopProxy("rds-resource:" + refreshed.getDbInstanceArn());+ var events = rdsService.describeEvents("dead-db", "db-instance", null, null, 60);+ assertEquals(1, events.size());+ assertEquals(List.of("availability"), events.getFirst().eventCategories());+ assertEquals(refreshed.getDbInstanceArn(), events.getFirst().sourceArn());++ // Repeated health reads do not duplicate the transition event.+ rdsService.refreshDbInstanceRuntimeHealth(refreshed);+ assertEquals(1, rdsService.describeEvents("dead-db", "db-instance", null, null, 60).size());+ }+@Testvoid optionGroupsUseTheInjectedAccountAwareStorage() {String accountId = "123456789012";