Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(rds): report backing container failures (#3455)

Franklin G. Mendoza68c783b576cc
diff --git a/docs/services/rds.md b/docs/services/rds.md
index 6a30cd725d..4f0dc3a3d7 100644
--- a/docs/services/rds.md
+++ b/docs/services/rds.md
@@ -19,6 +19,7 @@ RDS Data API (`rds-data`) is documented separately because it uses REST JSON rou
| `ModifyDBInstance` | Update instance settings |
| `RebootDBInstance` | Restart a database instance |
| `DescribeOrderableDBInstanceOptions` | List deterministic instance class options |
+| `DescribeEvents` | - |
| `CreateDBSubnetGroup` | Create a DB subnet group; tags given here are readable through `ListTagsForResource` |
| `DescribeDBSubnetGroups` | List DB subnet groups |
| `ModifyDBSubnetGroup` | Update DB subnet group description and subnet list |
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.java
index 33cc5aa8e2..653cbedb96 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.java
@@ -22,6 +22,7 @@ import io.github.hectorvent.floci.services.rds.model.DbProxyTargetGroup;
import io.github.hectorvent.floci.services.rds.model.DbSubnetGroup;
import io.github.hectorvent.floci.services.rds.model.OptionGroup;
import io.github.hectorvent.floci.services.rds.model.OptionGroupOption;
+import io.github.hectorvent.floci.services.rds.model.RdsEvent;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import jakarta.ws.rs.core.MultivaluedMap;
@@ -70,6 +71,7 @@ public class RdsQueryHandler {
case "ModifyDBInstance" -> handleModifyDbInstance(params, region);
case "RebootDBInstance" -> handleRebootDbInstance(params, region);
case "DescribeOrderableDBInstanceOptions" -> handleDescribeOrderableDbInstanceOptions(params);
+ case "DescribeEvents" -> handleDescribeEvents(params, region);
case "CreateDBSubnetGroup" -> handleCreateDbSubnetGroup(params, region);
case "DescribeDBSubnetGroups" -> handleDescribeDbSubnetGroups(params, region);
case "ModifyDBSubnetGroup" -> handleModifyDbSubnetGroup(params, region);
@@ -201,8 +203,12 @@ public class RdsQueryHandler {
List<String> engines = engineFilter(params);
XmlBuilder xml = new XmlBuilder().start("DBInstances");
for (DbInstance i : result) {
- if (engines.isEmpty() || engines.contains(instanceEngine(i))) {
- xml.start("DBInstance").raw(dbInstanceInnerXml(i)).end("DBInstance");
+ DbInstance reconciled = service.refreshDbInstanceRuntimeHealth(i);
+ if (reconciled == null) {
+ reconciled = i;
+ }
+ if (engines.isEmpty() || engines.contains(instanceEngine(reconciled))) {
+ xml.start("DBInstance").raw(dbInstanceInnerXml(reconciled)).end("DBInstance");
}
}
boolean listForm = (identifier == null || identifier.isBlank())
@@ -224,6 +230,99 @@ public class RdsQueryHandler {
}
}
+ private Response handleDescribeEvents(MultivaluedMap<String, String> params, String region) {
+ String sourceIdentifier = params.getFirst("SourceIdentifier");
+ String sourceType = params.getFirst("SourceType");
+ if (sourceIdentifier != null && (sourceType == null || sourceType.isBlank())) {
+ throw new AwsException("InvalidParameterCombination",
+ "SourceType must be provided when SourceIdentifier is specified.", 400);
+ }
+ if (sourceType != null && !List.of("db-instance", "db-parameter-group", "db-security-group",
+ "db-snapshot", "db-cluster", "db-cluster-snapshot", "custom-engine-version",
+ "db-proxy", "blue-green-deployment", "db-shard-group", "zero-etl").contains(sourceType)) {
+ throw new AwsException("InvalidParameterValue", "SourceType is invalid.", 400);
+ }
+ Integer duration = parseOptionalInt(params.getFirst("Duration"));
+ if (duration != null && (duration < 1 || duration > 20_160)) {
+ throw new AwsException("InvalidParameterValue", "Duration must be between 1 and 20160.", 400);
+ }
+ Integer maxRecords = parseOptionalInt(params.getFirst("MaxRecords"));
+ if (maxRecords != null && (maxRecords < 20 || maxRecords > 100)) {
+ throw new AwsException("InvalidParameterValue", "MaxRecords must be between 20 and 100.", 400);
+ }
+ java.time.Instant start = parseOptionalInstant(params.getFirst("StartTime"));
+ java.time.Instant end = parseOptionalInstant(params.getFirst("EndTime"));
+ if (start != null && end != null && start.isAfter(end)) {
+ throw new AwsException("InvalidParameterCombination", "StartTime must be before EndTime.", 400);
+ }
+
+ // Reconcile current instance health before returning the event history. This mirrors the
+ // same control-plane observation used by DescribeDBInstances and records the transition once.
+ for (DbInstance instance : service.listDbInstances(null, region)) {
+ service.refreshDbInstanceRuntimeHealth(instance);
+ }
+ List<RdsEvent> all = service.describeEvents(sourceIdentifier, sourceType, start, end, duration);
+ int offset = parseMarker(params.getFirst("Marker"));
+ int limit = maxRecords != null ? maxRecords : 100;
+ int from = Math.min(offset, all.size());
+ int to = Math.min(from + limit, all.size());
+ XmlBuilder xml = new XmlBuilder().start("Events");
+ for (RdsEvent event : all.subList(from, to)) {
+ xml.start("Event")
+ .elem("SourceIdentifier", event.sourceIdentifier())
+ .elem("SourceType", event.sourceType())
+ .elem("Message", event.message())
+ .start("EventCategories");
+ for (String category : event.eventCategories()) {
+ xml.elem("EventCategory", category);
+ }
+ xml.end("EventCategories")
+ .elem("Date", event.date().toString())
+ .elem("SourceArn", event.sourceArn())
+ .end("Event");
+ }
+ xml.end("Events");
+ if (to < all.size()) {
+ xml.elem("Marker", String.valueOf(to));
+ }
+ return Response.ok(AwsQueryResponse.envelope("DescribeEvents", AwsNamespaces.RDS, xml.build())).build();
+ }
+
+ private static Integer parseOptionalInt(String value) {
+ if (value == null || value.isBlank()) {
+ return null;
+ }
+ try {
+ return Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ throw new AwsException("InvalidParameterValue", "The parameter must be an integer.", 400);
+ }
+ }
+
+ private static java.time.Instant parseOptionalInstant(String value) {
+ if (value == null || value.isBlank()) {
+ return null;
+ }
+ try {
+ return java.time.Instant.parse(value);
+ } catch (java.time.format.DateTimeParseException e) {
+ throw new AwsException("InvalidParameterValue", "Invalid timestamp.", 400);
+ }
+ }
+
+ private static int parseMarker(String marker) {
+ if (marker == null || marker.isBlank()) {
+ return 0;
+ }
+ try {
+ int value = Integer.parseInt(marker);
+ if (value < 0) throw new NumberFormatException();
+ return value;
+ } catch (NumberFormatException e) {
+ throw new AwsException("InvalidParameterValue", "Marker is invalid.", 400);
+ }
+ }
+
private Response handleDeleteDbInstance(
MultivaluedMap<String, String> params, String region) {
String id = params.getFirst("DBInstanceIdentifier");
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
index 8a8cae22d1..5633251271 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
@@ -35,6 +35,7 @@ import io.github.hectorvent.floci.services.rds.model.DbProxy;
import io.github.hectorvent.floci.services.rds.model.DbProxyAuth;
import io.github.hectorvent.floci.services.rds.model.DbProxyTarget;
import io.github.hectorvent.floci.services.rds.model.DbProxyTargetGroup;
+import io.github.hectorvent.floci.services.rds.model.RdsEvent;
import io.github.hectorvent.floci.services.rds.model.DbSnapshot;
import io.github.hectorvent.floci.services.rds.model.DbSubnetGroup;
import io.github.hectorvent.floci.services.rds.model.OptionGroup;
@@ -140,6 +141,7 @@ public class RdsService implements Resettable, ResourceProvider {
private final StorageBackend<String, DbProxyTargetGroup> proxyTargetGroups;
private final StorageBackend<String, DbSnapshot> snapshots;
private final StorageBackend<String, String> snapshotData;
+ private StorageBackend<String, RdsEvent> events = new InMemoryStorage<>();
private final RdsContainerManager containerManager;
private final RdsProxyManager proxyManager;
// CreateDBCluster/CreateDBInstance register the new resource only after the
@@ -221,6 +223,8 @@ public class RdsService implements Resettable, ResourceProvider {
new TypeReference<Map<String, DbSnapshot>>() {});
this.snapshotData = storageFactory.create("rds", "rds-snapshot-data.json",
new TypeReference<Map<String, String>>() {});
+ this.events = storageFactory.create("rds", "rds-events.json",
+ new TypeReference<Map<String, RdsEvent>>() {});
}
RdsService(RdsContainerManager containerManager,
@@ -1232,6 +1236,65 @@ public class RdsService implements Resettable, ResourceProvider {
return unique.values();
}
+ /**
+ * Reconciles the control-plane status with the Docker runtime before a describe response.
+ * AWS does not keep reporting an instance as available after its database process is gone.
+ */
+ public synchronized DbInstance refreshDbInstanceRuntimeHealth(DbInstance instance) {
+ if (instance == null
+ || config.services().rds().mock()
+ || instance.getStatus() != DbInstanceStatus.AVAILABLE
+ || instance.getContainerId() == null
+ || instance.getContainerId().isBlank()) {
+ return instance;
+ }
+ if (containerManager.isContainerRunning(instance.getContainerId())) {
+ return instance;
+ }
+
+ instance.setStatus(DbInstanceStatus.FAILED);
+ recordRuntimeFailureEvent(instance);
+ String accountId = accountIdFromArn(instance.getDbInstanceArn());
+ String region = regionFromArn(instance.getDbInstanceArn());
+ putInstanceForScope(accountId, region, instance.getDbInstanceIdentifier(), instance);
+ try {
+ proxyManager.stopProxy(rdsResourceRelayKey(
+ instance.getDbInstanceArn(), instance.getDbInstanceIdentifier()));
+ } catch (RuntimeException e) {
+ // Health reconciliation must still return the failed control-plane state even when
+ // closing the local relay encounters a stale or already-closed listener.
+ LOG.warnv(e, "Failed to stop RDS proxy for unhealthy instance {0}",
+ instance.getDbInstanceIdentifier());
+ }
+ LOG.warnv("RDS instance {0} backing container is no longer running; marking it failed",
+ instance.getDbInstanceIdentifier());
+ return instance;
+ }
+
+ private void recordRuntimeFailureEvent(DbInstance instance) {
+ String eventId = "runtime-failure:" + instance.getDbInstanceArn();
+ if (events.get(eventId).isPresent()) {
+ return;
+ }
+ events.put(eventId, new RdsEvent(
+ eventId, instance.getDbInstanceIdentifier(), "db-instance",
+ "DB instance backing database process is unavailable.",
+ List.of("availability"), Instant.now(), instance.getDbInstanceArn()));
+ }
+
+ public List<RdsEvent> describeEvents(String sourceIdentifier, String sourceType,
+ Instant startTime, Instant endTime, Integer durationMinutes) {
+ Instant effectiveEnd = endTime != null ? endTime : Instant.now();
+ Instant effectiveStart = startTime != null ? startTime
+ : effectiveEnd.minusSeconds((durationMinutes != null ? durationMinutes : 60) * 60L);
+ return events.scan(_ -> true).stream()
+ .filter(event -> sourceIdentifier == null || sourceIdentifier.equals(event.sourceIdentifier()))
+ .filter(event -> sourceType == null || sourceType.equals(event.sourceType()))
+ .filter(event -> !event.date().isBefore(effectiveStart) && !event.date().isAfter(effectiveEnd))
+ .sorted(java.util.Comparator.comparing(RdsEvent::date))
+ .toList();
+ }
+
public Collection<DbInstance> listDbInstancesByDbiResourceIds(Collection<String> resourceIds) {
return listDbInstancesByDbiResourceIds(resourceIds, regionResolver.getDefaultRegion());
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java
index 225fd25aac..88215f63f2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java
@@ -381,6 +381,13 @@ public class RdsContainerManager {
}
}
+ /** Returns whether a backing RDS container still exists and is running. */
+ public boolean isContainerRunning(String containerId) {
+ return containerId != null
+ && !containerId.isBlank()
+ && lifecycleManager.isContainerRunning(containerId);
+ }
+
/** Returns the retained runtime handle used to persist cleanup identity after a failed start. */
public RdsContainerHandle getActiveHandle(String runtimeId) {
if (runtimeId == null || runtimeId.isBlank()) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/model/RdsEvent.java b/src/main/java/io/github/hectorvent/floci/services/rds/model/RdsEvent.java
new file mode 100644
index 0000000000..8ca5c42b02
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/model/RdsEvent.java
@@ -0,0 +1,17 @@
+package io.github.hectorvent.floci.services.rds.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+import java.util.List;
+
+@RegisterForReflection
+public record RdsEvent(
+ String id,
+ String sourceIdentifier,
+ String sourceType,
+ String message,
+ List<String> eventCategories,
+ Instant date,
+ String sourceArn) {
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
index 899d9b9241..098dd3a51a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
@@ -5484,6 +5484,29 @@ class RdsServiceTest {
assertEquals("og1", modified.getOptionGroupName());
}
+ @Test
+ void refreshRuntimeHealthMarksDeadContainerFailedAndStopsProxy() {
+ when(rdsConfig.mock()).thenReturn(false);
+ when(containerManager.isContainerRunning("cont-id")).thenReturn(false);
+ DbInstance instance = rdsService.createDbInstance(
+ "dead-db", "postgres", "16", "admin", "password", "dbname",
+ "db.t3.micro", 20, false, null, null, null, null, false, false,
+ null, Map.of(), List.of(), null, null, true);
+
+ DbInstance refreshed = rdsService.refreshDbInstanceRuntimeHealth(instance);
+
+ assertEquals(DbInstanceStatus.FAILED, refreshed.getStatus());
+ verify(proxyManager).stopProxy("rds-resource:" + refreshed.getDbInstanceArn());
+ var events = rdsService.describeEvents("dead-db", "db-instance", null, null, 60);
+ assertEquals(1, events.size());
+ assertEquals(List.of("availability"), events.getFirst().eventCategories());
+ assertEquals(refreshed.getDbInstanceArn(), events.getFirst().sourceArn());
+
+ // Repeated health reads do not duplicate the transition event.
+ rdsService.refreshDbInstanceRuntimeHealth(refreshed);
+ assertEquals(1, rdsService.describeEvents("dead-db", "db-instance", null, null, 60).size());
+ }
+
@Test
void optionGroupsUseTheInjectedAccountAwareStorage() {
String accountId = "123456789012";