Light, fluffy, and always free - The AWS Local Emulator alternative
feat(rds-data): opt into retryable database resume errors (#5703)
Aniket Shuklafb6adc00d84f
diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.mdindex f698b73343..c3fe6a6c01 100644--- a/docs/configuration/advanced/application-yml.md+++ b/docs/configuration/advanced/application-yml.md@@ -220,6 +220,7 @@ floci:rds-data:enabled: truetransaction-ttl-seconds: 180+ database-resuming-exception-enabled: falseeventbridge:enabled: true@@ -348,6 +349,7 @@ All keys in this table are declared on `EmulatorConfig` and accept environment v| `FLOCI_SERVICES_DOCKER_NETWORK` | *(unset)* | Shared Docker network for Lambda, RDS, ElastiCache containers || `FLOCI_SERVICES_RDS_DATA_ENABLED` | `true` | Enable the RDS Data API service || `FLOCI_SERVICES_RDS_DATA_TRANSACTION_TTL_SECONDS` | `180` | Idle timeout, in seconds, before leaked RDS Data API transactions expire |+| `FLOCI_SERVICES_RDS_DATA_DATABASE_RESUMING_EXCEPTION_ENABLED` | `false` | Return HTTP 400 `DatabaseResumingException` during Aurora resume instead of waiting || `FLOCI_SERVICES_ECS_MOCK` | `false` | Skip Docker; tasks go straight to RUNNING (useful for CI) || `FLOCI_SERVICES_ECS_DOCKER_NETWORK` | *(unset)* | Docker network for ECS task containers || `FLOCI_SERVICES_ECS_DEFAULT_MEMORY_MB` | `512` | Default memory (MB) when task definition omits it |diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.mdindex f505ce8b0c..c00ba9b353 100644--- a/docs/configuration/environment-variables.md+++ b/docs/configuration/environment-variables.md@@ -438,6 +438,7 @@ These services spawn Docker containers. They require access to the Docker socket| `FLOCI_SERVICES_RDS_DOCKER_NETWORK` | _(none)_ | Docker network for RDS containers (overrides `FLOCI_SERVICES_DOCKER_NETWORK`) || `FLOCI_SERVICES_RDS_DATA_ENABLED` | `true` | Enable the RDS Data API service. Requires `FLOCI_SERVICES_RDS_ENABLED=true` || `FLOCI_SERVICES_RDS_DATA_TRANSACTION_TTL_SECONDS` | `180` | Idle timeout, in seconds, before leaked RDS Data API transactions expire |+| `FLOCI_SERVICES_RDS_DATA_DATABASE_RESUMING_EXCEPTION_ENABLED` | `false` | Return HTTP 400 `DatabaseResumingException` while an auto-paused Aurora backend resumes, instead of waiting |### OpenSearchdiff --git a/docs/services/rds-data.md b/docs/services/rds-data.mdindex 2a3b0be343..c91af02dda 100644--- a/docs/services/rds-data.md+++ b/docs/services/rds-data.md@@ -55,9 +55,17 @@ The deprecated `ExecuteSql` operation is recognized at `POST /ExecuteSql` and re|---|---|---|| `FLOCI_SERVICES_RDS_DATA_ENABLED` | `true` | Enable or disable the RDS Data API service || `FLOCI_SERVICES_RDS_DATA_TRANSACTION_TTL_SECONDS` | `180` | Idle timeout, in seconds, before leaked Data API transactions expire |+| `FLOCI_SERVICES_RDS_DATA_DATABASE_RESUMING_EXCEPTION_ENABLED` | `false` | Return `DatabaseResumingException` (HTTP 400) while an auto-paused Aurora backend resumes, instead of waiting for it |The RDS Data API also requires the RDS service itself to be enabled because it resolves `resourceArn` values to local RDS containers.+With `database-resuming-exception-enabled=true`, `ExecuteStatement`, `BatchExecuteStatement`,+and `BeginTransaction` initiate resume in the background and return the retryable error before+opening a database connection. Calls made while resume is in progress return the same error;+retry after a few seconds. Once ready, requests run normally. Existing transactions keep their+backend awake until commit, rollback, or expiry. The default remains blocking resume for local+compatibility; TCP database connections and internal RDS commands always wait for resume.+## Example```bashdiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 2387ecde53..ae7191e95e 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1712,6 +1712,9 @@ public interface EmulatorConfig {@WithDefault("180")long transactionTtlSeconds();++ @WithDefault("false")+ boolean databaseResumingExceptionEnabled();}interface RedshiftDataServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/AutoPauseController.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/AutoPauseController.javaindex 02a5dc1c10..7eb413a3cb 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/AutoPauseController.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/AutoPauseController.java@@ -81,6 +81,7 @@ final class AutoPauseController {// was cancelled or belongs to an earlier state, and does nothing.private Future<?> timer;private long timerToken;+ private boolean resumeRequested;private boolean pauseCanceled;private boolean closed;private Instant lastSignal = Instant.EPOCH;@@ -163,6 +164,10 @@ final class AutoPauseController {* @throws IllegalStateException when Docker cannot resume the container, which stays paused*/RdsBackendGate.Lease acquire() throws InterruptedException {+ return acquire(true);+ }++ RdsBackendGate.Lease acquire(boolean waitForResume) throws InterruptedException {lock.lock();try {while (true) {@@ -174,12 +179,19 @@ final class AutoPauseController {cancelTimer();return new ControllerLease();}+ if (state == State.PAUSING) {+ pauseCanceled = true;+ }+ if (!waitForResume) {+ if (state == State.PAUSED && !resumeRequested) {+ startTimer(this::resumeInBackground, Duration.ZERO);+ resumeRequested = true;+ }+ throw new RdsBackendGate.ResumingException();+ }if (state == State.PAUSED) {resume();} else {- if (state == State.PAUSING) {- pauseCanceled = true;- }settled.await();}}@@ -312,8 +324,8 @@ final class AutoPauseController {if (thawFailure != null) {LOG.warnv(thawFailure, "Could not undo the auto-pause of RDS container {0}", containerId);state = State.PAUSED;- armPausedReport();settled.signalAll();+ armPausedReport();return;}}@@ -334,15 +346,21 @@ final class AutoPauseController {RuntimeException failure = withoutLock(() -> freezer.unpause(containerId));if (failure != null) {state = State.PAUSED;- armPausedReport();settled.signalAll();+ armPausedReport();throw new IllegalStateException("Could not resume auto-paused RDS container " + containerId, failure);}if (closed || resumeDelay.isZero()) {finishResume();} else {- startTimer(this::resumeDelayElapsed, resumeDelay);+ try {+ startTimer(this::resumeDelayElapsed, resumeDelay);+ } catch (IllegalStateException e) {+ // Docker has thawed the backend; a stopped timer must not strand its waiters.+ finishResume();+ throw e;+ }}}@@ -383,12 +401,12 @@ final class AutoPauseController {if (closed || state != State.RUNNING || leases > 0 || secondsUntilAutoPause == null) {return;}- startTimer(this::idleTimerFired, Duration.ofSeconds(secondsUntilAutoPause));+ startBestEffortTimer(this::idleTimerFired, Duration.ofSeconds(secondsUntilAutoPause));}private void armPausedReport() {if (!closed) {- startTimer(this::pausedReportDue, PAUSED_REPORT_INTERVAL);+ startBestEffortTimer(this::pausedReportDue, PAUSED_REPORT_INTERVAL);}}@@ -404,6 +422,15 @@ final class AutoPauseController {}}+ private void startBestEffortTimer(LongConsumer task, Duration delay) {+ try {+ startTimer(task, delay);+ } catch (IllegalStateException e) {+ // Shutdown must not turn a completed request's lease release into a failure.+ LOG.debugv(e, "Could not schedule best-effort RDS timer for {0}", containerId);+ }+ }+private void startTimer(LongConsumer task, Duration delay) {cancelTimer();long token = timerToken;@@ -411,6 +438,7 @@ final class AutoPauseController {}private void cancelTimer() {+ resumeRequested = false;if (timer != null) {timer.cancel(false);timer = null;diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsBackendGate.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsBackendGate.javaindex 2104e52191..ffe1b37194 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsBackendGate.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsBackendGate.java@@ -19,6 +19,19 @@ public interface RdsBackendGate {*/Lease enter(String host, int port) throws InterruptedException;+ /**+ * Like {@link #enter(String, int)}, but with {@code waitForResume=false} starts resume in the+ * background and throws {@link ResumingException} if the backend is not yet ready.+ * Backends without auto-pause may use this default implementation.+ */+ default Lease enter(String host, int port, boolean waitForResume) throws InterruptedException {+ return enter(host, port);+ }++ /** The backend is being resumed; no lease was acquired and the caller must retry. */+ class ResumingException extends RuntimeException {+ }+/** Held for as long as a client uses the backend; closing it more than once is harmless. */interface Lease extends AutoCloseable {diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.javaindex 5fed84b696..4cba52b0ac 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java@@ -37,7 +37,6 @@ import java.util.List;import java.util.Map;import java.util.Optional;import java.util.Set;-import java.util.concurrent.CompletableFuture;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;@@ -486,12 +485,17 @@ public class RdsContainerManager implements RdsBackendGate, Resettable {@Overridepublic Lease enter(String host, int port) throws InterruptedException {+ return enter(host, port, true);+ }++ @Override+ public Lease enter(String host, int port, boolean waitForResume) throws InterruptedException {if (host == null) {return Lease.NONE;}for (AutoPauseController controller : autoPauses.values()) {if (controller.serves(host, port)) {- return controller.acquire();+ return controller.acquire(waitForResume);}}return Lease.NONE;@@ -577,8 +581,7 @@ public class RdsContainerManager implements RdsBackendGate, Resettable {() -> Thread.ofVirtual().name("rds-auto-pause").start(task),delay.toMillis(), TimeUnit.MILLISECONDS);} catch (RejectedExecutionException e) {- LOG.debugv("RDS auto-pause timer is shut down; not scheduling: {0}", e.getMessage());- return CompletableFuture.completedFuture(null);+ throw new IllegalStateException("RDS auto-pause timer is shut down", e);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.javaindex 6773a1e4dd..7057497e69 100644--- a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java+++ b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java@@ -72,6 +72,7 @@ public class RdsDataService implements Resettable {private final ConcurrentMap<String, TransactionContext> transactions = new ConcurrentHashMap<>();private final ScheduledExecutorService transactionCleanupExecutor;private final RdsBackendGate backendGate;+ private final boolean databaseResumingExceptionEnabled;@Injectpublic RdsDataService(RdsDataResourceResolver resourceResolver,@@ -81,7 +82,8 @@ public class RdsDataService implements Resettable {EmulatorConfig config,RdsBackendGate backendGate) {this(resourceResolver, secretsManagerService, objectMapper, connectionFactory,- Duration.ofSeconds(config.services().rdsData().transactionTtlSeconds()), backendGate);+ Duration.ofSeconds(config.services().rdsData().transactionTtlSeconds()), backendGate,+ config.services().rdsData().databaseResumingExceptionEnabled());}RdsDataService(RdsDataResourceResolver resourceResolver,@@ -99,12 +101,23 @@ public class RdsDataService implements Resettable {RdsDataConnectionFactory connectionFactory,Duration transactionTtl,RdsBackendGate backendGate) {+ this(resourceResolver, secretsManagerService, objectMapper, connectionFactory, transactionTtl, backendGate, false);+ }++ RdsDataService(RdsDataResourceResolver resourceResolver,+ SecretsManagerService secretsManagerService,+ ObjectMapper objectMapper,+ RdsDataConnectionFactory connectionFactory,+ Duration transactionTtl,+ RdsBackendGate backendGate,+ boolean databaseResumingExceptionEnabled) {this.resourceResolver = resourceResolver;this.secretsManagerService = secretsManagerService;this.objectMapper = objectMapper;this.connectionFactory = connectionFactory;this.transactionTtl = transactionTtl;this.backendGate = backendGate;+ this.databaseResumingExceptionEnabled = databaseResumingExceptionEnabled;this.transactionCleanupExecutor = Executors.newSingleThreadScheduledExecutor(r -> {Thread thread = new Thread(r, "rds-data-transaction-cleanup");thread.setDaemon(true);@@ -298,12 +311,15 @@ public class RdsDataService implements Resettable {}/**- * Resumes an auto-paused Aurora cluster before the Data API uses its database, as Aurora- * does, and keeps it from pausing again until the returned lease is closed.+ * Keeps a ready backend awake. By default resume is awaited; the opt-in AWS error mode+ * initiates resume and asks the caller to retry instead, without opening a connection.*/private RdsBackendGate.Lease enterBackend(RdsDataResourceResolver.DatabaseTarget target) {try {- return backendGate.enter(target.host(), target.port());+ return backendGate.enter(target.host(), target.port(), !databaseResumingExceptionEnabled);+ } catch (RdsBackendGate.ResumingException e) {+ throw new AwsException("DatabaseResumingException",+ "The Aurora Serverless v2 DB instance is resuming. Wait a few seconds and try again.", 400);} catch (InterruptedException e) {Thread.currentThread().interrupt();throw new AwsException("InternalServerErrorException",diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex cdf74de48e..e6c19b2511 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -504,6 +504,7 @@ floci:rds-data:enabled: truetransaction-ttl-seconds: 180+ database-resuming-exception-enabled: false # FLOCI_SERVICES_RDS_DATA_DATABASE_RESUMING_EXCEPTION_ENABLEDredshift-data:enabled: trueresult-ttl-hours: 24diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/container/AutoPauseControllerTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/container/AutoPauseControllerTest.javaindex 2fa57ce614..0aff3e4461 100644--- a/src/test/java/io/github/hectorvent/floci/services/rds/container/AutoPauseControllerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rds/container/AutoPauseControllerTest.java@@ -21,10 +21,12 @@ import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicInteger;import java.util.function.BooleanSupplier;+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertSame;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.junit.jupiter.api.Assertions.fail;@@ -90,6 +92,139 @@ class AutoPauseControllerTest {assertEquals(State.PAUSED, controller.state(), "idle again after the connection closed");}+ @Test+ void nonBlockingAcquisitionStartsOneBackgroundResumeWithoutTakingALease() throws Exception {+ AutoPauseController controller = pausedController(Duration.ofSeconds(15));++ assertTimeoutPreemptively(Duration.ofSeconds(2),+ () -> assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false)));+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));+ assertEquals(0, controller.leaseCount());+ assertEquals(0, freezer.unpauses.get(), "Docker resume must not block the Data API request");++ scheduler.advance(Duration.ZERO);+ assertEquals(State.RESUMING, controller.state());+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));+ assertEquals(1, freezer.unpauses.get());++ scheduler.advance(Duration.ofSeconds(15));+ try (RdsBackendGate.Lease lease = controller.acquire(false)) {+ assertEquals(1, controller.leaseCount());+ assertFalse(freezer.paused);+ }+ assertEquals(0, controller.leaseCount());+ }++ @Test+ void closingBeforeTheQueuedResumeThawsTheContainerOnce() throws Exception {+ AutoPauseController controller = pausedController(Duration.ofSeconds(15));+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));++ controller.close();+ scheduler.advance(Duration.ofDays(1));++ assertFalse(freezer.paused);+ assertEquals(1, freezer.unpauses.get());+ assertSame(RdsBackendGate.Lease.NONE, controller.acquire(false));+ }++ @Test+ void nonBlockingArrivalDuringPauseCancelsItWithoutAcquiringALease() throws Exception {+ AutoPauseController controller = controller(Duration.ZERO);+ controller.configure(300, listener);+ freezer.blockPause();+ Thread pausing = Thread.ofVirtual().start(() -> scheduler.advance(IDLE));+ freezer.awaitPauseEntered();+ try {+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));+ assertEquals(0, controller.leaseCount());+ } finally {+ freezer.releasePause();+ pausing.join(5000);+ }+ assertEquals(State.RUNNING, controller.state());+ assertFalse(freezer.paused);+ }++ @Test+ void leaseReleaseStillSucceedsAfterTheTimerStops() throws Exception {+ AutoPauseController controller = controller(Duration.ZERO);+ controller.configure(300, listener);+ RdsBackendGate.Lease lease = controller.acquire();+ scheduler.reject = true;++ assertDoesNotThrow(lease::close);+ assertEquals(0, controller.leaseCount());+ assertEquals(State.RUNNING, controller.state());+ }++ @Test+ void pausedReportingIsBestEffortButResumeSchedulingMustSucceed() throws Exception {+ AutoPauseController controller = controller(Duration.ZERO);+ controller.configure(300, listener);+ scheduler.reject = true;++ assertTrue(assertDoesNotThrow(controller::pauseIfIdle));+ assertEquals(State.PAUSED, controller.state());+ assertThrows(IllegalStateException.class, () -> controller.acquire(false));+ scheduler.reject = false;+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));+ scheduler.advance(Duration.ZERO);+ assertEquals(State.RUNNING, controller.state());+ }++ @Test+ void rejectedBackgroundResumeCanBeRetried() throws Exception {+ AutoPauseController controller = pausedController(Duration.ZERO);+ scheduler.reject = true;+ assertThrows(IllegalStateException.class, () -> controller.acquire(false));+ assertEquals(State.PAUSED, controller.state());+ assertEquals(0, controller.leaseCount());++ scheduler.reject = false;+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));+ scheduler.advance(Duration.ZERO);+ assertEquals(State.RUNNING, controller.state());+ assertEquals(1, freezer.unpauses.get());+ }++ @Test+ void rejectedResumeDelayReleasesBlockingWaitersAfterDockerThaws() throws Exception {+ AutoPauseController controller = pausedController(Duration.ofSeconds(15));+ freezer.blockUnpause();+ Acquirer first = new Acquirer(controller, freezer);+ freezer.awaitUnpauseEntered();+ Acquirer second = new Acquirer(controller, freezer);+ second.awaitWaiting();+ scheduler.reject = true;+ freezer.releaseUnpause();++ assertThrows(Exception.class, first::lease);+ scheduler.reject = false;+ second.lease().close();+ assertEquals(State.RUNNING, controller.state());+ assertEquals(0, controller.leaseCount());+ }++ @Test+ void failedBackgroundResumeCanBeRetriedWithoutLeakingALease() throws Exception {+ AutoPauseController controller = pausedController(Duration.ZERO);+ freezer.unpauseFailure = new IllegalStateException("daemon unavailable");+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));+ scheduler.advance(Duration.ZERO);+ assertEquals(State.PAUSED, controller.state());+ assertEquals(0, controller.leaseCount());++ freezer.unpauseFailure = null;+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));+ scheduler.advance(Duration.ZERO);+ try (RdsBackendGate.Lease lease = controller.acquire(false)) {+ assertFalse(freezer.paused);+ }+ assertEquals(0, controller.leaseCount());+ assertEquals(2, freezer.unpauses.get());+ }+@Testvoid resumeDelayHoldsConnectionsIncludingOnesThatArriveWhileResuming() throws Exception {AutoPauseController controller = pausedController(Duration.ofSeconds(15));@@ -375,9 +510,13 @@ class AutoPauseControllerTest {private Instant now = Instant.parse("2026-01-01T00:00:00Z");private long sequence;private final List<Scheduled> tasks = new ArrayList<>();+ private volatile boolean reject;@Overridepublic synchronized Future<?> schedule(Runnable task, Duration delay) {+ if (reject) {+ throw new IllegalStateException("scheduler stopped");+ }FutureTask<Void> future = new FutureTask<>(task, null);tasks.add(new Scheduled(now.plus(delay), sequence++, future));return future;diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.javaindex f9d58d2059..862f1265fb 100644--- a/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java@@ -1234,6 +1234,7 @@ class RdsContainerManagerTest {assertTrue(manager.pauseIfIdle(runtimeId));verify(pause).exec();+ assertThrows(RdsBackendGate.ResumingException.class, () -> manager.enter("db1", 3306, false));manager.enter("db1", 3306).close();verify(unpause).exec();diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsDataResumingIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsDataResumingIntegrationTest.javanew file mode 100644index 0000000000..b98fd0517a--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsDataResumingIntegrationTest.java@@ -0,0 +1,102 @@+package io.github.hectorvent.floci.services.rds.container;++import com.github.dockerjava.api.DockerClient;+import io.github.hectorvent.floci.services.rds.RdsService;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import io.restassured.specification.RequestSpecification;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Tag;+import org.junit.jupiter.api.Test;++import java.time.Duration;+import java.util.List;+import java.util.Map;++import static io.restassured.RestAssured.given;+import static io.restassured.http.ContentType.URLENC;+import static org.awaitility.Awaitility.await;+import static org.hamcrest.Matchers.equalTo;++@QuarkusTest+@TestProfile(RdsDataResumingIntegrationTest.ResumingProfile.class)+@Tag("docker")+class RdsDataResumingIntegrationTest {++ private static final String RDS_AUTH =+ "AWS4-HMAC-SHA256 Credential=test/20261015/us-east-1/rds/aws4_request";++ @Inject+ DockerClient dockerClient;++ @Inject+ RdsService rdsService;++ @Inject+ RdsContainerManager containerManager;++ @Test+ void pausedAuroraReturnsTypedHttpErrorsAndExecutesTheRetry() {+ dockerClient.pingCmd().exec();+ RestAssuredJsonUtils.configureAwsContentTypes();+ String clusterId = "data-resuming-" + Long.toString(System.nanoTime(), 36);+ String instanceId = clusterId + "-1";+ String secretArn = given().contentType("application/x-amz-json-1.1")+ .header("X-Amz-Target", "secretsmanager.CreateSecret")+ .body(Map.of("Name", clusterId, "SecretString", "{\"username\":\"master\",\"password\":\"Master123!\"}"))+ .post("/").then().statusCode(200).extract().path("ARN");+ try {+ rds("CreateDBCluster").formParam("DBClusterIdentifier", clusterId)+ .formParam("Engine", "aurora-postgresql")+ .formParam("MasterUsername", "master").formParam("MasterUserPassword", "Master123!")+ .formParam("DatabaseName", "appdb")+ .formParam("ServerlessV2ScalingConfiguration.MinCapacity", "0")+ .formParam("ServerlessV2ScalingConfiguration.MaxCapacity", "2")+ .formParam("ServerlessV2ScalingConfiguration.SecondsUntilAutoPause", "300")+ .post("/").then().statusCode(200);+ rds("CreateDBInstance").formParam("DBInstanceIdentifier", instanceId)+ .formParam("DBClusterIdentifier", clusterId).formParam("Engine", "aurora-postgresql")+ .formParam("DBInstanceClass", "db.serverless").post("/").then().statusCode(200);+ String clusterArn = rdsService.getDbCluster(clusterId).getDbClusterArn();+ await().atMost(Duration.ofSeconds(10)).until(() -> containerManager.pauseIfIdle(clusterArn));+ Map<String, Object> request = Map.of("resourceArn", clusterArn, "secretArn", secretArn,+ "database", "appdb", "sql", "select 1", "parameterSets", List.of(List.of()));++ for (String operation : List.of("Execute", "BeginTransaction", "BatchExecute")) {+ given().contentType("application/json").body(request).post("/" + operation)+ .then().statusCode(400)+ .header("X-Amzn-Errortype", "DatabaseResumingException")+ .body("__type", equalTo("DatabaseResumingException"));+ }++ await().atMost(Duration.ofSeconds(20)).untilAsserted(() ->+ given().contentType("application/json").body(request).post("/Execute")+ .then().statusCode(200).body("records[0][0].longValue", equalTo(1)));+ given().contentType("application/json").body(request).post("/Execute")+ .then().statusCode(200).body("records[0][0].longValue", equalTo(1));+ } finally {+ rds("DeleteDBInstance").formParam("DBInstanceIdentifier", instanceId)+ .formParam("SkipFinalSnapshot", "true").post("/");+ rds("DeleteDBCluster").formParam("DBClusterIdentifier", clusterId)+ .formParam("SkipFinalSnapshot", "true").post("/");+ given().contentType("application/x-amz-json-1.1")+ .header("X-Amz-Target", "secretsmanager.DeleteSecret")+ .body(Map.of("SecretId", secretArn, "ForceDeleteWithoutRecovery", true)).post("/");+ }+ }++ private static RequestSpecification rds(String action) {+ return given().header("Authorization", RDS_AUTH).contentType(URLENC)+ .formParam("Action", action).formParam("Version", "2014-10-31");+ }++ public static class ResumingProfile implements QuarkusTestProfile {+ @Override+ public Map<String, String> getConfigOverrides() {+ return Map.of("floci.services.rds-data.database-resuming-exception-enabled", "true",+ "floci.services.rds.aurora-resume-delay-millis", "5000");+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataServiceTest.javaindex e7f124d81f..1f82788cf9 100644--- a/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataServiceTest.java@@ -11,6 +11,8 @@ import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;import org.h2.Driver;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;import java.lang.reflect.Proxy;import java.sql.Connection;@@ -1146,11 +1148,70 @@ class RdsDataServiceTest {});}+ @ParameterizedTest+ @ValueSource(strings = {"ExecuteStatement", "BatchExecuteStatement", "BeginTransaction"})+ void optedInCallsRejectResumingBackendsBeforeOpeningJdbcAndSucceedOnRetry(String operation) throws Exception {+ TestHarness harness = new TestHarness(true);+ harness.createEventsTable();+ harness.gate.resuming = true;++ AwsException error = assertThrows(AwsException.class, () -> call(harness, operation));++ assertEquals("DatabaseResumingException", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertEquals(0, harness.connectionsOpened.get());+ assertEquals(0, harness.gate.held.get());++ harness.gate.resuming = false;+ call(harness, operation);+ assertEquals(1, harness.connectionsOpened.get());+ if ("BatchExecuteStatement".equals(operation)) {+ assertEquals(1, harness.service.executeStatement(+ harness.request("select count(*) from data_api_events"), REGION)+ .path("records").get(0).get(0).path("longValue").asInt());+ }+ harness.service.shutdown();+ assertEquals(0, harness.gate.held.get());+ }++ @Test+ void defaultConfigurationStillWaitsForResumingBackends() {+ TestHarness harness = new TestHarness();+ harness.gate.resuming = true;++ assertEquals(1, harness.service.executeStatement(harness.request("select 1"), REGION)+ .path("records").get(0).get(0).path("longValue").asInt());+ assertEquals(0, harness.gate.held.get());+ harness.service.shutdown();+ }++ private void call(TestHarness harness, String operation) {+ switch (operation) {+ case "ExecuteStatement" -> harness.service.executeStatement(harness.request("select 1"), REGION);+ case "BatchExecuteStatement" -> {+ ObjectNode request = harness.request("insert into data_api_events(name) values ('retry')");+ request.set("parameterSets", objectMapper.createArrayNode().add(objectMapper.createArrayNode()));+ harness.service.batchExecuteStatement(request, REGION);+ }+ case "BeginTransaction" -> harness.service.beginTransaction(harness.beginRequest(), REGION);+ default -> throw new AssertionError(operation);+ }+ }+/** Counts the backends the Data API enters and the holds still open on them. */private static final class CountingGate implements RdsBackendGate {private final List<String> entered = new CopyOnWriteArrayList<>();private final AtomicInteger held = new AtomicInteger();private volatile IllegalStateException failure;+ private boolean resuming;++ @Override+ public Lease enter(String host, int port, boolean waitForResume) {+ if (!waitForResume && resuming) {+ throw new ResumingException();+ }+ return enter(host, port);+ }@Overridepublic Lease enter(String host, int port) {@@ -1173,12 +1234,17 @@ class RdsDataServiceTest {private final RdsDataResourceResolver resolver;private final RdsDataResourceResolver.DatabaseTarget target;private final CountingGate gate = new CountingGate();+ private final AtomicInteger connectionsOpened = new AtomicInteger();private final RdsDataService service;private TestHarness() {this(DatabaseEngine.MYSQL, Duration.ofSeconds(60));}+ private TestHarness(boolean resumingExceptionEnabled) {+ this(defaultSecrets(), DatabaseEngine.POSTGRES, Duration.ofSeconds(60), resumingExceptionEnabled);+ }+private TestHarness(SecretsManagerService secrets) {this(secrets, DatabaseEngine.MYSQL, Duration.ofSeconds(60));}@@ -1196,6 +1262,11 @@ class RdsDataServiceTest {}private TestHarness(SecretsManagerService secrets, DatabaseEngine engine, Duration transactionTtl) {+ this(secrets, engine, transactionTtl, false);+ }++ private TestHarness(SecretsManagerService secrets, DatabaseEngine engine, Duration transactionTtl,+ boolean resumingExceptionEnabled) {jdbcUrl = "jdbc:h2:mem:rdsdata_" + UUID.randomUUID() + ";MODE=" + h2Mode(engine)+ ";DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1";resolver = mock(RdsDataResourceResolver.class);@@ -1211,10 +1282,12 @@ class RdsDataServiceTest {String username,String password,String database) throws SQLException {+ connectionsOpened.incrementAndGet();return getConnection();}};- service = new RdsDataService(resolver, secrets, objectMapper, connectionFactory, transactionTtl, gate);+ service = new RdsDataService(resolver, secrets, objectMapper, connectionFactory,+ transactionTtl, gate, resumingExceptionEnabled);}private Connection getConnection() throws SQLException {diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 1a60b2f551..f418fa26da 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -219,6 +219,7 @@ floci:rds-data:enabled: truetransaction-ttl-seconds: 180+ database-resuming-exception-enabled: falseredshift-data:enabled: trueresult-ttl-hours: 24