Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(rds-data): opt into retryable database resume errors (#5703)

Aniket Shuklafb6adc00d84f
diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.md
index f698b73343..c3fe6a6c01 100644
--- a/docs/configuration/advanced/application-yml.md
+++ b/docs/configuration/advanced/application-yml.md
@@ -220,6 +220,7 @@ floci:
rds-data:
enabled: true
transaction-ttl-seconds: 180
+ database-resuming-exception-enabled: false
eventbridge:
enabled: true
@@ -348,6 +349,7 @@ All keys in this table are declared on `EmulatorConfig` and accept environment v
| `FLOCI_SERVICES_DOCKER_NETWORK` | *(unset)* | Shared Docker network for Lambda, RDS, ElastiCache containers |
| `FLOCI_SERVICES_RDS_DATA_ENABLED` | `true` | Enable the RDS Data API service |
| `FLOCI_SERVICES_RDS_DATA_TRANSACTION_TTL_SECONDS` | `180` | Idle timeout, in seconds, before leaked RDS Data API transactions expire |
+| `FLOCI_SERVICES_RDS_DATA_DATABASE_RESUMING_EXCEPTION_ENABLED` | `false` | Return HTTP 400 `DatabaseResumingException` during Aurora resume instead of waiting |
| `FLOCI_SERVICES_ECS_MOCK` | `false` | Skip Docker; tasks go straight to RUNNING (useful for CI) |
| `FLOCI_SERVICES_ECS_DOCKER_NETWORK` | *(unset)* | Docker network for ECS task containers |
| `FLOCI_SERVICES_ECS_DEFAULT_MEMORY_MB` | `512` | Default memory (MB) when task definition omits it |
diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md
index f505ce8b0c..c00ba9b353 100644
--- a/docs/configuration/environment-variables.md
+++ b/docs/configuration/environment-variables.md
@@ -438,6 +438,7 @@ These services spawn Docker containers. They require access to the Docker socket
| `FLOCI_SERVICES_RDS_DOCKER_NETWORK` | _(none)_ | Docker network for RDS containers (overrides `FLOCI_SERVICES_DOCKER_NETWORK`) |
| `FLOCI_SERVICES_RDS_DATA_ENABLED` | `true` | Enable the RDS Data API service. Requires `FLOCI_SERVICES_RDS_ENABLED=true` |
| `FLOCI_SERVICES_RDS_DATA_TRANSACTION_TTL_SECONDS` | `180` | Idle timeout, in seconds, before leaked RDS Data API transactions expire |
+| `FLOCI_SERVICES_RDS_DATA_DATABASE_RESUMING_EXCEPTION_ENABLED` | `false` | Return HTTP 400 `DatabaseResumingException` while an auto-paused Aurora backend resumes, instead of waiting |
### OpenSearch
diff --git a/docs/services/rds-data.md b/docs/services/rds-data.md
index 2a3b0be343..c91af02dda 100644
--- a/docs/services/rds-data.md
+++ b/docs/services/rds-data.md
@@ -55,9 +55,17 @@ The deprecated `ExecuteSql` operation is recognized at `POST /ExecuteSql` and re
|---|---|---|
| `FLOCI_SERVICES_RDS_DATA_ENABLED` | `true` | Enable or disable the RDS Data API service |
| `FLOCI_SERVICES_RDS_DATA_TRANSACTION_TTL_SECONDS` | `180` | Idle timeout, in seconds, before leaked Data API transactions expire |
+| `FLOCI_SERVICES_RDS_DATA_DATABASE_RESUMING_EXCEPTION_ENABLED` | `false` | Return `DatabaseResumingException` (HTTP 400) while an auto-paused Aurora backend resumes, instead of waiting for it |
The RDS Data API also requires the RDS service itself to be enabled because it resolves `resourceArn` values to local RDS containers.
+With `database-resuming-exception-enabled=true`, `ExecuteStatement`, `BatchExecuteStatement`,
+and `BeginTransaction` initiate resume in the background and return the retryable error before
+opening a database connection. Calls made while resume is in progress return the same error;
+retry after a few seconds. Once ready, requests run normally. Existing transactions keep their
+backend awake until commit, rollback, or expiry. The default remains blocking resume for local
+compatibility; TCP database connections and internal RDS commands always wait for resume.
+
## Example
```bash
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 2387ecde53..ae7191e95e 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1712,6 +1712,9 @@ public interface EmulatorConfig {
@WithDefault("180")
long transactionTtlSeconds();
+
+ @WithDefault("false")
+ boolean databaseResumingExceptionEnabled();
}
interface RedshiftDataServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/AutoPauseController.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/AutoPauseController.java
index 02a5dc1c10..7eb413a3cb 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/AutoPauseController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/AutoPauseController.java
@@ -81,6 +81,7 @@ final class AutoPauseController {
// was cancelled or belongs to an earlier state, and does nothing.
private Future<?> timer;
private long timerToken;
+ private boolean resumeRequested;
private boolean pauseCanceled;
private boolean closed;
private Instant lastSignal = Instant.EPOCH;
@@ -163,6 +164,10 @@ final class AutoPauseController {
* @throws IllegalStateException when Docker cannot resume the container, which stays paused
*/
RdsBackendGate.Lease acquire() throws InterruptedException {
+ return acquire(true);
+ }
+
+ RdsBackendGate.Lease acquire(boolean waitForResume) throws InterruptedException {
lock.lock();
try {
while (true) {
@@ -174,12 +179,19 @@ final class AutoPauseController {
cancelTimer();
return new ControllerLease();
}
+ if (state == State.PAUSING) {
+ pauseCanceled = true;
+ }
+ if (!waitForResume) {
+ if (state == State.PAUSED && !resumeRequested) {
+ startTimer(this::resumeInBackground, Duration.ZERO);
+ resumeRequested = true;
+ }
+ throw new RdsBackendGate.ResumingException();
+ }
if (state == State.PAUSED) {
resume();
} else {
- if (state == State.PAUSING) {
- pauseCanceled = true;
- }
settled.await();
}
}
@@ -312,8 +324,8 @@ final class AutoPauseController {
if (thawFailure != null) {
LOG.warnv(thawFailure, "Could not undo the auto-pause of RDS container {0}", containerId);
state = State.PAUSED;
- armPausedReport();
settled.signalAll();
+ armPausedReport();
return;
}
}
@@ -334,15 +346,21 @@ final class AutoPauseController {
RuntimeException failure = withoutLock(() -> freezer.unpause(containerId));
if (failure != null) {
state = State.PAUSED;
- armPausedReport();
settled.signalAll();
+ armPausedReport();
throw new IllegalStateException(
"Could not resume auto-paused RDS container " + containerId, failure);
}
if (closed || resumeDelay.isZero()) {
finishResume();
} else {
- startTimer(this::resumeDelayElapsed, resumeDelay);
+ try {
+ startTimer(this::resumeDelayElapsed, resumeDelay);
+ } catch (IllegalStateException e) {
+ // Docker has thawed the backend; a stopped timer must not strand its waiters.
+ finishResume();
+ throw e;
+ }
}
}
@@ -383,12 +401,12 @@ final class AutoPauseController {
if (closed || state != State.RUNNING || leases > 0 || secondsUntilAutoPause == null) {
return;
}
- startTimer(this::idleTimerFired, Duration.ofSeconds(secondsUntilAutoPause));
+ startBestEffortTimer(this::idleTimerFired, Duration.ofSeconds(secondsUntilAutoPause));
}
private void armPausedReport() {
if (!closed) {
- startTimer(this::pausedReportDue, PAUSED_REPORT_INTERVAL);
+ startBestEffortTimer(this::pausedReportDue, PAUSED_REPORT_INTERVAL);
}
}
@@ -404,6 +422,15 @@ final class AutoPauseController {
}
}
+ private void startBestEffortTimer(LongConsumer task, Duration delay) {
+ try {
+ startTimer(task, delay);
+ } catch (IllegalStateException e) {
+ // Shutdown must not turn a completed request's lease release into a failure.
+ LOG.debugv(e, "Could not schedule best-effort RDS timer for {0}", containerId);
+ }
+ }
+
private void startTimer(LongConsumer task, Duration delay) {
cancelTimer();
long token = timerToken;
@@ -411,6 +438,7 @@ final class AutoPauseController {
}
private void cancelTimer() {
+ resumeRequested = false;
if (timer != null) {
timer.cancel(false);
timer = null;
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsBackendGate.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsBackendGate.java
index 2104e52191..ffe1b37194 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsBackendGate.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsBackendGate.java
@@ -19,6 +19,19 @@ public interface RdsBackendGate {
*/
Lease enter(String host, int port) throws InterruptedException;
+ /**
+ * Like {@link #enter(String, int)}, but with {@code waitForResume=false} starts resume in the
+ * background and throws {@link ResumingException} if the backend is not yet ready.
+ * Backends without auto-pause may use this default implementation.
+ */
+ default Lease enter(String host, int port, boolean waitForResume) throws InterruptedException {
+ return enter(host, port);
+ }
+
+ /** The backend is being resumed; no lease was acquired and the caller must retry. */
+ class ResumingException extends RuntimeException {
+ }
+
/** Held for as long as a client uses the backend; closing it more than once is harmless. */
interface Lease extends AutoCloseable {
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java
index 5fed84b696..4cba52b0ac 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java
@@ -37,7 +37,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -486,12 +485,17 @@ public class RdsContainerManager implements RdsBackendGate, Resettable {
@Override
public Lease enter(String host, int port) throws InterruptedException {
+ return enter(host, port, true);
+ }
+
+ @Override
+ public Lease enter(String host, int port, boolean waitForResume) throws InterruptedException {
if (host == null) {
return Lease.NONE;
}
for (AutoPauseController controller : autoPauses.values()) {
if (controller.serves(host, port)) {
- return controller.acquire();
+ return controller.acquire(waitForResume);
}
}
return Lease.NONE;
@@ -577,8 +581,7 @@ public class RdsContainerManager implements RdsBackendGate, Resettable {
() -> Thread.ofVirtual().name("rds-auto-pause").start(task),
delay.toMillis(), TimeUnit.MILLISECONDS);
} catch (RejectedExecutionException e) {
- LOG.debugv("RDS auto-pause timer is shut down; not scheduling: {0}", e.getMessage());
- return CompletableFuture.completedFuture(null);
+ throw new IllegalStateException("RDS auto-pause timer is shut down", e);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java
index 6773a1e4dd..7057497e69 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java
@@ -72,6 +72,7 @@ public class RdsDataService implements Resettable {
private final ConcurrentMap<String, TransactionContext> transactions = new ConcurrentHashMap<>();
private final ScheduledExecutorService transactionCleanupExecutor;
private final RdsBackendGate backendGate;
+ private final boolean databaseResumingExceptionEnabled;
@Inject
public RdsDataService(RdsDataResourceResolver resourceResolver,
@@ -81,7 +82,8 @@ public class RdsDataService implements Resettable {
EmulatorConfig config,
RdsBackendGate backendGate) {
this(resourceResolver, secretsManagerService, objectMapper, connectionFactory,
- Duration.ofSeconds(config.services().rdsData().transactionTtlSeconds()), backendGate);
+ Duration.ofSeconds(config.services().rdsData().transactionTtlSeconds()), backendGate,
+ config.services().rdsData().databaseResumingExceptionEnabled());
}
RdsDataService(RdsDataResourceResolver resourceResolver,
@@ -99,12 +101,23 @@ public class RdsDataService implements Resettable {
RdsDataConnectionFactory connectionFactory,
Duration transactionTtl,
RdsBackendGate backendGate) {
+ this(resourceResolver, secretsManagerService, objectMapper, connectionFactory, transactionTtl, backendGate, false);
+ }
+
+ RdsDataService(RdsDataResourceResolver resourceResolver,
+ SecretsManagerService secretsManagerService,
+ ObjectMapper objectMapper,
+ RdsDataConnectionFactory connectionFactory,
+ Duration transactionTtl,
+ RdsBackendGate backendGate,
+ boolean databaseResumingExceptionEnabled) {
this.resourceResolver = resourceResolver;
this.secretsManagerService = secretsManagerService;
this.objectMapper = objectMapper;
this.connectionFactory = connectionFactory;
this.transactionTtl = transactionTtl;
this.backendGate = backendGate;
+ this.databaseResumingExceptionEnabled = databaseResumingExceptionEnabled;
this.transactionCleanupExecutor = Executors.newSingleThreadScheduledExecutor(r -> {
Thread thread = new Thread(r, "rds-data-transaction-cleanup");
thread.setDaemon(true);
@@ -298,12 +311,15 @@ public class RdsDataService implements Resettable {
}
/**
- * Resumes an auto-paused Aurora cluster before the Data API uses its database, as Aurora
- * does, and keeps it from pausing again until the returned lease is closed.
+ * Keeps a ready backend awake. By default resume is awaited; the opt-in AWS error mode
+ * initiates resume and asks the caller to retry instead, without opening a connection.
*/
private RdsBackendGate.Lease enterBackend(RdsDataResourceResolver.DatabaseTarget target) {
try {
- return backendGate.enter(target.host(), target.port());
+ return backendGate.enter(target.host(), target.port(), !databaseResumingExceptionEnabled);
+ } catch (RdsBackendGate.ResumingException e) {
+ throw new AwsException("DatabaseResumingException",
+ "The Aurora Serverless v2 DB instance is resuming. Wait a few seconds and try again.", 400);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new AwsException("InternalServerErrorException",
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index cdf74de48e..e6c19b2511 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -504,6 +504,7 @@ floci:
rds-data:
enabled: true
transaction-ttl-seconds: 180
+ database-resuming-exception-enabled: false # FLOCI_SERVICES_RDS_DATA_DATABASE_RESUMING_EXCEPTION_ENABLED
redshift-data:
enabled: true
result-ttl-hours: 24
diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/container/AutoPauseControllerTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/container/AutoPauseControllerTest.java
index 2fa57ce614..0aff3e4461 100644
--- a/src/test/java/io/github/hectorvent/floci/services/rds/container/AutoPauseControllerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/rds/container/AutoPauseControllerTest.java
@@ -21,10 +21,12 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BooleanSupplier;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@ -90,6 +92,139 @@ class AutoPauseControllerTest {
assertEquals(State.PAUSED, controller.state(), "idle again after the connection closed");
}
+ @Test
+ void nonBlockingAcquisitionStartsOneBackgroundResumeWithoutTakingALease() throws Exception {
+ AutoPauseController controller = pausedController(Duration.ofSeconds(15));
+
+ assertTimeoutPreemptively(Duration.ofSeconds(2),
+ () -> assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false)));
+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));
+ assertEquals(0, controller.leaseCount());
+ assertEquals(0, freezer.unpauses.get(), "Docker resume must not block the Data API request");
+
+ scheduler.advance(Duration.ZERO);
+ assertEquals(State.RESUMING, controller.state());
+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));
+ assertEquals(1, freezer.unpauses.get());
+
+ scheduler.advance(Duration.ofSeconds(15));
+ try (RdsBackendGate.Lease lease = controller.acquire(false)) {
+ assertEquals(1, controller.leaseCount());
+ assertFalse(freezer.paused);
+ }
+ assertEquals(0, controller.leaseCount());
+ }
+
+ @Test
+ void closingBeforeTheQueuedResumeThawsTheContainerOnce() throws Exception {
+ AutoPauseController controller = pausedController(Duration.ofSeconds(15));
+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));
+
+ controller.close();
+ scheduler.advance(Duration.ofDays(1));
+
+ assertFalse(freezer.paused);
+ assertEquals(1, freezer.unpauses.get());
+ assertSame(RdsBackendGate.Lease.NONE, controller.acquire(false));
+ }
+
+ @Test
+ void nonBlockingArrivalDuringPauseCancelsItWithoutAcquiringALease() throws Exception {
+ AutoPauseController controller = controller(Duration.ZERO);
+ controller.configure(300, listener);
+ freezer.blockPause();
+ Thread pausing = Thread.ofVirtual().start(() -> scheduler.advance(IDLE));
+ freezer.awaitPauseEntered();
+ try {
+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));
+ assertEquals(0, controller.leaseCount());
+ } finally {
+ freezer.releasePause();
+ pausing.join(5000);
+ }
+ assertEquals(State.RUNNING, controller.state());
+ assertFalse(freezer.paused);
+ }
+
+ @Test
+ void leaseReleaseStillSucceedsAfterTheTimerStops() throws Exception {
+ AutoPauseController controller = controller(Duration.ZERO);
+ controller.configure(300, listener);
+ RdsBackendGate.Lease lease = controller.acquire();
+ scheduler.reject = true;
+
+ assertDoesNotThrow(lease::close);
+ assertEquals(0, controller.leaseCount());
+ assertEquals(State.RUNNING, controller.state());
+ }
+
+ @Test
+ void pausedReportingIsBestEffortButResumeSchedulingMustSucceed() throws Exception {
+ AutoPauseController controller = controller(Duration.ZERO);
+ controller.configure(300, listener);
+ scheduler.reject = true;
+
+ assertTrue(assertDoesNotThrow(controller::pauseIfIdle));
+ assertEquals(State.PAUSED, controller.state());
+ assertThrows(IllegalStateException.class, () -> controller.acquire(false));
+ scheduler.reject = false;
+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));
+ scheduler.advance(Duration.ZERO);
+ assertEquals(State.RUNNING, controller.state());
+ }
+
+ @Test
+ void rejectedBackgroundResumeCanBeRetried() throws Exception {
+ AutoPauseController controller = pausedController(Duration.ZERO);
+ scheduler.reject = true;
+ assertThrows(IllegalStateException.class, () -> controller.acquire(false));
+ assertEquals(State.PAUSED, controller.state());
+ assertEquals(0, controller.leaseCount());
+
+ scheduler.reject = false;
+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));
+ scheduler.advance(Duration.ZERO);
+ assertEquals(State.RUNNING, controller.state());
+ assertEquals(1, freezer.unpauses.get());
+ }
+
+ @Test
+ void rejectedResumeDelayReleasesBlockingWaitersAfterDockerThaws() throws Exception {
+ AutoPauseController controller = pausedController(Duration.ofSeconds(15));
+ freezer.blockUnpause();
+ Acquirer first = new Acquirer(controller, freezer);
+ freezer.awaitUnpauseEntered();
+ Acquirer second = new Acquirer(controller, freezer);
+ second.awaitWaiting();
+ scheduler.reject = true;
+ freezer.releaseUnpause();
+
+ assertThrows(Exception.class, first::lease);
+ scheduler.reject = false;
+ second.lease().close();
+ assertEquals(State.RUNNING, controller.state());
+ assertEquals(0, controller.leaseCount());
+ }
+
+ @Test
+ void failedBackgroundResumeCanBeRetriedWithoutLeakingALease() throws Exception {
+ AutoPauseController controller = pausedController(Duration.ZERO);
+ freezer.unpauseFailure = new IllegalStateException("daemon unavailable");
+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));
+ scheduler.advance(Duration.ZERO);
+ assertEquals(State.PAUSED, controller.state());
+ assertEquals(0, controller.leaseCount());
+
+ freezer.unpauseFailure = null;
+ assertThrows(RdsBackendGate.ResumingException.class, () -> controller.acquire(false));
+ scheduler.advance(Duration.ZERO);
+ try (RdsBackendGate.Lease lease = controller.acquire(false)) {
+ assertFalse(freezer.paused);
+ }
+ assertEquals(0, controller.leaseCount());
+ assertEquals(2, freezer.unpauses.get());
+ }
+
@Test
void resumeDelayHoldsConnectionsIncludingOnesThatArriveWhileResuming() throws Exception {
AutoPauseController controller = pausedController(Duration.ofSeconds(15));
@@ -375,9 +510,13 @@ class AutoPauseControllerTest {
private Instant now = Instant.parse("2026-01-01T00:00:00Z");
private long sequence;
private final List<Scheduled> tasks = new ArrayList<>();
+ private volatile boolean reject;
@Override
public synchronized Future<?> schedule(Runnable task, Duration delay) {
+ if (reject) {
+ throw new IllegalStateException("scheduler stopped");
+ }
FutureTask<Void> future = new FutureTask<>(task, null);
tasks.add(new Scheduled(now.plus(delay), sequence++, future));
return future;
diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java
index f9d58d2059..862f1265fb 100644
--- a/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java
@@ -1234,6 +1234,7 @@ class RdsContainerManagerTest {
assertTrue(manager.pauseIfIdle(runtimeId));
verify(pause).exec();
+ assertThrows(RdsBackendGate.ResumingException.class, () -> manager.enter("db1", 3306, false));
manager.enter("db1", 3306).close();
verify(unpause).exec();
diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsDataResumingIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsDataResumingIntegrationTest.java
new file mode 100644
index 0000000000..b98fd0517a
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsDataResumingIntegrationTest.java
@@ -0,0 +1,102 @@
+package io.github.hectorvent.floci.services.rds.container;
+
+import com.github.dockerjava.api.DockerClient;
+import io.github.hectorvent.floci.services.rds.RdsService;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import io.restassured.specification.RequestSpecification;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+
+import static io.restassured.RestAssured.given;
+import static io.restassured.http.ContentType.URLENC;
+import static org.awaitility.Awaitility.await;
+import static org.hamcrest.Matchers.equalTo;
+
+@QuarkusTest
+@TestProfile(RdsDataResumingIntegrationTest.ResumingProfile.class)
+@Tag("docker")
+class RdsDataResumingIntegrationTest {
+
+ private static final String RDS_AUTH =
+ "AWS4-HMAC-SHA256 Credential=test/20261015/us-east-1/rds/aws4_request";
+
+ @Inject
+ DockerClient dockerClient;
+
+ @Inject
+ RdsService rdsService;
+
+ @Inject
+ RdsContainerManager containerManager;
+
+ @Test
+ void pausedAuroraReturnsTypedHttpErrorsAndExecutesTheRetry() {
+ dockerClient.pingCmd().exec();
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ String clusterId = "data-resuming-" + Long.toString(System.nanoTime(), 36);
+ String instanceId = clusterId + "-1";
+ String secretArn = given().contentType("application/x-amz-json-1.1")
+ .header("X-Amz-Target", "secretsmanager.CreateSecret")
+ .body(Map.of("Name", clusterId, "SecretString", "{\"username\":\"master\",\"password\":\"Master123!\"}"))
+ .post("/").then().statusCode(200).extract().path("ARN");
+ try {
+ rds("CreateDBCluster").formParam("DBClusterIdentifier", clusterId)
+ .formParam("Engine", "aurora-postgresql")
+ .formParam("MasterUsername", "master").formParam("MasterUserPassword", "Master123!")
+ .formParam("DatabaseName", "appdb")
+ .formParam("ServerlessV2ScalingConfiguration.MinCapacity", "0")
+ .formParam("ServerlessV2ScalingConfiguration.MaxCapacity", "2")
+ .formParam("ServerlessV2ScalingConfiguration.SecondsUntilAutoPause", "300")
+ .post("/").then().statusCode(200);
+ rds("CreateDBInstance").formParam("DBInstanceIdentifier", instanceId)
+ .formParam("DBClusterIdentifier", clusterId).formParam("Engine", "aurora-postgresql")
+ .formParam("DBInstanceClass", "db.serverless").post("/").then().statusCode(200);
+ String clusterArn = rdsService.getDbCluster(clusterId).getDbClusterArn();
+ await().atMost(Duration.ofSeconds(10)).until(() -> containerManager.pauseIfIdle(clusterArn));
+ Map<String, Object> request = Map.of("resourceArn", clusterArn, "secretArn", secretArn,
+ "database", "appdb", "sql", "select 1", "parameterSets", List.of(List.of()));
+
+ for (String operation : List.of("Execute", "BeginTransaction", "BatchExecute")) {
+ given().contentType("application/json").body(request).post("/" + operation)
+ .then().statusCode(400)
+ .header("X-Amzn-Errortype", "DatabaseResumingException")
+ .body("__type", equalTo("DatabaseResumingException"));
+ }
+
+ await().atMost(Duration.ofSeconds(20)).untilAsserted(() ->
+ given().contentType("application/json").body(request).post("/Execute")
+ .then().statusCode(200).body("records[0][0].longValue", equalTo(1)));
+ given().contentType("application/json").body(request).post("/Execute")
+ .then().statusCode(200).body("records[0][0].longValue", equalTo(1));
+ } finally {
+ rds("DeleteDBInstance").formParam("DBInstanceIdentifier", instanceId)
+ .formParam("SkipFinalSnapshot", "true").post("/");
+ rds("DeleteDBCluster").formParam("DBClusterIdentifier", clusterId)
+ .formParam("SkipFinalSnapshot", "true").post("/");
+ given().contentType("application/x-amz-json-1.1")
+ .header("X-Amz-Target", "secretsmanager.DeleteSecret")
+ .body(Map.of("SecretId", secretArn, "ForceDeleteWithoutRecovery", true)).post("/");
+ }
+ }
+
+ private static RequestSpecification rds(String action) {
+ return given().header("Authorization", RDS_AUTH).contentType(URLENC)
+ .formParam("Action", action).formParam("Version", "2014-10-31");
+ }
+
+ public static class ResumingProfile implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of("floci.services.rds-data.database-resuming-exception-enabled", "true",
+ "floci.services.rds.aurora-resume-delay-millis", "5000");
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataServiceTest.java
index e7f124d81f..1f82788cf9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataServiceTest.java
@@ -11,6 +11,8 @@ import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;
import org.h2.Driver;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.lang.reflect.Proxy;
import java.sql.Connection;
@@ -1146,11 +1148,70 @@ class RdsDataServiceTest {
});
}
+ @ParameterizedTest
+ @ValueSource(strings = {"ExecuteStatement", "BatchExecuteStatement", "BeginTransaction"})
+ void optedInCallsRejectResumingBackendsBeforeOpeningJdbcAndSucceedOnRetry(String operation) throws Exception {
+ TestHarness harness = new TestHarness(true);
+ harness.createEventsTable();
+ harness.gate.resuming = true;
+
+ AwsException error = assertThrows(AwsException.class, () -> call(harness, operation));
+
+ assertEquals("DatabaseResumingException", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ assertEquals(0, harness.connectionsOpened.get());
+ assertEquals(0, harness.gate.held.get());
+
+ harness.gate.resuming = false;
+ call(harness, operation);
+ assertEquals(1, harness.connectionsOpened.get());
+ if ("BatchExecuteStatement".equals(operation)) {
+ assertEquals(1, harness.service.executeStatement(
+ harness.request("select count(*) from data_api_events"), REGION)
+ .path("records").get(0).get(0).path("longValue").asInt());
+ }
+ harness.service.shutdown();
+ assertEquals(0, harness.gate.held.get());
+ }
+
+ @Test
+ void defaultConfigurationStillWaitsForResumingBackends() {
+ TestHarness harness = new TestHarness();
+ harness.gate.resuming = true;
+
+ assertEquals(1, harness.service.executeStatement(harness.request("select 1"), REGION)
+ .path("records").get(0).get(0).path("longValue").asInt());
+ assertEquals(0, harness.gate.held.get());
+ harness.service.shutdown();
+ }
+
+ private void call(TestHarness harness, String operation) {
+ switch (operation) {
+ case "ExecuteStatement" -> harness.service.executeStatement(harness.request("select 1"), REGION);
+ case "BatchExecuteStatement" -> {
+ ObjectNode request = harness.request("insert into data_api_events(name) values ('retry')");
+ request.set("parameterSets", objectMapper.createArrayNode().add(objectMapper.createArrayNode()));
+ harness.service.batchExecuteStatement(request, REGION);
+ }
+ case "BeginTransaction" -> harness.service.beginTransaction(harness.beginRequest(), REGION);
+ default -> throw new AssertionError(operation);
+ }
+ }
+
/** Counts the backends the Data API enters and the holds still open on them. */
private static final class CountingGate implements RdsBackendGate {
private final List<String> entered = new CopyOnWriteArrayList<>();
private final AtomicInteger held = new AtomicInteger();
private volatile IllegalStateException failure;
+ private boolean resuming;
+
+ @Override
+ public Lease enter(String host, int port, boolean waitForResume) {
+ if (!waitForResume && resuming) {
+ throw new ResumingException();
+ }
+ return enter(host, port);
+ }
@Override
public Lease enter(String host, int port) {
@@ -1173,12 +1234,17 @@ class RdsDataServiceTest {
private final RdsDataResourceResolver resolver;
private final RdsDataResourceResolver.DatabaseTarget target;
private final CountingGate gate = new CountingGate();
+ private final AtomicInteger connectionsOpened = new AtomicInteger();
private final RdsDataService service;
private TestHarness() {
this(DatabaseEngine.MYSQL, Duration.ofSeconds(60));
}
+ private TestHarness(boolean resumingExceptionEnabled) {
+ this(defaultSecrets(), DatabaseEngine.POSTGRES, Duration.ofSeconds(60), resumingExceptionEnabled);
+ }
+
private TestHarness(SecretsManagerService secrets) {
this(secrets, DatabaseEngine.MYSQL, Duration.ofSeconds(60));
}
@@ -1196,6 +1262,11 @@ class RdsDataServiceTest {
}
private TestHarness(SecretsManagerService secrets, DatabaseEngine engine, Duration transactionTtl) {
+ this(secrets, engine, transactionTtl, false);
+ }
+
+ private TestHarness(SecretsManagerService secrets, DatabaseEngine engine, Duration transactionTtl,
+ boolean resumingExceptionEnabled) {
jdbcUrl = "jdbc:h2:mem:rdsdata_" + UUID.randomUUID() + ";MODE=" + h2Mode(engine)
+ ";DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1";
resolver = mock(RdsDataResourceResolver.class);
@@ -1211,10 +1282,12 @@ class RdsDataServiceTest {
String username,
String password,
String database) throws SQLException {
+ connectionsOpened.incrementAndGet();
return getConnection();
}
};
- service = new RdsDataService(resolver, secrets, objectMapper, connectionFactory, transactionTtl, gate);
+ service = new RdsDataService(resolver, secrets, objectMapper, connectionFactory,
+ transactionTtl, gate, resumingExceptionEnabled);
}
private Connection getConnection() throws SQLException {
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index 1a60b2f551..f418fa26da 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -219,6 +219,7 @@ floci:
rds-data:
enabled: true
transaction-ttl-seconds: 180
+ database-resuming-exception-enabled: false
redshift-data:
enabled: true
result-ttl-hours: 24