Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(redshift): stay queued for backend ownership until the deadline (#6634)

Jackson Price8b13c75fa265
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
index b22fbc5335..6174373b07 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
@@ -56,7 +56,7 @@ public class RedshiftInterceptingBridge {
private static final int PUMP_READ_TIMEOUT_MS = 200;
private static final int CLIENT_READ_TIMEOUT_MS = 10_000;
- private static final long PUMP_PARK_WAIT_MS = 2_000L;
+ static final long PUMP_PARK_WAIT_MS = 2_000L;
/** Bounded read timeout while an exchange owns the backend, so a stalled backend cannot hang the session. */
private static final int EXCHANGE_READ_TIMEOUT_MS = 60_000;
private static final long BUSY_BACKOFF_NANOS = 2_000_000L;
@@ -82,7 +82,9 @@ public class RedshiftInterceptingBridge {
private final Map<String, ExtendedSpectrumExchange.PortalCursor> spectrumCursors = new HashMap<>();
private final BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);
- private final ReentrantLock backendLock = new ReentrantLock(true);
+ private final ReentrantLock backendLock;
+ /** Budget of {@link #runWithBackendOwned} only; the Extended Query paths keep {@link #PUMP_PARK_WAIT_MS}. */
+ private final long ownershipWaitMillis;
private volatile boolean pumpBetweenMessages = true;
private volatile boolean pumpFinished = false;
private boolean extendedSpectrumError;
@@ -132,6 +134,26 @@ public class RedshiftInterceptingBridge {
SpectrumInterceptor spectrumInterceptor, String clusterKey, String databaseName,
String defaultIamRoleArn, RedshiftAuditLogPublisher auditLogPublisher,
String databaseUser) {
+ this(client, backend, s3Service, iamService, clusterAccountId, iamRoleArns, spectrumInterceptor,
+ clusterKey, databaseName, defaultIamRoleArn, auditLogPublisher, databaseUser, new ReentrantLock(true),
+ PUMP_PARK_WAIT_MS);
+ }
+
+ /**
+ * For tests: a bridge with no services, whose fair backend lock and {@link #runWithBackendOwned}
+ * budget the caller supplies.
+ */
+ RedshiftInterceptingBridge(Socket client, Socket backend, ReentrantLock backendLock, long ownershipWaitMillis) {
+ this(client, backend, null, null, null, List.of(), null, null, null, null, null, null, backendLock,
+ ownershipWaitMillis);
+ }
+
+ private RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService,
+ String clusterAccountId, List<String> iamRoleArns,
+ SpectrumInterceptor spectrumInterceptor, String clusterKey, String databaseName,
+ String defaultIamRoleArn, RedshiftAuditLogPublisher auditLogPublisher,
+ String databaseUser, ReentrantLock backendLock,
+ long ownershipWaitMillis) {
this.client = client;
this.backend = backend;
this.s3Service = s3Service;
@@ -145,6 +167,8 @@ public class RedshiftInterceptingBridge {
this.auditLogPublisher = auditLogPublisher;
this.databaseUser = databaseUser;
this.relationalBackend = new PostgresExtendedBackendSession(backend);
+ this.backendLock = backendLock;
+ this.ownershipWaitMillis = ownershipWaitMillis;
}
private SpectrumSession spectrumSession() {
@@ -774,11 +798,12 @@ public class RedshiftInterceptingBridge {
/**
* Take exclusive ownership of an idle backend at a wire-message boundary, run {@code exchange},
- * then release it. Returns {@code false} if the backend could not be caught idle within
- * {@link #PUMP_PARK_WAIT_MS}; the caller must then forward the original {@code 'Q'} unmodified.
+ * then release it. Returns {@code false} if the backend could not be caught idle within the
+ * ownership budget ({@link #PUMP_PARK_WAIT_MS} outside tests); the caller must then forward the
+ * original {@code 'Q'} unmodified.
*/
boolean runWithBackendOwned(BackendExchange exchange) throws IOException {
- long deadlineNanos = System.nanoTime() + PUMP_PARK_WAIT_MS * 1_000_000L;
+ long deadlineNanos = System.nanoTime() + ownershipWaitMillis * 1_000_000L;
while (true) {
if (pumpFinished) {
return false;
@@ -797,17 +822,18 @@ public class RedshiftInterceptingBridge {
}
boolean locked;
try {
- locked = backendLock.tryLock(PUMP_READ_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+ // One queued wait for the rest of the budget: a fair-lock waiter that times out and retries
+ // can miss every handoff to an idle pump, which takes the lock back as soon as it releases it.
+ long remainingNanos = Math.max(0L, deadlineNanos - System.nanoTime());
+ locked = backendLock.tryLock(remainingNanos, TimeUnit.NANOSECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException("interrupted while acquiring the backend lock", e);
}
if (!locked) {
- if (System.nanoTime() >= deadlineNanos) {
- LOG.warn("could not take the backend stream in time; forwarding the COPY unintercepted");
- return false;
- }
- continue;
+ // A timed tryLock returns false only once the whole remaining budget has elapsed.
+ LOG.warn("could not take the backend stream in time; forwarding the COPY unintercepted");
+ return false;
}
boolean backendBusy;
try {
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftBackendOwnershipTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftBackendOwnershipTest.java
new file mode 100644
index 0000000000..333b07736c
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftBackendOwnershipTest.java
@@ -0,0 +1,133 @@
+package io.github.hectorvent.floci.services.redshift.proxy;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+import java.io.IOException;
+import java.net.Socket;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class RedshiftBackendOwnershipTest {
+
+ // Taking ownership only sets the backend socket's read timeout, so unconnected sockets will do.
+ private final Socket client = new Socket();
+ private final Socket backend = new Socket();
+
+ @AfterEach
+ void closeSockets() throws IOException {
+ client.close();
+ backend.close();
+ }
+
+ /**
+ * The pump holds the backend lock for each idle read and takes it back as soon as it releases it. A
+ * waiter whose timed attempt runs out while a read is in progress resumes only after the pump has
+ * started its next read, so it must stay queued across that read to be handed the backend.
+ */
+ @Test
+ @Timeout(10)
+ void retainsItsPlaceWhileAnIdlePumpFinishesReading() throws Exception {
+ DelayedWaiterLock lock = new DelayedWaiterLock();
+ RedshiftInterceptingBridge bridge = new RedshiftInterceptingBridge(client, backend, lock,
+ RedshiftInterceptingBridge.PUMP_PARK_WAIT_MS);
+ AtomicBoolean stopped = new AtomicBoolean();
+ AtomicReference<Throwable> pumpFailure = new AtomicReference<>();
+ Thread pump = Thread.ofVirtual().start(() -> {
+ try {
+ while (!stopped.get()) {
+ lock.lockInterruptibly();
+ try {
+ lock.readStarted.release();
+ // Outlasts one 200 ms lock attempt (PUMP_READ_TIMEOUT_MS), well inside the 2 s deadline.
+ lock.endRead.tryAcquire(350, TimeUnit.MILLISECONDS);
+ } finally {
+ lock.unlock();
+ }
+ }
+ } catch (InterruptedException expected) {
+ // The test stops the pump by interrupting it.
+ Thread.currentThread().interrupt();
+ } catch (Throwable failure) {
+ pumpFailure.set(failure);
+ }
+ });
+ try {
+ assertTrue(lock.readStarted.tryAcquire(1, TimeUnit.SECONDS));
+ assertTrue(bridge.runWithBackendOwned(() -> true),
+ "an idle pump must yield the backend before the ownership deadline");
+ assertNull(pumpFailure.get());
+ } finally {
+ stopped.set(true);
+ pump.interrupt();
+ pump.join(1000);
+ assertFalse(pump.isAlive());
+ }
+ }
+
+ @Test
+ @Timeout(10)
+ void doesNotRunAnExchangeWhenTheBackendRemainsOwned() throws Exception {
+ ReentrantLock lock = new ReentrantLock(true);
+ // A short budget is enough to show the call gives up; the handoff case keeps the production one.
+ RedshiftInterceptingBridge bridge = new RedshiftInterceptingBridge(client, backend, lock, 100);
+ CountDownLatch held = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ Thread pump = Thread.ofVirtual().start(() -> {
+ lock.lock();
+ try {
+ held.countDown();
+ release.await();
+ } catch (InterruptedException expected) {
+ // Only an interrupt from outside the test could end the wait early; keep its status.
+ Thread.currentThread().interrupt();
+ } finally {
+ lock.unlock();
+ }
+ });
+ try {
+ assertTrue(held.await(1, TimeUnit.SECONDS));
+ AtomicBoolean exchanged = new AtomicBoolean();
+ assertFalse(bridge.runWithBackendOwned(() -> {
+ exchanged.set(true);
+ return true;
+ }));
+ assertFalse(exchanged.get());
+ } finally {
+ release.countDown();
+ pump.join(1000);
+ assertFalse(pump.isAlive());
+ }
+ }
+
+ private static final class DelayedWaiterLock extends ReentrantLock {
+ private final Semaphore readStarted = new Semaphore(0);
+ private final Semaphore endRead = new Semaphore(0);
+
+ private DelayedWaiterLock() {
+ super(true);
+ }
+
+ @Override
+ public boolean tryLock(long timeout, TimeUnit unit) throws InterruptedException {
+ boolean acquired = super.tryLock(timeout, unit);
+ if (!acquired) {
+ // The timed-out waiter resumes only after the pump has finished this read and started the next.
+ endRead.release();
+ if (!readStarted.tryAcquire(1, TimeUnit.SECONDS)) {
+ throw new AssertionError("pump did not resume");
+ }
+ }
+ return acquired;
+ }
+ }
+}