Light, fluffy, and always free - The AWS Local Emulator alternative
fix(redshift): stay queued for backend ownership until the deadline (#6634)
Jackson Price8b13c75fa265
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.javaindex b22fbc5335..6174373b07 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java@@ -56,7 +56,7 @@ public class RedshiftInterceptingBridge {private static final int PUMP_READ_TIMEOUT_MS = 200;private static final int CLIENT_READ_TIMEOUT_MS = 10_000;- private static final long PUMP_PARK_WAIT_MS = 2_000L;+ static final long PUMP_PARK_WAIT_MS = 2_000L;/** Bounded read timeout while an exchange owns the backend, so a stalled backend cannot hang the session. */private static final int EXCHANGE_READ_TIMEOUT_MS = 60_000;private static final long BUSY_BACKOFF_NANOS = 2_000_000L;@@ -82,7 +82,9 @@ public class RedshiftInterceptingBridge {private final Map<String, ExtendedSpectrumExchange.PortalCursor> spectrumCursors = new HashMap<>();private final BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);- private final ReentrantLock backendLock = new ReentrantLock(true);+ private final ReentrantLock backendLock;+ /** Budget of {@link #runWithBackendOwned} only; the Extended Query paths keep {@link #PUMP_PARK_WAIT_MS}. */+ private final long ownershipWaitMillis;private volatile boolean pumpBetweenMessages = true;private volatile boolean pumpFinished = false;private boolean extendedSpectrumError;@@ -132,6 +134,26 @@ public class RedshiftInterceptingBridge {SpectrumInterceptor spectrumInterceptor, String clusterKey, String databaseName,String defaultIamRoleArn, RedshiftAuditLogPublisher auditLogPublisher,String databaseUser) {+ this(client, backend, s3Service, iamService, clusterAccountId, iamRoleArns, spectrumInterceptor,+ clusterKey, databaseName, defaultIamRoleArn, auditLogPublisher, databaseUser, new ReentrantLock(true),+ PUMP_PARK_WAIT_MS);+ }++ /**+ * For tests: a bridge with no services, whose fair backend lock and {@link #runWithBackendOwned}+ * budget the caller supplies.+ */+ RedshiftInterceptingBridge(Socket client, Socket backend, ReentrantLock backendLock, long ownershipWaitMillis) {+ this(client, backend, null, null, null, List.of(), null, null, null, null, null, null, backendLock,+ ownershipWaitMillis);+ }++ private RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService,+ String clusterAccountId, List<String> iamRoleArns,+ SpectrumInterceptor spectrumInterceptor, String clusterKey, String databaseName,+ String defaultIamRoleArn, RedshiftAuditLogPublisher auditLogPublisher,+ String databaseUser, ReentrantLock backendLock,+ long ownershipWaitMillis) {this.client = client;this.backend = backend;this.s3Service = s3Service;@@ -145,6 +167,8 @@ public class RedshiftInterceptingBridge {this.auditLogPublisher = auditLogPublisher;this.databaseUser = databaseUser;this.relationalBackend = new PostgresExtendedBackendSession(backend);+ this.backendLock = backendLock;+ this.ownershipWaitMillis = ownershipWaitMillis;}private SpectrumSession spectrumSession() {@@ -774,11 +798,12 @@ public class RedshiftInterceptingBridge {/*** Take exclusive ownership of an idle backend at a wire-message boundary, run {@code exchange},- * then release it. Returns {@code false} if the backend could not be caught idle within- * {@link #PUMP_PARK_WAIT_MS}; the caller must then forward the original {@code 'Q'} unmodified.+ * then release it. Returns {@code false} if the backend could not be caught idle within the+ * ownership budget ({@link #PUMP_PARK_WAIT_MS} outside tests); the caller must then forward the+ * original {@code 'Q'} unmodified.*/boolean runWithBackendOwned(BackendExchange exchange) throws IOException {- long deadlineNanos = System.nanoTime() + PUMP_PARK_WAIT_MS * 1_000_000L;+ long deadlineNanos = System.nanoTime() + ownershipWaitMillis * 1_000_000L;while (true) {if (pumpFinished) {return false;@@ -797,17 +822,18 @@ public class RedshiftInterceptingBridge {}boolean locked;try {- locked = backendLock.tryLock(PUMP_READ_TIMEOUT_MS, TimeUnit.MILLISECONDS);+ // One queued wait for the rest of the budget: a fair-lock waiter that times out and retries+ // can miss every handoff to an idle pump, which takes the lock back as soon as it releases it.+ long remainingNanos = Math.max(0L, deadlineNanos - System.nanoTime());+ locked = backendLock.tryLock(remainingNanos, TimeUnit.NANOSECONDS);} catch (InterruptedException e) {Thread.currentThread().interrupt();throw new IOException("interrupted while acquiring the backend lock", e);}if (!locked) {- if (System.nanoTime() >= deadlineNanos) {- LOG.warn("could not take the backend stream in time; forwarding the COPY unintercepted");- return false;- }- continue;+ // A timed tryLock returns false only once the whole remaining budget has elapsed.+ LOG.warn("could not take the backend stream in time; forwarding the COPY unintercepted");+ return false;}boolean backendBusy;try {diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftBackendOwnershipTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftBackendOwnershipTest.javanew file mode 100644index 0000000000..333b07736c--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftBackendOwnershipTest.java@@ -0,0 +1,133 @@+package io.github.hectorvent.floci.services.redshift.proxy;++import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;++import java.io.IOException;+import java.net.Socket;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.Semaphore;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicReference;+import java.util.concurrent.locks.ReentrantLock;++import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++class RedshiftBackendOwnershipTest {++ // Taking ownership only sets the backend socket's read timeout, so unconnected sockets will do.+ private final Socket client = new Socket();+ private final Socket backend = new Socket();++ @AfterEach+ void closeSockets() throws IOException {+ client.close();+ backend.close();+ }++ /**+ * The pump holds the backend lock for each idle read and takes it back as soon as it releases it. A+ * waiter whose timed attempt runs out while a read is in progress resumes only after the pump has+ * started its next read, so it must stay queued across that read to be handed the backend.+ */+ @Test+ @Timeout(10)+ void retainsItsPlaceWhileAnIdlePumpFinishesReading() throws Exception {+ DelayedWaiterLock lock = new DelayedWaiterLock();+ RedshiftInterceptingBridge bridge = new RedshiftInterceptingBridge(client, backend, lock,+ RedshiftInterceptingBridge.PUMP_PARK_WAIT_MS);+ AtomicBoolean stopped = new AtomicBoolean();+ AtomicReference<Throwable> pumpFailure = new AtomicReference<>();+ Thread pump = Thread.ofVirtual().start(() -> {+ try {+ while (!stopped.get()) {+ lock.lockInterruptibly();+ try {+ lock.readStarted.release();+ // Outlasts one 200 ms lock attempt (PUMP_READ_TIMEOUT_MS), well inside the 2 s deadline.+ lock.endRead.tryAcquire(350, TimeUnit.MILLISECONDS);+ } finally {+ lock.unlock();+ }+ }+ } catch (InterruptedException expected) {+ // The test stops the pump by interrupting it.+ Thread.currentThread().interrupt();+ } catch (Throwable failure) {+ pumpFailure.set(failure);+ }+ });+ try {+ assertTrue(lock.readStarted.tryAcquire(1, TimeUnit.SECONDS));+ assertTrue(bridge.runWithBackendOwned(() -> true),+ "an idle pump must yield the backend before the ownership deadline");+ assertNull(pumpFailure.get());+ } finally {+ stopped.set(true);+ pump.interrupt();+ pump.join(1000);+ assertFalse(pump.isAlive());+ }+ }++ @Test+ @Timeout(10)+ void doesNotRunAnExchangeWhenTheBackendRemainsOwned() throws Exception {+ ReentrantLock lock = new ReentrantLock(true);+ // A short budget is enough to show the call gives up; the handoff case keeps the production one.+ RedshiftInterceptingBridge bridge = new RedshiftInterceptingBridge(client, backend, lock, 100);+ CountDownLatch held = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ Thread pump = Thread.ofVirtual().start(() -> {+ lock.lock();+ try {+ held.countDown();+ release.await();+ } catch (InterruptedException expected) {+ // Only an interrupt from outside the test could end the wait early; keep its status.+ Thread.currentThread().interrupt();+ } finally {+ lock.unlock();+ }+ });+ try {+ assertTrue(held.await(1, TimeUnit.SECONDS));+ AtomicBoolean exchanged = new AtomicBoolean();+ assertFalse(bridge.runWithBackendOwned(() -> {+ exchanged.set(true);+ return true;+ }));+ assertFalse(exchanged.get());+ } finally {+ release.countDown();+ pump.join(1000);+ assertFalse(pump.isAlive());+ }+ }++ private static final class DelayedWaiterLock extends ReentrantLock {+ private final Semaphore readStarted = new Semaphore(0);+ private final Semaphore endRead = new Semaphore(0);++ private DelayedWaiterLock() {+ super(true);+ }++ @Override+ public boolean tryLock(long timeout, TimeUnit unit) throws InterruptedException {+ boolean acquired = super.tryLock(timeout, unit);+ if (!acquired) {+ // The timed-out waiter resumes only after the pump has finished this read and started the next.+ endRead.release();+ if (!readStarted.tryAcquire(1, TimeUnit.SECONDS)) {+ throw new AssertionError("pump did not resume");+ }+ }+ return acquired;+ }+ }+}