Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(redshift): raise postgres max_connections to 500 (#6421)

Vũ Thế Huyfac6378abd1b
diff --git a/docs/services/redshift.md b/docs/services/redshift.md
index f68e1a4a90..383213e261 100644
--- a/docs/services/redshift.md
+++ b/docs/services/redshift.md
@@ -286,7 +286,7 @@ For `AWS::Redshift::ClusterParameterGroup`, `Parameters` is applied via `ModifyC
| `FLOCI_SERVICES_REDSHIFT_ENDPOINT_HOST` | _(unset)_ | Hostname advertised in `DescribeClusters`; unset resolves from the Docker host |
| `FLOCI_SERVICES_REDSHIFT_PROXY_HANDSHAKE_TIMEOUT_MILLIS` | `10000` | Max time a client has to complete the startup/auth handshake before the proxy drops it |
| `FLOCI_SERVICES_REDSHIFT_PROXY_BACKEND_CONNECT_TIMEOUT_MILLIS` | `5000` | Max time the proxy waits for the backend TCP connect |
-| `FLOCI_SERVICES_REDSHIFT_PROXY_MAX_CONNECTIONS` | `100` | Max concurrent connections per proxy before new ones are refused |
+| `FLOCI_SERVICES_REDSHIFT_PROXY_MAX_CONNECTIONS` | `2000` | Max concurrent connections per proxy before new ones are refused |
Redshift needs the Docker socket so it can launch PostgreSQL containers. Each cluster's container is published on a dynamically assigned host port, returned by `DescribeClusters`.
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 4837631ef9..34ec80ebf6 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1729,7 +1729,7 @@ public interface EmulatorConfig {
int proxyHandshakeTimeoutMillis();
@WithDefault("5000")
int proxyBackendConnectTimeoutMillis();
- @WithDefault("100")
+ @WithDefault("2000")
int proxyMaxConnections();
@WithDefault("1000000")
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
index 3f135255fa..42995a1448 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
@@ -175,7 +175,8 @@ public class RedshiftService {
try {
LOG.infov("Recovering container for persisted cluster: {0}", cluster.getClusterIdentifier());
RedshiftContainerHandle handle = containerManager.adoptOrStart(
- entry.accountId(), cluster.getClusterIdentifier(), cluster.getMasterUsername(), password);
+ entry.accountId(), cluster.getClusterIdentifier(), cluster.getMasterUsername(), password,
+ CLUSTER_DB_NAME, cluster.getNodeType());
// A cluster persisted before the auth proxy existed has proxyPort == 0. Allocate one
// now; its endpoint changes exactly once after this upgrade. Existing clusters keep
@@ -272,7 +273,8 @@ public class RedshiftService {
int proxyPort = -1;
try {
String accountId = clusters.accountId();
- RedshiftContainerHandle handle = containerManager.start(accountId, identifier, username, password);
+ RedshiftContainerHandle handle = containerManager.start(
+ accountId, identifier, username, password, CLUSTER_DB_NAME, cluster.getNodeType());
proxyPort = allocateProxyPort();
Endpoint endpoint = proxyEndpoint(proxyPort);
cluster.setProxyPort(proxyPort);
@@ -994,7 +996,8 @@ public class RedshiftService {
String password = cluster.getMasterPassword() != null ? cluster.getMasterPassword() : "admin";
RedshiftContainerHandle handle = containerManager.start(
- accountId, clusterIdentifier, cluster.getMasterUsername(), password);
+ accountId, clusterIdentifier, cluster.getMasterUsername(), password,
+ CLUSTER_DB_NAME, cluster.getNodeType());
// Reuse the stored proxy port so the advertised endpoint is unchanged by a reboot.
if (proxyPort < 0) {
@@ -1309,7 +1312,8 @@ public class RedshiftService {
int proxyPort = -1;
try {
String accountId = clusters.accountId();
- RedshiftContainerHandle handle = containerManager.start(accountId, clusterIdentifier, username, password);
+ RedshiftContainerHandle handle = containerManager.start(
+ accountId, clusterIdentifier, username, password, CLUSTER_DB_NAME, cluster.getNodeType());
proxyPort = allocateProxyPort();
Endpoint endpoint = proxyEndpoint(proxyPort);
cluster.setProxyPort(proxyPort);
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java
index d5c1a68630..5b24fe40e8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java
@@ -35,6 +35,8 @@ public class RedshiftContainerManager {
private static final Logger LOG = Logger.getLogger(RedshiftContainerManager.class);
private static final String DEFAULT_DATABASE = "dev";
+ static final int DEFAULT_MAX_CONNECTIONS = 500;
+ static final int LARGE_CLUSTER_MAX_CONNECTIONS = 2000;
private final ContainerBuilder containerBuilder;
private final ContainerLifecycleManager lifecycleManager;
@@ -69,27 +71,34 @@ public class RedshiftContainerManager {
}
public RedshiftContainerHandle start(String accountId, String clusterIdentifier, String masterUsername, String masterPassword) {
- return start(accountId, clusterIdentifier, masterUsername, masterPassword, DEFAULT_DATABASE);
+ return start(accountId, clusterIdentifier, masterUsername, masterPassword, DEFAULT_DATABASE, null);
}
/** Starts the container with {@code databaseName} as its one database, instead of the default {@code dev}. */
public RedshiftContainerHandle start(String accountId, String clusterIdentifier, String masterUsername,
String masterPassword, String databaseName) {
+ return start(accountId, clusterIdentifier, masterUsername, masterPassword, databaseName, null);
+ }
+
+ public RedshiftContainerHandle start(String accountId, String clusterIdentifier, String masterUsername,
+ String masterPassword, String databaseName, String nodeType) {
String image = config.services().redshift().imageVersion();
String containerName = containerName(accountId, clusterIdentifier);
// Only the current name: adoptOrStart does not look for legacy-named survivors.
stop(accountId, clusterIdentifier);
+ String effectiveDb = (databaseName != null && !databaseName.isBlank()) ? databaseName : DEFAULT_DATABASE;
List<String> envVars = List.of(
"POSTGRES_USER=" + masterUsername,
"POSTGRES_PASSWORD=" + masterPassword,
- "POSTGRES_DB=" + databaseName
+ "POSTGRES_DB=" + effectiveDb
);
+ int maxConnections = maxConnections(clusterIdentifier, nodeType);
ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image)
.withName(containerName)
.withEnv(envVars)
- .withCmd(List.of("postgres", "-c", "allow_system_table_mods=on"))
+ .withCmd(List.of("postgres", "-c", "allow_system_table_mods=on", "-c", "max_connections=" + maxConnections))
.withDockerNetwork(config.services().redshift().dockerNetwork())
.withLogRotation();
@@ -103,7 +112,7 @@ public class RedshiftContainerManager {
ContainerSpec spec = specBuilder.build();
ContainerInfo info = lifecycleManager.createAndStartEphemeral(spec);
return initializeAndRegisterHandle(info, accountId, clusterIdentifier, masterUsername, containerName, enginePort,
- databaseName);
+ effectiveDb);
}
/**
@@ -117,20 +126,26 @@ public class RedshiftContainerManager {
* project's decision not to provision reusable named volumes for Redshift containers.
*/
public RedshiftContainerHandle adoptOrStart(String accountId, String clusterIdentifier, String masterUsername, String masterPassword) {
- return adoptOrStart(accountId, clusterIdentifier, masterUsername, masterPassword, DEFAULT_DATABASE);
+ return adoptOrStart(accountId, clusterIdentifier, masterUsername, masterPassword, DEFAULT_DATABASE, null);
}
/** As {@link #adoptOrStart(String, String, String, String)} for a container whose database is not {@code dev}. */
public RedshiftContainerHandle adoptOrStart(String accountId, String clusterIdentifier, String masterUsername,
String masterPassword, String databaseName) {
+ return adoptOrStart(accountId, clusterIdentifier, masterUsername, masterPassword, databaseName, null);
+ }
+
+ public RedshiftContainerHandle adoptOrStart(String accountId, String clusterIdentifier, String masterUsername,
+ String masterPassword, String databaseName, String nodeType) {
String containerName = containerName(accountId, clusterIdentifier);
Optional<Container> existing = lifecycleManager.findByName(containerName);
if (existing.isEmpty()) {
LOG.warnv("No surviving container for cluster {0}; starting a fresh empty one: the previous"
+ " contents are not recoverable (no reusable named volume backs Redshift containers)", clusterIdentifier);
- return start(accountId, clusterIdentifier, masterUsername, masterPassword, databaseName);
+ return start(accountId, clusterIdentifier, masterUsername, masterPassword, databaseName, nodeType);
}
+ String effectiveDb = (databaseName != null && !databaseName.isBlank()) ? databaseName : DEFAULT_DATABASE;
adoptedContainers.put(containerKey(accountId, clusterIdentifier), existing.get().getId());
LOG.infov("Adopting existing container {0} for cluster {1} to avoid discarding its data",
containerName, clusterIdentifier);
@@ -138,7 +153,24 @@ public class RedshiftContainerManager {
ContainerInfo info = lifecycleManager.adopt(existing.get().getId(), List.of(enginePort),
containerBuilder.resolveDockerNetwork(config.services().redshift().dockerNetwork()));
return initializeAndRegisterHandle(info, accountId, clusterIdentifier, masterUsername, containerName, enginePort,
- databaseName);
+ effectiveDb);
+ }
+
+ static int maxConnections(String clusterIdentifier, String nodeType) {
+ return maxConnectionsForNodeType(nodeType);
+ }
+
+ static int maxConnectionsForNodeType(String nodeType) {
+ if (nodeType == null || nodeType.isBlank()) {
+ return DEFAULT_MAX_CONNECTIONS;
+ }
+ String normalized = nodeType.toLowerCase().trim();
+ if (normalized.startsWith("ra3") || normalized.startsWith("rg")
+ || normalized.endsWith("8xlarge") || normalized.endsWith("16xlarge")
+ || normalized.equals("serverless")) {
+ return LARGE_CLUSTER_MAX_CONNECTIONS;
+ }
+ return DEFAULT_MAX_CONNECTIONS;
}
private RedshiftContainerHandle initializeAndRegisterHandle(
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java b/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java
index 78fa4003eb..4334572af7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java
@@ -72,8 +72,8 @@ public class RedshiftServerlessRuntime {
String backendId = backendId(region, workgroupName);
try {
RedshiftContainerHandle handle = adopt
- ? containerManager.adoptOrStart(accountId, backendId, masterUsername, masterPassword, dbName)
- : containerManager.start(accountId, backendId, masterUsername, masterPassword, dbName);
+ ? containerManager.adoptOrStart(accountId, backendId, masterUsername, masterPassword, dbName, "serverless")
+ : containerManager.start(accountId, backendId, masterUsername, masterPassword, dbName, "serverless");
proxyManager.startProxy(relayKey(accountId, backendId), endpoint.getPort(),
handle.getHost(), handle.getPort(), endpoint.getAddress(),
masterUsername, masterPassword, dbName,
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index bc68b88a7c..492685bc89 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -527,7 +527,7 @@ floci:
# endpoint-host: localhost # Hostname clients use; empty -> DockerHostResolver
proxy-handshake-timeout-millis: 10000 # Max time a client has to complete the startup/auth handshake
proxy-backend-connect-timeout-millis: 5000 # Max time to wait for the backend TCP connect
- proxy-max-connections: 100 # Max concurrent proxy connections before new ones are refused
+ proxy-max-connections: 2000 # Max concurrent proxy connections before new ones are refused
spectrum-max-rows: 1000000 # FLOCI_SERVICES_REDSHIFT_SPECTRUM_MAX_ROWS; rows loaded per external table per query
rds:
enabled: true
diff --git a/src/test/java/io/github/hectorvent/floci/config/RedshiftProxyConfigTest.java b/src/test/java/io/github/hectorvent/floci/config/RedshiftProxyConfigTest.java
index 94be59c2fa..044e46f727 100644
--- a/src/test/java/io/github/hectorvent/floci/config/RedshiftProxyConfigTest.java
+++ b/src/test/java/io/github/hectorvent/floci/config/RedshiftProxyConfigTest.java
@@ -29,6 +29,6 @@ class RedshiftProxyConfigTest {
void redshiftProxyBoundsHaveDefaults() {
assertEquals(10000, config.services().redshift().proxyHandshakeTimeoutMillis());
assertEquals(5000, config.services().redshift().proxyBackendConnectTimeoutMillis());
- assertEquals(100, config.services().redshift().proxyMaxConnections());
+ assertEquals(2000, config.services().redshift().proxyMaxConnections());
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/RedshiftSnapshotCfnIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/RedshiftSnapshotCfnIntegrationTest.java
index a90bd78ba4..89f318da64 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/RedshiftSnapshotCfnIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/RedshiftSnapshotCfnIntegrationTest.java
@@ -53,7 +53,7 @@ class RedshiftSnapshotCfnIntegrationTest {
@BeforeEach
void prepareRuntime() {
- when(containers.start(anyString(), anyString(), anyString(), anyString()))
+ when(containers.start(anyString(), anyString(), anyString(), anyString(), any(), any()))
.thenAnswer(invocation -> new RedshiftContainerHandle("mock-container",
invocation.getArgument(1), "localhost", 5432));
doAnswer(invocation -> {
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDefaultIamRoleIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDefaultIamRoleIntegrationTest.java
index 8e4ea13825..39b28d4fb5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDefaultIamRoleIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDefaultIamRoleIntegrationTest.java
@@ -19,6 +19,7 @@ import java.net.URI;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
@@ -37,7 +38,7 @@ class RedshiftDefaultIamRoleIntegrationTest {
String clusterId = "sdk-default-role";
String first = "arn:aws:iam::000000000000:role/first";
String second = "arn:aws:iam::000000000000:role/second";
- when(containerManager.start(anyString(), anyString(), anyString(), anyString()))
+ when(containerManager.start(anyString(), anyString(), anyString(), anyString(), any(), any()))
.thenReturn(new RedshiftContainerHandle("mock-container", clusterId, "localhost", 5432));
try (RedshiftClient client = RedshiftClient.builder()
.endpointOverride(endpoint)
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotIntegrationTest.java
index 36f0998617..a5c7a54ab0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotIntegrationTest.java
@@ -47,7 +47,7 @@ class RedshiftFinalSnapshotIntegrationTest {
@BeforeEach
void prepareRuntime() {
- when(containerManager.start(anyString(), anyString(), anyString(), anyString()))
+ when(containerManager.start(anyString(), anyString(), anyString(), anyString(), any(), any()))
.thenReturn(new RedshiftContainerHandle("mock-container", "cluster", "localhost", 5432));
doAnswer(invocation -> {
Files.writeString(invocation.getArgument(3, Path.class), DUMP);
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotStorageTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotStorageTest.java
index 1d87060be0..32cc6235d5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotStorageTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftFinalSnapshotStorageTest.java
@@ -108,7 +108,7 @@ class RedshiftFinalSnapshotStorageTest {
when(access.storageMode("redshift")).thenReturn("persistent");
StorageFactory factory = new StorageFactory(config, access);
RedshiftContainerManager manager = mock(RedshiftContainerManager.class);
- when(manager.start(anyString(), anyString(), anyString(), anyString()))
+ when(manager.start(anyString(), anyString(), anyString(), anyString(), any(), any()))
.thenReturn(new RedshiftContainerHandle("container", "restored", "localhost", 5432));
doAnswer(invocation -> {
Files.writeString(invocation.getArgument(3, Path.class), "backup contents");
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java
index cbb4a3bcb4..2c40312aba 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java
@@ -167,14 +167,14 @@ public class RedshiftOperationsTest {
@Test
@Order(2)
void clusterAndSnapshotLifecycle() {
- when(containerManager.start(any(), eq("cluster-src"), any(), any()))
+ when(containerManager.start(any(), eq("cluster-src"), any(), any(), any(), any()))
.thenReturn(new RedshiftContainerHandle("c1", "cluster-src", "localhost", 5439));
doAnswer(invocation -> {
Path p = invocation.getArgument(3);
Files.writeString(p, "-- dump sql table test_data;");
return null;
}).when(containerManager).takeSnapshot(any(), eq("cluster-src"), eq("admin"), any(Path.class));
- when(containerManager.start(any(), eq("cluster-restored"), any(), any()))
+ when(containerManager.start(any(), eq("cluster-restored"), any(), any(), any(), any()))
.thenReturn(new RedshiftContainerHandle("c2", "cluster-restored", "localhost", 5440));
// 1. CreateCluster
@@ -842,7 +842,7 @@ public class RedshiftOperationsTest {
@Test
@Order(3)
void tagLifecycle() {
- when(containerManager.start(any(), eq("cluster-tags"), any(), any()))
+ when(containerManager.start(any(), eq("cluster-tags"), any(), any(), any(), any()))
.thenReturn(new RedshiftContainerHandle("c3", "cluster-tags", "localhost", 5441));
given()
@@ -1019,7 +1019,7 @@ public class RedshiftOperationsTest {
@Test
@Order(9)
void snapshotCopyLifecycle() {
- when(containerManager.start(any(), eq("cluster-copy"), any(), any()))
+ when(containerManager.start(any(), eq("cluster-copy"), any(), any(), any(), any()))
.thenReturn(new RedshiftContainerHandle("c9", "cluster-copy", "localhost", 5449));
assertEquals(200, redshift("CreateCluster", "ClusterIdentifier", "cluster-copy", "NodeType", "dc2.large",
"MasterUsername", "admin", "MasterUserPassword", "password123").statusCode());
@@ -1098,7 +1098,7 @@ public class RedshiftOperationsTest {
@Test
@Order(10)
void snapshotScheduleLifecycle() {
- when(containerManager.start(any(), eq("cluster-sched"), any(), any()))
+ when(containerManager.start(any(), eq("cluster-sched"), any(), any(), any(), any()))
.thenReturn(new RedshiftContainerHandle("c10", "cluster-sched", "localhost", 5450));
assertEquals(200, redshift("CreateCluster", "ClusterIdentifier", "cluster-sched", "NodeType", "dc2.large",
"MasterUsername", "admin", "MasterUserPassword", "password123").statusCode());
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
index 971f0742e6..b59d06864d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
@@ -484,6 +484,15 @@ class RedshiftServiceTest {
verify(clusterBackend, atLeastOnce()).flush();
}
+ @Test
+ void createClusterPassesNodeTypeToContainerManager() {
+ when(cm.start("111111111111", "c-ra3", "admin", "pw"))
+ .thenReturn(new RedshiftContainerHandle("c-ra3", "c-ra3", "localhost", 5432));
+ service.createCluster("c-ra3", "ra3.xlplus", "admin", "pw",
+ null, List.of(), List.of(), null);
+ verify(cm).start("111111111111", "c-ra3", "admin", "pw", "dev", "ra3.xlplus");
+ }
+
private StorageFactory sf;
private AccountAwareStorageBackend<Cluster> clusterBackend;
private AccountAwareStorageBackend<Snapshot> snapshotBackend;
@@ -515,6 +524,10 @@ class RedshiftServiceTest {
snapshotScheduleBackend = mock(AccountAwareStorageBackend.class);
integrationBackend = mock(AccountAwareStorageBackend.class);
cm = mock(RedshiftContainerManager.class);
+ when(cm.start(any(), any(), any(), any(), any(), any())).thenAnswer(inv ->
+ cm.start(inv.getArgument(0), inv.getArgument(1), inv.getArgument(2), inv.getArgument(3)));
+ when(cm.adoptOrStart(any(), any(), any(), any(), any(), any())).thenAnswer(inv ->
+ cm.adoptOrStart(inv.getArgument(0), inv.getArgument(1), inv.getArgument(2), inv.getArgument(3)));
proxyManager = mock(RedshiftProxyManager.class);
dockerHostResolver = mock(DockerHostResolver.class);
when(dockerHostResolver.resolve()).thenReturn("localhost");
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManagerTest.java
index 908bc2fbaf..092ee6acc6 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManagerTest.java
@@ -35,6 +35,7 @@ import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -384,6 +385,61 @@ class RedshiftContainerManagerTest {
verify(lifecycleManager, never()).removeIfExistsStrict("floci-redshift-" + ACCOUNT_ID + "-test-cluster", true);
}
+ @Test
+ void startConfiguresMaxConnectionsByNodeTypeAndServerless() {
+ ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
+ when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
+ ContainerInfo info = new ContainerInfo("cont-123", Map.of(5432, new EndpointInfo("localhost", 5432)));
+ when(lifecycleManager.createAndStartEphemeral(any())).thenReturn(info);
+
+ manager.start(ACCOUNT_ID, "test-cluster", "admin", "pass");
+ verify(specBuilder).withCmd(List.of("postgres", "-c", "allow_system_table_mods=on", "-c", "max_connections=500"));
+ clearInvocations(specBuilder);
+
+ manager.start(ACCOUNT_ID, "dc2-cluster", "admin", "pass", "dev", "dc2.large");
+ verify(specBuilder).withCmd(List.of("postgres", "-c", "allow_system_table_mods=on", "-c", "max_connections=500"));
+ clearInvocations(specBuilder);
+
+ manager.start(ACCOUNT_ID, "dc2-8x-cluster", "admin", "pass", "dev", "dc2.8xlarge");
+ verify(specBuilder).withCmd(List.of("postgres", "-c", "allow_system_table_mods=on", "-c", "max_connections=2000"));
+ clearInvocations(specBuilder);
+
+ manager.start(ACCOUNT_ID, "ra3-cluster", "admin", "pass", "dev", "ra3.xlplus");
+ verify(specBuilder).withCmd(List.of("postgres", "-c", "allow_system_table_mods=on", "-c", "max_connections=2000"));
+ clearInvocations(specBuilder);
+
+ manager.start(ACCOUNT_ID, "ra3-4x-cluster", "admin", "pass", "dev", "ra3.4xlplus");
+ verify(specBuilder).withCmd(List.of("postgres", "-c", "allow_system_table_mods=on", "-c", "max_connections=2000"));
+ clearInvocations(specBuilder);
+
+ manager.start(ACCOUNT_ID, "ra3-16x-cluster", "admin", "pass", "dev", "ra3.16xlarge");
+ verify(specBuilder).withCmd(List.of("postgres", "-c", "allow_system_table_mods=on", "-c", "max_connections=2000"));
+ clearInvocations(specBuilder);
+
+ manager.start(ACCOUNT_ID, "rg-cluster", "admin", "pass", "dev", "rg.xlarge");
+ verify(specBuilder).withCmd(List.of("postgres", "-c", "allow_system_table_mods=on", "-c", "max_connections=2000"));
+ clearInvocations(specBuilder);
+
+ manager.start(ACCOUNT_ID, "serverless_us-east-1_analytics-wg", "admin", "pass", "analytics", "serverless");
+ verify(specBuilder).withCmd(List.of("postgres", "-c", "allow_system_table_mods=on", "-c", "max_connections=2000"));
+ }
+
+ @Test
+ void maxConnectionsResolvesCorrectLimits() {
+ assertEquals(500, RedshiftContainerManager.maxConnections("my-cluster", null));
+ assertEquals(500, RedshiftContainerManager.maxConnections("my-cluster", ""));
+ assertEquals(500, RedshiftContainerManager.maxConnections("my-cluster", "dc2.large"));
+ assertEquals(500, RedshiftContainerManager.maxConnections("my-cluster", "ds2.xlarge"));
+ assertEquals(2000, RedshiftContainerManager.maxConnections("my-cluster", "dc2.8xlarge"));
+ assertEquals(2000, RedshiftContainerManager.maxConnections("my-cluster", "ra3.xlplus"));
+ assertEquals(2000, RedshiftContainerManager.maxConnections("my-cluster", "ra3.4xlplus"));
+ assertEquals(2000, RedshiftContainerManager.maxConnections("my-cluster", "ra3.4xlarge"));
+ assertEquals(2000, RedshiftContainerManager.maxConnections("my-cluster", "ra3.16xlarge"));
+ assertEquals(2000, RedshiftContainerManager.maxConnections("my-cluster", "rg.xlarge"));
+ assertEquals(2000, RedshiftContainerManager.maxConnections("my-cluster", "rg2.xlarge"));
+ assertEquals(2000, RedshiftContainerManager.maxConnections("my-cluster", "serverless"));
+ }
+
@Test
void sameClusterIdentifierAcrossAccountsDoesNotCollide() {
ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, org.mockito.Mockito.RETURNS_SELF);
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java
index 304f5def0d..a9a76020fd 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java
@@ -15,6 +15,7 @@ import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Semaphore;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -45,7 +46,7 @@ class RedshiftProxyManagerTest {
EmulatorConfig.RedshiftServiceConfig redshiftConfig = mock(EmulatorConfig.RedshiftServiceConfig.class);
when(redshiftConfig.proxyHandshakeTimeoutMillis()).thenReturn(5000);
when(redshiftConfig.proxyBackendConnectTimeoutMillis()).thenReturn(5000);
- when(redshiftConfig.proxyMaxConnections()).thenReturn(100);
+ when(redshiftConfig.proxyMaxConnections()).thenReturn(2000);
EmulatorConfig.ServicesConfig servicesConfig = mock(EmulatorConfig.ServicesConfig.class);
when(servicesConfig.redshift()).thenReturn(redshiftConfig);
EmulatorConfig config = mock(EmulatorConfig.class);
@@ -71,6 +72,22 @@ class RedshiftProxyManagerTest {
}
}
+ @Test
+ void startConfiguresProxyMaxConnections() throws Exception {
+ RedshiftProxyManager manager = newManager();
+ int port = FreePorts.anyInterfacePort();
+ try {
+ start(manager, KEY, port);
+ RedshiftAuthProxy proxy = registry(manager).get(KEY);
+ Field permitsField = RedshiftAuthProxy.class.getDeclaredField("connectionPermits");
+ permitsField.setAccessible(true);
+ Semaphore permits = (Semaphore) permitsField.get(proxy);
+ assertEquals(2000, permits.availablePermits());
+ } finally {
+ manager.stopAll();
+ }
+ }
+
@Test
void stopProxyRemovesTheKeyAndReleasesThePort() throws Exception {
RedshiftProxyManager manager = newManager();
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntimeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntimeTest.java
index 80f6a8592f..b6ededf487 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntimeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntimeTest.java
@@ -49,7 +49,7 @@ class RedshiftServerlessRuntimeTest {
}
private PasswordValidator startAndCaptureValidator(RedshiftServerlessRuntime.AdminCredentials admin) {
- when(containers.start(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics")).thenReturn(handle());
+ when(containers.start(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics", "serverless")).thenReturn(handle());
runtime.start(ACCOUNT, REGION, "my-wg", "root", "Secret123", "analytics", ENDPOINT, false,
() -> Optional.ofNullable(admin));
ArgumentCaptor<PasswordValidator> validator = ArgumentCaptor.forClass(PasswordValidator.class);
@@ -60,17 +60,17 @@ class RedshiftServerlessRuntimeTest {
@Test
void startRunsTheContainerThenFrontsItWithTheProxyOnTheAllocatedEndpoint() {
- when(containers.start(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics")).thenReturn(handle());
+ when(containers.start(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics", "serverless")).thenReturn(handle());
RedshiftServerlessRuntime.Backend backend = runtime.start(ACCOUNT, REGION, "my-wg", "root", "Secret123",
"analytics", ENDPOINT, false, Optional::empty);
assertEquals(new RedshiftServerlessRuntime.Backend("172.17.0.5", 5432), backend);
InOrder order = inOrder(containers, proxies);
- order.verify(containers).start(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics");
+ order.verify(containers).start(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics", "serverless");
order.verify(proxies).startProxy(eq(RELAY_KEY), eq(7100), eq("172.17.0.5"), eq(5432), eq("localhost"),
eq("root"), eq("Secret123"), eq("analytics"), any(PasswordValidator.class), eq(List.of()));
- verify(containers, never()).adoptOrStart(any(), any(), any(), any(), any());
+ verify(containers, never()).adoptOrStart(any(), any(), any(), any(), any(), any());
}
@Test
@@ -110,7 +110,7 @@ class RedshiftServerlessRuntimeTest {
@Test
void failedAdoptedProxyUsesRecoveryRollback() {
- when(containers.adoptOrStart(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics")).thenReturn(handle());
+ when(containers.adoptOrStart(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics", "serverless")).thenReturn(handle());
doThrow(new RuntimeException("port in use")).when(proxies).startProxy(any(), anyInt(), any(), anyInt(),
any(), any(), any(), any(), any(PasswordValidator.class), any());
@@ -124,18 +124,18 @@ class RedshiftServerlessRuntimeTest {
@Test
void adoptReattachesToASurvivingContainer() {
- when(containers.adoptOrStart(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics")).thenReturn(handle());
+ when(containers.adoptOrStart(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics", "serverless")).thenReturn(handle());
runtime.start(ACCOUNT, REGION, "my-wg", "root", "Secret123", "analytics", ENDPOINT, true, Optional::empty);
- verify(containers, never()).start(any(), any(), any(), any(), any());
+ verify(containers, never()).start(any(), any(), any(), any(), any(), any());
verify(proxies).startProxy(eq(RELAY_KEY), eq(7100), any(), anyInt(), any(), any(), any(), any(),
any(PasswordValidator.class), any());
}
@Test
void aProxyThatFailsToStartRollsBackTheContainerAndCredentials() {
- when(containers.start(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics")).thenReturn(handle());
+ when(containers.start(ACCOUNT, BACKEND_ID, "root", "Secret123", "analytics", "serverless")).thenReturn(handle());
doThrow(new RuntimeException("port in use")).when(proxies).startProxy(any(), anyInt(), any(), anyInt(),
any(), any(), any(), any(), any(PasswordValidator.class), any());
broker.issue(ACCOUNT, BACKEND_ID, "IAM:alice", List.of(), 900);
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index 3efa98900c..642bc49535 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -216,7 +216,7 @@ floci:
default-credential-duration-seconds: 900
proxy-handshake-timeout-millis: 10000
proxy-backend-connect-timeout-millis: 5000
- proxy-max-connections: 100
+ proxy-max-connections: 2000
spectrum-max-rows: 1000000
rds:
enabled: true