Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dsql): refuse a startup packet or client message outside DSQL's limits (#6258)
Alessandro Lattao564ae350a1a0
diff --git a/docs/services/dsql.md b/docs/services/dsql.mdindex c3ac80b7f7..beb8cf13c3 100644--- a/docs/services/dsql.md+++ b/docs/services/dsql.md@@ -47,6 +47,12 @@ TLS is mandatory. The certificate covers `*.dsql.<region>.localhost.floci.io` fois signed by Floci's local CA (`GET /_floci/ca.pem`, see [TLS](../configuration/tls.md)), which thecontainers Floci starts already trust, so `sslmode=verify-full` works.+The limits are DSQL's: a startup packet of 10,000 bytes or more, sent over TLS, ends the connection+with `FATAL 08P01 invalid length of startup packet`, and a client message over 10 MiB with+`FATAL 08P01 invalid message length`, as soon as its length arrives. A startup packet sent before+TLS is refused with `FATAL 08P01 unable to accept connection, SSL is mandatory. AWS Authentication+is required.`, whatever its length.+The password is an IAM authentication token for the endpoint, generated by the SDKs and the CLIexactly as for AWS:diff --git a/src/main/java/io/github/hectorvent/floci/core/common/PostgresWire.java b/src/main/java/io/github/hectorvent/floci/core/common/PostgresWire.javaindex 4ebb67ba73..631cd87bf2 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/PostgresWire.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/PostgresWire.java@@ -36,6 +36,16 @@ public final class PostgresWire {public record CString(String value, int next) {}+ /**+ * A packet whose length is outside what its reader takes. Nothing after the length was read, so+ * a server can still answer the way PostgreSQL does before it closes the connection.+ */+ public static final class InvalidLengthException extends IOException {+ InvalidLengthException(String message) {+ super(message);+ }+ }+// ── Framing ───────────────────────────────────────────────────────────────/** Reads a startup-phase packet: its length, refused outside 8 to {@code maxLength}, code and body. */@@ -89,14 +99,14 @@ public final class PostgresWire {return out.toByteArray();}- /** {@code length}, refused when below {@code minimum} or above {@code maximum}. */+ /** {@code length}, refused with an {@link InvalidLengthException} when below {@code minimum} or above {@code maximum}. */public static int checkedLength(int length, int minimum, int maximum, String packetName) throws IOException {if (length < minimum) {- throw new IOException("PostgreSQL " + packetName + " length is below the "+ throw new InvalidLengthException("PostgreSQL " + packetName + " length is below the "+ minimum + " byte minimum: " + length);}if (length > maximum) {- throw new IOException("PostgreSQL " + packetName + " length exceeds the "+ throw new InvalidLengthException("PostgreSQL " + packetName + " length exceeds the "+ maximum + " byte limit: " + length);}return length;diff --git a/src/main/java/io/github/hectorvent/floci/services/dsql/dataplane/DsqlSession.java b/src/main/java/io/github/hectorvent/floci/services/dsql/dataplane/DsqlSession.javaindex a503ff1ab2..40a0e11265 100644--- a/src/main/java/io/github/hectorvent/floci/services/dsql/dataplane/DsqlSession.java+++ b/src/main/java/io/github/hectorvent/floci/services/dsql/dataplane/DsqlSession.java@@ -43,9 +43,14 @@ import java.util.stream.Stream;final class DsqlSession {static final Duration CONNECTION_LIFETIME = Duration.ofHours(1);- // A startup packet is refused above PostgreSQL's own limit, 10,000 bytes, and a message above 64 MB.- private static final int MAX_STARTUP = 10_000;- private static final int MAX_MESSAGE = 64 * 1024 * 1024;+ // DSQL's (measured): a startup packet of 10,000 bytes or more, and a client message over 10 MiB,+ // are refused with a FATAL before the connection closes. What the database sends is not DSQL's+ // limit to apply, and is read up to 64 MB.+ private static final int MAX_STARTUP = 9_999;+ private static final int MAX_CLIENT_MESSAGE = 10 * 1024 * 1024;+ private static final int MAX_DATABASE_MESSAGE = 64 * 1024 * 1024;+ private static final String INVALID_STARTUP_LENGTH = "invalid length of startup packet";+ private static final String INVALID_MESSAGE_LENGTH = "invalid message length";private static final String SERIALIZATION_FAILURE = "40001";private static final String QUERY_CANCELED = "57014";private static final String ACCESS_DENIED = "unable to accept connection, access denied";@@ -103,7 +108,7 @@ final class DsqlSession {socket.setSoTimeout(endpoint.handshakeTimeoutMillis());InputStream rawIn = socket.getInputStream();OutputStream rawOut = socket.getOutputStream();- PostgresWire.Startup first = PostgresWire.readStartup(rawIn, MAX_STARTUP, "startup message");+ PostgresWire.Startup first = readPlaintextPacket(rawIn);if (first.code() == PostgresWire.CANCEL_REQUEST) {endpoint.forwardCancel(first.body());return;@@ -111,7 +116,7 @@ final class DsqlSession {if (first.code() == PostgresWire.GSSENC_REQUEST) {rawOut.write('N');rawOut.flush();- first = PostgresWire.readStartup(rawIn, MAX_STARTUP, "startup message");+ first = readPlaintextPacket(rawIn);}if (first.code() != PostgresWire.SSL_REQUEST) {refuse(rawOut, "08P01", "unable to accept connection, SSL is mandatory. AWS Authentication is required.",@@ -139,7 +144,13 @@ final class DsqlSession {InputStream in = new BufferedInputStream(tls.getInputStream());OutputStream out = new BufferedOutputStream(tls.getOutputStream());- PostgresWire.Startup startup = PostgresWire.readStartup(in, MAX_STARTUP, "startup message");+ PostgresWire.Startup startup;+ try {+ startup = PostgresWire.readStartup(in, MAX_STARTUP, "startup message");+ } catch (PostgresWire.InvalidLengthException e) {+ refuse(out, "08P01", INVALID_STARTUP_LENGTH, null);+ return;+ }if (startup.code() == PostgresWire.CANCEL_REQUEST) {endpoint.forwardCancel(startup.body());return;@@ -154,7 +165,13 @@ final class DsqlSession {PostgresWire.write(out, (byte) 'R', PostgresWire.int32(3));out.flush();- PostgresWire.Message password = PostgresWire.read(in, MAX_MESSAGE, "message");+ PostgresWire.Message password;+ try {+ password = PostgresWire.read(in, MAX_CLIENT_MESSAGE, "message");+ } catch (PostgresWire.InvalidLengthException e) {+ refuse(out, "08P01", INVALID_MESSAGE_LENGTH, null);+ return;+ }if (password.type() != 'p') {refuse(out, "08006", ACCESS_DENIED, null);return;@@ -228,7 +245,13 @@ final class DsqlSession {private void relayToBackend(InputStream in, OutputStream backendOut) throws IOException {while (true) {- PostgresWire.Message message = PostgresWire.read(in, MAX_MESSAGE, "message");+ PostgresWire.Message message;+ try {+ message = PostgresWire.read(in, MAX_CLIENT_MESSAGE, "message");+ } catch (PostgresWire.InvalidLengthException e) {+ close("FATAL", "08P01", INVALID_MESSAGE_LENGTH, "Session Id: " + sessionId);+ return;+ }if (message.type() != 'X') {lastClientMessage = now();if (transactionStatus == 'T' && pastAgeLimit()) {@@ -579,7 +602,7 @@ final class DsqlSession {List<PostgresWire.Message> startup = new ArrayList<>();boolean started = false;while (true) {- PostgresWire.Message message = PostgresWire.read(backendIn, MAX_MESSAGE, "message");+ PostgresWire.Message message = PostgresWire.read(backendIn, MAX_DATABASE_MESSAGE, "message");if (!started) {// DSQL refuses a connection instead of authenticating it, so what the database// answers the startup waits for its ReadyForQuery, and a refusal takes its place.@@ -749,6 +772,23 @@ final class DsqlSession {return PostgresWire.fields(fields);}+ /**+ * A packet sent before TLS. DSQL refuses one that is no SSLRequest, GSSENCRequest or+ * CancelRequest as not TLS whatever length it claims, and ends the connection without a word on+ * one under 8 bytes (measured). So a length past what a startup packet may hold is refused with+ * its body left unread; a shorter body is read, so the refusal reaches the client in order.+ */+ private static PostgresWire.Startup readPlaintextPacket(InputStream in) throws IOException {+ int length = PostgresWire.checkedLength(PostgresWire.readInt32(in), 8, Integer.MAX_VALUE, "startup message");+ int code = PostgresWire.readInt32(in);+ if (length > MAX_STARTUP) {+ return new PostgresWire.Startup(code, new byte[0]);+ }+ byte[] body = new byte[length - 8];+ PostgresWire.readFully(in, body);+ return new PostgresWire.Startup(code, body);+ }+/** An ErrorResponse body of {@code severity} (FATAL, ERROR), as DSQL orders its fields. */static byte[] error(String severity, String state, String message, String detail, String hint) {Map<Character, String> fields = new LinkedHashMap<>();diff --git a/src/test/java/io/github/hectorvent/floci/services/dsql/DsqlDataPlaneIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dsql/DsqlDataPlaneIntegrationTest.javaindex 61ae7780e7..be2ada1500 100644--- a/src/test/java/io/github/hectorvent/floci/services/dsql/DsqlDataPlaneIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dsql/DsqlDataPlaneIntegrationTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.dsql;import com.github.dockerjava.api.DockerClient;+import io.github.hectorvent.floci.core.common.PostgresWire;import io.github.hectorvent.floci.services.dsql.dataplane.DsqlEndpoint;import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.testing.EnforcementFixtures;@@ -18,13 +19,19 @@ import org.junit.jupiter.api.extension.RegisterExtension;import org.postgresql.util.PSQLException;import org.postgresql.util.PSQLWarning;+import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.Socket;import java.sql.Connection;import java.sql.DriverManager;import java.sql.ResultSet;import java.sql.SQLException;import java.sql.Statement;import java.time.Instant;+import java.util.LinkedHashMap;import java.util.List;+import java.util.Map;import java.util.Properties;import java.util.concurrent.CompletableFuture;import java.util.concurrent.TimeUnit;@@ -137,6 +144,85 @@ class DsqlDataPlaneIntegrationTest {assertEquals("55000", otherDatabase.getSQLState());}+ /**+ * DSQL's limits (measured): a startup packet of 10,000 bytes or more, or under 8, and a client+ * message over 10 MiB or under 4, are refused as soon as their length arrives, before signing in+ * and after, and the connection closes.+ */+ @Test+ void aPacketOutsideDsqlsLimitsIsRefusedAsItsLengthArrives() throws Exception {+ try (DsqlWireClient client = DsqlWireClient.tls(endpoint.port(), cluster.host())) {+ client.send(startupOfLength(9_999));+ PostgresWire.Message password = client.next();+ assertEquals('R', password.type());+ assertEquals(3, PostgresWire.int32(password.body(), 0));+ }+ try (DsqlWireClient client = DsqlWireClient.tls(endpoint.port(), cluster.host())) {+ client.send(startupOfLength(10_000));+ assertRefused(client, "invalid length of startup packet");+ }+ try (DsqlWireClient client = DsqlWireClient.tls(endpoint.port(), cluster.host())) {+ client.send(PostgresWire.int32(7), new byte[3]);+ assertRefused(client, "invalid length of startup packet");+ }+ try (DsqlWireClient client = DsqlWireClient.tls(endpoint.port(), cluster.host())) {+ client.send(startupOfLength(100));+ assertEquals('R', client.next().type());+ client.send(header('p', 10 * 1024 * 1024 + 1));+ assertRefused(client, "invalid message length");+ }+ for (int length : new int[] {10 * 1024 * 1024 + 1, 3}) {+ try (DsqlWireClient client = cluster.wireClient("admin", cluster.adminToken())) {+ client.send(header('Q', length));+ assertRefused(client, "invalid message length");+ }+ }+ }++ /**+ * Before TLS, DSQL reads no more of a packet than its length and code (measured): a startup+ * packet is refused as not TLS whatever length it claims, after a GSSENCRequest too, and one+ * under 8 bytes ends the connection without a word.+ */+ @Test+ void aPacketBeforeTlsIsRefusedAsNotTlsWhateverItsLength() throws Exception {+ for (int length : new int[] {9_999, 10_000, 20_000}) {+ for (boolean gssFirst : new boolean[] {false, true}) {+ try (Socket socket = new Socket("127.0.0.1", endpoint.port())) {+ InputStream in = socket.getInputStream();+ OutputStream out = socket.getOutputStream();+ if (gssFirst) {+ out.write(PostgresWire.int32(8));+ out.write(PostgresWire.int32(PostgresWire.GSSENC_REQUEST));+ assertEquals('N', in.read());+ }+ out.write(startupOfLength(length));+ PostgresWire.Message error = PostgresWire.read(in, Integer.MAX_VALUE, "message");+ Map<Character, String> fields = PostgresWire.parseFields(error.body(), 0);+ assertEquals("08P01", fields.get('C'), length + (gssFirst ? " after GSSENCRequest" : ""));+ assertEquals("unable to accept connection, SSL is mandatory. AWS Authentication is required.",+ fields.get('M'));+ }+ }+ }+ try (Socket socket = new Socket("127.0.0.1", endpoint.port())) {+ socket.getOutputStream().write(PostgresWire.int32(7));+ socket.getOutputStream().write(new byte[3]);+ assertEquals(-1, socket.getInputStream().read());+ }+ }++ @Test+ void aMessageOfTenMibIsTaken() throws Exception {+ try (DsqlWireClient client = cluster.wireClient("admin", cluster.adminToken())) {+ String padding = "x".repeat(10 * 1024 * 1024 - 4 - 1 - "SELECT 1 /**/".length());+ client.send(query("SELECT 1 /*" + padding + "*/"));+ assertEquals(List.of("SELECT 1"), client.untilReady());+ client.send(query("SELECT 2"));+ assertEquals(List.of("SELECT 1"), client.untilReady());+ }+ }+@Testvoid aRoleLogsInOnlyForTheIamRoleMappedToIt() throws Exception {String accessKeyId = "ASIA" + Long.toString(System.nanoTime(), 36).toUpperCase();@@ -627,4 +713,34 @@ class DsqlDataPlaneIntegrationTest {return EnforcementFixtures.createUser(cleanup, httpPort, EnforcementFixtures.auth(account, "iam"),"dsql-" + System.nanoTime());}++ /** A startup packet signing in as {@code admin}, its application_name padded to make {@code length} bytes. */+ private static byte[] startupOfLength(int length) {+ Map<String, String> parameters = new LinkedHashMap<>();+ parameters.put("user", "admin");+ parameters.put("database", "postgres");+ parameters.put("application_name", "");+ parameters.put("application_name", "a".repeat(length - PostgresWire.startupMessage(parameters).length));+ return PostgresWire.startupMessage(parameters);+ }++ /** A message's type and the length it announces, without any of its body. */+ private static byte[] header(char type, int length) {+ byte[] header = new byte[5];+ header[0] = (byte) type;+ System.arraycopy(PostgresWire.int32(length), 0, header, 1, 4);+ return header;+ }++ /** The endpoint answers DSQL's FATAL with {@code message} and the session's id, then closes. */+ private static void assertRefused(DsqlWireClient client, String message) throws IOException {+ PostgresWire.Message error = client.next();+ assertEquals('E', error.type());+ Map<Character, String> fields = PostgresWire.parseFields(error.body(), 0);+ assertEquals("FATAL", fields.get('S'));+ assertEquals("08P01", fields.get('C'));+ assertEquals(message, fields.get('M'));+ assertTrue(fields.get('D').startsWith("Session Id: "), fields.get('D'));+ assertThrows(IOException.class, client::next);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/dsql/DsqlWireClient.java b/src/test/java/io/github/hectorvent/floci/services/dsql/DsqlWireClient.javaindex c8a0a3f8c0..13fd989a75 100644--- a/src/test/java/io/github/hectorvent/floci/services/dsql/DsqlWireClient.java+++ b/src/test/java/io/github/hectorvent/floci/services/dsql/DsqlWireClient.java@@ -28,13 +28,7 @@ public final class DsqlWireClient implements AutoCloseable {/** Signs in to the endpoint on {@code port} as {@code user}, naming {@code host} as the server. */public DsqlWireClient(int port, String host, String user, String password) throws Exception {- Socket plain = new Socket("127.0.0.1", port);- plain.getOutputStream().write(PostgresWire.int32(8));- plain.getOutputStream().write(PostgresWire.int32(PostgresWire.SSL_REQUEST));- assertEquals('S', plain.getInputStream().read());- socket = new NonValidatingFactory(null).createSocket(plain, host, port, true);- in = new BufferedInputStream(socket.getInputStream());- out = socket.getOutputStream();+ this(port, host);Map<String, String> startup = new LinkedHashMap<>();startup.put("user", user);startup.put("database", "postgres");@@ -44,6 +38,26 @@ public final class DsqlWireClient implements AutoCloseable {untilReady();}+ private DsqlWireClient(int port, String host) throws Exception {+ Socket plain = new Socket("127.0.0.1", port);+ plain.getOutputStream().write(PostgresWire.int32(8));+ plain.getOutputStream().write(PostgresWire.int32(PostgresWire.SSL_REQUEST));+ assertEquals('S', plain.getInputStream().read());+ socket = new NonValidatingFactory(null).createSocket(plain, host, port, true);+ in = new BufferedInputStream(socket.getInputStream());+ out = socket.getOutputStream();+ }++ /** TLS to the endpoint on {@code port}, naming {@code host}, and nothing sent on it yet: no startup packet. */+ public static DsqlWireClient tls(int port, String host) throws Exception {+ return new DsqlWireClient(port, host);+ }++ /** The next message the endpoint sends; an {@link IOException} once it has closed the connection. */+ public PostgresWire.Message next() throws IOException {+ return PostgresWire.read(in, Integer.MAX_VALUE, "message");+ }+public void send(byte[]... messages) throws IOException {for (byte[] message : messages) {out.write(message);