Light, fluffy, and always free - The AWS Local Emulator alternative
feat(redshift): authorize COPY and UNLOAD with IAM_ROLE (#3815)
Vũ Thế Huy677cdbe053c9
diff --git a/docs/services/redshift.md b/docs/services/redshift.mdindex 724863367b..ddfcb2f28c 100644--- a/docs/services/redshift.md+++ b/docs/services/redshift.md@@ -197,11 +197,14 @@ order) through its own S3 service and streams the rows into the backing PostgreSa comma default delimiter.- `IGNOREHEADER` and `HEADER` skip lines from the first resolved object only.- `GZIP` is the only input compression recognized; `BZIP2`, `LZOP` and `ZSTD` are not.-- S3 access is authorized as an unsigned request: with `FLOCI_SERVICES_S3_ENFORCE_AUTH` off it is- unrestricted; with it on, bucket policy and public access settings apply.+- `IAM_ROLE '<role-arn>'` is supported. The role must be associated with the cluster, exist in+ the local IAM service, and trust Redshift to assume it. With+ `FLOCI_SERVICES_S3_ENFORCE_AUTH` off, S3 policy checks are skipped. With it on, the role's+ identity policy must allow the required S3 actions, and any bucket policy must not deny the+ request. `IAM_ROLE default` is not supported.- Any other clause (`FIXEDWIDTH`, `JSON`, `PARQUET`, `AVRO`, `ORC`, `MANIFEST`, `MAXERROR`,`DATEFORMAT`, `TIMEFORMAT`, `REGION`, `ENCODING`, `ESCAPE`, `REMOVEQUOTES`, `BLANKSASNULL`,- `EMPTYASNULL`, `TRUNCATECOLUMNS`, `ACCEPTINVCHARS`, credentials clauses, and so on) is not+ `EMPTYASNULL`, `TRUNCATECOLUMNS`, `ACCEPTINVCHARS`, `CREDENTIALS`, and so on) is notrecognized: the statement is forwarded unchanged and PostgreSQL returns its own error.- A multi-statement query whose COPY is followed by another statement is not intercepted; send theCOPY on its own.@@ -248,8 +251,12 @@ the result to S3 as one or more objects under `<prefix>`.`ALLOWOVERWRITE` a failed UNLOAD leaves its objects in place (they may have replaced prior data,so they are not deleted); a `MANIFEST` request that fails this way can leave data objects withouta manifest, and rerunning the same statement overwrites them.-- S3 access is authorized as an unsigned request, like COPY from S3.-- Any other option (`PARQUET`, `ENCRYPTED`, `REGION`, `IAM_ROLE` / `CREDENTIALS`,+- `IAM_ROLE '<role-arn>'` is supported. The role must be associated with the cluster, exist in+ the local IAM service, and trust Redshift to assume it. With+ `FLOCI_SERVICES_S3_ENFORCE_AUTH` off, S3 policy checks are skipped. With it on, the role's+ identity policy must allow the required S3 actions, and any bucket policy must not deny the+ request. `IAM_ROLE default` is not supported.+- Any other option (`PARQUET`, `ENCRYPTED`, `REGION`, `CREDENTIALS`,`ZSTD`, `EXTENSION`, `CLEANPATH`, `PARTITION`, and so on) is not intercepted; thestatement is forwarded and PostgreSQL reports its own error.- Extended Query UNLOAD is supported when the complete statement is present in `Parse` and has no@@ -287,5 +294,6 @@ These views expose the documented Redshift column names, types, and ordering map- Parameter groups apply no real engine settings; values are stored and echoed back only.- Subnet groups, VPC routing, and security groups are metadata only.- Resize, pause/resume, IAM authentication, snapshot schedules, and cross-region snapshot copy.+- `IAM_ROLE default` and cross-account role ARNs are not supported for COPY or UNLOAD.- The auth proxy validates the master user's password and any live `GetClusterCredentials` credential. Other non-master users pass straight through to PostgreSQL, which remains the authority for their credentials.- IAM database authentication over the wire, and `sslmode=verify-full` against the self-signed proxy certificate.diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluator.javaindex 6ee57597ef..f59e522ee4 100644--- a/src/main/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluator.java+++ b/src/main/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluator.java@@ -7,6 +7,7 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.util.regex.Matcher;import java.util.regex.Pattern;/**@@ -74,6 +75,35 @@ public class AssumeRolePolicyEvaluator {return allow;}+ /**+ * Returns true if the trust policy allows the named AWS service principal to assume the role.+ */+ public boolean allowsService(String trustPolicyDocument, String servicePrincipal) {+ if (trustPolicyDocument == null || trustPolicyDocument.isBlank()) {+ return false;+ }+ JsonNode statements;+ try {+ statements = objectMapper.readTree(trustPolicyDocument).path("Statement");+ } catch (Exception e) {+ LOG.warnv("Failed to parse trust policy: {0}", e.getMessage());+ return false;+ }+ boolean allow = false;+ if (statements.isArray()) {+ for (JsonNode statement : statements) {+ switch (evaluateServiceStatement(statement, servicePrincipal)) {+ case DENY -> { return false; }+ case ALLOW -> allow = true;+ case NO_MATCH -> { }+ }+ }+ } else if (statements.isObject()) {+ return evaluateServiceStatement(statements, servicePrincipal) == Match.ALLOW;+ }+ return allow;+ }+private enum Match { ALLOW, DENY, NO_MATCH }private Match evaluateStatement(JsonNode stmt, String callerArn, String callerAccount) {@@ -86,6 +116,34 @@ public class AssumeRolePolicyEvaluator {return "Deny".equalsIgnoreCase(stmt.path("Effect").asText("Allow")) ? Match.DENY : Match.ALLOW;}+ private Match evaluateServiceStatement(JsonNode stmt, String servicePrincipal) {+ if (!actionApplies(stmt) || !matchesServicePrincipal(stmt.get("Principal"), servicePrincipal)) {+ return Match.NO_MATCH;+ }+ return "Deny".equalsIgnoreCase(stmt.path("Effect").asText("Allow")) ? Match.DENY : Match.ALLOW;+ }++ private boolean matchesServicePrincipal(JsonNode principalNode, String servicePrincipal) {+ if (principalNode == null || servicePrincipal == null) {+ return false;+ }+ JsonNode service = principalNode.isObject() ? principalNode.get("Service") : null;+ if (service == null) {+ return false;+ }+ if (service.isTextual()) {+ return IamPolicyEvaluator.globMatches(service.asText(), servicePrincipal);+ }+ if (service.isArray()) {+ for (JsonNode entry : service) {+ if (entry.isTextual() && IamPolicyEvaluator.globMatches(entry.asText(), servicePrincipal)) {+ return true;+ }+ }+ }+ return false;+ }+/*** True if the statement's action element applies to {@code sts:AssumeRole}. An {@code Action}* element applies when any of its patterns match; a {@code NotAction} element applies when none@@ -161,7 +219,7 @@ public class AssumeRolePolicyEvaluator {if (principal.matches("\\d{12}")) {return principal.equals(callerAccount);}- var rootMatcher = ACCOUNT_ROOT_ARN.matcher(principal);+ Matcher rootMatcher = ACCOUNT_ROOT_ARN.matcher(principal);if (rootMatcher.matches()) {return rootMatcher.group(1).equals(callerAccount);}@@ -191,7 +249,7 @@ public class AssumeRolePolicyEvaluator {* be denied a role they are entitled to, with nothing in the response saying why.*/private static String assumedRoleToRoleArn(String arn) {- var m = ASSUMED_ROLE_ARN.matcher(arn);+ Matcher m = ASSUMED_ROLE_ARN.matcher(arn);if (!m.matches()) {return null;}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.javaindex 91a2f6f5e9..3c524da572 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java@@ -67,9 +67,13 @@ public class RedshiftQueryHandler {String masterUserPassword = params.getFirst("MasterUserPassword");String clusterSubnetGroupName = params.getFirst("ClusterSubnetGroupName");List<String> vpcSecurityGroupIds = memberList(params, "VpcSecurityGroupIds");+ List<String> iamRoleArns = memberList(params, "IamRoles");- Cluster cluster = service.createCluster(identifier, nodeType, masterUsername, masterUserPassword,- clusterSubnetGroupName, vpcSecurityGroupIds);+ Cluster cluster = iamRoleArns.isEmpty()+ ? service.createCluster(identifier, nodeType, masterUsername, masterUserPassword,+ clusterSubnetGroupName, vpcSecurityGroupIds)+ : service.createCluster(identifier, nodeType, masterUsername, masterUserPassword,+ clusterSubnetGroupName, vpcSecurityGroupIds, iamRoleArns);String xml = new XmlBuilder().start("CreateClusterResponse").start("CreateClusterResult")@@ -622,6 +626,14 @@ public class RedshiftQueryHandler {builder.end("VpcSecurityGroups");}+ if (cluster.getIamRoleArns() != null && !cluster.getIamRoleArns().isEmpty()) {+ builder.start("IamRoles");+ for (String iamRoleArn : cluster.getIamRoleArns()) {+ builder.start("IamRole").elem("IamRoleArn", iamRoleArn).end("IamRole");+ }+ builder.end("IamRoles");+ }+if (cluster.getClusterParameterGroupName() != null) {builder.start("ClusterParameterGroups").start("ClusterParameterGroup")@@ -848,6 +860,7 @@ public class RedshiftQueryHandler {return switch (baseName) {case "SubnetIds" -> quoted + "(\\.member|\\.SubnetIdentifier)?\\.\\d+";case "VpcSecurityGroupIds" -> quoted + "(\\.member|\\.VpcSecurityGroupId)?\\.\\d+";+ case "IamRoles" -> quoted + "(\\.member|\\.IamRoleArn)?\\.\\d+";case "TagKeys" -> quoted + "(\\.member|\\.TagKey)?\\.\\d+";case "DbGroups" -> quoted + "(\\.member|\\.DbGroup)?\\.\\d+";default -> quoted + "(\\.member)?\\.\\d+";diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.javaindex 5fdf85377f..989a09977b 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java@@ -117,7 +117,8 @@ public class RedshiftService {relayKey(entry.accountId(), cluster.getClusterIdentifier()), proxyPort,handle.getHost(), handle.getPort(), endpoint.getAddress(),cluster.getMasterUsername(), password, CLUSTER_DB_NAME,- passwordValidatorFor(entry.accountId(), cluster.getClusterIdentifier()));+ passwordValidatorFor(entry.accountId(), cluster.getClusterIdentifier()),+ cluster.getIamRoleArns());cluster.setContainerHost(handle.getHost());cluster.setContainerPort(handle.getPort());cluster.setEndpoint(endpoint);@@ -143,13 +144,20 @@ public class RedshiftService {}public Cluster createCluster(String identifier, String nodeType, String username, String password) {- return createCluster(identifier, nodeType, username, password, null, List.of());+ return createCluster(identifier, nodeType, username, password, null, List.of(), List.of());}// synchronized like modify/reboot: the container + proxy + port steps must not// interleave with another admin call on the same cluster.public synchronized Cluster createCluster(String identifier, String nodeType, String username, String password,String clusterSubnetGroupName, List<String> vpcSecurityGroupIds) {+ return createCluster(identifier, nodeType, username, password, clusterSubnetGroupName,+ vpcSecurityGroupIds, List.of());+ }++ public synchronized Cluster createCluster(String identifier, String nodeType, String username, String password,+ String clusterSubnetGroupName, List<String> vpcSecurityGroupIds,+ List<String> iamRoleArns) {if (clusters.get(identifier).isPresent()) {throw new AwsException("ClusterAlreadyExists", "Cluster " + identifier + " already exists", 400);}@@ -164,6 +172,7 @@ public class RedshiftService {cluster.setMasterPassword(password);cluster.setClusterSubnetGroupName(clusterSubnetGroupName);cluster.setVpcSecurityGroupIds(vpcSecurityGroupIds != null ? vpcSecurityGroupIds : List.of());+ cluster.setIamRoleArns(iamRoleArns != null ? List.copyOf(iamRoleArns) : List.of());cluster.setClusterStatus("creating");clusters.put(identifier, cluster);clusters.flush();@@ -181,7 +190,7 @@ public class RedshiftService {proxyManager.startProxy(relayKey(accountId, identifier), proxyPort,handle.getHost(), handle.getPort(), endpoint.getAddress(),username, password, CLUSTER_DB_NAME,- passwordValidatorFor(accountId, identifier));+ passwordValidatorFor(accountId, identifier), cluster.getIamRoleArns());cluster.setContainerHost(handle.getHost());cluster.setContainerPort(handle.getPort());cluster.setEndpoint(endpoint);@@ -497,7 +506,7 @@ public class RedshiftService {cluster.setProxyPort(proxyPort);proxyManager.startProxy(key, proxyPort, handle.getHost(), handle.getPort(),endpoint.getAddress(), cluster.getMasterUsername(), password, CLUSTER_DB_NAME,- passwordValidatorFor(accountId, clusterIdentifier));+ passwordValidatorFor(accountId, clusterIdentifier), cluster.getIamRoleArns());cluster.setContainerHost(handle.getHost());cluster.setContainerPort(handle.getPort());cluster.setEndpoint(endpoint);@@ -727,7 +736,7 @@ public class RedshiftService {proxyManager.startProxy(relayKey(accountId, clusterIdentifier), proxyPort,handle.getHost(), handle.getPort(), endpoint.getAddress(),username, password, CLUSTER_DB_NAME,- passwordValidatorFor(accountId, clusterIdentifier));+ passwordValidatorFor(accountId, clusterIdentifier), cluster.getIamRoleArns());cluster.setContainerHost(handle.getHost());cluster.setContainerPort(handle.getPort());cluster.setEndpoint(endpoint);diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/model/Cluster.java b/src/main/java/io/github/hectorvent/floci/services/redshift/model/Cluster.javaindex 24c2107b5d..d857d5c5e0 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/model/Cluster.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/model/Cluster.java@@ -17,6 +17,7 @@ public class Cluster {private String clusterSubnetGroupName;private String clusterParameterGroupName;private List<String> vpcSecurityGroupIds = new ArrayList<>();+ private List<String> iamRoleArns = new ArrayList<>();public String getClusterIdentifier() { return clusterIdentifier; }public void setClusterIdentifier(String clusterIdentifier) { this.clusterIdentifier = clusterIdentifier; }@@ -36,6 +37,8 @@ public class Cluster {public void setClusterParameterGroupName(String clusterParameterGroupName) { this.clusterParameterGroupName = clusterParameterGroupName; }public List<String> getVpcSecurityGroupIds() { return vpcSecurityGroupIds; }public void setVpcSecurityGroupIds(List<String> vpcSecurityGroupIds) { this.vpcSecurityGroupIds = vpcSecurityGroupIds; }+ public List<String> getIamRoleArns() { return iamRoleArns; }+ public void setIamRoleArns(List<String> iamRoleArns) { this.iamRoleArns = iamRoleArns; }// Real backend address of this cluster's PostgreSQL container. `endpoint` now points at the// auth proxy, not the container, so the container address is kept here for proxy wiring anddiff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.javaindex 1fe4cb6eeb..bb5d52c626 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java@@ -25,7 +25,8 @@ public final class CopyStatementParser {int headerLines,boolean gzip,boolean csv,- String nullAs) implements S3Statement {+ String nullAs,+ String iamRoleArn) implements S3Statement {}public record S3Unload(@@ -41,7 +42,8 @@ public final class CopyStatementParser {boolean manifest,boolean allowOverwrite,boolean parallel,- long maxFileSizeBytes) implements S3Statement {+ long maxFileSizeBytes,+ String iamRoleArn) implements S3Statement {}private static final Pattern COPY_PATTERN = Pattern.compile(@@ -67,7 +69,7 @@ public final class CopyStatementParser {*/private static final Pattern UNLOAD_UNSUPPORTED_CLAUSE = Pattern.compile("(?i)\\b(FIXEDWIDTH|PARQUET|AVRO|ORC|JSON|SHAPEFILE|BZIP2|LZOP|ZSTD"- + "|ENCRYPTED|ENCODING|REGION|CREDENTIALS|IAM_ROLE|ACCESS_KEY_ID"+ + "|ENCRYPTED|ENCODING|REGION|CREDENTIALS|ACCESS_KEY_ID"+ "|SECRET_ACCESS_KEY|SESSION_TOKEN|MASTER_SYMMETRIC_KEY|KMS_KEY_ID"+ "|EXTENSION|CLEANPATH|PARTITION|MAXFILESIZE\\s+\\d+\\s*(?:TB|PB))\\b");@@ -79,6 +81,8 @@ public final class CopyStatementParser {"(?i)\\bDELIMITER\\s+(?:AS\\s+)?(?:'((?:[^']|'')*)'|\"([^\"]*)\"|([^\\s;]+))");private static final Pattern NULL_AS_PATTERN = Pattern.compile("(?i)\\bNULL\\s+(?:AS\\s+)?(?:'((?:[^']|'')*)'|\"([^\"]*)\")");+ private static final Pattern IAM_ROLE_PATTERN = Pattern.compile(+ "(?i)\\bIAM_ROLE\\s+(?:'((?:[^']|'')*)'|\"([^\"]*)\")");private static final Pattern IGNOREHEADER_PATTERN = Pattern.compile("(?i)\\bIGNOREHEADER\\s+(?:AS\\s+)?(\\d+)\\b");private static final Pattern HEADER_PATTERN = Pattern.compile("(?i)\\bHEADER\\b");@@ -92,7 +96,7 @@ public final class CopyStatementParser {*/private static final Pattern UNSUPPORTED_CLAUSE = Pattern.compile("(?i)\\b(FIXEDWIDTH|PARQUET|AVRO|ORC|JSON|SHAPEFILE|BZIP2|LZOP|ZSTD|MANIFEST|MAXERROR"- + "|DATEFORMAT|TIMEFORMAT|ENCRYPTED|ENCODING|REGION|CREDENTIALS|IAM_ROLE"+ + "|DATEFORMAT|TIMEFORMAT|ENCRYPTED|ENCODING|REGION|CREDENTIALS"+ "|ACCESS_KEY_ID|SECRET_ACCESS_KEY|SESSION_TOKEN|MASTER_SYMMETRIC_KEY|KMS_KEY_ID"+ "|ACCEPTINVCHARS|ACCEPTANYDATE|BLANKSASNULL|EMPTYASNULL|FILLRECORD|TRIMBLANKS"+ "|TRUNCATECOLUMNS|IGNOREBLANKLINES|ESCAPE|REMOVEQUOTES|EXPLICIT_IDS|COMPUPDATE"@@ -151,6 +155,7 @@ public final class CopyStatementParser {String nullAs = null;String delimiter = null;long maxFileSizeBytes = 0L;+ String iamRoleArn = null;boolean seenCsv = false;boolean seenGzip = false;@@ -162,6 +167,7 @@ public final class CopyStatementParser {boolean seenNull = false;boolean seenDelimiter = false;boolean seenMaxFileSize = false;+ boolean seenIamRole = false;Matcher csvM = CSV_PATTERN.matcher(options);Matcher gzipM = GZIP_PATTERN.matcher(options);@@ -173,6 +179,7 @@ public final class CopyStatementParser {Matcher delimiterM = DELIMITER_PATTERN.matcher(options);Matcher nullM = NULL_AS_PATTERN.matcher(options);Matcher maxFileSizeM = MAXFILESIZE_PATTERN.matcher(options);+ Matcher iamRoleM = IAM_ROLE_PATTERN.matcher(options);int offset = 0;int len = options.length();@@ -259,6 +266,13 @@ public final class CopyStatementParser {seenNull = true;nullAs = extractNullValue(nullM);offset = nullM.end();+ } else if (matchClause(iamRoleM, offset, len)) {+ if (seenIamRole) {+ return null;+ }+ seenIamRole = true;+ iamRoleArn = extractNullValue(iamRoleM);+ offset = iamRoleM.end();} else {return null;}@@ -268,7 +282,7 @@ public final class CopyStatementParser {delimiter = csv ? "," : "|";}return new S3Unload(select, bucket, prefix, delimiter, header, gzip, csv,- addQuotes, nullAs, manifest, allowOverwrite, parallel, maxFileSizeBytes);+ addQuotes, nullAs, manifest, allowOverwrite, parallel, maxFileSizeBytes, iamRoleArn);}private static S3CopyFrom parseCopy(String cleaned) {@@ -319,11 +333,14 @@ public final class CopyStatementParser {String delimiter = null;int headerLines = 0;+ String iamRoleArn = null;+boolean seenCsv = false;boolean seenGzip = false;boolean seenNull = false;boolean seenDelimiter = false;boolean seenHeader = false;+ boolean seenIamRole = false;Matcher csvMatcher = CSV_PATTERN.matcher(options);Matcher gzipMatcher = GZIP_PATTERN.matcher(options);@@ -331,6 +348,7 @@ public final class CopyStatementParser {Matcher headerMatcher = HEADER_PATTERN.matcher(options);Matcher delimiterMatcher = DELIMITER_PATTERN.matcher(options);Matcher nullMatcher = NULL_AS_PATTERN.matcher(options);+ Matcher iamRoleMatcher = IAM_ROLE_PATTERN.matcher(options);int offset = 0;int len = options.length();@@ -384,6 +402,13 @@ public final class CopyStatementParser {seenNull = true;nullAs = extractNullValue(nullMatcher);offset = nullMatcher.end();+ } else if (matchClause(iamRoleMatcher, offset, len)) {+ if (seenIamRole) {+ return null;+ }+ seenIamRole = true;+ iamRoleArn = extractNullValue(iamRoleMatcher);+ offset = iamRoleMatcher.end();} else {return null;}@@ -393,7 +418,7 @@ public final class CopyStatementParser {delimiter = csv ? "," : "|";}- return new S3CopyFrom(table, columns, bucket, keyOrPrefix, delimiter, headerLines, gzip, csv, nullAs);+ return new S3CopyFrom(table, columns, bucket, keyOrPrefix, delimiter, headerLines, gzip, csv, nullAs, iamRoleArn);}private static boolean matchClause(Matcher m, int start, int end) {diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.javaindex 09d67cf398..074d14f33c 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.java@@ -1,9 +1,11 @@package io.github.hectorvent.floci.services.redshift.proxy;+import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.s3.S3Service;import java.io.IOException;import java.io.OutputStream;import java.net.Socket;+import java.util.List;final class ExtendedS3Exchange {@@ -15,8 +17,16 @@ final class ExtendedS3Exchange {static void execute(Socket client, Socket backend,PostgresWireDecoder.FrontendMessage executeFrame,- CopyStatementParser.S3Statement statement, S3Service s3Service,+ CopyStatementParser.S3Statement statement, S3Service s3Service, IamService iamService,BackendResponseCoordinator coordinator, BackendResponseCoordinator.Ticket ticket) throws IOException {+ execute(client, backend, executeFrame, statement, s3Service, iamService, null, List.of(), coordinator, ticket);+ }++ static void execute(Socket client, Socket backend,+ PostgresWireDecoder.FrontendMessage executeFrame,+ CopyStatementParser.S3Statement statement, S3Service s3Service, IamService iamService,+ String clusterAccountId, List<String> iamRoleArns, BackendResponseCoordinator coordinator,+ BackendResponseCoordinator.Ticket ticket) throws IOException {BackendResponseCoordinator.GateResult gate;try {gate = coordinator.awaitExtendedExecuteTurn(ticket);@@ -33,12 +43,14 @@ final class ExtendedS3Exchange {try {switch (statement) {case CopyStatementParser.S3CopyFrom copy -> {- CopyResult result = runCopy(client, backend, executeFrame, copy, s3Service, coordinator);+ CopyResult result = runCopy(client, backend, executeFrame, copy, s3Service, iamService,+ clusterAccountId, iamRoleArns, coordinator);failed = !result.succeeded();deferredReadyForQuery = result.deferredReadyForQuery();}case CopyStatementParser.S3Unload unload -> {- CopyResult result = runUnload(client, backend, executeFrame, unload, s3Service, coordinator);+ CopyResult result = runUnload(client, backend, executeFrame, unload, s3Service, iamService,+ clusterAccountId, iamRoleArns, coordinator);failed = !result.succeeded();deferredReadyForQuery = result.deferredReadyForQuery();}@@ -57,8 +69,8 @@ final class ExtendedS3Exchange {private static CopyResult runCopy(Socket client, Socket backend,PostgresWireDecoder.FrontendMessage executeFrame,- CopyStatementParser.S3CopyFrom spec, S3Service s3Service,- BackendResponseCoordinator coordinator) throws IOException {+ CopyStatementParser.S3CopyFrom spec, S3Service s3Service, IamService iamService,+ String clusterAccountId, List<String> iamRoleArns, BackendResponseCoordinator coordinator) throws IOException {OutputStream backendOut = backend.getOutputStream();backendOut.write(executeFrame.toPacketBytes());backendOut.flush();@@ -77,7 +89,7 @@ final class ExtendedS3Exchange {S3CopySimulator.CopyInput input;try {- input = S3CopySimulator.prepareCopy(spec, s3Service);+ input = S3CopySimulator.prepareCopy(spec, s3Service, iamService, clusterAccountId, iamRoleArns);} catch (S3CopySimulator.S3TransferException e) {S3CopySimulator.writeCopyFail(backendOut, e.getMessage());forwardClientSyncToBackend(sync, backendOut, coordinator);@@ -97,6 +109,8 @@ final class ExtendedS3Exchange {String detail = e.getMessage() != null ? e.getMessage() : e.toString();sendError(client, "XX000", "S3 COPY failed: " + detail);return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));+ } finally {+ S3CopySimulator.releaseCopySession(input);}forwardClientSyncToBackend(sync, backendOut, coordinator);@@ -106,8 +120,8 @@ final class ExtendedS3Exchange {private static CopyResult runUnload(Socket client, Socket backend,PostgresWireDecoder.FrontendMessage executeFrame,- CopyStatementParser.S3Unload spec, S3Service s3Service,- BackendResponseCoordinator coordinator) throws IOException {+ CopyStatementParser.S3Unload spec, S3Service s3Service, IamService iamService,+ String clusterAccountId, List<String> iamRoleArns, BackendResponseCoordinator coordinator) throws IOException {OutputStream backendOut = backend.getOutputStream();backendOut.write(executeFrame.toPacketBytes());backendOut.flush();@@ -125,7 +139,7 @@ final class ExtendedS3Exchange {S3CopySimulator.UnloadCollector collector;try {- collector = S3CopySimulator.prepareUnload(spec, s3Service);+ collector = S3CopySimulator.prepareUnload(spec, s3Service, iamService, clusterAccountId, iamRoleArns);} catch (S3CopySimulator.S3TransferException e) {S3CopySimulator.writeCopyFail(backendOut, e.getMessage());drainExecute(client, decoder, coordinator, false);diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.javaindex 6dab1791a3..ac06e346d3 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.redshift.proxy;+import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;import io.github.hectorvent.floci.services.rds.proxy.PostgresProtocolHandler;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;@@ -12,6 +13,7 @@ import java.net.BindException;import java.net.InetSocketAddress;import java.net.ServerSocket;import java.net.Socket;+import java.util.List;import java.util.concurrent.Semaphore;/**@@ -35,6 +37,9 @@ public class RedshiftAuthProxy {private final RdsProxyTlsCertificates tlsCertificates;private final PasswordValidator passwordValidator;private final S3Service s3Service;+ private final IamService iamService;+ private final String clusterAccountId;+ private final List<String> iamRoleArns;private final int handshakeTimeoutMillis;private final int backendConnectTimeoutMillis;private final Semaphore connectionPermits;@@ -46,7 +51,22 @@ public class RedshiftAuthProxy {String masterUsername, String masterPassword, String dbName,RdsSigV4Validator sigV4, RdsProxyTlsCertificates tlsCertificates,PasswordValidator passwordValidator,- S3Service s3Service,+ S3Service s3Service, IamService iamService,+ int handshakeTimeoutMillis, int backendConnectTimeoutMillis,+ int maxConnections) {+ this(clusterKey, backendHost, backendPort, masterUsername, masterPassword, dbName, sigV4,+ tlsCertificates, passwordValidator, s3Service, iamService, null,+ List.of(),+ handshakeTimeoutMillis, backendConnectTimeoutMillis, maxConnections);+ }++ public RedshiftAuthProxy(String clusterKey, String backendHost, int backendPort,+ String masterUsername, String masterPassword, String dbName,+ RdsSigV4Validator sigV4, RdsProxyTlsCertificates tlsCertificates,+ PasswordValidator passwordValidator,+ S3Service s3Service, IamService iamService,+ String clusterAccountId,+ List<String> iamRoleArns,int handshakeTimeoutMillis, int backendConnectTimeoutMillis,int maxConnections) {this.clusterKey = clusterKey;@@ -59,6 +79,9 @@ public class RedshiftAuthProxy {this.tlsCertificates = tlsCertificates;this.passwordValidator = passwordValidator;this.s3Service = s3Service;+ this.iamService = iamService;+ this.clusterAccountId = clusterAccountId;+ this.iamRoleArns = iamRoleArns == null ? List.of() : List.copyOf(iamRoleArns);this.handshakeTimeoutMillis = handshakeTimeoutMillis;this.backendConnectTimeoutMillis = backendConnectTimeoutMillis;this.connectionPermits = new Semaphore(Math.max(1, maxConnections));@@ -175,7 +198,8 @@ public class RedshiftAuthProxy {if (session != null) {// Redshift-only DDL (DISTKEY/SORTKEY/ENCODE/...) is rewritten for the plain// PostgreSQL backend on the way through; every other message is relayed verbatim.- new RedshiftInterceptingBridge(session.client(), session.backend(), s3Service).run();+ new RedshiftInterceptingBridge(session.client(), session.backend(), s3Service, iamService,+ clusterAccountId, iamRoleArns).run();}} catch (Exception e) {LOG.debugv("Redshift connection error for cluster {0}: {1}", clusterKey, e.getMessage());diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.javaindex 5b07d11637..553b75bd2b 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.redshift.proxy;+import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.s3.S3Service;import org.jboss.logging.Logger;@@ -8,6 +9,7 @@ import java.io.InputStream;import java.io.OutputStream;import java.net.Socket;import java.net.SocketTimeoutException;+import java.util.List;import java.util.concurrent.TimeUnit;import java.util.concurrent.locks.LockSupport;import java.util.concurrent.locks.ReentrantLock;@@ -41,6 +43,9 @@ public class RedshiftInterceptingBridge {private final Socket client;private final Socket backend;private final S3Service s3Service;+ private final IamService iamService;+ private final String clusterAccountId;+ private final List<String> iamRoleArns;private final ExtendedQuerySession session = new ExtendedQuerySession();private final BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);@@ -48,10 +53,23 @@ public class RedshiftInterceptingBridge {private volatile boolean pumpBetweenMessages = true;private volatile boolean pumpFinished = false;- public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service) {+ public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService) {+ this(client, backend, s3Service, iamService, null);+ }++ public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService,+ String clusterAccountId) {+ this(client, backend, s3Service, iamService, clusterAccountId, List.of());+ }++ public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService,+ String clusterAccountId, List<String> iamRoleArns) {this.client = client;this.backend = backend;this.s3Service = s3Service;+ this.iamService = iamService;+ this.clusterAccountId = clusterAccountId;+ this.iamRoleArns = iamRoleArns == null ? List.of() : List.copyOf(iamRoleArns);}@FunctionalInterface@@ -138,10 +156,10 @@ public class RedshiftInterceptingBridge {CopyStatementParser.S3Statement statement = parsed;boolean intercepted = runWithBackendOwned(() -> switch (statement) {case CopyStatementParser.S3CopyFrom copy -> S3CopySimulator.runCopyFrom(- client, backend, copy, s3Service, coordinator.lastReadyStatus(),+ client, backend, copy, s3Service, iamService, clusterAccountId, iamRoleArns, coordinator.lastReadyStatus(),status -> coordinator.onBackendFrame('Z', new byte[]{(byte) status}));case CopyStatementParser.S3Unload unload -> S3CopySimulator.runUnload(- client, backend, unload, s3Service, coordinator.lastReadyStatus(),+ client, backend, unload, s3Service, iamService, clusterAccountId, iamRoleArns, coordinator.lastReadyStatus(),status -> coordinator.onBackendFrame('Z', new byte[]{(byte) status}));});if (intercepted) {@@ -276,7 +294,8 @@ public class RedshiftInterceptingBridge {}try {backend.setSoTimeout(EXCHANGE_READ_TIMEOUT_MS);- ExtendedS3Exchange.execute(client, backend, executeFrame, statement, s3Service, coordinator, ticket);+ ExtendedS3Exchange.execute(client, backend, executeFrame, statement, s3Service, iamService,+ clusterAccountId, iamRoleArns, coordinator, ticket);} finally {try {backend.setSoTimeout(PUMP_READ_TIMEOUT_MS);diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.javaindex 3f8e06d092..f45f026d8f 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.redshift.proxy;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;@@ -12,6 +13,7 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.io.IOException;+import java.util.List;import java.util.concurrent.ConcurrentHashMap;/**@@ -27,6 +29,7 @@ public class RedshiftProxyManager {private final RdsSigV4Validator sigV4Validator;private final RdsProxyTlsCertificates tlsCertificates;private final S3Service s3Service;+ private final IamService iamService;private final EmulatorConfig config;private final ConcurrentHashMap<String, RedshiftAuthProxy> proxies = new ConcurrentHashMap<>();/**@@ -39,10 +42,11 @@ public class RedshiftProxyManager {@Injectpublic RedshiftProxyManager(RdsSigV4Validator sigV4Validator, RdsProxyTlsCertificates tlsCertificates,- S3Service s3Service, EmulatorConfig config) {+ S3Service s3Service, IamService iamService, EmulatorConfig config) {this.sigV4Validator = sigV4Validator;this.tlsCertificates = tlsCertificates;this.s3Service = s3Service;+ this.iamService = iamService;this.config = config;}@@ -50,15 +54,26 @@ public class RedshiftProxyManager {String backendHost, int backendPort, String advertisedHost,String masterUsername, String masterPassword, String dbName,PasswordValidator passwordValidator) {+ startProxy(relayKey, proxyPort, backendHost, backendPort, advertisedHost, masterUsername,+ masterPassword, dbName, passwordValidator, List.of());+ }++ public synchronized void startProxy(String relayKey, int proxyPort,+ String backendHost, int backendPort, String advertisedHost,+ String masterUsername, String masterPassword, String dbName,+ PasswordValidator passwordValidator, List<String> iamRoleArns) {// A prior unclosable entry for this key is left in place: its listener may still// be bound, and only a successful stop (never a fresh start) may drop it.// Make sure the self-signed proxy certificate covers the host clients will connect to,// so sslmode=prefer/require handshakes succeed.tlsCertificates.ensureHost(advertisedHost);EmulatorConfig.RedshiftServiceConfig redshiftConfig = config.services().redshift();+ String clusterAccountId = relayKey.substring(0, relayKey.indexOf(':'));RedshiftAuthProxy proxy = new RedshiftAuthProxy(relayKey, backendHost, backendPort, masterUsername, masterPassword, dbName,- sigV4Validator, tlsCertificates, passwordValidator, s3Service,+ sigV4Validator, tlsCertificates, passwordValidator, s3Service, iamService,+ clusterAccountId,+ iamRoleArns,redshiftConfig.proxyHandshakeTimeoutMillis(), redshiftConfig.proxyBackendConnectTimeoutMillis(),redshiftConfig.proxyMaxConnections());try {diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.javaindex 98624f8c1e..12e4fdf01e 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java@@ -1,6 +1,13 @@package io.github.hectorvent.floci.services.redshift.proxy;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.iam.AssumeRolePolicyEvaluator;+import io.github.hectorvent.floci.services.iam.IamPolicyEvaluator;+import io.github.hectorvent.floci.services.iam.IamService;+import io.github.hectorvent.floci.services.iam.model.CallerContext;+import io.github.hectorvent.floci.services.iam.model.IamRole;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.S3Object;import org.jboss.logging.Logger;@@ -12,9 +19,13 @@ import java.io.InputStream;import java.io.OutputStream;import java.net.Socket;import java.nio.charset.StandardCharsets;+import java.security.SecureRandom;+import java.time.Duration;+import java.time.Instant;import java.util.ArrayList;import java.util.List;import java.util.Map;+import java.util.Optional;import java.util.concurrent.Semaphore;import java.util.function.IntConsumer;import java.util.zip.GZIPInputStream;@@ -55,10 +66,129 @@ public final class S3CopySimulator {private static final String SQLSTATE_INSUFFICIENT_PRIVILEGE = "42501";private static final String SQLSTATE_INTERNAL = "XX000";+ private static final SecureRandom SECURE_RANDOM = new SecureRandom();+ private static final String UPPER_ALPHANUMERIC = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";+ private static final String SECRET_CHARACTERS =+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";+ /** COPY/UNLOAD is synchronous end-to-end; this only needs to outlive one statement. */+ private static final Duration ROLE_SESSION_TTL = Duration.ofMinutes(5);++ // S3Service's signed-request authorization only checks the bucket's resource policy: for a+ // genuine HTTP request the identity-policy gate already happened upstream in+ // IamEnforcementFilter before the request ever reached S3Service. This in-process call never+ // goes through that filter, so the role's identity policy is evaluated here explicitly.+ // Stateless (only needs an ObjectMapper), so a local instance avoids threading a new+ // dependency through the whole proxy chain for this one check.+ private static final IamPolicyEvaluator ROLE_POLICY_EVALUATOR = new IamPolicyEvaluator(new ObjectMapper());+ private static final AssumeRolePolicyEvaluator ROLE_TRUST_POLICY_EVALUATOR =+ new AssumeRolePolicyEvaluator(new ObjectMapper());+private S3CopySimulator() {}- record CopyInput(CopyStatementParser.S3CopyFrom spec, List<String> keys, S3Service s3) {+ record CopyInput(CopyStatementParser.S3CopyFrom spec, List<String> keys, S3Service s3,+ IamService iamService, RoleSession roleSession) {+ }++ private record RoleSession(String accessKeyId, String sessionToken) {+ }++ /**+ * Validates the role ARN, its account and Redshift trust policy, then mints a short-lived session.+ */+ private static RoleSession resolveRoleSession(String iamRoleArn, IamService iamService,+ String clusterAccountId, List<String> associatedRoleArns) {+ AwsArnUtils.Arn parsed;+ try {+ parsed = AwsArnUtils.parse(iamRoleArn);+ } catch (IllegalArgumentException e) {+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,+ "IAM Role '" + iamRoleArn + "' could not be assumed: malformed ARN", e);+ }+ if (!"iam".equals(parsed.service()) || !parsed.resource().startsWith("role/")) {+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,+ "IAM Role '" + iamRoleArn + "' could not be assumed: not an IAM role ARN", null);+ }+ if (clusterAccountId != null && !clusterAccountId.equals(parsed.accountId())) {+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,+ "IAM Role '" + iamRoleArn + "' could not be assumed: cross-account role ARNs are not supported", null);+ }+ if (clusterAccountId != null && associatedRoleArns != null && !associatedRoleArns.contains(iamRoleArn)) {+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,+ "IAM Role '" + iamRoleArn+ + "' could not be assumed: role is not associated with the Redshift cluster", null);+ }+ String roleName = parsed.resource().substring(parsed.resource().lastIndexOf('/') + 1);+ Optional<IamRole> role = iamService.findRole(parsed.accountId(), roleName);+ if (role.isEmpty()) {+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,+ "IAM Role '" + iamRoleArn + "' could not be assumed: role does not exist", null);+ }+ if (clusterAccountId != null+ && !ROLE_TRUST_POLICY_EVALUATOR.allowsService(+ role.get().getAssumeRolePolicyDocument(), "redshift.amazonaws.com")) {+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,+ "IAM Role '" + iamRoleArn + "' could not be assumed: trust policy does not allow redshift.amazonaws.com",+ null);+ }++ String accessKeyId = "ASIA" + randomString(UPPER_ALPHANUMERIC, 16);+ // A session token is required: IamService.findSecretKey(accessKeyId, sessionToken) treats a+ // null token on either side as a non-match, so a tokenless session can never be recognised+ // as a known access key and every signed authorization call would fail closed.+ String sessionToken = randomString(SECRET_CHARACTERS, 200);+ iamService.registerSessionForAccount(parsed.accountId(), accessKeyId,+ randomString(SECRET_CHARACTERS, 40), sessionToken, iamRoleArn,+ Instant.now().plus(ROLE_SESSION_TTL), null);+ return new RoleSession(accessKeyId, sessionToken);+ }++ private static void releaseRoleSession(RoleSession roleSession, String iamRoleArn, IamService iamService) {+ if (roleSession == null) {+ return;+ }+ AwsArnUtils.Arn parsed = AwsArnUtils.parse(iamRoleArn);+ iamService.unregisterSession(parsed.accountId(), roleSession.accessKeyId());+ }++ /**+ * Evaluates the role's identity-based policy for one S3 action, skipped entirely when+ * {@code FLOCI_SERVICES_S3_ENFORCE_AUTH} is off (matching the anonymous path's behavior).+ */+ private static void authorizeRoleAction(S3Service s3, IamService iamService, String roleArn,+ String action, String resourceArn) {+ if (!s3.isAuthEnforced()) {+ return;+ }+ CallerContext caller;+ try {+ caller = iamService.resolvePrincipalContext(roleArn);+ } catch (AwsException e) {+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,+ "IAM Role '" + roleArn + "' could not be assumed: " + e.getMessage(), e);+ }+ IamPolicyEvaluator.SimulationDecision decision =+ ROLE_POLICY_EVALUATOR.simulatePrincipalPolicy(caller, action, resourceArn, Map.of());+ if (decision != IamPolicyEvaluator.SimulationDecision.ALLOWED) {+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,+ "S3 access denied for IAM Role '" + roleArn + "': " + action + " on " + resourceArn, null);+ }+ }++ private static String bucketArn(String bucket) {+ return "arn:aws:s3:::" + bucket;+ }++ private static String objectArn(String bucket, String key) {+ return "arn:aws:s3:::" + bucket + "/" + key;+ }++ private static String randomString(String characters, int length) {+ StringBuilder value = new StringBuilder(length);+ for (int i = 0; i < length; i++) {+ value.append(characters.charAt(SECURE_RANDOM.nextInt(characters.length())));+ }+ return value.toString();}interface UnloadCollector extends AutoCloseable {@@ -85,35 +215,63 @@ public final class S3CopySimulator {}}- static CopyInput prepareCopy(CopyStatementParser.S3CopyFrom spec, S3Service s3) {- try {- s3.authorizeAnonymousListBucket(spec.bucket());- } catch (AwsException e) {- throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,- "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);- }+ static CopyInput prepareCopy(CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService) {+ return prepareCopy(spec, s3, iamService, null);+ }- List<String> keys;- try {- keys = resolveKeys(spec, s3);- } catch (AwsException e) {- throw new S3TransferException(SQLSTATE_INTERNAL,- "S3 COPY could not list s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);- }- if (keys.isEmpty()) {- throw new S3TransferException(SQLSTATE_INTERNAL,- "S3 object s3://" + spec.bucket() + "/" + spec.keyOrPrefix() + " not found", null);- }+ static CopyInput prepareCopy(CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService,+ String clusterAccountId) {+ return prepareCopy(spec, s3, iamService, clusterAccountId, null);+ }+ static CopyInput prepareCopy(CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService,+ String clusterAccountId, List<String> associatedRoleArns) {+ RoleSession roleSession = spec.iamRoleArn() != null+ ? resolveRoleSession(spec.iamRoleArn(), iamService, clusterAccountId, associatedRoleArns)+ : null;try {- for (String key : keys) {- s3.authorizeAnonymousGetObject(spec.bucket(), key);+ try {+ if (roleSession != null) {+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:ListBucket", bucketArn(spec.bucket()));+ s3.authorizeSignedListBucket(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket());+ } else {+ s3.authorizeAnonymousListBucket(spec.bucket());+ }+ } catch (AwsException e) {+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,+ "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);}- } catch (AwsException e) {- throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,- "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);++ List<String> keys;+ try {+ keys = resolveKeys(spec, s3);+ } catch (AwsException e) {+ throw new S3TransferException(SQLSTATE_INTERNAL,+ "S3 COPY could not list s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);+ }+ if (keys.isEmpty()) {+ throw new S3TransferException(SQLSTATE_INTERNAL,+ "S3 object s3://" + spec.bucket() + "/" + spec.keyOrPrefix() + " not found", null);+ }++ try {+ for (String key : keys) {+ if (roleSession != null) {+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:GetObject", objectArn(spec.bucket(), key));+ s3.authorizeSignedGetObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), key);+ } else {+ s3.authorizeAnonymousGetObject(spec.bucket(), key);+ }+ }+ } catch (AwsException e) {+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,+ "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);+ }+ return new CopyInput(spec, List.copyOf(keys), s3, iamService, roleSession);+ } catch (RuntimeException e) {+ releaseRoleSession(roleSession, spec.iamRoleArn(), iamService);+ throw e;}- return new CopyInput(spec, List.copyOf(keys), s3);}static String copyBackendSql(CopyStatementParser.S3CopyFrom spec) {@@ -121,36 +279,78 @@ public final class S3CopySimulator {}static void streamCopyInput(CopyInput input, OutputStream backendOut) throws IOException {- streamObjects(input.spec(), input.s3(), input.keys(), backendOut);+ streamObjects(input.spec(), input.s3(), input.iamService(), input.roleSession(), input.keys(), backendOut);+ }++ static void releaseCopySession(CopyInput input) {+ releaseRoleSession(input.roleSession(), input.spec().iamRoleArn(), input.iamService());+ }++ static UnloadCollector prepareUnload(CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService) {+ return prepareUnload(spec, s3, iamService, null);+ }++ static UnloadCollector prepareUnload(CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService,+ String clusterAccountId) {+ return prepareUnload(spec, s3, iamService, clusterAccountId, null);}- static UnloadCollector prepareUnload(CopyStatementParser.S3Unload spec, S3Service s3) {+ static UnloadCollector prepareUnload(CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService,+ String clusterAccountId, List<String> associatedRoleArns) {+ RoleSession roleSession = spec.iamRoleArn() != null+ ? resolveRoleSession(spec.iamRoleArn(), iamService, clusterAccountId, associatedRoleArns)+ : null;String probeKey = unloadDataKey(spec, 0);try {- s3.authorizeAnonymousPutObject(spec.bucket(), probeKey);- if (spec.manifest()) {- s3.authorizeAnonymousPutObject(spec.bucket(), spec.prefix() + "manifest");- }- if (!spec.allowOverwrite()) {- s3.authorizeAnonymousListBucket(spec.bucket());- if (targetPrefixHasObjects(spec, s3)) {- throw new S3TransferException(SQLSTATE_INTERNAL,- "S3 prefix s3://" + spec.bucket() + "/" + spec.prefix()- + " is not empty; specify ALLOWOVERWRITE to overwrite", null);+ if (roleSession != null) {+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:PutObject", objectArn(spec.bucket(), probeKey));+ s3.authorizeSignedPutObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), probeKey);+ if (spec.manifest()) {+ String manifestKey = spec.prefix() + "manifest";+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:PutObject", objectArn(spec.bucket(), manifestKey));+ s3.authorizeSignedPutObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), manifestKey);+ }+ if (!spec.allowOverwrite()) {+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:ListBucket", bucketArn(spec.bucket()));+ s3.authorizeSignedListBucket(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket());+ if (targetPrefixHasObjects(spec, s3)) {+ throw new S3TransferException(SQLSTATE_INTERNAL,+ "S3 prefix s3://" + spec.bucket() + "/" + spec.prefix()+ + " is not empty; specify ALLOWOVERWRITE to overwrite", null);+ }+ }+ } else {+ s3.authorizeAnonymousPutObject(spec.bucket(), probeKey);+ if (spec.manifest()) {+ s3.authorizeAnonymousPutObject(spec.bucket(), spec.prefix() + "manifest");+ }+ if (!spec.allowOverwrite()) {+ s3.authorizeAnonymousListBucket(spec.bucket());+ if (targetPrefixHasObjects(spec, s3)) {+ throw new S3TransferException(SQLSTATE_INTERNAL,+ "S3 prefix s3://" + spec.bucket() + "/" + spec.prefix()+ + " is not empty; specify ALLOWOVERWRITE to overwrite", null);+ }}}} catch (AwsException e) {+ releaseRoleSession(roleSession, spec.iamRoleArn(), iamService);throw new S3TransferException(unloadWriteSqlState(e), unloadWriteMessage(e, spec), e);+ } catch (RuntimeException e) {+ releaseRoleSession(roleSession, spec.iamRoleArn(), iamService);+ throw e;}if (!UNLOAD_HEAP_MIB.tryAcquire(UNLOAD_INITIAL_MIB)) {+ releaseRoleSession(roleSession, spec.iamRoleArn(), iamService);throw new S3TransferException(SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,"UNLOAD memory budget exhausted; retry shortly", null);}try {- return new S3UnloadCollector(spec, s3);+ return new S3UnloadCollector(spec, s3, iamService, roleSession);} catch (IOException e) {UNLOAD_HEAP_MIB.release(UNLOAD_INITIAL_MIB);+ releaseRoleSession(roleSession, spec.iamRoleArn(), iamService);throw new S3TransferException(SQLSTATE_INTERNAL, "UNLOAD failed: " + e.getMessage(), e);}}@@ -167,65 +367,82 @@ public final class S3CopySimulator {* client). Part 4 always handles; the value exists so a later interceptor can decline.*/public static boolean runCopyFrom(Socket client, Socket backend,- CopyStatementParser.S3CopyFrom spec, S3Service s3,+ CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService,char txStatus) throws IOException {- return runCopyFrom(client, backend, spec, s3, txStatus, null);+ return runCopyFrom(client, backend, spec, s3, iamService, txStatus, null);+ }++ public static boolean runCopyFrom(Socket client, Socket backend,+ CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService,+ char txStatus, IntConsumer onStatusChange) throws IOException {+ return runCopyFrom(client, backend, spec, s3, iamService, null, txStatus, onStatusChange);+ }++ public static boolean runCopyFrom(Socket client, Socket backend,+ CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService,+ String clusterAccountId, char txStatus, IntConsumer onStatusChange) throws IOException {+ return runCopyFrom(client, backend, spec, s3, iamService, clusterAccountId, null, txStatus, onStatusChange);}public static boolean runCopyFrom(Socket client, Socket backend,- CopyStatementParser.S3CopyFrom spec, S3Service s3,+ CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService,+ String clusterAccountId, List<String> associatedRoleArns,char txStatus, IntConsumer onStatusChange) throws IOException {CopyInput input;try {- input = prepareCopy(spec, s3);+ input = prepareCopy(spec, s3, iamService, clusterAccountId, associatedRoleArns);} catch (S3TransferException e) {LOG.debugv(e, "COPY preparation failed for s3://{0}/{1}", spec.bucket(), spec.keyOrPrefix());sendError(client, backend, e.sqlState(), e.getMessage(), txStatus, onStatusChange);return true;}- OutputStream backendOut = backend.getOutputStream();- backendOut.write(PostgresWireDecoder.encodeQuery(copyBackendSql(spec)));- backendOut.flush();-- PostgresWireDecoder backendDecoder = new PostgresWireDecoder(backend.getInputStream());- PostgresWireDecoder.FrontendMessage first;try {- first = nextNonAsync(backendDecoder, client);- } catch (IOException e) {- LOG.warnv(e, "backend read failed while awaiting CopyInResponse");- closeQuietly(backend);- sendError(client, null, SQLSTATE_INTERNAL, "S3 COPY failed: backend closed or timed out", txStatus, onStatusChange);- closeQuietly(client);- return true;- }- if (first == null) {- LOG.warn("backend closed before answering the fabricated COPY");- closeQuietly(backend);- sendError(client, null, SQLSTATE_INTERNAL, "S3 COPY failed: backend closed before COPY started", txStatus, onStatusChange);- closeQuietly(client);- return true;- }- if (first.type() != 'G') {- // Backend rejected the COPY itself (e.g. no such table). Its ErrorResponse and the- // ReadyForQuery that follows are the client's one response.- forward(client, first);- drainToReadyForQuery(backendDecoder, client, onStatusChange);- return true;- }+ OutputStream backendOut = backend.getOutputStream();+ backendOut.write(PostgresWireDecoder.encodeQuery(copyBackendSql(spec)));+ backendOut.flush();- // The CopyIn stream is open. Any failure from here is resolved with exactly one response:- // a CopyFail to the backend, whose ErrorResponse/ReadyForQuery is relayed to the client;- // or, if the backend is unreachable, one synthesized ErrorResponse/ReadyForQuery.- try {- streamCopyInput(input, backendOut);- writeCopyDone(backendOut);- drainToReadyForQuery(backendDecoder, client, onStatusChange);- } catch (RuntimeException | IOException e) {- LOG.warnv(e, "S3 COPY streaming failed; aborting the open CopyIn");- abortOpenCopyIn(client, backend, backendOut, backendDecoder, e, txStatus, onStatusChange);+ PostgresWireDecoder backendDecoder = new PostgresWireDecoder(backend.getInputStream());+ PostgresWireDecoder.FrontendMessage first;+ try {+ first = nextNonAsync(backendDecoder, client);+ } catch (IOException e) {+ LOG.warnv(e, "backend read failed while awaiting CopyInResponse");+ closeQuietly(backend);+ sendError(client, null, SQLSTATE_INTERNAL, "S3 COPY failed: backend closed or timed out", txStatus, onStatusChange);+ closeQuietly(client);+ return true;+ }+ if (first == null) {+ LOG.warn("backend closed before answering the fabricated COPY");+ closeQuietly(backend);+ sendError(client, null, SQLSTATE_INTERNAL, "S3 COPY failed: backend closed before COPY started", txStatus, onStatusChange);+ closeQuietly(client);+ return true;+ }+ if (first.type() != 'G') {+ // Backend rejected the COPY itself (e.g. no such table). Its ErrorResponse and the+ // ReadyForQuery that follows are the client's one response.+ forward(client, first);+ drainToReadyForQuery(backendDecoder, client, onStatusChange);+ return true;+ }++ // The CopyIn stream is open. Any failure from here is resolved with exactly one response:+ // a CopyFail to the backend, whose ErrorResponse/ReadyForQuery is relayed to the client;+ // or, if the backend is unreachable, one synthesized ErrorResponse/ReadyForQuery.+ try {+ streamCopyInput(input, backendOut);+ writeCopyDone(backendOut);+ drainToReadyForQuery(backendDecoder, client, onStatusChange);+ } catch (RuntimeException | IOException e) {+ LOG.warnv(e, "S3 COPY streaming failed; aborting the open CopyIn");+ abortOpenCopyIn(client, backend, backendOut, backendDecoder, e, txStatus, onStatusChange);+ }+ return true;+ } finally {+ releaseCopySession(input);}- return true;}private static void abortOpenCopyIn(Socket client, Socket backend, OutputStream backendOut,@@ -290,9 +507,14 @@ public final class S3CopySimulator {}private static void streamObjects(CopyStatementParser.S3CopyFrom spec, S3Service s3,+ IamService iamService, RoleSession roleSession,List<String> keys, OutputStream backendOut) throws IOException {byte[] buffer = new byte[CHUNK];for (int i = 0; i < keys.size(); i++) {+ if (roleSession != null) {+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:GetObject", objectArn(spec.bucket(), keys.get(i)));+ s3.authorizeSignedGetObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), keys.get(i));+ }S3Object object = s3.getObject(spec.bucket(), keys.get(i));byte[] data = object != null && object.getData() != null ? object.getData() : new byte[0];InputStream in = new ByteArrayInputStream(data);@@ -492,16 +714,28 @@ public final class S3CopySimulator {}public static boolean runUnload(Socket client, Socket backend,- CopyStatementParser.S3Unload spec, S3Service s3, char txStatus) throws IOException {- return runUnload(client, backend, spec, s3, txStatus, null);+ CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService, char txStatus) throws IOException {+ return runUnload(client, backend, spec, s3, iamService, txStatus, null);}public static boolean runUnload(Socket client, Socket backend,- CopyStatementParser.S3Unload spec, S3Service s3, char txStatus,+ CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService, char txStatus,IntConsumer onStatusChange) throws IOException {+ return runUnload(client, backend, spec, s3, iamService, null, txStatus, onStatusChange);+ }++ public static boolean runUnload(Socket client, Socket backend,+ CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService, String clusterAccountId,+ char txStatus, IntConsumer onStatusChange) throws IOException {+ return runUnload(client, backend, spec, s3, iamService, clusterAccountId, null, txStatus, onStatusChange);+ }++ public static boolean runUnload(Socket client, Socket backend,+ CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService, String clusterAccountId,+ List<String> associatedRoleArns, char txStatus, IntConsumer onStatusChange) throws IOException {UnloadCollector collector;try {- collector = prepareUnload(spec, s3);+ collector = prepareUnload(spec, s3, iamService, clusterAccountId, associatedRoleArns);} catch (S3TransferException e) {LOG.debugv(e, "UNLOAD preparation failed for s3://{0}/{1}", spec.bucket(), spec.prefix());sendError(client, backend, e.sqlState(), e.getMessage(), txStatus, onStatusChange);@@ -645,6 +879,8 @@ public final class S3CopySimulator {private static final class S3UnloadCollector implements UnloadCollector {private final CopyStatementParser.S3Unload spec;private final S3Service s3;+ private final IamService iamService;+ private final RoleSession roleSession;private final long threshold;private final String contentType;private final List<String> writtenKeys = new ArrayList<>();@@ -666,9 +902,12 @@ public final class S3CopySimulator {private int sliceIndex;private int heldMib = UNLOAD_INITIAL_MIB;- private S3UnloadCollector(CopyStatementParser.S3Unload spec, S3Service s3) throws IOException {+ private S3UnloadCollector(CopyStatementParser.S3Unload spec, S3Service s3,+ IamService iamService, RoleSession roleSession) throws IOException {this.spec = spec;this.s3 = s3;+ this.iamService = iamService;+ this.roleSession = roleSession;threshold = spec.maxFileSizeBytes() > 0 ? spec.maxFileSizeBytes() : UNLOAD_TARGET_FILE_BYTES;contentType = spec.gzip() ? "application/gzip" : "text/plain";acc = newAccumulator();@@ -734,7 +973,12 @@ public final class S3CopySimulator {if (spec.manifest()) {try {String key = spec.prefix() + "manifest";- s3.authorizeAnonymousPutObject(spec.bucket(), key);+ if (roleSession != null) {+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:PutObject", objectArn(spec.bucket(), key));+ s3.authorizeSignedPutObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), key);+ } else {+ s3.authorizeAnonymousPutObject(spec.bucket(), key);+ }s3.putObject(spec.bucket(), key,manifestJson(spec.bucket(), writtenKeys, writtenLengths).getBytes(StandardCharsets.UTF_8),@@ -753,7 +997,7 @@ public final class S3CopySimulator {}closeAcc(acc, sink);if (spec.manifest()) {- deleteWritten(s3, spec, writtenKeys);+ deleteWritten(s3, iamService, spec, writtenKeys, roleSession);}aborted = true;}@@ -768,6 +1012,7 @@ public final class S3CopySimulator {}UNLOAD_HEAP_MIB.release(heldMib);heldMib = 0;+ releaseRoleSession(roleSession, spec.iamRoleArn(), iamService);closed = true;}@@ -791,7 +1036,12 @@ public final class S3CopySimulator {private void writePayload(byte[] payload, int index) {String key = unloadDataKey(spec, index);try {- s3.authorizeAnonymousPutObject(spec.bucket(), key);+ if (roleSession != null) {+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:PutObject", objectArn(spec.bucket(), key));+ s3.authorizeSignedPutObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), key);+ } else {+ s3.authorizeAnonymousPutObject(spec.bucket(), key);+ }s3.putObject(spec.bucket(), key, payload, contentType, Map.of());} catch (AwsException e) {fail(unloadWriteSqlState(e), unloadWriteMessage(e, spec), e);@@ -894,7 +1144,9 @@ public final class S3CopySimulator {}}- private static void deleteWritten(S3Service s3, CopyStatementParser.S3Unload spec, List<String> keys) {+ private static void deleteWritten(S3Service s3, IamService iamService,+ CopyStatementParser.S3Unload spec, List<String> keys,+ RoleSession roleSession) {if (spec.allowOverwrite()) {// Under ALLOWOVERWRITE a slice key may have replaced a pre-existing object, and there is no// reliable way to tell an overwrite from a fresh write (the check and the write are not@@ -908,7 +1160,12 @@ public final class S3CopySimulator {// created by this operation and is safe to remove.for (String k : keys) {try {- s3.authorizeAnonymousDeleteObject(spec.bucket(), k);+ if (roleSession != null) {+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:DeleteObject", objectArn(spec.bucket(), k));+ s3.authorizeSignedDeleteObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), k);+ } else {+ s3.authorizeAnonymousDeleteObject(spec.bucket(), k);+ }s3.deleteObject(spec.bucket(), k);} catch (RuntimeException e) {LOG.debugv(e, "could not remove partial UNLOAD object s3://{0}/{1}", spec.bucket(), k);diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex e3bc6e2d25..92123cd98e 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -735,6 +735,30 @@ public class S3Service implements Resettable, ResourceProvider {authorizeDeleteObject(bucketName, key, null, RequestAuthorization.unsigned());}+ /**+ * Authorize {@code s3:GetObject} as a signed principal (an IAM role session's access key),+ * reusing the identity-policy + resource-policy evaluation a genuine SigV4 request goes+ * through. Used for Redshift {@code COPY ... IAM_ROLE '<arn>'}.+ */+ public void authorizeSignedGetObject(String accessKeyId, String sessionToken, String bucketName, String key) {+ authorizeGetObject(bucketName, key, null, new RequestAuthorization(true, accessKeyId, sessionToken));+ }++ /** Authorize a signed {@code s3:PutObject}; see {@link #authorizeSignedGetObject}. */+ public void authorizeSignedPutObject(String accessKeyId, String sessionToken, String bucketName, String key) {+ authorizePutObject(bucketName, key, new RequestAuthorization(true, accessKeyId, sessionToken));+ }++ /** Authorize a signed {@code s3:ListBucket}; see {@link #authorizeSignedGetObject}. */+ public void authorizeSignedListBucket(String accessKeyId, String sessionToken, String bucketName) {+ authorizeListBucket(bucketName, new RequestAuthorization(true, accessKeyId, sessionToken));+ }++ /** Authorize a signed {@code s3:DeleteObject}; see {@link #authorizeSignedGetObject}. */+ public void authorizeSignedDeleteObject(String accessKeyId, String sessionToken, String bucketName, String key) {+ authorizeDeleteObject(bucketName, key, null, new RequestAuthorization(true, accessKeyId, sessionToken));+ }+public void authorizeCloudFrontOacGetObject(String bucketName, String key, String distributionArn) {authorizeCloudFrontGetObject(@@ -929,7 +953,7 @@ public class S3Service implements Resettable, ResourceProvider {.orElse(false);}- boolean isAuthEnforced() {+ public boolean isAuthEnforced() {return enforceAuth;}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 210dcd0907..d90818cbea 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -103,6 +103,7 @@ quarkus:- --initialize-at-run-time=io.github.hectorvent.floci.services.datasync.DataSyncService- --initialize-at-run-time=io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCredentials- --initialize-at-run-time=io.github.hectorvent.floci.services.ssoportal.SsoPortalService+ - --initialize-at-run-time=io.github.hectorvent.floci.services.redshift.proxy.S3CopySimulator# snappy-java (Firehose Snappy/HADOOP_SNAPPY delivery) extracts and loads a# bundled JNI library from the static initializers of these two classes.- --initialize-at-run-time=org.xerial.snappy.Snappydiff --git a/src/test/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluatorTest.java b/src/test/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluatorTest.javaindex 766e6644a7..b4676f4eee 100644--- a/src/test/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluatorTest.java@@ -196,6 +196,28 @@ class AssumeRolePolicyEvaluatorTest {trust("{\"Service\":\"lambda.amazonaws.com\"}"), CALLER_ARN, CALLER_ACCOUNT));}+ @Test+ void allowsMatchingServicePrincipal() {+ assertTrue(evaluator.allowsService(trust("{\"Service\":\"redshift.amazonaws.com\"}"),+ "redshift.amazonaws.com"));+ }++ @Test+ void deniesDifferentServicePrincipal() {+ assertFalse(evaluator.allowsService(trust("{\"Service\":\"lambda.amazonaws.com\"}"),+ "redshift.amazonaws.com"));+ }++ @Test+ void explicitServiceDenyOverridesAllow() {+ String doc = """+ {"Version":"2012-10-17","Statement":[+ {"Effect":"Allow","Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"},+ {"Effect":"Deny","Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}+ """;+ assertFalse(evaluator.allowsService(doc, "redshift.amazonaws.com"));+ }+@Testvoid deniesBlankOrMalformedDocument() {assertFalse(evaluator.allows(null, CALLER_ARN, CALLER_ACCOUNT));diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIamRoleEnforcedIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIamRoleEnforcedIntegrationTest.javanew file mode 100644index 0000000000..0d972fda03--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIamRoleEnforcedIntegrationTest.java@@ -0,0 +1,141 @@+package io.github.hectorvent.floci.services.redshift;++import io.github.hectorvent.floci.services.iam.IamService;+import io.github.hectorvent.floci.services.redshift.model.Cluster;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import jakarta.inject.Inject;+import org.awaitility.Awaitility;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.Assumptions;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.nio.charset.StandardCharsets;+import java.sql.Connection;+import java.sql.DriverManager;+import java.sql.PreparedStatement;+import java.sql.SQLException;+import java.sql.Statement;+import java.time.Duration;+import java.util.List;+import java.util.Map;+import java.util.Objects;++import static org.junit.jupiter.api.Assertions.assertThrows;++/**+ * Covers COPY IAM_ROLE authorization when {@code FLOCI_SERVICES_S3_ENFORCE_AUTH} is on, so the+ * role's identity-based policy actually gates S3 access. Kept in its own {@code @QuarkusTest}+ * class (own {@link EnforceAuthProfile}) because Quarkus restarts the application per distinct+ * test profile; {@link RedshiftInterceptorIntegrationTest} runs with enforcement off.+ */+@QuarkusTest+@TestProfile(RedshiftInterceptorIamRoleEnforcedIntegrationTest.EnforceAuthProfile.class)+class RedshiftInterceptorIamRoleEnforcedIntegrationTest {++ private static final String REDSHIFT_TRUST_POLICY = """+ {"Version":"2012-10-17","Statement":[+ {"Effect":"Allow","Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}+ """;++ @Inject+ RedshiftService service;++ @Inject+ S3Service s3;++ @Inject+ IamService iamService;++ private String clusterId;++ @BeforeAll+ static void requireDocker() {+ Assumptions.assumeTrue(isDockerAvailable(), "Docker daemon must be available for Redshift interceptor integration tests");+ }++ private static boolean isDockerAvailable() {+ try {+ Process process = new ProcessBuilder("docker", "version", "--format", "{{.Server.Version}}")+ .redirectErrorStream(true)+ .start();+ return process.waitFor() == 0;+ } catch (Exception e) {+ return false;+ }+ }++ @AfterEach+ void cleanUp() {+ if (clusterId != null) {+ service.deleteCluster(clusterId);+ }+ }++ private static Connection waitForConnection(Cluster cluster) throws SQLException {+ String url = "jdbc:postgresql://127.0.0.1:" + cluster.getEndpoint().getPort() + "/dev";+ return Awaitility.await()+ .atMost(Duration.ofSeconds(30))+ .pollInterval(Duration.ofMillis(500))+ .ignoreExceptions()+ .until(() -> DriverManager.getConnection(url, "admin", "Secret123"), Objects::nonNull);+ }++ @Test+ void copyWithAllowingRolePolicySucceeds() throws Exception {+ clusterId = "it-copy-iam-role-allow";+ String bucket = "redshift-iam-role-allow";+ s3.createBucket(bucket, "us-east-1");+ s3.putObject(bucket, "p1.txt", "1|alice\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());+ iamService.createRole("CopyRoleAllow", "/", REDSHIFT_TRUST_POLICY, null, 0, null);+ iamService.putRolePolicy("CopyRoleAllow", "AllowS3", """+ {"Version":"2012-10-17","Statement":[+ {"Effect":"Allow","Action":["s3:GetObject","s3:ListBucket"],"Resource":"*"}+ ]}+ """);+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123", null, List.of(),+ List.of("arn:aws:iam::000000000000:role/CopyRoleAllow"));++ try (Connection connection = waitForConnection(cluster);+ Statement ddl = connection.createStatement()) {+ ddl.execute("CREATE TABLE people (id int, name text)");+ try (PreparedStatement copy = connection.prepareStatement(+ "COPY people FROM 's3://" + bucket + "/p1.txt' "+ + "IAM_ROLE 'arn:aws:iam::000000000000:role/CopyRoleAllow'")) {+ copy.execute();+ }+ }+ }++ @Test+ void copyWithNoMatchingRolePolicyFails() throws Exception {+ clusterId = "it-copy-iam-role-deny";+ String bucket = "redshift-iam-role-deny";+ s3.createBucket(bucket, "us-east-1");+ s3.putObject(bucket, "p1.txt", "1|alice\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());+ iamService.createRole("CopyRoleDeny", "/", REDSHIFT_TRUST_POLICY, null, 0, null);+ // No policy attached: implicit deny.+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123", null, List.of(),+ List.of("arn:aws:iam::000000000000:role/CopyRoleDeny"));++ try (Connection connection = waitForConnection(cluster);+ Statement ddl = connection.createStatement()) {+ ddl.execute("CREATE TABLE people (id int, name text)");+ try (PreparedStatement copy = connection.prepareStatement(+ "COPY people FROM 's3://" + bucket + "/p1.txt' "+ + "IAM_ROLE 'arn:aws:iam::000000000000:role/CopyRoleDeny'")) {+ assertThrows(SQLException.class, copy::execute);+ }+ }+ }++ public static final class EnforceAuthProfile implements QuarkusTestProfile {+ @Override+ public Map<String, String> getConfigOverrides() {+ return Map.of("floci.services.s3.enforce-auth", "true");+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.javaindex ba0db3196e..f6071fd8e4 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.redshift;+import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.redshift.model.Cluster;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.S3Object;@@ -39,12 +40,20 @@ import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTestclass RedshiftInterceptorIntegrationTest {+ private static final String REDSHIFT_TRUST_POLICY = """+ {"Version":"2012-10-17","Statement":[+ {"Effect":"Allow","Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}+ """;+@InjectRedshiftService service;@InjectS3Service s3;+ @Inject+ IamService iamService;+private String clusterId;@BeforeAll@@ -131,6 +140,32 @@ class RedshiftInterceptorIntegrationTest {s3.getObject(bucket, objects.get(0).getKey()).getData(), StandardCharsets.UTF_8));}+ @Test+ void copyWithIamRoleSucceedsWhenEnforceAuthIsOffEvenWithoutAPolicy() throws Exception {+ clusterId = "it-copy-iam-role-no-enforce";+ String bucket = "redshift-iam-role-no-enforce";+ s3.createBucket(bucket, "us-east-1");+ s3.putObject(bucket, "people/p1.txt",+ "1|alice\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());+ iamService.createRole("CopyRoleNoPolicy", "/", REDSHIFT_TRUST_POLICY, null, 0, null);+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123", null, List.of(),+ List.of("arn:aws:iam::000000000000:role/CopyRoleNoPolicy"));++ try (Connection connection = waitForConnection(cluster, "admin", "Secret123");+ Statement ddl = connection.createStatement()) {+ ddl.execute("CREATE TABLE people (id int, name text)");+ try (PreparedStatement copy = connection.prepareStatement(+ "COPY people FROM 's3://" + bucket + "/people/p1.txt' "+ + "IAM_ROLE 'arn:aws:iam::000000000000:role/CopyRoleNoPolicy'")) {+ copy.execute();+ }+ try (ResultSet rows = connection.createStatement().executeQuery("SELECT count(*) FROM people")) {+ assertTrue(rows.next());+ assertEquals(1, rows.getInt(1));+ }+ }+ }+@Testvoid namedPreparedCopyAndUnloadCanBeExecutedTwice() throws Exception {clusterId = "it-extended-named";diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.javaindex 8fcb69fa50..67474c7c85 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java@@ -76,6 +76,28 @@ class RedshiftQueryHandlerTest {assertTrue(xml.contains("<ClusterStatus>available</ClusterStatus>"));assertTrue(xml.contains("<RequestId>test-req-id</RequestId>"));}++ @Test+ void createClusterParsesIamRoleArnLocationName() {+ MultivaluedMap<String, String> params = new MultivaluedHashMap<>();+ params.putSingle("ClusterIdentifier", "test-cluster");+ params.putSingle("NodeType", "dc2.large");+ params.putSingle("MasterUsername", "admin");+ params.putSingle("MasterUserPassword", "password123");+ params.putSingle("IamRoles.IamRoleArn.2", "arn:aws:iam::000000000000:role/second");+ params.putSingle("IamRoles.IamRoleArn.1", "arn:aws:iam::000000000000:role/first");++ Cluster cluster = new Cluster();+ cluster.setClusterIdentifier("test-cluster");+ cluster.setClusterStatus("available");+ when(service.createCluster(any(), any(), any(), any(), any(), any(), any())).thenReturn(cluster);++ handler.handle("CreateCluster", params);++ verify(service).createCluster(eq("test-cluster"), eq("dc2.large"), eq("admin"), eq("password123"),+ isNull(), eq(List.of()), eq(List.of("arn:aws:iam::000000000000:role/first",+ "arn:aws:iam::000000000000:role/second")));+ }@Testvoid testDescribeClusters() {diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.javaindex db33ae3fab..6a1fafd742 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java@@ -192,7 +192,7 @@ class RedshiftServiceTest {verify(proxyManager).startProxy(eq("111111111111:c1"), eq(7108),eq("172.17.0.12"), eq(32830), eq("localhost"),- eq("admin"), eq("Secret123"), eq("dev"), any());+ eq("admin"), eq("Secret123"), eq("dev"), any(), any());}@Test@@ -240,7 +240,7 @@ class RedshiftServiceTest {verify(proxyManager).startProxy(eq("111111111111:c1"), eq(cluster.getProxyPort()),eq("172.17.0.9"), eq(32800), eq("localhost"),- eq("admin"), eq("Secret123"), eq("dev"), any());+ eq("admin"), eq("Secret123"), eq("dev"), any(), any());}@Test@@ -456,7 +456,7 @@ class RedshiftServiceTest {verify(proxyManager).stopProxy("111111111111:c1");verify(proxyManager).startProxy(eq("111111111111:c1"), eq(7107),eq("172.17.0.11"), eq(32820), eq("localhost"),- eq("admin"), eq("Secret123"), eq("dev"), any());+ eq("admin"), eq("Secret123"), eq("dev"), any(), any());}@Test@@ -487,7 +487,7 @@ class RedshiftServiceTest {// replacement container must be stopped too — once before the restart, once in// rollback — so it is not left running behind a "failed" cluster.verify(proxyManager).startProxy(eq("111111111111:c1"), eq(7107), any(), anyInt(),- any(), any(), any(), any(), any());+ any(), any(), any(), any(), any(), any());verify(proxyManager, times(2)).stopProxy("111111111111:c1");verify(cm, times(2)).stop("111111111111", "c1");}@@ -860,7 +860,7 @@ class RedshiftServiceTest {assertEquals(restored.getEndpoint().getPort(), restored.getProxyPort());verify(proxyManager).startProxy(eq("111111111111:restored"), anyInt(),eq("172.17.0.10"), eq(32810), eq("localhost"),- eq("admin"), eq("Secret123"), eq("dev"), any());+ eq("admin"), eq("Secret123"), eq("dev"), any(), any());}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.javaindex a8d02b3020..ae88d7d8b5 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java@@ -78,6 +78,50 @@ class CopyStatementParserTest {assertEquals("t", c.targetTable());}+ @Test+ void parsesIamRoleOnCopy() {+ CopyStatementParser.S3CopyFrom c = copyFrom(+ "COPY t FROM 's3://b/k' IAM_ROLE 'arn:aws:iam::000000000000:role/CopyRole'");+ assertEquals("arn:aws:iam::000000000000:role/CopyRole", c.iamRoleArn());+ }++ @Test+ void parsesIamRoleCombinedWithOtherOptions() {+ CopyStatementParser.S3CopyFrom c = copyFrom(+ "COPY t FROM 's3://b/k' IAM_ROLE 'arn:aws:iam::000000000000:role/CopyRole' "+ + "GZIP DELIMITER ','");+ assertEquals("arn:aws:iam::000000000000:role/CopyRole", c.iamRoleArn());+ assertTrue(c.gzip());+ assertEquals(",", c.delimiter());+ }++ @Test+ void copyWithoutIamRoleLeavesItNull() {+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY t FROM 's3://b/k'");+ assertNull(c.iamRoleArn());+ }++ @Test+ void iamRoleDefaultKeywordIsStillUnsupported() {+ // Bare `default` (no quotes) does not match the quoted-ARN clause, so it falls through+ // to the existing catch-all and the whole statement is rejected (fail-open), same as today.+ assertNull(CopyStatementParser.parse("COPY t FROM 's3://b/k' IAM_ROLE default"));+ }++ @Test+ void parsesIamRoleOnUnload() {+ CopyStatementParser.S3Unload u = (CopyStatementParser.S3Unload) CopyStatementParser.parse(+ "UNLOAD ('select 1') TO 's3://b/p' IAM_ROLE 'arn:aws:iam::000000000000:role/UnloadRole'");+ assertEquals("arn:aws:iam::000000000000:role/UnloadRole", u.iamRoleArn());+ }++ @Test+ void unloadWithoutIamRoleLeavesItNull() {+ CopyStatementParser.S3Unload u = (CopyStatementParser.S3Unload) CopyStatementParser.parse(+ "UNLOAD ('select 1') TO 's3://b/p'");+ assertNull(u.iamRoleArn());+ }+@Testvoid rejectsInjectionInTableName() {assertNull(CopyStatementParser.parse("COPY t; DROP TABLE u; FROM 's3://b/k'"));@@ -122,7 +166,6 @@ class CopyStatementParserTest {assertNull(CopyStatementParser.parse("COPY t FROM 's3://b/k' JSON 'auto'"));assertNull(CopyStatementParser.parse("COPY t FROM 's3://b/k' FIXEDWIDTH 'a:1,b:2'"));assertNull(CopyStatementParser.parse("COPY t FROM 's3://b/k' GZIP MAXERROR 10"));- assertNull(CopyStatementParser.parse("COPY t FROM 's3://b/k' IAM_ROLE 'arn:aws:iam::0:role/r'"));assertNull(CopyStatementParser.parse("COPY t FROM 's3://b/k' DATEFORMAT 'YYYY-MM-DD'"));}@@ -263,7 +306,6 @@ class CopyStatementParserTest {void unloadRejectsUnsupportedOptions() {assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' PARQUET"));assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' ENCRYPTED"));- assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' IAM_ROLE 'arn:aws:iam::0:role/r'"));assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' REGION 'us-west-2'"));assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' EXTENSION 'csv'"));assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' ZSTD"));diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.javaindex 468ab11255..389a7747bf 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.redshift.proxy;+import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.S3Object;import org.junit.jupiter.api.AfterEach;@@ -34,6 +35,7 @@ class ExtendedS3ExchangeTest {private Socket testClient;private Socket testBackend;private S3Service s3;+ private IamService iamService;private final AtomicReference<Throwable> backendFailure = new AtomicReference<>();@BeforeEach@@ -48,6 +50,7 @@ class ExtendedS3ExchangeTest {testBackend = backendListener.accept();backendListener.close();s3 = mock(S3Service.class);+ iamService = mock(IamService.class);}@AfterEach@@ -65,7 +68,7 @@ class ExtendedS3ExchangeTest {when(s3.getObject("b", "in/data")).thenReturn(new S3Object("b", "in/data", "1|a\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));CopyStatementParser.S3CopyFrom copy = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "b", "in/data", "|", 0, false, false, null);+ "t", List.of(), "b", "in/data", "|", 0, false, false, null, null);BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());BackendResponseCoordinator.Ticket ticket = coordinator.register(BackendResponseCoordinator.Operation.EXECUTE, null);@@ -73,7 +76,7 @@ class ExtendedS3ExchangeTest {writeSync();Thread backend = backendThread(() -> playCopyIn(copied));- ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), copy, s3, coordinator, ticket);+ ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), copy, s3, iamService, coordinator, ticket);joinBackend(backend);PostgresWireDecoder clientDecoder = new PostgresWireDecoder(testClient.getInputStream());@@ -94,14 +97,14 @@ class ExtendedS3ExchangeTest {});CopyStatementParser.S3Unload unload = new CopyStatementParser.S3Unload("select a from t", "b", "out/", "|", false, false, false,- false, null, false, true, false, 0);+ false, null, false, true, false, 0, null);BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());BackendResponseCoordinator.Ticket ticket = coordinator.register(BackendResponseCoordinator.Operation.EXECUTE, null);writeSync();Thread backend = backendThread(this::playCopyOut);- ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), unload, s3, coordinator, ticket);+ ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), unload, s3, iamService, coordinator, ticket);joinBackend(backend);assertEquals("1\n2\n", new String(written.get("out/000"), StandardCharsets.US_ASCII));@@ -116,7 +119,7 @@ class ExtendedS3ExchangeTest {@Testvoid missingCopyInputSendsCopyFailAndWaitsForSyncRecovery() throws Exception {CopyStatementParser.S3CopyFrom copy = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "b", "missing", "|", 0, false, false, null);+ "t", List.of(), "b", "missing", "|", 0, false, false, null, null);BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());BackendResponseCoordinator.Ticket ticket = coordinator.register(BackendResponseCoordinator.Operation.EXECUTE, null);@@ -134,7 +137,7 @@ class ExtendedS3ExchangeTest {testBackend.getOutputStream().flush();});- ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), copy, s3, coordinator, ticket);+ ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), copy, s3, iamService, coordinator, ticket);joinBackend(backend);testClient.setSoTimeout(1_000);diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.javaindex da27d39748..9998be6fc9 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.redshift.proxy;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.services.acm.CertificateGenerator;+import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;@@ -74,7 +75,7 @@ class RedshiftAuthProxyTest {proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),"admin", "Secret123", "dev",mock(RdsSigV4Validator.class), realTls(), (user, pw) -> PasswordValidator.AuthResult.MASTER_EQUIVALENT,- mock(S3Service.class), 5000, 5000, 100);+ mock(S3Service.class), mock(IamService.class), 5000, 5000, 100);proxy.start(proxyPort);try (Socket client = new Socket("localhost", proxyPort)) {@@ -117,7 +118,7 @@ class RedshiftAuthProxyTest {proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),"admin", "Secret123", "dev",mock(RdsSigV4Validator.class), realTls(), (user, pw) -> PasswordValidator.AuthResult.MASTER_EQUIVALENT,- mock(S3Service.class), 5000, 5000, 100);+ mock(S3Service.class), mock(IamService.class), 5000, 5000, 100);proxy.start(proxyPort);// Connect, then drop without ever sending a startup packet.@@ -134,7 +135,7 @@ class RedshiftAuthProxyTest {proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),"admin", "Secret123", "dev",mock(RdsSigV4Validator.class), realTls(), (user, pw) -> PasswordValidator.AuthResult.MASTER_EQUIVALENT,- mock(S3Service.class), 200, 5000, 100);+ mock(S3Service.class), mock(IamService.class), 200, 5000, 100);proxy.start(proxyPort);try (Socket client = new Socket("localhost", proxyPort)) {@@ -153,7 +154,7 @@ class RedshiftAuthProxyTest {proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),"admin", "Secret123", "dev",mock(RdsSigV4Validator.class), realTls(), (user, pw) -> PasswordValidator.AuthResult.MASTER_EQUIVALENT,- mock(S3Service.class), 5000, 5000, 1);+ mock(S3Service.class), mock(IamService.class), 5000, 5000, 1);proxy.start(proxyPort);// Hold the proxy's single connection permit from the test thread itself, so the@@ -192,7 +193,7 @@ class RedshiftAuthProxyTest {proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),"admin", "Secret123", "dev",mock(RdsSigV4Validator.class), realTls(), (user, pw) -> PasswordValidator.AuthResult.MASTER_EQUIVALENT,- mock(S3Service.class), 5000, 5000, 100);+ mock(S3Service.class), mock(IamService.class), 5000, 5000, 100);proxy.start(proxyPort); // must not throw despite the port being busy at firstassertTrue(portAccepts(proxyPort), "proxy never bound the port after the squatter released it");@@ -205,7 +206,7 @@ class RedshiftAuthProxyTest {proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),"admin", "old", "dev",mock(RdsSigV4Validator.class), realTls(), (user, pw) -> PasswordValidator.AuthResult.MASTER_EQUIVALENT,- mock(S3Service.class), 5000, 5000, 100);+ mock(S3Service.class), mock(IamService.class), 5000, 5000, 100);proxy.start(proxyPort);proxy.updateMasterPassword("rotated");diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.javaindex 2214d9476b..4cb26e418c 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.redshift.proxy;+import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.S3Object;import org.jboss.logging.Logger;@@ -47,7 +48,8 @@ class RedshiftInterceptingBridgeTest {testBackendEnd = backendListener.accept();s3Stub = Mockito.mock(S3Service.class);- RedshiftInterceptingBridge bridge = new RedshiftInterceptingBridge(bridgeClientEnd, bridgeBackendEnd, s3Stub);+ RedshiftInterceptingBridge bridge = new RedshiftInterceptingBridge(+ bridgeClientEnd, bridgeBackendEnd, s3Stub, Mockito.mock(IamService.class));bridgeThread = Thread.ofVirtual().name("bridge-under-test").start(bridge::run);}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.javaindex 6830b41dd0..7bee189f13 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.redshift.proxy;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;@@ -29,10 +30,13 @@ import static org.mockito.Mockito.when;class RedshiftProxyManagerTest {+ private static final String KEY = "111111111111:c1";+ private static final String SECOND_KEY = "111111111111:c2";+private RedshiftProxyManager newManager() {return new RedshiftProxyManager(mock(RdsSigV4Validator.class), mock(RdsProxyTlsCertificates.class),- mock(S3Service.class), testConfig());+ mock(S3Service.class), mock(IamService.class), testConfig());}private static EmulatorConfig testConfig() {@@ -57,8 +61,8 @@ class RedshiftProxyManagerTest {RedshiftProxyManager manager = newManager();int port = availablePort();try {- start(manager, "111111111111:c1", port);- assertTrue(registry(manager).containsKey("111111111111:c1"));+ start(manager, KEY, port);+ assertTrue(registry(manager).containsKey(KEY));assertPortUnavailable(port);} finally {manager.stopAll();@@ -69,11 +73,11 @@ class RedshiftProxyManagerTest {void stopProxyRemovesTheKeyAndReleasesThePort() throws Exception {RedshiftProxyManager manager = newManager();int port = availablePort();- start(manager, "k", port);+ start(manager, KEY, port);- manager.stopProxy("k");+ manager.stopProxy(KEY);- assertFalse(registry(manager).containsKey("k"));+ assertFalse(registry(manager).containsKey(KEY));assertPortAvailable(port);}@@ -82,10 +86,10 @@ class RedshiftProxyManagerTest {RedshiftProxyManager manager = newManager();int firstPort = availablePort();try {- start(manager, "k", firstPort);+ start(manager, KEY, firstPort);int replacementPort = availablePort();- start(manager, "k", replacementPort);+ start(manager, KEY, replacementPort);assertPortAvailable(firstPort);assertPortUnavailable(replacementPort);@@ -106,9 +110,9 @@ class RedshiftProxyManagerTest {RedshiftProxyManager manager = newManager();int port = availablePort();try {- start(manager, "k", port);- manager.updateMasterPassword("k", "rotated");- assertEquals("rotated", masterPassword(registry(manager).get("k")));+ start(manager, KEY, port);+ manager.updateMasterPassword(KEY, "rotated");+ assertEquals("rotated", masterPassword(registry(manager).get(KEY)));} finally {manager.stopAll();}@@ -118,9 +122,9 @@ class RedshiftProxyManagerTest {void stopAllReleasesEveryListenerAndIsIdempotent() throws IOException {RedshiftProxyManager manager = newManager();int a = availablePort();- start(manager, "a", a);+ start(manager, KEY, a);int b = availablePort();- start(manager, "b", b);+ start(manager, SECOND_KEY, b);manager.stopAll();@@ -134,8 +138,8 @@ class RedshiftProxyManagerTest {RedshiftProxyManager manager = newManager();try (ServerSocket occupied = new ServerSocket(0)) {assertThrows(RuntimeException.class,- () -> start(manager, "k", occupied.getLocalPort()));- assertFalse(registry(manager).containsKey("k"));+ () -> start(manager, KEY, occupied.getLocalPort()));+ assertFalse(registry(manager).containsKey(KEY));} finally {manager.stopAll();}@@ -146,12 +150,12 @@ class RedshiftProxyManagerTest {RedshiftProxyManager manager = newManager();RedshiftAuthProxy badProxy = mock(RedshiftAuthProxy.class);doThrow(new RuntimeException("close failed")).when(badProxy).stop();- registry(manager).put("k", badProxy);+ registry(manager).put(KEY, badProxy);- assertThrows(RuntimeException.class, () -> manager.stopProxy("k"));+ assertThrows(RuntimeException.class, () -> manager.stopProxy(KEY));// Proxy stays registered so a later cleanup attempt can still reach it and retry.- assertSame(badProxy, registry(manager).get("k"));+ assertSame(badProxy, registry(manager).get(KEY));}@Test@@ -159,12 +163,12 @@ class RedshiftProxyManagerTest {RedshiftProxyManager manager = newManager();RedshiftAuthProxy proxy = mock(RedshiftAuthProxy.class);doThrow(new RuntimeException("close failed")).doNothing().when(proxy).stop();- registry(manager).put("k", proxy);+ registry(manager).put(KEY, proxy);- assertThrows(RuntimeException.class, () -> manager.stopProxy("k"));- assertDoesNotThrow(() -> manager.stopProxy("k"));+ assertThrows(RuntimeException.class, () -> manager.stopProxy(KEY));+ assertDoesNotThrow(() -> manager.stopProxy(KEY));- assertFalse(registry(manager).containsKey("k"));+ assertFalse(registry(manager).containsKey(KEY));verify(proxy, times(2)).stop();}@@ -173,12 +177,12 @@ class RedshiftProxyManagerTest {RedshiftProxyManager manager = newManager();RedshiftAuthProxy stuck = mock(RedshiftAuthProxy.class);doThrow(new RuntimeException("close failed")).when(stuck).stop();- unclosable(manager).put("k", stuck);+ unclosable(manager).put(KEY, stuck);- assertThrows(RuntimeException.class, () -> manager.stopProxy("k"));+ assertThrows(RuntimeException.class, () -> manager.stopProxy(KEY));// Reference retained so the next attempt can retry the close.- assertSame(stuck, unclosable(manager).get("k"));+ assertSame(stuck, unclosable(manager).get(KEY));}@Test@@ -186,13 +190,13 @@ class RedshiftProxyManagerTest {RedshiftProxyManager manager = newManager();RedshiftAuthProxy stuck = mock(RedshiftAuthProxy.class);doThrow(new RuntimeException("close failed")).when(stuck).stop();- unclosable(manager).put("k", stuck);+ unclosable(manager).put(KEY, stuck);- assertThrows(RuntimeException.class, () -> manager.stopProxy("k"));- assertThrows(RuntimeException.class, () -> manager.stopProxy("k"));+ assertThrows(RuntimeException.class, () -> manager.stopProxy(KEY));+ assertThrows(RuntimeException.class, () -> manager.stopProxy(KEY));verify(stuck, times(2)).stop();- assertTrue(unclosable(manager).containsKey("k"));+ assertTrue(unclosable(manager).containsKey(KEY));}@Test@@ -200,12 +204,12 @@ class RedshiftProxyManagerTest {RedshiftProxyManager manager = newManager();RedshiftAuthProxy recovering = mock(RedshiftAuthProxy.class);doThrow(new RuntimeException("close failed")).doNothing().when(recovering).stop();- unclosable(manager).put("k", recovering);+ unclosable(manager).put(KEY, recovering);- assertThrows(RuntimeException.class, () -> manager.stopProxy("k"));- assertDoesNotThrow(() -> manager.stopProxy("k"));+ assertThrows(RuntimeException.class, () -> manager.stopProxy(KEY));+ assertDoesNotThrow(() -> manager.stopProxy(KEY));- assertFalse(unclosable(manager).containsKey("k"));+ assertFalse(unclosable(manager).containsKey(KEY));}@Test@@ -213,12 +217,12 @@ class RedshiftProxyManagerTest {RedshiftProxyManager manager = newManager();RedshiftAuthProxy stuck = mock(RedshiftAuthProxy.class);doThrow(new RuntimeException("close failed")).when(stuck).stop();- unclosable(manager).put("k", stuck);+ unclosable(manager).put(KEY, stuck);int port = availablePort();try {// A fresh start for the same key must not silently drop the leaked listener.- start(manager, "k", port);- assertSame(stuck, unclosable(manager).get("k"));+ start(manager, KEY, port);+ assertSame(stuck, unclosable(manager).get(KEY));} finally {manager.stopAll();}@@ -230,13 +234,13 @@ class RedshiftProxyManagerTest {RedshiftAuthProxy recovered = mock(RedshiftAuthProxy.class);RedshiftAuthProxy stillStuck = mock(RedshiftAuthProxy.class);doThrow(new RuntimeException("close failed")).when(stillStuck).stop();- unclosable(manager).put("a", recovered);- unclosable(manager).put("b", stillStuck);+ unclosable(manager).put(KEY, recovered);+ unclosable(manager).put(SECOND_KEY, stillStuck);manager.stopAll();- assertFalse(unclosable(manager).containsKey("a"));- assertTrue(unclosable(manager).containsKey("b"));+ assertFalse(unclosable(manager).containsKey(KEY));+ assertTrue(unclosable(manager).containsKey(SECOND_KEY));}// --- reflection + port helpers copied from RdsProxyManagerTest ---diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.javaindex b80d175b94..9aad9aca8f 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java@@ -1,6 +1,8 @@package io.github.hectorvent.floci.services.redshift.proxy;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.iam.IamService;+import io.github.hectorvent.floci.services.iam.model.IamRole;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.S3Object;import org.junit.jupiter.api.AfterEach;@@ -35,6 +37,10 @@ import static org.mockito.ArgumentMatchers.eq;import static org.mockito.ArgumentMatchers.isNull;import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.verifyNoInteractions;import static org.mockito.Mockito.when;class S3CopySimulatorTest {@@ -140,7 +146,7 @@ class S3CopySimulatorTest {private CopyStatementParser.S3Unload unloadSpec(String bucket, String prefix,boolean gzip, boolean manifest, boolean allowOverwrite, boolean parallel, long maxFileSize) {return new CopyStatementParser.S3Unload("select a,b from t", bucket, prefix,- "|", false, gzip, false, false, null, manifest, allowOverwrite, parallel, maxFileSize);+ "|", false, gzip, false, false, null, manifest, allowOverwrite, parallel, maxFileSize, null);}@Test@@ -154,9 +160,9 @@ class S3CopySimulatorTest {new S3Object("b", "prefix/a", new byte[0], "text/plain")),List.of(), false, null));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "b", "prefix/", null, 0, false, true, null);+ "t", List.of(), "b", "prefix/", null, 0, false, true, null, null);- S3CopySimulator.CopyInput input = S3CopySimulator.prepareCopy(spec, s3);+ S3CopySimulator.CopyInput input = S3CopySimulator.prepareCopy(spec, s3, null);assertEquals(List.of("prefix/a", "prefix/z"), input.keys());assertEquals("COPY t FROM STDIN WITH (FORMAT csv, DELIMITER ',')",@@ -166,16 +172,140 @@ class S3CopySimulatorTest {@Testvoid preparedCopyReportsMissingObject() {CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "b", "missing", "|", 0, false, false, null);+ "t", List.of(), "b", "missing", "|", 0, false, false, null, null);S3CopySimulator.S3TransferException error = assertThrows(S3CopySimulator.S3TransferException.class,- () -> S3CopySimulator.prepareCopy(spec, s3));+ () -> S3CopySimulator.prepareCopy(spec, s3, null));assertEquals("XX000", error.sqlState());assertEquals("S3 object s3://b/missing not found", error.getMessage());}+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/CopyRole";++ @Test+ void prepareCopyWithMalformedIamRoleArnFailsBeforeTouchingS3() {+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(+ "t", List.of(), "b", "k", "|", 0, false, false, null, "not-an-arn");+ IamService iamService = mock(IamService.class);++ S3CopySimulator.S3TransferException error = assertThrows(+ S3CopySimulator.S3TransferException.class,+ () -> S3CopySimulator.prepareCopy(spec, s3, iamService));++ assertEquals("42501", error.sqlState());+ verifyNoInteractions(s3);+ }++ @Test+ void prepareCopyWithUnknownRoleFailsBeforeTouchingS3() {+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(+ "t", List.of(), "b", "k", "|", 0, false, false, null, ROLE_ARN);+ IamService iamService = mock(IamService.class);+ when(iamService.findRole(any(), any())).thenReturn(java.util.Optional.empty());++ S3CopySimulator.S3TransferException error = assertThrows(+ S3CopySimulator.S3TransferException.class,+ () -> S3CopySimulator.prepareCopy(spec, s3, iamService));++ assertEquals("42501", error.sqlState());+ verifyNoInteractions(s3);+ }++ @Test+ void prepareCopyRejectsCrossAccountRole() {+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(+ "t", List.of(), "b", "k", "|", 0, false, false, null, ROLE_ARN);+ IamService iamService = mock(IamService.class);++ S3CopySimulator.S3TransferException error = assertThrows(+ S3CopySimulator.S3TransferException.class,+ () -> S3CopySimulator.prepareCopy(spec, s3, iamService, "111111111111"));++ assertEquals("42501", error.sqlState());+ assertTrue(error.getMessage().contains("cross-account"));+ verifyNoInteractions(s3);+ verifyNoInteractions(iamService);+ }++ @Test+ void prepareCopyRejectsRoleNotAssociatedWithCluster() {+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(+ "t", List.of(), "b", "k", "|", 0, false, false, null, ROLE_ARN);+ IamService iamService = mock(IamService.class);++ S3CopySimulator.S3TransferException error = assertThrows(+ S3CopySimulator.S3TransferException.class,+ () -> S3CopySimulator.prepareCopy(spec, s3, iamService, "000000000000", List.of()));++ assertEquals("42501", error.sqlState());+ assertTrue(error.getMessage().contains("not associated"));+ verifyNoInteractions(s3);+ verifyNoInteractions(iamService);+ }++ @Test+ void prepareCopyRejectsRoleWithoutRedshiftTrust() {+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(+ "t", List.of(), "b", "k", "|", 0, false, false, null, ROLE_ARN);+ IamService iamService = mock(IamService.class);+ IamRole role = mock(IamRole.class);+ when(iamService.findRole("000000000000", "CopyRole"))+ .thenReturn(java.util.Optional.of(role));+ when(role.getAssumeRolePolicyDocument()).thenReturn(""+ + "{\"Statement\":[{\"Effect\":\"Allow\","+ + "\"Principal\":{\"Service\":\"lambda.amazonaws.com\"},"+ + "\"Action\":\"sts:AssumeRole\"}]}" );++ S3CopySimulator.S3TransferException error = assertThrows(+ S3CopySimulator.S3TransferException.class,+ () -> S3CopySimulator.prepareCopy(spec, s3, iamService, "000000000000"));++ assertEquals("42501", error.sqlState());+ assertTrue(error.getMessage().contains("trust policy"));+ verifyNoInteractions(s3);+ }++ @Test+ void prepareCopyWithValidRoleCallsSignedAuthorization() {+ when(s3.objectExists("b", "k")).thenReturn(true);+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(+ "t", List.of(), "b", "k", "|", 0, false, false, null, ROLE_ARN);+ IamService iamService = mock(IamService.class);+ when(iamService.findRole(any(), any()))+ .thenReturn(java.util.Optional.of(mock(io.github.hectorvent.floci.services.iam.model.IamRole.class)));++ S3CopySimulator.CopyInput input = S3CopySimulator.prepareCopy(spec, s3, iamService);++ verify(s3, never()).authorizeAnonymousGetObject(any(), any());+ verify(s3, never()).authorizeAnonymousListBucket(any());+ verify(s3).authorizeSignedListBucket(any(), any(), eq("b"));+ verify(s3).authorizeSignedGetObject(any(), any(), eq("b"), eq("k"));+ verify(iamService).registerSessionForAccount(+ eq("000000000000"), any(), any(), any(), eq(ROLE_ARN), any(), isNull());+ verify(iamService, never()).unregisterSession(any(), any());+ }++ @Test+ void copyStreamingUsesRoleSessionForObjectAuthorization() throws Exception {+ when(s3.objectExists("b", "k")).thenReturn(true);+ when(s3.getObject("b", "k"))+ .thenReturn(new S3Object("b", "k", "1|alice\n".getBytes(StandardCharsets.UTF_8), "text/plain"));+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(+ "t", List.of(), "b", "k", "|", 0, false, false, null, ROLE_ARN);+ IamService iamService = mock(IamService.class);+ when(iamService.findRole(any(), any()))+ .thenReturn(java.util.Optional.of(mock(io.github.hectorvent.floci.services.iam.model.IamRole.class)));++ S3CopySimulator.CopyInput input = S3CopySimulator.prepareCopy(spec, s3, iamService);+ S3CopySimulator.streamCopyInput(input, new ByteArrayOutputStream());++ verify(s3, times(2))+ .authorizeSignedGetObject(any(), any(), eq("b"), eq("k"));+ verify(s3, never()).authorizeAnonymousGetObject(any(), any());+ }+@Testvoid unloadCollectorWritesOneEmptyObjectForZeroRows() throws Exception {Map<String, byte[]> written = new ConcurrentHashMap<>();@@ -185,7 +315,7 @@ class S3CopySimulatorTest {});CopyStatementParser.S3Unload spec = unloadSpec("b", "out/", false, false, true, false, 0);- try (S3CopySimulator.UnloadCollector collector = S3CopySimulator.prepareUnload(spec, s3)) {+ try (S3CopySimulator.UnloadCollector collector = S3CopySimulator.prepareUnload(spec, s3, null)) {collector.complete();}@@ -208,7 +338,7 @@ class S3CopySimulatorTest {CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);Thread backend = backendThread(() -> playUnloadBackend("1|alice\n", "2|bob\n"));- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertTrue(handled);@@ -236,10 +366,10 @@ class S3CopySimulatorTest {CopyStatementParser.S3Unload spec = new CopyStatementParser.S3Unload("select a,b from t", "wh", "out/", "|", true, false, true, false, null,- false, false, true, 0);+ false, false, true, 0, null);// The fake backend stands in for PostgreSQL: the first CopyData frame is the header row.Thread backend = backendThread(() -> playUnloadBackend("h1|h2\n", "1|a\n", "2|b\n", "3|c\n"));- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertEquals(3, written.size(), written.keySet().toString());@@ -267,7 +397,7 @@ class S3CopySimulatorTest {CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);Thread backend = backendThread(() -> playUnloadBackend("1|alice\n", "2|bob\n", "3|carol\n"));- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertTrue(written.containsKey("out/0000_part_00"));@@ -295,7 +425,7 @@ class S3CopySimulatorTest {CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, false, 0);Thread backend = backendThread(() -> playUnloadBackend("1|a\n"));- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertTrue(written.containsKey("out/000"), written.keySet().toString());@@ -313,7 +443,7 @@ class S3CopySimulatorTest {CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", true, false, false, true, 0);Thread backend = backendThread(() -> playUnloadBackend("1|a\n", "2|b\n"));- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertTrue(written.containsKey("out/0000_part_00.gz"), written.keySet().toString());@@ -340,7 +470,7 @@ class S3CopySimulatorTest {CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, false, true, 0);Thread backend = backendThread(() -> playUnloadBackend("1|alice\n", "2|bob\n", "3|carol\n"));- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);String manifest = new String(written.get("out/manifest"), StandardCharsets.UTF_8);@@ -360,7 +490,7 @@ class S3CopySimulatorTest {CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);// No backend thread: the select must never run.- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');assertTrue(handled);PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());@@ -386,7 +516,7 @@ class S3CopySimulatorTest {out.flush();});- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());@@ -423,7 +553,7 @@ class S3CopySimulatorTest {}});- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());@@ -458,7 +588,7 @@ class S3CopySimulatorTest {out.flush();});- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'T');+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'T');joinBackend(backend);PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());@@ -503,7 +633,7 @@ class S3CopySimulatorTest {out.flush();});- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertTrue(handled);@@ -527,7 +657,7 @@ class S3CopySimulatorTest {S3CopySimulator.UNLOAD_HEAP_MIB.acquire(192);try {- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');assertTrue(handled);PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());@@ -547,7 +677,7 @@ class S3CopySimulatorTest {when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any())).thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));CopyStatementParser.S3Unload spec = new CopyStatementParser.S3Unload(- "select a from t", "wh", "out/", "\t", true, false, false, true, "\\N", false, true, true, 0);+ "select a from t", "wh", "out/", "\t", true, false, false, true, "\\N", false, true, true, 0, null);AtomicReference<String> seenSql = new AtomicReference<>();Thread backend = backendThread(() -> {@@ -565,7 +695,7 @@ class S3CopySimulatorTest {out.flush();});- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);String sql = seenSql.get();@@ -602,7 +732,7 @@ class S3CopySimulatorTest {out.flush();});- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertTrue(handled);@@ -617,7 +747,7 @@ class S3CopySimulatorTest {.thenThrow(new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');assertTrue(handled);PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());@@ -648,7 +778,7 @@ class S3CopySimulatorTest {CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, false, true, 0);Thread backend = backendThread(() -> playUnloadBackend("1|a\n"));- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertEquals(1, deleted.size());@@ -680,7 +810,7 @@ class S3CopySimulatorTest {CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, true, true, 0);Thread backend = backendThread(() -> playUnloadBackend("1|a\n", "2|b\n"));- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);// Under ALLOWOVERWRITE a written key may have replaced pre-existing (or another@@ -701,7 +831,7 @@ class S3CopySimulatorTest {.when(s3).authorizeAnonymousListBucket(eq("wh"));CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');assertTrue(handled);PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());@@ -741,7 +871,7 @@ class S3CopySimulatorTest {testBackend.close();});- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertEquals(1, deleted.size());@@ -809,12 +939,12 @@ class S3CopySimulatorTest {new S3Object("wh", "d/a.txt", "1|alice\n2|bob\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "people", List.of(), "wh", "d/a.txt", "|", 0, false, false, null);+ "people", List.of(), "wh", "d/a.txt", "|", 0, false, false, null, null);ByteArrayOutputStream copyData = new ByteArrayOutputStream();Thread backend = backendThread(() -> playHappyBackend(copyData));- boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertTrue(handled);@@ -830,12 +960,12 @@ class S3CopySimulatorTest {when(s3.getObject("wh", "k")).thenReturn(new S3Object("wh", "k", "x\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of("a", "b"), "wh", "k", "|", 0, false, false, null);+ "t", List.of("a", "b"), "wh", "k", "|", 0, false, false, null, null);AtomicReference<String> fabricated = new AtomicReference<>();Thread backend = backendThread(() -> fabricated.set(captureFabricatedThenComplete()));- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);String sql = fabricated.get();@@ -851,12 +981,12 @@ class S3CopySimulatorTest {when(s3.getObject("wh", "k")).thenReturn(new S3Object("wh", "k", "x\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of("a", "b"), "wh", "k", null, 0, false, true, null);+ "t", List.of("a", "b"), "wh", "k", null, 0, false, true, null, null);AtomicReference<String> fabricated = new AtomicReference<>();Thread backend = backendThread(() -> fabricated.set(captureFabricatedThenComplete()));- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);String sql = fabricated.get();@@ -871,12 +1001,12 @@ class S3CopySimulatorTest {when(s3.getObject("wh", "k")).thenReturn(new S3Object("wh", "k", "x\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "wh", "k", "|", 0, false, false, "\\N");+ "t", List.of(), "wh", "k", "|", 0, false, false, "\\N", null);AtomicReference<String> fabricated = new AtomicReference<>();Thread backend = backendThread(() -> fabricated.set(captureFabricatedThenComplete()));- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);String sql = fabricated.get();@@ -898,11 +1028,11 @@ class S3CopySimulatorTest {new S3Object("wh", "p/2", "2|b\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "wh", "p/", "|", 1, false, false, null);+ "t", List.of(), "wh", "p/", "|", 1, false, false, null, null);ByteArrayOutputStream copyData = new ByteArrayOutputStream();Thread backend = backendThread(() -> playHappyBackend(copyData));- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertEquals("1|a\n2|b\n", copyData.toString(StandardCharsets.US_ASCII));@@ -914,11 +1044,11 @@ class S3CopySimulatorTest {when(s3.getObject("wh", "k.gz")).thenReturn(new S3Object("wh", "k.gz", gzip("1|a\n2|b\n"), "application/gzip"));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "wh", "k.gz", "|", 0, true, false, null);+ "t", List.of(), "wh", "k.gz", "|", 0, true, false, null, null);ByteArrayOutputStream copyData = new ByteArrayOutputStream();Thread backend = backendThread(() -> playHappyBackend(copyData));- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertEquals("1|a\n2|b\n", copyData.toString(StandardCharsets.US_ASCII));@@ -929,9 +1059,9 @@ class S3CopySimulatorTest {doThrow(new AwsException("AccessDenied", "no", 403)).when(s3).authorizeAnonymousListBucket("wh");CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "wh", "k", "|", 0, false, false, null);+ "t", List.of(), "wh", "k", "|", 0, false, false, null, null);- boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');assertTrue(handled);PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());@@ -947,9 +1077,9 @@ class S3CopySimulatorTest {when(s3.listObjectsWithPrefixes(eq("wh"), eq("missing"), isNull(), anyInt(), any(), any())).thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "wh", "missing", "|", 0, false, false, null);+ "t", List.of(), "wh", "missing", "|", 0, false, false, null, null);- boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');assertTrue(handled);PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());@@ -965,7 +1095,7 @@ class S3CopySimulatorTest {when(s3.getObject("wh", "k")).thenReturn(new S3Object("wh", "k", "1\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "nosuch", List.of(), "wh", "k", "|", 0, false, false, null);+ "nosuch", List.of(), "wh", "k", "|", 0, false, false, null, null);Thread backend = backendThread(() -> {new PostgresWireDecoder(testBackend.getInputStream()).nextMessage();@@ -978,7 +1108,7 @@ class S3CopySimulatorTest {out.flush();});- boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertTrue(handled);@@ -992,7 +1122,7 @@ class S3CopySimulatorTest {when(s3.objectExists("wh", "k")).thenReturn(true);when(s3.getObject("wh", "k")).thenThrow(new AwsException("InternalError", "boom", 500));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "wh", "k", "|", 0, false, false, null);+ "t", List.of(), "wh", "k", "|", 0, false, false, null, null);Thread backend = backendThread(() -> {OutputStream out = testBackend.getOutputStream();@@ -1014,7 +1144,7 @@ class S3CopySimulatorTest {out.flush();});- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());@@ -1031,14 +1161,14 @@ class S3CopySimulatorTest {when(s3.getObject("wh", "k")).thenReturn(new S3Object("wh", "k", "1\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "wh", "k", "|", 0, false, false, null);+ "t", List.of(), "wh", "k", "|", 0, false, false, null, null);Thread backend = backendThread(() -> {new PostgresWireDecoder(testBackend.getInputStream()).nextMessage();testBackend.close();});- boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertTrue(handled);@@ -1053,7 +1183,7 @@ class S3CopySimulatorTest {when(s3.listObjectsWithPrefixes(eq("wh"), eq("missing"), isNull(), anyInt(), any(), any())).thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "wh", "missing", "|", 0, false, false, null);+ "t", List.of(), "wh", "missing", "|", 0, false, false, null, null);Thread backend = backendThread(() -> {PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());@@ -1069,7 +1199,7 @@ class S3CopySimulatorTest {out.flush();});- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'T');+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'T');joinBackend(backend);PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());@@ -1097,11 +1227,11 @@ class S3CopySimulatorTest {new S3Object("wh", "multi/2", "2|b\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "wh", "multi/", "|", 0, false, false, null);+ "t", List.of(), "wh", "multi/", "|", 0, false, false, null, null);ByteArrayOutputStream copyData = new ByteArrayOutputStream();Thread backend = backendThread(() -> playHappyBackend(copyData));- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertEquals("1|a\n2|b\n", copyData.toString(StandardCharsets.US_ASCII));@@ -1113,7 +1243,7 @@ class S3CopySimulatorTest {when(s3.getObject("wh", "k")).thenReturn(new S3Object("wh", "k", "1\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "wh", "k", "|", 0, false, false, null);+ "t", List.of(), "wh", "k", "|", 0, false, false, null, null);simBackend.setSoTimeout(100);@@ -1126,7 +1256,7 @@ class S3CopySimulatorTest {}});- boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');+ boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');joinBackend(backend);assertTrue(handled);@@ -1143,7 +1273,7 @@ class S3CopySimulatorTest {when(s3.listObjectsWithPrefixes(eq("wh"), eq("missing"), isNull(), anyInt(), any(), any())).thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "wh", "missing", "|", 0, false, false, null);+ "t", List.of(), "wh", "missing", "|", 0, false, false, null, null);// Backend closes socket when abort query arrives, simulating dropped connectionThread backend = backendThread(() -> {@@ -1154,7 +1284,7 @@ class S3CopySimulatorTest {testBackend.close();});- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'T');+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'T');joinBackend(backend);assertTrue(simBackend.isClosed(), "simBackend must be closed upon synchronization failure");@@ -1172,7 +1302,7 @@ class S3CopySimulatorTest {when(s3.listObjectsWithPrefixes(eq("wh"), eq("missing"), isNull(), anyInt(), any(), any())).thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(- "t", List.of(), "wh", "missing", "|", 0, false, false, null);+ "t", List.of(), "wh", "missing", "|", 0, false, false, null, null);// Fake backend returns ReadyForQuery with 'T' (e.g. abort query unexpectedly succeeded)Thread backend = backendThread(() -> {@@ -1185,7 +1315,7 @@ class S3CopySimulatorTest {out.flush();});- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'T');+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'T');joinBackend(backend);assertTrue(simBackend.isClosed(), "simBackend must be closed when abort returns non-E status");diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.javaindex a974d6f7a0..8f48075ca0 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java@@ -1222,5 +1222,25 @@ class S3ServiceTest {s3Service.createBucket("anon-del-bucket", "us-east-1");assertDoesNotThrow(() -> s3Service.authorizeAnonymousDeleteObject("anon-del-bucket", "some/key"));}++ @Test+ void authorizeSignedGetObjectIsANoOpWhenEnforceAuthIsOff() {+ assertDoesNotThrow(() -> s3Service.authorizeSignedGetObject("ASIAFAKEKEY00000001", "sessiontoken", "signed-get-bucket", "some/key"));+ }++ @Test+ void authorizeSignedPutObjectIsANoOpWhenEnforceAuthIsOff() {+ assertDoesNotThrow(() -> s3Service.authorizeSignedPutObject("ASIAFAKEKEY00000001", "sessiontoken", "signed-put-bucket", "some/key"));+ }++ @Test+ void authorizeSignedListBucketIsANoOpWhenEnforceAuthIsOff() {+ assertDoesNotThrow(() -> s3Service.authorizeSignedListBucket("ASIAFAKEKEY00000001", "sessiontoken", "signed-list-bucket"));+ }++ @Test+ void authorizeSignedDeleteObjectIsANoOpWhenEnforceAuthIsOff() {+ assertDoesNotThrow(() -> s3Service.authorizeSignedDeleteObject("ASIAFAKEKEY00000001", "sessiontoken", "signed-del-bucket", "some/key"));+ }}diff --git a/tools/docs/services.yaml b/tools/docs/services.yamlindex 9b98d0dc89..bc46c62091 100644--- a/tools/docs/services.yaml+++ b/tools/docs/services.yaml@@ -281,10 +281,10 @@ services:doc: docs/services/redshift.mdsources:- { path: src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java, mode: switch }- # SubnetIds / VpcSecurityGroupIds / TagKeys / DbGroups are labels of an inner+ # SubnetIds / VpcSecurityGroupIds / IamRoles / TagKeys / DbGroups are labels of an inner# member-name switch (memberKeyRegex), not actions, same shape as the rds# exclusions above.- exclude_actions: [SubnetIds, VpcSecurityGroupIds, TagKeys, DbGroups]+ exclude_actions: [SubnetIds, VpcSecurityGroupIds, IamRoles, TagKeys, DbGroups]- service: redshiftdatadoc: docs/services/redshift-data.mdsources: