Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(redshift): authorize COPY and UNLOAD with IAM_ROLE (#3815)

Vũ Thế Huy677cdbe053c9
diff --git a/docs/services/redshift.md b/docs/services/redshift.md
index 724863367b..ddfcb2f28c 100644
--- a/docs/services/redshift.md
+++ b/docs/services/redshift.md
@@ -197,11 +197,14 @@ order) through its own S3 service and streams the rows into the backing PostgreS
a comma default delimiter.
- `IGNOREHEADER` and `HEADER` skip lines from the first resolved object only.
- `GZIP` is the only input compression recognized; `BZIP2`, `LZOP` and `ZSTD` are not.
-- S3 access is authorized as an unsigned request: with `FLOCI_SERVICES_S3_ENFORCE_AUTH` off it is
- unrestricted; with it on, bucket policy and public access settings apply.
+- `IAM_ROLE '<role-arn>'` is supported. The role must be associated with the cluster, exist in
+ the local IAM service, and trust Redshift to assume it. With
+ `FLOCI_SERVICES_S3_ENFORCE_AUTH` off, S3 policy checks are skipped. With it on, the role's
+ identity policy must allow the required S3 actions, and any bucket policy must not deny the
+ request. `IAM_ROLE default` is not supported.
- Any other clause (`FIXEDWIDTH`, `JSON`, `PARQUET`, `AVRO`, `ORC`, `MANIFEST`, `MAXERROR`,
`DATEFORMAT`, `TIMEFORMAT`, `REGION`, `ENCODING`, `ESCAPE`, `REMOVEQUOTES`, `BLANKSASNULL`,
- `EMPTYASNULL`, `TRUNCATECOLUMNS`, `ACCEPTINVCHARS`, credentials clauses, and so on) is not
+ `EMPTYASNULL`, `TRUNCATECOLUMNS`, `ACCEPTINVCHARS`, `CREDENTIALS`, and so on) is not
recognized: the statement is forwarded unchanged and PostgreSQL returns its own error.
- A multi-statement query whose COPY is followed by another statement is not intercepted; send the
COPY on its own.
@@ -248,8 +251,12 @@ the result to S3 as one or more objects under `<prefix>`.
`ALLOWOVERWRITE` a failed UNLOAD leaves its objects in place (they may have replaced prior data,
so they are not deleted); a `MANIFEST` request that fails this way can leave data objects without
a manifest, and rerunning the same statement overwrites them.
-- S3 access is authorized as an unsigned request, like COPY from S3.
-- Any other option (`PARQUET`, `ENCRYPTED`, `REGION`, `IAM_ROLE` / `CREDENTIALS`,
+- `IAM_ROLE '<role-arn>'` is supported. The role must be associated with the cluster, exist in
+ the local IAM service, and trust Redshift to assume it. With
+ `FLOCI_SERVICES_S3_ENFORCE_AUTH` off, S3 policy checks are skipped. With it on, the role's
+ identity policy must allow the required S3 actions, and any bucket policy must not deny the
+ request. `IAM_ROLE default` is not supported.
+- Any other option (`PARQUET`, `ENCRYPTED`, `REGION`, `CREDENTIALS`,
`ZSTD`, `EXTENSION`, `CLEANPATH`, `PARTITION`, and so on) is not intercepted; the
statement is forwarded and PostgreSQL reports its own error.
- Extended Query UNLOAD is supported when the complete statement is present in `Parse` and has no
@@ -287,5 +294,6 @@ These views expose the documented Redshift column names, types, and ordering map
- Parameter groups apply no real engine settings; values are stored and echoed back only.
- Subnet groups, VPC routing, and security groups are metadata only.
- Resize, pause/resume, IAM authentication, snapshot schedules, and cross-region snapshot copy.
+- `IAM_ROLE default` and cross-account role ARNs are not supported for COPY or UNLOAD.
- The auth proxy validates the master user's password and any live `GetClusterCredentials` credential. Other non-master users pass straight through to PostgreSQL, which remains the authority for their credentials.
- IAM database authentication over the wire, and `sslmode=verify-full` against the self-signed proxy certificate.
diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluator.java
index 6ee57597ef..f59e522ee4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluator.java
@@ -7,6 +7,7 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
@@ -74,6 +75,35 @@ public class AssumeRolePolicyEvaluator {
return allow;
}
+ /**
+ * Returns true if the trust policy allows the named AWS service principal to assume the role.
+ */
+ public boolean allowsService(String trustPolicyDocument, String servicePrincipal) {
+ if (trustPolicyDocument == null || trustPolicyDocument.isBlank()) {
+ return false;
+ }
+ JsonNode statements;
+ try {
+ statements = objectMapper.readTree(trustPolicyDocument).path("Statement");
+ } catch (Exception e) {
+ LOG.warnv("Failed to parse trust policy: {0}", e.getMessage());
+ return false;
+ }
+ boolean allow = false;
+ if (statements.isArray()) {
+ for (JsonNode statement : statements) {
+ switch (evaluateServiceStatement(statement, servicePrincipal)) {
+ case DENY -> { return false; }
+ case ALLOW -> allow = true;
+ case NO_MATCH -> { }
+ }
+ }
+ } else if (statements.isObject()) {
+ return evaluateServiceStatement(statements, servicePrincipal) == Match.ALLOW;
+ }
+ return allow;
+ }
+
private enum Match { ALLOW, DENY, NO_MATCH }
private Match evaluateStatement(JsonNode stmt, String callerArn, String callerAccount) {
@@ -86,6 +116,34 @@ public class AssumeRolePolicyEvaluator {
return "Deny".equalsIgnoreCase(stmt.path("Effect").asText("Allow")) ? Match.DENY : Match.ALLOW;
}
+ private Match evaluateServiceStatement(JsonNode stmt, String servicePrincipal) {
+ if (!actionApplies(stmt) || !matchesServicePrincipal(stmt.get("Principal"), servicePrincipal)) {
+ return Match.NO_MATCH;
+ }
+ return "Deny".equalsIgnoreCase(stmt.path("Effect").asText("Allow")) ? Match.DENY : Match.ALLOW;
+ }
+
+ private boolean matchesServicePrincipal(JsonNode principalNode, String servicePrincipal) {
+ if (principalNode == null || servicePrincipal == null) {
+ return false;
+ }
+ JsonNode service = principalNode.isObject() ? principalNode.get("Service") : null;
+ if (service == null) {
+ return false;
+ }
+ if (service.isTextual()) {
+ return IamPolicyEvaluator.globMatches(service.asText(), servicePrincipal);
+ }
+ if (service.isArray()) {
+ for (JsonNode entry : service) {
+ if (entry.isTextual() && IamPolicyEvaluator.globMatches(entry.asText(), servicePrincipal)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
/**
* True if the statement's action element applies to {@code sts:AssumeRole}. An {@code Action}
* element applies when any of its patterns match; a {@code NotAction} element applies when none
@@ -161,7 +219,7 @@ public class AssumeRolePolicyEvaluator {
if (principal.matches("\\d{12}")) {
return principal.equals(callerAccount);
}
- var rootMatcher = ACCOUNT_ROOT_ARN.matcher(principal);
+ Matcher rootMatcher = ACCOUNT_ROOT_ARN.matcher(principal);
if (rootMatcher.matches()) {
return rootMatcher.group(1).equals(callerAccount);
}
@@ -191,7 +249,7 @@ public class AssumeRolePolicyEvaluator {
* be denied a role they are entitled to, with nothing in the response saying why.
*/
private static String assumedRoleToRoleArn(String arn) {
- var m = ASSUMED_ROLE_ARN.matcher(arn);
+ Matcher m = ASSUMED_ROLE_ARN.matcher(arn);
if (!m.matches()) {
return null;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java
index 91a2f6f5e9..3c524da572 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java
@@ -67,9 +67,13 @@ public class RedshiftQueryHandler {
String masterUserPassword = params.getFirst("MasterUserPassword");
String clusterSubnetGroupName = params.getFirst("ClusterSubnetGroupName");
List<String> vpcSecurityGroupIds = memberList(params, "VpcSecurityGroupIds");
+ List<String> iamRoleArns = memberList(params, "IamRoles");
- Cluster cluster = service.createCluster(identifier, nodeType, masterUsername, masterUserPassword,
- clusterSubnetGroupName, vpcSecurityGroupIds);
+ Cluster cluster = iamRoleArns.isEmpty()
+ ? service.createCluster(identifier, nodeType, masterUsername, masterUserPassword,
+ clusterSubnetGroupName, vpcSecurityGroupIds)
+ : service.createCluster(identifier, nodeType, masterUsername, masterUserPassword,
+ clusterSubnetGroupName, vpcSecurityGroupIds, iamRoleArns);
String xml = new XmlBuilder()
.start("CreateClusterResponse")
.start("CreateClusterResult")
@@ -622,6 +626,14 @@ public class RedshiftQueryHandler {
builder.end("VpcSecurityGroups");
}
+ if (cluster.getIamRoleArns() != null && !cluster.getIamRoleArns().isEmpty()) {
+ builder.start("IamRoles");
+ for (String iamRoleArn : cluster.getIamRoleArns()) {
+ builder.start("IamRole").elem("IamRoleArn", iamRoleArn).end("IamRole");
+ }
+ builder.end("IamRoles");
+ }
+
if (cluster.getClusterParameterGroupName() != null) {
builder.start("ClusterParameterGroups")
.start("ClusterParameterGroup")
@@ -848,6 +860,7 @@ public class RedshiftQueryHandler {
return switch (baseName) {
case "SubnetIds" -> quoted + "(\\.member|\\.SubnetIdentifier)?\\.\\d+";
case "VpcSecurityGroupIds" -> quoted + "(\\.member|\\.VpcSecurityGroupId)?\\.\\d+";
+ case "IamRoles" -> quoted + "(\\.member|\\.IamRoleArn)?\\.\\d+";
case "TagKeys" -> quoted + "(\\.member|\\.TagKey)?\\.\\d+";
case "DbGroups" -> quoted + "(\\.member|\\.DbGroup)?\\.\\d+";
default -> quoted + "(\\.member)?\\.\\d+";
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
index 5fdf85377f..989a09977b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
@@ -117,7 +117,8 @@ public class RedshiftService {
relayKey(entry.accountId(), cluster.getClusterIdentifier()), proxyPort,
handle.getHost(), handle.getPort(), endpoint.getAddress(),
cluster.getMasterUsername(), password, CLUSTER_DB_NAME,
- passwordValidatorFor(entry.accountId(), cluster.getClusterIdentifier()));
+ passwordValidatorFor(entry.accountId(), cluster.getClusterIdentifier()),
+ cluster.getIamRoleArns());
cluster.setContainerHost(handle.getHost());
cluster.setContainerPort(handle.getPort());
cluster.setEndpoint(endpoint);
@@ -143,13 +144,20 @@ public class RedshiftService {
}
public Cluster createCluster(String identifier, String nodeType, String username, String password) {
- return createCluster(identifier, nodeType, username, password, null, List.of());
+ return createCluster(identifier, nodeType, username, password, null, List.of(), List.of());
}
// synchronized like modify/reboot: the container + proxy + port steps must not
// interleave with another admin call on the same cluster.
public synchronized Cluster createCluster(String identifier, String nodeType, String username, String password,
String clusterSubnetGroupName, List<String> vpcSecurityGroupIds) {
+ return createCluster(identifier, nodeType, username, password, clusterSubnetGroupName,
+ vpcSecurityGroupIds, List.of());
+ }
+
+ public synchronized Cluster createCluster(String identifier, String nodeType, String username, String password,
+ String clusterSubnetGroupName, List<String> vpcSecurityGroupIds,
+ List<String> iamRoleArns) {
if (clusters.get(identifier).isPresent()) {
throw new AwsException("ClusterAlreadyExists", "Cluster " + identifier + " already exists", 400);
}
@@ -164,6 +172,7 @@ public class RedshiftService {
cluster.setMasterPassword(password);
cluster.setClusterSubnetGroupName(clusterSubnetGroupName);
cluster.setVpcSecurityGroupIds(vpcSecurityGroupIds != null ? vpcSecurityGroupIds : List.of());
+ cluster.setIamRoleArns(iamRoleArns != null ? List.copyOf(iamRoleArns) : List.of());
cluster.setClusterStatus("creating");
clusters.put(identifier, cluster);
clusters.flush();
@@ -181,7 +190,7 @@ public class RedshiftService {
proxyManager.startProxy(relayKey(accountId, identifier), proxyPort,
handle.getHost(), handle.getPort(), endpoint.getAddress(),
username, password, CLUSTER_DB_NAME,
- passwordValidatorFor(accountId, identifier));
+ passwordValidatorFor(accountId, identifier), cluster.getIamRoleArns());
cluster.setContainerHost(handle.getHost());
cluster.setContainerPort(handle.getPort());
cluster.setEndpoint(endpoint);
@@ -497,7 +506,7 @@ public class RedshiftService {
cluster.setProxyPort(proxyPort);
proxyManager.startProxy(key, proxyPort, handle.getHost(), handle.getPort(),
endpoint.getAddress(), cluster.getMasterUsername(), password, CLUSTER_DB_NAME,
- passwordValidatorFor(accountId, clusterIdentifier));
+ passwordValidatorFor(accountId, clusterIdentifier), cluster.getIamRoleArns());
cluster.setContainerHost(handle.getHost());
cluster.setContainerPort(handle.getPort());
cluster.setEndpoint(endpoint);
@@ -727,7 +736,7 @@ public class RedshiftService {
proxyManager.startProxy(relayKey(accountId, clusterIdentifier), proxyPort,
handle.getHost(), handle.getPort(), endpoint.getAddress(),
username, password, CLUSTER_DB_NAME,
- passwordValidatorFor(accountId, clusterIdentifier));
+ passwordValidatorFor(accountId, clusterIdentifier), cluster.getIamRoleArns());
cluster.setContainerHost(handle.getHost());
cluster.setContainerPort(handle.getPort());
cluster.setEndpoint(endpoint);
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/model/Cluster.java b/src/main/java/io/github/hectorvent/floci/services/redshift/model/Cluster.java
index 24c2107b5d..d857d5c5e0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/model/Cluster.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/model/Cluster.java
@@ -17,6 +17,7 @@ public class Cluster {
private String clusterSubnetGroupName;
private String clusterParameterGroupName;
private List<String> vpcSecurityGroupIds = new ArrayList<>();
+ private List<String> iamRoleArns = new ArrayList<>();
public String getClusterIdentifier() { return clusterIdentifier; }
public void setClusterIdentifier(String clusterIdentifier) { this.clusterIdentifier = clusterIdentifier; }
@@ -36,6 +37,8 @@ public class Cluster {
public void setClusterParameterGroupName(String clusterParameterGroupName) { this.clusterParameterGroupName = clusterParameterGroupName; }
public List<String> getVpcSecurityGroupIds() { return vpcSecurityGroupIds; }
public void setVpcSecurityGroupIds(List<String> vpcSecurityGroupIds) { this.vpcSecurityGroupIds = vpcSecurityGroupIds; }
+ public List<String> getIamRoleArns() { return iamRoleArns; }
+ public void setIamRoleArns(List<String> iamRoleArns) { this.iamRoleArns = iamRoleArns; }
// Real backend address of this cluster's PostgreSQL container. `endpoint` now points at the
// auth proxy, not the container, so the container address is kept here for proxy wiring and
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java
index 1fe4cb6eeb..bb5d52c626 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java
@@ -25,7 +25,8 @@ public final class CopyStatementParser {
int headerLines,
boolean gzip,
boolean csv,
- String nullAs) implements S3Statement {
+ String nullAs,
+ String iamRoleArn) implements S3Statement {
}
public record S3Unload(
@@ -41,7 +42,8 @@ public final class CopyStatementParser {
boolean manifest,
boolean allowOverwrite,
boolean parallel,
- long maxFileSizeBytes) implements S3Statement {
+ long maxFileSizeBytes,
+ String iamRoleArn) implements S3Statement {
}
private static final Pattern COPY_PATTERN = Pattern.compile(
@@ -67,7 +69,7 @@ public final class CopyStatementParser {
*/
private static final Pattern UNLOAD_UNSUPPORTED_CLAUSE = Pattern.compile(
"(?i)\\b(FIXEDWIDTH|PARQUET|AVRO|ORC|JSON|SHAPEFILE|BZIP2|LZOP|ZSTD"
- + "|ENCRYPTED|ENCODING|REGION|CREDENTIALS|IAM_ROLE|ACCESS_KEY_ID"
+ + "|ENCRYPTED|ENCODING|REGION|CREDENTIALS|ACCESS_KEY_ID"
+ "|SECRET_ACCESS_KEY|SESSION_TOKEN|MASTER_SYMMETRIC_KEY|KMS_KEY_ID"
+ "|EXTENSION|CLEANPATH|PARTITION|MAXFILESIZE\\s+\\d+\\s*(?:TB|PB))\\b");
@@ -79,6 +81,8 @@ public final class CopyStatementParser {
"(?i)\\bDELIMITER\\s+(?:AS\\s+)?(?:'((?:[^']|'')*)'|\"([^\"]*)\"|([^\\s;]+))");
private static final Pattern NULL_AS_PATTERN = Pattern.compile(
"(?i)\\bNULL\\s+(?:AS\\s+)?(?:'((?:[^']|'')*)'|\"([^\"]*)\")");
+ private static final Pattern IAM_ROLE_PATTERN = Pattern.compile(
+ "(?i)\\bIAM_ROLE\\s+(?:'((?:[^']|'')*)'|\"([^\"]*)\")");
private static final Pattern IGNOREHEADER_PATTERN = Pattern.compile(
"(?i)\\bIGNOREHEADER\\s+(?:AS\\s+)?(\\d+)\\b");
private static final Pattern HEADER_PATTERN = Pattern.compile("(?i)\\bHEADER\\b");
@@ -92,7 +96,7 @@ public final class CopyStatementParser {
*/
private static final Pattern UNSUPPORTED_CLAUSE = Pattern.compile(
"(?i)\\b(FIXEDWIDTH|PARQUET|AVRO|ORC|JSON|SHAPEFILE|BZIP2|LZOP|ZSTD|MANIFEST|MAXERROR"
- + "|DATEFORMAT|TIMEFORMAT|ENCRYPTED|ENCODING|REGION|CREDENTIALS|IAM_ROLE"
+ + "|DATEFORMAT|TIMEFORMAT|ENCRYPTED|ENCODING|REGION|CREDENTIALS"
+ "|ACCESS_KEY_ID|SECRET_ACCESS_KEY|SESSION_TOKEN|MASTER_SYMMETRIC_KEY|KMS_KEY_ID"
+ "|ACCEPTINVCHARS|ACCEPTANYDATE|BLANKSASNULL|EMPTYASNULL|FILLRECORD|TRIMBLANKS"
+ "|TRUNCATECOLUMNS|IGNOREBLANKLINES|ESCAPE|REMOVEQUOTES|EXPLICIT_IDS|COMPUPDATE"
@@ -151,6 +155,7 @@ public final class CopyStatementParser {
String nullAs = null;
String delimiter = null;
long maxFileSizeBytes = 0L;
+ String iamRoleArn = null;
boolean seenCsv = false;
boolean seenGzip = false;
@@ -162,6 +167,7 @@ public final class CopyStatementParser {
boolean seenNull = false;
boolean seenDelimiter = false;
boolean seenMaxFileSize = false;
+ boolean seenIamRole = false;
Matcher csvM = CSV_PATTERN.matcher(options);
Matcher gzipM = GZIP_PATTERN.matcher(options);
@@ -173,6 +179,7 @@ public final class CopyStatementParser {
Matcher delimiterM = DELIMITER_PATTERN.matcher(options);
Matcher nullM = NULL_AS_PATTERN.matcher(options);
Matcher maxFileSizeM = MAXFILESIZE_PATTERN.matcher(options);
+ Matcher iamRoleM = IAM_ROLE_PATTERN.matcher(options);
int offset = 0;
int len = options.length();
@@ -259,6 +266,13 @@ public final class CopyStatementParser {
seenNull = true;
nullAs = extractNullValue(nullM);
offset = nullM.end();
+ } else if (matchClause(iamRoleM, offset, len)) {
+ if (seenIamRole) {
+ return null;
+ }
+ seenIamRole = true;
+ iamRoleArn = extractNullValue(iamRoleM);
+ offset = iamRoleM.end();
} else {
return null;
}
@@ -268,7 +282,7 @@ public final class CopyStatementParser {
delimiter = csv ? "," : "|";
}
return new S3Unload(select, bucket, prefix, delimiter, header, gzip, csv,
- addQuotes, nullAs, manifest, allowOverwrite, parallel, maxFileSizeBytes);
+ addQuotes, nullAs, manifest, allowOverwrite, parallel, maxFileSizeBytes, iamRoleArn);
}
private static S3CopyFrom parseCopy(String cleaned) {
@@ -319,11 +333,14 @@ public final class CopyStatementParser {
String delimiter = null;
int headerLines = 0;
+ String iamRoleArn = null;
+
boolean seenCsv = false;
boolean seenGzip = false;
boolean seenNull = false;
boolean seenDelimiter = false;
boolean seenHeader = false;
+ boolean seenIamRole = false;
Matcher csvMatcher = CSV_PATTERN.matcher(options);
Matcher gzipMatcher = GZIP_PATTERN.matcher(options);
@@ -331,6 +348,7 @@ public final class CopyStatementParser {
Matcher headerMatcher = HEADER_PATTERN.matcher(options);
Matcher delimiterMatcher = DELIMITER_PATTERN.matcher(options);
Matcher nullMatcher = NULL_AS_PATTERN.matcher(options);
+ Matcher iamRoleMatcher = IAM_ROLE_PATTERN.matcher(options);
int offset = 0;
int len = options.length();
@@ -384,6 +402,13 @@ public final class CopyStatementParser {
seenNull = true;
nullAs = extractNullValue(nullMatcher);
offset = nullMatcher.end();
+ } else if (matchClause(iamRoleMatcher, offset, len)) {
+ if (seenIamRole) {
+ return null;
+ }
+ seenIamRole = true;
+ iamRoleArn = extractNullValue(iamRoleMatcher);
+ offset = iamRoleMatcher.end();
} else {
return null;
}
@@ -393,7 +418,7 @@ public final class CopyStatementParser {
delimiter = csv ? "," : "|";
}
- return new S3CopyFrom(table, columns, bucket, keyOrPrefix, delimiter, headerLines, gzip, csv, nullAs);
+ return new S3CopyFrom(table, columns, bucket, keyOrPrefix, delimiter, headerLines, gzip, csv, nullAs, iamRoleArn);
}
private static boolean matchClause(Matcher m, int start, int end) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.java
index 09d67cf398..074d14f33c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.java
@@ -1,9 +1,11 @@
package io.github.hectorvent.floci.services.redshift.proxy;
+import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.s3.S3Service;
import java.io.IOException;
import java.io.OutputStream;
import java.net.Socket;
+import java.util.List;
final class ExtendedS3Exchange {
@@ -15,8 +17,16 @@ final class ExtendedS3Exchange {
static void execute(Socket client, Socket backend,
PostgresWireDecoder.FrontendMessage executeFrame,
- CopyStatementParser.S3Statement statement, S3Service s3Service,
+ CopyStatementParser.S3Statement statement, S3Service s3Service, IamService iamService,
BackendResponseCoordinator coordinator, BackendResponseCoordinator.Ticket ticket) throws IOException {
+ execute(client, backend, executeFrame, statement, s3Service, iamService, null, List.of(), coordinator, ticket);
+ }
+
+ static void execute(Socket client, Socket backend,
+ PostgresWireDecoder.FrontendMessage executeFrame,
+ CopyStatementParser.S3Statement statement, S3Service s3Service, IamService iamService,
+ String clusterAccountId, List<String> iamRoleArns, BackendResponseCoordinator coordinator,
+ BackendResponseCoordinator.Ticket ticket) throws IOException {
BackendResponseCoordinator.GateResult gate;
try {
gate = coordinator.awaitExtendedExecuteTurn(ticket);
@@ -33,12 +43,14 @@ final class ExtendedS3Exchange {
try {
switch (statement) {
case CopyStatementParser.S3CopyFrom copy -> {
- CopyResult result = runCopy(client, backend, executeFrame, copy, s3Service, coordinator);
+ CopyResult result = runCopy(client, backend, executeFrame, copy, s3Service, iamService,
+ clusterAccountId, iamRoleArns, coordinator);
failed = !result.succeeded();
deferredReadyForQuery = result.deferredReadyForQuery();
}
case CopyStatementParser.S3Unload unload -> {
- CopyResult result = runUnload(client, backend, executeFrame, unload, s3Service, coordinator);
+ CopyResult result = runUnload(client, backend, executeFrame, unload, s3Service, iamService,
+ clusterAccountId, iamRoleArns, coordinator);
failed = !result.succeeded();
deferredReadyForQuery = result.deferredReadyForQuery();
}
@@ -57,8 +69,8 @@ final class ExtendedS3Exchange {
private static CopyResult runCopy(Socket client, Socket backend,
PostgresWireDecoder.FrontendMessage executeFrame,
- CopyStatementParser.S3CopyFrom spec, S3Service s3Service,
- BackendResponseCoordinator coordinator) throws IOException {
+ CopyStatementParser.S3CopyFrom spec, S3Service s3Service, IamService iamService,
+ String clusterAccountId, List<String> iamRoleArns, BackendResponseCoordinator coordinator) throws IOException {
OutputStream backendOut = backend.getOutputStream();
backendOut.write(executeFrame.toPacketBytes());
backendOut.flush();
@@ -77,7 +89,7 @@ final class ExtendedS3Exchange {
S3CopySimulator.CopyInput input;
try {
- input = S3CopySimulator.prepareCopy(spec, s3Service);
+ input = S3CopySimulator.prepareCopy(spec, s3Service, iamService, clusterAccountId, iamRoleArns);
} catch (S3CopySimulator.S3TransferException e) {
S3CopySimulator.writeCopyFail(backendOut, e.getMessage());
forwardClientSyncToBackend(sync, backendOut, coordinator);
@@ -97,6 +109,8 @@ final class ExtendedS3Exchange {
String detail = e.getMessage() != null ? e.getMessage() : e.toString();
sendError(client, "XX000", "S3 COPY failed: " + detail);
return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));
+ } finally {
+ S3CopySimulator.releaseCopySession(input);
}
forwardClientSyncToBackend(sync, backendOut, coordinator);
@@ -106,8 +120,8 @@ final class ExtendedS3Exchange {
private static CopyResult runUnload(Socket client, Socket backend,
PostgresWireDecoder.FrontendMessage executeFrame,
- CopyStatementParser.S3Unload spec, S3Service s3Service,
- BackendResponseCoordinator coordinator) throws IOException {
+ CopyStatementParser.S3Unload spec, S3Service s3Service, IamService iamService,
+ String clusterAccountId, List<String> iamRoleArns, BackendResponseCoordinator coordinator) throws IOException {
OutputStream backendOut = backend.getOutputStream();
backendOut.write(executeFrame.toPacketBytes());
backendOut.flush();
@@ -125,7 +139,7 @@ final class ExtendedS3Exchange {
S3CopySimulator.UnloadCollector collector;
try {
- collector = S3CopySimulator.prepareUnload(spec, s3Service);
+ collector = S3CopySimulator.prepareUnload(spec, s3Service, iamService, clusterAccountId, iamRoleArns);
} catch (S3CopySimulator.S3TransferException e) {
S3CopySimulator.writeCopyFail(backendOut, e.getMessage());
drainExecute(client, decoder, coordinator, false);
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java
index 6dab1791a3..ac06e346d3 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.redshift.proxy;
+import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;
import io.github.hectorvent.floci.services.rds.proxy.PostgresProtocolHandler;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;
@@ -12,6 +13,7 @@ import java.net.BindException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
+import java.util.List;
import java.util.concurrent.Semaphore;
/**
@@ -35,6 +37,9 @@ public class RedshiftAuthProxy {
private final RdsProxyTlsCertificates tlsCertificates;
private final PasswordValidator passwordValidator;
private final S3Service s3Service;
+ private final IamService iamService;
+ private final String clusterAccountId;
+ private final List<String> iamRoleArns;
private final int handshakeTimeoutMillis;
private final int backendConnectTimeoutMillis;
private final Semaphore connectionPermits;
@@ -46,7 +51,22 @@ public class RedshiftAuthProxy {
String masterUsername, String masterPassword, String dbName,
RdsSigV4Validator sigV4, RdsProxyTlsCertificates tlsCertificates,
PasswordValidator passwordValidator,
- S3Service s3Service,
+ S3Service s3Service, IamService iamService,
+ int handshakeTimeoutMillis, int backendConnectTimeoutMillis,
+ int maxConnections) {
+ this(clusterKey, backendHost, backendPort, masterUsername, masterPassword, dbName, sigV4,
+ tlsCertificates, passwordValidator, s3Service, iamService, null,
+ List.of(),
+ handshakeTimeoutMillis, backendConnectTimeoutMillis, maxConnections);
+ }
+
+ public RedshiftAuthProxy(String clusterKey, String backendHost, int backendPort,
+ String masterUsername, String masterPassword, String dbName,
+ RdsSigV4Validator sigV4, RdsProxyTlsCertificates tlsCertificates,
+ PasswordValidator passwordValidator,
+ S3Service s3Service, IamService iamService,
+ String clusterAccountId,
+ List<String> iamRoleArns,
int handshakeTimeoutMillis, int backendConnectTimeoutMillis,
int maxConnections) {
this.clusterKey = clusterKey;
@@ -59,6 +79,9 @@ public class RedshiftAuthProxy {
this.tlsCertificates = tlsCertificates;
this.passwordValidator = passwordValidator;
this.s3Service = s3Service;
+ this.iamService = iamService;
+ this.clusterAccountId = clusterAccountId;
+ this.iamRoleArns = iamRoleArns == null ? List.of() : List.copyOf(iamRoleArns);
this.handshakeTimeoutMillis = handshakeTimeoutMillis;
this.backendConnectTimeoutMillis = backendConnectTimeoutMillis;
this.connectionPermits = new Semaphore(Math.max(1, maxConnections));
@@ -175,7 +198,8 @@ public class RedshiftAuthProxy {
if (session != null) {
// Redshift-only DDL (DISTKEY/SORTKEY/ENCODE/...) is rewritten for the plain
// PostgreSQL backend on the way through; every other message is relayed verbatim.
- new RedshiftInterceptingBridge(session.client(), session.backend(), s3Service).run();
+ new RedshiftInterceptingBridge(session.client(), session.backend(), s3Service, iamService,
+ clusterAccountId, iamRoleArns).run();
}
} catch (Exception e) {
LOG.debugv("Redshift connection error for cluster {0}: {1}", clusterKey, e.getMessage());
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
index 5b07d11637..553b75bd2b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.redshift.proxy;
+import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.s3.S3Service;
import org.jboss.logging.Logger;
@@ -8,6 +9,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
import java.net.SocketTimeoutException;
+import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.LockSupport;
import java.util.concurrent.locks.ReentrantLock;
@@ -41,6 +43,9 @@ public class RedshiftInterceptingBridge {
private final Socket client;
private final Socket backend;
private final S3Service s3Service;
+ private final IamService iamService;
+ private final String clusterAccountId;
+ private final List<String> iamRoleArns;
private final ExtendedQuerySession session = new ExtendedQuerySession();
private final BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);
@@ -48,10 +53,23 @@ public class RedshiftInterceptingBridge {
private volatile boolean pumpBetweenMessages = true;
private volatile boolean pumpFinished = false;
- public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service) {
+ public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService) {
+ this(client, backend, s3Service, iamService, null);
+ }
+
+ public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService,
+ String clusterAccountId) {
+ this(client, backend, s3Service, iamService, clusterAccountId, List.of());
+ }
+
+ public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService,
+ String clusterAccountId, List<String> iamRoleArns) {
this.client = client;
this.backend = backend;
this.s3Service = s3Service;
+ this.iamService = iamService;
+ this.clusterAccountId = clusterAccountId;
+ this.iamRoleArns = iamRoleArns == null ? List.of() : List.copyOf(iamRoleArns);
}
@FunctionalInterface
@@ -138,10 +156,10 @@ public class RedshiftInterceptingBridge {
CopyStatementParser.S3Statement statement = parsed;
boolean intercepted = runWithBackendOwned(() -> switch (statement) {
case CopyStatementParser.S3CopyFrom copy -> S3CopySimulator.runCopyFrom(
- client, backend, copy, s3Service, coordinator.lastReadyStatus(),
+ client, backend, copy, s3Service, iamService, clusterAccountId, iamRoleArns, coordinator.lastReadyStatus(),
status -> coordinator.onBackendFrame('Z', new byte[]{(byte) status}));
case CopyStatementParser.S3Unload unload -> S3CopySimulator.runUnload(
- client, backend, unload, s3Service, coordinator.lastReadyStatus(),
+ client, backend, unload, s3Service, iamService, clusterAccountId, iamRoleArns, coordinator.lastReadyStatus(),
status -> coordinator.onBackendFrame('Z', new byte[]{(byte) status}));
});
if (intercepted) {
@@ -276,7 +294,8 @@ public class RedshiftInterceptingBridge {
}
try {
backend.setSoTimeout(EXCHANGE_READ_TIMEOUT_MS);
- ExtendedS3Exchange.execute(client, backend, executeFrame, statement, s3Service, coordinator, ticket);
+ ExtendedS3Exchange.execute(client, backend, executeFrame, statement, s3Service, iamService,
+ clusterAccountId, iamRoleArns, coordinator, ticket);
} finally {
try {
backend.setSoTimeout(PUMP_READ_TIMEOUT_MS);
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java
index 3f8e06d092..f45f026d8f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.redshift.proxy;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;
import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;
@@ -12,6 +13,7 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.io.IOException;
+import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -27,6 +29,7 @@ public class RedshiftProxyManager {
private final RdsSigV4Validator sigV4Validator;
private final RdsProxyTlsCertificates tlsCertificates;
private final S3Service s3Service;
+ private final IamService iamService;
private final EmulatorConfig config;
private final ConcurrentHashMap<String, RedshiftAuthProxy> proxies = new ConcurrentHashMap<>();
/**
@@ -39,10 +42,11 @@ public class RedshiftProxyManager {
@Inject
public RedshiftProxyManager(RdsSigV4Validator sigV4Validator, RdsProxyTlsCertificates tlsCertificates,
- S3Service s3Service, EmulatorConfig config) {
+ S3Service s3Service, IamService iamService, EmulatorConfig config) {
this.sigV4Validator = sigV4Validator;
this.tlsCertificates = tlsCertificates;
this.s3Service = s3Service;
+ this.iamService = iamService;
this.config = config;
}
@@ -50,15 +54,26 @@ public class RedshiftProxyManager {
String backendHost, int backendPort, String advertisedHost,
String masterUsername, String masterPassword, String dbName,
PasswordValidator passwordValidator) {
+ startProxy(relayKey, proxyPort, backendHost, backendPort, advertisedHost, masterUsername,
+ masterPassword, dbName, passwordValidator, List.of());
+ }
+
+ public synchronized void startProxy(String relayKey, int proxyPort,
+ String backendHost, int backendPort, String advertisedHost,
+ String masterUsername, String masterPassword, String dbName,
+ PasswordValidator passwordValidator, List<String> iamRoleArns) {
// A prior unclosable entry for this key is left in place: its listener may still
// be bound, and only a successful stop (never a fresh start) may drop it.
// Make sure the self-signed proxy certificate covers the host clients will connect to,
// so sslmode=prefer/require handshakes succeed.
tlsCertificates.ensureHost(advertisedHost);
EmulatorConfig.RedshiftServiceConfig redshiftConfig = config.services().redshift();
+ String clusterAccountId = relayKey.substring(0, relayKey.indexOf(':'));
RedshiftAuthProxy proxy = new RedshiftAuthProxy(
relayKey, backendHost, backendPort, masterUsername, masterPassword, dbName,
- sigV4Validator, tlsCertificates, passwordValidator, s3Service,
+ sigV4Validator, tlsCertificates, passwordValidator, s3Service, iamService,
+ clusterAccountId,
+ iamRoleArns,
redshiftConfig.proxyHandshakeTimeoutMillis(), redshiftConfig.proxyBackendConnectTimeoutMillis(),
redshiftConfig.proxyMaxConnections());
try {
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java
index 98624f8c1e..12e4fdf01e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java
@@ -1,6 +1,13 @@
package io.github.hectorvent.floci.services.redshift.proxy;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.iam.AssumeRolePolicyEvaluator;
+import io.github.hectorvent.floci.services.iam.IamPolicyEvaluator;
+import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.iam.model.CallerContext;
+import io.github.hectorvent.floci.services.iam.model.IamRole;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.jboss.logging.Logger;
@@ -12,9 +19,13 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
+import java.security.SecureRandom;
+import java.time.Duration;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.Semaphore;
import java.util.function.IntConsumer;
import java.util.zip.GZIPInputStream;
@@ -55,10 +66,129 @@ public final class S3CopySimulator {
private static final String SQLSTATE_INSUFFICIENT_PRIVILEGE = "42501";
private static final String SQLSTATE_INTERNAL = "XX000";
+ private static final SecureRandom SECURE_RANDOM = new SecureRandom();
+ private static final String UPPER_ALPHANUMERIC = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+ private static final String SECRET_CHARACTERS =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+ /** COPY/UNLOAD is synchronous end-to-end; this only needs to outlive one statement. */
+ private static final Duration ROLE_SESSION_TTL = Duration.ofMinutes(5);
+
+ // S3Service's signed-request authorization only checks the bucket's resource policy: for a
+ // genuine HTTP request the identity-policy gate already happened upstream in
+ // IamEnforcementFilter before the request ever reached S3Service. This in-process call never
+ // goes through that filter, so the role's identity policy is evaluated here explicitly.
+ // Stateless (only needs an ObjectMapper), so a local instance avoids threading a new
+ // dependency through the whole proxy chain for this one check.
+ private static final IamPolicyEvaluator ROLE_POLICY_EVALUATOR = new IamPolicyEvaluator(new ObjectMapper());
+ private static final AssumeRolePolicyEvaluator ROLE_TRUST_POLICY_EVALUATOR =
+ new AssumeRolePolicyEvaluator(new ObjectMapper());
+
private S3CopySimulator() {
}
- record CopyInput(CopyStatementParser.S3CopyFrom spec, List<String> keys, S3Service s3) {
+ record CopyInput(CopyStatementParser.S3CopyFrom spec, List<String> keys, S3Service s3,
+ IamService iamService, RoleSession roleSession) {
+ }
+
+ private record RoleSession(String accessKeyId, String sessionToken) {
+ }
+
+ /**
+ * Validates the role ARN, its account and Redshift trust policy, then mints a short-lived session.
+ */
+ private static RoleSession resolveRoleSession(String iamRoleArn, IamService iamService,
+ String clusterAccountId, List<String> associatedRoleArns) {
+ AwsArnUtils.Arn parsed;
+ try {
+ parsed = AwsArnUtils.parse(iamRoleArn);
+ } catch (IllegalArgumentException e) {
+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "IAM Role '" + iamRoleArn + "' could not be assumed: malformed ARN", e);
+ }
+ if (!"iam".equals(parsed.service()) || !parsed.resource().startsWith("role/")) {
+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "IAM Role '" + iamRoleArn + "' could not be assumed: not an IAM role ARN", null);
+ }
+ if (clusterAccountId != null && !clusterAccountId.equals(parsed.accountId())) {
+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "IAM Role '" + iamRoleArn + "' could not be assumed: cross-account role ARNs are not supported", null);
+ }
+ if (clusterAccountId != null && associatedRoleArns != null && !associatedRoleArns.contains(iamRoleArn)) {
+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "IAM Role '" + iamRoleArn
+ + "' could not be assumed: role is not associated with the Redshift cluster", null);
+ }
+ String roleName = parsed.resource().substring(parsed.resource().lastIndexOf('/') + 1);
+ Optional<IamRole> role = iamService.findRole(parsed.accountId(), roleName);
+ if (role.isEmpty()) {
+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "IAM Role '" + iamRoleArn + "' could not be assumed: role does not exist", null);
+ }
+ if (clusterAccountId != null
+ && !ROLE_TRUST_POLICY_EVALUATOR.allowsService(
+ role.get().getAssumeRolePolicyDocument(), "redshift.amazonaws.com")) {
+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "IAM Role '" + iamRoleArn + "' could not be assumed: trust policy does not allow redshift.amazonaws.com",
+ null);
+ }
+
+ String accessKeyId = "ASIA" + randomString(UPPER_ALPHANUMERIC, 16);
+ // A session token is required: IamService.findSecretKey(accessKeyId, sessionToken) treats a
+ // null token on either side as a non-match, so a tokenless session can never be recognised
+ // as a known access key and every signed authorization call would fail closed.
+ String sessionToken = randomString(SECRET_CHARACTERS, 200);
+ iamService.registerSessionForAccount(parsed.accountId(), accessKeyId,
+ randomString(SECRET_CHARACTERS, 40), sessionToken, iamRoleArn,
+ Instant.now().plus(ROLE_SESSION_TTL), null);
+ return new RoleSession(accessKeyId, sessionToken);
+ }
+
+ private static void releaseRoleSession(RoleSession roleSession, String iamRoleArn, IamService iamService) {
+ if (roleSession == null) {
+ return;
+ }
+ AwsArnUtils.Arn parsed = AwsArnUtils.parse(iamRoleArn);
+ iamService.unregisterSession(parsed.accountId(), roleSession.accessKeyId());
+ }
+
+ /**
+ * Evaluates the role's identity-based policy for one S3 action, skipped entirely when
+ * {@code FLOCI_SERVICES_S3_ENFORCE_AUTH} is off (matching the anonymous path's behavior).
+ */
+ private static void authorizeRoleAction(S3Service s3, IamService iamService, String roleArn,
+ String action, String resourceArn) {
+ if (!s3.isAuthEnforced()) {
+ return;
+ }
+ CallerContext caller;
+ try {
+ caller = iamService.resolvePrincipalContext(roleArn);
+ } catch (AwsException e) {
+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "IAM Role '" + roleArn + "' could not be assumed: " + e.getMessage(), e);
+ }
+ IamPolicyEvaluator.SimulationDecision decision =
+ ROLE_POLICY_EVALUATOR.simulatePrincipalPolicy(caller, action, resourceArn, Map.of());
+ if (decision != IamPolicyEvaluator.SimulationDecision.ALLOWED) {
+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "S3 access denied for IAM Role '" + roleArn + "': " + action + " on " + resourceArn, null);
+ }
+ }
+
+ private static String bucketArn(String bucket) {
+ return "arn:aws:s3:::" + bucket;
+ }
+
+ private static String objectArn(String bucket, String key) {
+ return "arn:aws:s3:::" + bucket + "/" + key;
+ }
+
+ private static String randomString(String characters, int length) {
+ StringBuilder value = new StringBuilder(length);
+ for (int i = 0; i < length; i++) {
+ value.append(characters.charAt(SECURE_RANDOM.nextInt(characters.length())));
+ }
+ return value.toString();
}
interface UnloadCollector extends AutoCloseable {
@@ -85,35 +215,63 @@ public final class S3CopySimulator {
}
}
- static CopyInput prepareCopy(CopyStatementParser.S3CopyFrom spec, S3Service s3) {
- try {
- s3.authorizeAnonymousListBucket(spec.bucket());
- } catch (AwsException e) {
- throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
- "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);
- }
+ static CopyInput prepareCopy(CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService) {
+ return prepareCopy(spec, s3, iamService, null);
+ }
- List<String> keys;
- try {
- keys = resolveKeys(spec, s3);
- } catch (AwsException e) {
- throw new S3TransferException(SQLSTATE_INTERNAL,
- "S3 COPY could not list s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);
- }
- if (keys.isEmpty()) {
- throw new S3TransferException(SQLSTATE_INTERNAL,
- "S3 object s3://" + spec.bucket() + "/" + spec.keyOrPrefix() + " not found", null);
- }
+ static CopyInput prepareCopy(CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService,
+ String clusterAccountId) {
+ return prepareCopy(spec, s3, iamService, clusterAccountId, null);
+ }
+ static CopyInput prepareCopy(CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService,
+ String clusterAccountId, List<String> associatedRoleArns) {
+ RoleSession roleSession = spec.iamRoleArn() != null
+ ? resolveRoleSession(spec.iamRoleArn(), iamService, clusterAccountId, associatedRoleArns)
+ : null;
try {
- for (String key : keys) {
- s3.authorizeAnonymousGetObject(spec.bucket(), key);
+ try {
+ if (roleSession != null) {
+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:ListBucket", bucketArn(spec.bucket()));
+ s3.authorizeSignedListBucket(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket());
+ } else {
+ s3.authorizeAnonymousListBucket(spec.bucket());
+ }
+ } catch (AwsException e) {
+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);
}
- } catch (AwsException e) {
- throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
- "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);
+
+ List<String> keys;
+ try {
+ keys = resolveKeys(spec, s3);
+ } catch (AwsException e) {
+ throw new S3TransferException(SQLSTATE_INTERNAL,
+ "S3 COPY could not list s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);
+ }
+ if (keys.isEmpty()) {
+ throw new S3TransferException(SQLSTATE_INTERNAL,
+ "S3 object s3://" + spec.bucket() + "/" + spec.keyOrPrefix() + " not found", null);
+ }
+
+ try {
+ for (String key : keys) {
+ if (roleSession != null) {
+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:GetObject", objectArn(spec.bucket(), key));
+ s3.authorizeSignedGetObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), key);
+ } else {
+ s3.authorizeAnonymousGetObject(spec.bucket(), key);
+ }
+ }
+ } catch (AwsException e) {
+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);
+ }
+ return new CopyInput(spec, List.copyOf(keys), s3, iamService, roleSession);
+ } catch (RuntimeException e) {
+ releaseRoleSession(roleSession, spec.iamRoleArn(), iamService);
+ throw e;
}
- return new CopyInput(spec, List.copyOf(keys), s3);
}
static String copyBackendSql(CopyStatementParser.S3CopyFrom spec) {
@@ -121,36 +279,78 @@ public final class S3CopySimulator {
}
static void streamCopyInput(CopyInput input, OutputStream backendOut) throws IOException {
- streamObjects(input.spec(), input.s3(), input.keys(), backendOut);
+ streamObjects(input.spec(), input.s3(), input.iamService(), input.roleSession(), input.keys(), backendOut);
+ }
+
+ static void releaseCopySession(CopyInput input) {
+ releaseRoleSession(input.roleSession(), input.spec().iamRoleArn(), input.iamService());
+ }
+
+ static UnloadCollector prepareUnload(CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService) {
+ return prepareUnload(spec, s3, iamService, null);
+ }
+
+ static UnloadCollector prepareUnload(CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService,
+ String clusterAccountId) {
+ return prepareUnload(spec, s3, iamService, clusterAccountId, null);
}
- static UnloadCollector prepareUnload(CopyStatementParser.S3Unload spec, S3Service s3) {
+ static UnloadCollector prepareUnload(CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService,
+ String clusterAccountId, List<String> associatedRoleArns) {
+ RoleSession roleSession = spec.iamRoleArn() != null
+ ? resolveRoleSession(spec.iamRoleArn(), iamService, clusterAccountId, associatedRoleArns)
+ : null;
String probeKey = unloadDataKey(spec, 0);
try {
- s3.authorizeAnonymousPutObject(spec.bucket(), probeKey);
- if (spec.manifest()) {
- s3.authorizeAnonymousPutObject(spec.bucket(), spec.prefix() + "manifest");
- }
- if (!spec.allowOverwrite()) {
- s3.authorizeAnonymousListBucket(spec.bucket());
- if (targetPrefixHasObjects(spec, s3)) {
- throw new S3TransferException(SQLSTATE_INTERNAL,
- "S3 prefix s3://" + spec.bucket() + "/" + spec.prefix()
- + " is not empty; specify ALLOWOVERWRITE to overwrite", null);
+ if (roleSession != null) {
+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:PutObject", objectArn(spec.bucket(), probeKey));
+ s3.authorizeSignedPutObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), probeKey);
+ if (spec.manifest()) {
+ String manifestKey = spec.prefix() + "manifest";
+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:PutObject", objectArn(spec.bucket(), manifestKey));
+ s3.authorizeSignedPutObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), manifestKey);
+ }
+ if (!spec.allowOverwrite()) {
+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:ListBucket", bucketArn(spec.bucket()));
+ s3.authorizeSignedListBucket(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket());
+ if (targetPrefixHasObjects(spec, s3)) {
+ throw new S3TransferException(SQLSTATE_INTERNAL,
+ "S3 prefix s3://" + spec.bucket() + "/" + spec.prefix()
+ + " is not empty; specify ALLOWOVERWRITE to overwrite", null);
+ }
+ }
+ } else {
+ s3.authorizeAnonymousPutObject(spec.bucket(), probeKey);
+ if (spec.manifest()) {
+ s3.authorizeAnonymousPutObject(spec.bucket(), spec.prefix() + "manifest");
+ }
+ if (!spec.allowOverwrite()) {
+ s3.authorizeAnonymousListBucket(spec.bucket());
+ if (targetPrefixHasObjects(spec, s3)) {
+ throw new S3TransferException(SQLSTATE_INTERNAL,
+ "S3 prefix s3://" + spec.bucket() + "/" + spec.prefix()
+ + " is not empty; specify ALLOWOVERWRITE to overwrite", null);
+ }
}
}
} catch (AwsException e) {
+ releaseRoleSession(roleSession, spec.iamRoleArn(), iamService);
throw new S3TransferException(unloadWriteSqlState(e), unloadWriteMessage(e, spec), e);
+ } catch (RuntimeException e) {
+ releaseRoleSession(roleSession, spec.iamRoleArn(), iamService);
+ throw e;
}
if (!UNLOAD_HEAP_MIB.tryAcquire(UNLOAD_INITIAL_MIB)) {
+ releaseRoleSession(roleSession, spec.iamRoleArn(), iamService);
throw new S3TransferException(SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,
"UNLOAD memory budget exhausted; retry shortly", null);
}
try {
- return new S3UnloadCollector(spec, s3);
+ return new S3UnloadCollector(spec, s3, iamService, roleSession);
} catch (IOException e) {
UNLOAD_HEAP_MIB.release(UNLOAD_INITIAL_MIB);
+ releaseRoleSession(roleSession, spec.iamRoleArn(), iamService);
throw new S3TransferException(SQLSTATE_INTERNAL, "UNLOAD failed: " + e.getMessage(), e);
}
}
@@ -167,65 +367,82 @@ public final class S3CopySimulator {
* client). Part 4 always handles; the value exists so a later interceptor can decline.
*/
public static boolean runCopyFrom(Socket client, Socket backend,
- CopyStatementParser.S3CopyFrom spec, S3Service s3,
+ CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService,
char txStatus) throws IOException {
- return runCopyFrom(client, backend, spec, s3, txStatus, null);
+ return runCopyFrom(client, backend, spec, s3, iamService, txStatus, null);
+ }
+
+ public static boolean runCopyFrom(Socket client, Socket backend,
+ CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService,
+ char txStatus, IntConsumer onStatusChange) throws IOException {
+ return runCopyFrom(client, backend, spec, s3, iamService, null, txStatus, onStatusChange);
+ }
+
+ public static boolean runCopyFrom(Socket client, Socket backend,
+ CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService,
+ String clusterAccountId, char txStatus, IntConsumer onStatusChange) throws IOException {
+ return runCopyFrom(client, backend, spec, s3, iamService, clusterAccountId, null, txStatus, onStatusChange);
}
public static boolean runCopyFrom(Socket client, Socket backend,
- CopyStatementParser.S3CopyFrom spec, S3Service s3,
+ CopyStatementParser.S3CopyFrom spec, S3Service s3, IamService iamService,
+ String clusterAccountId, List<String> associatedRoleArns,
char txStatus, IntConsumer onStatusChange) throws IOException {
CopyInput input;
try {
- input = prepareCopy(spec, s3);
+ input = prepareCopy(spec, s3, iamService, clusterAccountId, associatedRoleArns);
} catch (S3TransferException e) {
LOG.debugv(e, "COPY preparation failed for s3://{0}/{1}", spec.bucket(), spec.keyOrPrefix());
sendError(client, backend, e.sqlState(), e.getMessage(), txStatus, onStatusChange);
return true;
}
- OutputStream backendOut = backend.getOutputStream();
- backendOut.write(PostgresWireDecoder.encodeQuery(copyBackendSql(spec)));
- backendOut.flush();
-
- PostgresWireDecoder backendDecoder = new PostgresWireDecoder(backend.getInputStream());
- PostgresWireDecoder.FrontendMessage first;
try {
- first = nextNonAsync(backendDecoder, client);
- } catch (IOException e) {
- LOG.warnv(e, "backend read failed while awaiting CopyInResponse");
- closeQuietly(backend);
- sendError(client, null, SQLSTATE_INTERNAL, "S3 COPY failed: backend closed or timed out", txStatus, onStatusChange);
- closeQuietly(client);
- return true;
- }
- if (first == null) {
- LOG.warn("backend closed before answering the fabricated COPY");
- closeQuietly(backend);
- sendError(client, null, SQLSTATE_INTERNAL, "S3 COPY failed: backend closed before COPY started", txStatus, onStatusChange);
- closeQuietly(client);
- return true;
- }
- if (first.type() != 'G') {
- // Backend rejected the COPY itself (e.g. no such table). Its ErrorResponse and the
- // ReadyForQuery that follows are the client's one response.
- forward(client, first);
- drainToReadyForQuery(backendDecoder, client, onStatusChange);
- return true;
- }
+ OutputStream backendOut = backend.getOutputStream();
+ backendOut.write(PostgresWireDecoder.encodeQuery(copyBackendSql(spec)));
+ backendOut.flush();
- // The CopyIn stream is open. Any failure from here is resolved with exactly one response:
- // a CopyFail to the backend, whose ErrorResponse/ReadyForQuery is relayed to the client;
- // or, if the backend is unreachable, one synthesized ErrorResponse/ReadyForQuery.
- try {
- streamCopyInput(input, backendOut);
- writeCopyDone(backendOut);
- drainToReadyForQuery(backendDecoder, client, onStatusChange);
- } catch (RuntimeException | IOException e) {
- LOG.warnv(e, "S3 COPY streaming failed; aborting the open CopyIn");
- abortOpenCopyIn(client, backend, backendOut, backendDecoder, e, txStatus, onStatusChange);
+ PostgresWireDecoder backendDecoder = new PostgresWireDecoder(backend.getInputStream());
+ PostgresWireDecoder.FrontendMessage first;
+ try {
+ first = nextNonAsync(backendDecoder, client);
+ } catch (IOException e) {
+ LOG.warnv(e, "backend read failed while awaiting CopyInResponse");
+ closeQuietly(backend);
+ sendError(client, null, SQLSTATE_INTERNAL, "S3 COPY failed: backend closed or timed out", txStatus, onStatusChange);
+ closeQuietly(client);
+ return true;
+ }
+ if (first == null) {
+ LOG.warn("backend closed before answering the fabricated COPY");
+ closeQuietly(backend);
+ sendError(client, null, SQLSTATE_INTERNAL, "S3 COPY failed: backend closed before COPY started", txStatus, onStatusChange);
+ closeQuietly(client);
+ return true;
+ }
+ if (first.type() != 'G') {
+ // Backend rejected the COPY itself (e.g. no such table). Its ErrorResponse and the
+ // ReadyForQuery that follows are the client's one response.
+ forward(client, first);
+ drainToReadyForQuery(backendDecoder, client, onStatusChange);
+ return true;
+ }
+
+ // The CopyIn stream is open. Any failure from here is resolved with exactly one response:
+ // a CopyFail to the backend, whose ErrorResponse/ReadyForQuery is relayed to the client;
+ // or, if the backend is unreachable, one synthesized ErrorResponse/ReadyForQuery.
+ try {
+ streamCopyInput(input, backendOut);
+ writeCopyDone(backendOut);
+ drainToReadyForQuery(backendDecoder, client, onStatusChange);
+ } catch (RuntimeException | IOException e) {
+ LOG.warnv(e, "S3 COPY streaming failed; aborting the open CopyIn");
+ abortOpenCopyIn(client, backend, backendOut, backendDecoder, e, txStatus, onStatusChange);
+ }
+ return true;
+ } finally {
+ releaseCopySession(input);
}
- return true;
}
private static void abortOpenCopyIn(Socket client, Socket backend, OutputStream backendOut,
@@ -290,9 +507,14 @@ public final class S3CopySimulator {
}
private static void streamObjects(CopyStatementParser.S3CopyFrom spec, S3Service s3,
+ IamService iamService, RoleSession roleSession,
List<String> keys, OutputStream backendOut) throws IOException {
byte[] buffer = new byte[CHUNK];
for (int i = 0; i < keys.size(); i++) {
+ if (roleSession != null) {
+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:GetObject", objectArn(spec.bucket(), keys.get(i)));
+ s3.authorizeSignedGetObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), keys.get(i));
+ }
S3Object object = s3.getObject(spec.bucket(), keys.get(i));
byte[] data = object != null && object.getData() != null ? object.getData() : new byte[0];
InputStream in = new ByteArrayInputStream(data);
@@ -492,16 +714,28 @@ public final class S3CopySimulator {
}
public static boolean runUnload(Socket client, Socket backend,
- CopyStatementParser.S3Unload spec, S3Service s3, char txStatus) throws IOException {
- return runUnload(client, backend, spec, s3, txStatus, null);
+ CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService, char txStatus) throws IOException {
+ return runUnload(client, backend, spec, s3, iamService, txStatus, null);
}
public static boolean runUnload(Socket client, Socket backend,
- CopyStatementParser.S3Unload spec, S3Service s3, char txStatus,
+ CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService, char txStatus,
IntConsumer onStatusChange) throws IOException {
+ return runUnload(client, backend, spec, s3, iamService, null, txStatus, onStatusChange);
+ }
+
+ public static boolean runUnload(Socket client, Socket backend,
+ CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService, String clusterAccountId,
+ char txStatus, IntConsumer onStatusChange) throws IOException {
+ return runUnload(client, backend, spec, s3, iamService, clusterAccountId, null, txStatus, onStatusChange);
+ }
+
+ public static boolean runUnload(Socket client, Socket backend,
+ CopyStatementParser.S3Unload spec, S3Service s3, IamService iamService, String clusterAccountId,
+ List<String> associatedRoleArns, char txStatus, IntConsumer onStatusChange) throws IOException {
UnloadCollector collector;
try {
- collector = prepareUnload(spec, s3);
+ collector = prepareUnload(spec, s3, iamService, clusterAccountId, associatedRoleArns);
} catch (S3TransferException e) {
LOG.debugv(e, "UNLOAD preparation failed for s3://{0}/{1}", spec.bucket(), spec.prefix());
sendError(client, backend, e.sqlState(), e.getMessage(), txStatus, onStatusChange);
@@ -645,6 +879,8 @@ public final class S3CopySimulator {
private static final class S3UnloadCollector implements UnloadCollector {
private final CopyStatementParser.S3Unload spec;
private final S3Service s3;
+ private final IamService iamService;
+ private final RoleSession roleSession;
private final long threshold;
private final String contentType;
private final List<String> writtenKeys = new ArrayList<>();
@@ -666,9 +902,12 @@ public final class S3CopySimulator {
private int sliceIndex;
private int heldMib = UNLOAD_INITIAL_MIB;
- private S3UnloadCollector(CopyStatementParser.S3Unload spec, S3Service s3) throws IOException {
+ private S3UnloadCollector(CopyStatementParser.S3Unload spec, S3Service s3,
+ IamService iamService, RoleSession roleSession) throws IOException {
this.spec = spec;
this.s3 = s3;
+ this.iamService = iamService;
+ this.roleSession = roleSession;
threshold = spec.maxFileSizeBytes() > 0 ? spec.maxFileSizeBytes() : UNLOAD_TARGET_FILE_BYTES;
contentType = spec.gzip() ? "application/gzip" : "text/plain";
acc = newAccumulator();
@@ -734,7 +973,12 @@ public final class S3CopySimulator {
if (spec.manifest()) {
try {
String key = spec.prefix() + "manifest";
- s3.authorizeAnonymousPutObject(spec.bucket(), key);
+ if (roleSession != null) {
+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:PutObject", objectArn(spec.bucket(), key));
+ s3.authorizeSignedPutObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), key);
+ } else {
+ s3.authorizeAnonymousPutObject(spec.bucket(), key);
+ }
s3.putObject(spec.bucket(), key,
manifestJson(spec.bucket(), writtenKeys, writtenLengths)
.getBytes(StandardCharsets.UTF_8),
@@ -753,7 +997,7 @@ public final class S3CopySimulator {
}
closeAcc(acc, sink);
if (spec.manifest()) {
- deleteWritten(s3, spec, writtenKeys);
+ deleteWritten(s3, iamService, spec, writtenKeys, roleSession);
}
aborted = true;
}
@@ -768,6 +1012,7 @@ public final class S3CopySimulator {
}
UNLOAD_HEAP_MIB.release(heldMib);
heldMib = 0;
+ releaseRoleSession(roleSession, spec.iamRoleArn(), iamService);
closed = true;
}
@@ -791,7 +1036,12 @@ public final class S3CopySimulator {
private void writePayload(byte[] payload, int index) {
String key = unloadDataKey(spec, index);
try {
- s3.authorizeAnonymousPutObject(spec.bucket(), key);
+ if (roleSession != null) {
+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:PutObject", objectArn(spec.bucket(), key));
+ s3.authorizeSignedPutObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), key);
+ } else {
+ s3.authorizeAnonymousPutObject(spec.bucket(), key);
+ }
s3.putObject(spec.bucket(), key, payload, contentType, Map.of());
} catch (AwsException e) {
fail(unloadWriteSqlState(e), unloadWriteMessage(e, spec), e);
@@ -894,7 +1144,9 @@ public final class S3CopySimulator {
}
}
- private static void deleteWritten(S3Service s3, CopyStatementParser.S3Unload spec, List<String> keys) {
+ private static void deleteWritten(S3Service s3, IamService iamService,
+ CopyStatementParser.S3Unload spec, List<String> keys,
+ RoleSession roleSession) {
if (spec.allowOverwrite()) {
// Under ALLOWOVERWRITE a slice key may have replaced a pre-existing object, and there is no
// reliable way to tell an overwrite from a fresh write (the check and the write are not
@@ -908,7 +1160,12 @@ public final class S3CopySimulator {
// created by this operation and is safe to remove.
for (String k : keys) {
try {
- s3.authorizeAnonymousDeleteObject(spec.bucket(), k);
+ if (roleSession != null) {
+ authorizeRoleAction(s3, iamService, spec.iamRoleArn(), "s3:DeleteObject", objectArn(spec.bucket(), k));
+ s3.authorizeSignedDeleteObject(roleSession.accessKeyId(), roleSession.sessionToken(), spec.bucket(), k);
+ } else {
+ s3.authorizeAnonymousDeleteObject(spec.bucket(), k);
+ }
s3.deleteObject(spec.bucket(), k);
} catch (RuntimeException e) {
LOG.debugv(e, "could not remove partial UNLOAD object s3://{0}/{1}", spec.bucket(), k);
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index e3bc6e2d25..92123cd98e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -735,6 +735,30 @@ public class S3Service implements Resettable, ResourceProvider {
authorizeDeleteObject(bucketName, key, null, RequestAuthorization.unsigned());
}
+ /**
+ * Authorize {@code s3:GetObject} as a signed principal (an IAM role session's access key),
+ * reusing the identity-policy + resource-policy evaluation a genuine SigV4 request goes
+ * through. Used for Redshift {@code COPY ... IAM_ROLE '<arn>'}.
+ */
+ public void authorizeSignedGetObject(String accessKeyId, String sessionToken, String bucketName, String key) {
+ authorizeGetObject(bucketName, key, null, new RequestAuthorization(true, accessKeyId, sessionToken));
+ }
+
+ /** Authorize a signed {@code s3:PutObject}; see {@link #authorizeSignedGetObject}. */
+ public void authorizeSignedPutObject(String accessKeyId, String sessionToken, String bucketName, String key) {
+ authorizePutObject(bucketName, key, new RequestAuthorization(true, accessKeyId, sessionToken));
+ }
+
+ /** Authorize a signed {@code s3:ListBucket}; see {@link #authorizeSignedGetObject}. */
+ public void authorizeSignedListBucket(String accessKeyId, String sessionToken, String bucketName) {
+ authorizeListBucket(bucketName, new RequestAuthorization(true, accessKeyId, sessionToken));
+ }
+
+ /** Authorize a signed {@code s3:DeleteObject}; see {@link #authorizeSignedGetObject}. */
+ public void authorizeSignedDeleteObject(String accessKeyId, String sessionToken, String bucketName, String key) {
+ authorizeDeleteObject(bucketName, key, null, new RequestAuthorization(true, accessKeyId, sessionToken));
+ }
+
public void authorizeCloudFrontOacGetObject(
String bucketName, String key, String distributionArn) {
authorizeCloudFrontGetObject(
@@ -929,7 +953,7 @@ public class S3Service implements Resettable, ResourceProvider {
.orElse(false);
}
- boolean isAuthEnforced() {
+ public boolean isAuthEnforced() {
return enforceAuth;
}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 210dcd0907..d90818cbea 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -103,6 +103,7 @@ quarkus:
- --initialize-at-run-time=io.github.hectorvent.floci.services.datasync.DataSyncService
- --initialize-at-run-time=io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCredentials
- --initialize-at-run-time=io.github.hectorvent.floci.services.ssoportal.SsoPortalService
+ - --initialize-at-run-time=io.github.hectorvent.floci.services.redshift.proxy.S3CopySimulator
# snappy-java (Firehose Snappy/HADOOP_SNAPPY delivery) extracts and loads a
# bundled JNI library from the static initializers of these two classes.
- --initialize-at-run-time=org.xerial.snappy.Snappy
diff --git a/src/test/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluatorTest.java b/src/test/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluatorTest.java
index 766e6644a7..b4676f4eee 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluatorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iam/AssumeRolePolicyEvaluatorTest.java
@@ -196,6 +196,28 @@ class AssumeRolePolicyEvaluatorTest {
trust("{\"Service\":\"lambda.amazonaws.com\"}"), CALLER_ARN, CALLER_ACCOUNT));
}
+ @Test
+ void allowsMatchingServicePrincipal() {
+ assertTrue(evaluator.allowsService(trust("{\"Service\":\"redshift.amazonaws.com\"}"),
+ "redshift.amazonaws.com"));
+ }
+
+ @Test
+ void deniesDifferentServicePrincipal() {
+ assertFalse(evaluator.allowsService(trust("{\"Service\":\"lambda.amazonaws.com\"}"),
+ "redshift.amazonaws.com"));
+ }
+
+ @Test
+ void explicitServiceDenyOverridesAllow() {
+ String doc = """
+ {"Version":"2012-10-17","Statement":[
+ {"Effect":"Allow","Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"},
+ {"Effect":"Deny","Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}
+ """;
+ assertFalse(evaluator.allowsService(doc, "redshift.amazonaws.com"));
+ }
+
@Test
void deniesBlankOrMalformedDocument() {
assertFalse(evaluator.allows(null, CALLER_ARN, CALLER_ACCOUNT));
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIamRoleEnforcedIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIamRoleEnforcedIntegrationTest.java
new file mode 100644
index 0000000000..0d972fda03
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIamRoleEnforcedIntegrationTest.java
@@ -0,0 +1,141 @@
+package io.github.hectorvent.floci.services.redshift;
+
+import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.redshift.model.Cluster;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import org.awaitility.Awaitility;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Covers COPY IAM_ROLE authorization when {@code FLOCI_SERVICES_S3_ENFORCE_AUTH} is on, so the
+ * role's identity-based policy actually gates S3 access. Kept in its own {@code @QuarkusTest}
+ * class (own {@link EnforceAuthProfile}) because Quarkus restarts the application per distinct
+ * test profile; {@link RedshiftInterceptorIntegrationTest} runs with enforcement off.
+ */
+@QuarkusTest
+@TestProfile(RedshiftInterceptorIamRoleEnforcedIntegrationTest.EnforceAuthProfile.class)
+class RedshiftInterceptorIamRoleEnforcedIntegrationTest {
+
+ private static final String REDSHIFT_TRUST_POLICY = """
+ {"Version":"2012-10-17","Statement":[
+ {"Effect":"Allow","Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}
+ """;
+
+ @Inject
+ RedshiftService service;
+
+ @Inject
+ S3Service s3;
+
+ @Inject
+ IamService iamService;
+
+ private String clusterId;
+
+ @BeforeAll
+ static void requireDocker() {
+ Assumptions.assumeTrue(isDockerAvailable(), "Docker daemon must be available for Redshift interceptor integration tests");
+ }
+
+ private static boolean isDockerAvailable() {
+ try {
+ Process process = new ProcessBuilder("docker", "version", "--format", "{{.Server.Version}}")
+ .redirectErrorStream(true)
+ .start();
+ return process.waitFor() == 0;
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
+ @AfterEach
+ void cleanUp() {
+ if (clusterId != null) {
+ service.deleteCluster(clusterId);
+ }
+ }
+
+ private static Connection waitForConnection(Cluster cluster) throws SQLException {
+ String url = "jdbc:postgresql://127.0.0.1:" + cluster.getEndpoint().getPort() + "/dev";
+ return Awaitility.await()
+ .atMost(Duration.ofSeconds(30))
+ .pollInterval(Duration.ofMillis(500))
+ .ignoreExceptions()
+ .until(() -> DriverManager.getConnection(url, "admin", "Secret123"), Objects::nonNull);
+ }
+
+ @Test
+ void copyWithAllowingRolePolicySucceeds() throws Exception {
+ clusterId = "it-copy-iam-role-allow";
+ String bucket = "redshift-iam-role-allow";
+ s3.createBucket(bucket, "us-east-1");
+ s3.putObject(bucket, "p1.txt", "1|alice\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
+ iamService.createRole("CopyRoleAllow", "/", REDSHIFT_TRUST_POLICY, null, 0, null);
+ iamService.putRolePolicy("CopyRoleAllow", "AllowS3", """
+ {"Version":"2012-10-17","Statement":[
+ {"Effect":"Allow","Action":["s3:GetObject","s3:ListBucket"],"Resource":"*"}
+ ]}
+ """);
+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123", null, List.of(),
+ List.of("arn:aws:iam::000000000000:role/CopyRoleAllow"));
+
+ try (Connection connection = waitForConnection(cluster);
+ Statement ddl = connection.createStatement()) {
+ ddl.execute("CREATE TABLE people (id int, name text)");
+ try (PreparedStatement copy = connection.prepareStatement(
+ "COPY people FROM 's3://" + bucket + "/p1.txt' "
+ + "IAM_ROLE 'arn:aws:iam::000000000000:role/CopyRoleAllow'")) {
+ copy.execute();
+ }
+ }
+ }
+
+ @Test
+ void copyWithNoMatchingRolePolicyFails() throws Exception {
+ clusterId = "it-copy-iam-role-deny";
+ String bucket = "redshift-iam-role-deny";
+ s3.createBucket(bucket, "us-east-1");
+ s3.putObject(bucket, "p1.txt", "1|alice\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
+ iamService.createRole("CopyRoleDeny", "/", REDSHIFT_TRUST_POLICY, null, 0, null);
+ // No policy attached: implicit deny.
+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123", null, List.of(),
+ List.of("arn:aws:iam::000000000000:role/CopyRoleDeny"));
+
+ try (Connection connection = waitForConnection(cluster);
+ Statement ddl = connection.createStatement()) {
+ ddl.execute("CREATE TABLE people (id int, name text)");
+ try (PreparedStatement copy = connection.prepareStatement(
+ "COPY people FROM 's3://" + bucket + "/p1.txt' "
+ + "IAM_ROLE 'arn:aws:iam::000000000000:role/CopyRoleDeny'")) {
+ assertThrows(SQLException.class, copy::execute);
+ }
+ }
+ }
+
+ public static final class EnforceAuthProfile implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of("floci.services.s3.enforce-auth", "true");
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java
index ba0db3196e..f6071fd8e4 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.redshift;
+import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.redshift.model.Cluster;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
@@ -39,12 +40,20 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
@QuarkusTest
class RedshiftInterceptorIntegrationTest {
+ private static final String REDSHIFT_TRUST_POLICY = """
+ {"Version":"2012-10-17","Statement":[
+ {"Effect":"Allow","Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}
+ """;
+
@Inject
RedshiftService service;
@Inject
S3Service s3;
+ @Inject
+ IamService iamService;
+
private String clusterId;
@BeforeAll
@@ -131,6 +140,32 @@ class RedshiftInterceptorIntegrationTest {
s3.getObject(bucket, objects.get(0).getKey()).getData(), StandardCharsets.UTF_8));
}
+ @Test
+ void copyWithIamRoleSucceedsWhenEnforceAuthIsOffEvenWithoutAPolicy() throws Exception {
+ clusterId = "it-copy-iam-role-no-enforce";
+ String bucket = "redshift-iam-role-no-enforce";
+ s3.createBucket(bucket, "us-east-1");
+ s3.putObject(bucket, "people/p1.txt",
+ "1|alice\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
+ iamService.createRole("CopyRoleNoPolicy", "/", REDSHIFT_TRUST_POLICY, null, 0, null);
+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123", null, List.of(),
+ List.of("arn:aws:iam::000000000000:role/CopyRoleNoPolicy"));
+
+ try (Connection connection = waitForConnection(cluster, "admin", "Secret123");
+ Statement ddl = connection.createStatement()) {
+ ddl.execute("CREATE TABLE people (id int, name text)");
+ try (PreparedStatement copy = connection.prepareStatement(
+ "COPY people FROM 's3://" + bucket + "/people/p1.txt' "
+ + "IAM_ROLE 'arn:aws:iam::000000000000:role/CopyRoleNoPolicy'")) {
+ copy.execute();
+ }
+ try (ResultSet rows = connection.createStatement().executeQuery("SELECT count(*) FROM people")) {
+ assertTrue(rows.next());
+ assertEquals(1, rows.getInt(1));
+ }
+ }
+ }
+
@Test
void namedPreparedCopyAndUnloadCanBeExecutedTwice() throws Exception {
clusterId = "it-extended-named";
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java
index 8fcb69fa50..67474c7c85 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java
@@ -76,6 +76,28 @@ class RedshiftQueryHandlerTest {
assertTrue(xml.contains("<ClusterStatus>available</ClusterStatus>"));
assertTrue(xml.contains("<RequestId>test-req-id</RequestId>"));
}
+
+ @Test
+ void createClusterParsesIamRoleArnLocationName() {
+ MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+ params.putSingle("ClusterIdentifier", "test-cluster");
+ params.putSingle("NodeType", "dc2.large");
+ params.putSingle("MasterUsername", "admin");
+ params.putSingle("MasterUserPassword", "password123");
+ params.putSingle("IamRoles.IamRoleArn.2", "arn:aws:iam::000000000000:role/second");
+ params.putSingle("IamRoles.IamRoleArn.1", "arn:aws:iam::000000000000:role/first");
+
+ Cluster cluster = new Cluster();
+ cluster.setClusterIdentifier("test-cluster");
+ cluster.setClusterStatus("available");
+ when(service.createCluster(any(), any(), any(), any(), any(), any(), any())).thenReturn(cluster);
+
+ handler.handle("CreateCluster", params);
+
+ verify(service).createCluster(eq("test-cluster"), eq("dc2.large"), eq("admin"), eq("password123"),
+ isNull(), eq(List.of()), eq(List.of("arn:aws:iam::000000000000:role/first",
+ "arn:aws:iam::000000000000:role/second")));
+ }
@Test
void testDescribeClusters() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
index db33ae3fab..6a1fafd742 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
@@ -192,7 +192,7 @@ class RedshiftServiceTest {
verify(proxyManager).startProxy(eq("111111111111:c1"), eq(7108),
eq("172.17.0.12"), eq(32830), eq("localhost"),
- eq("admin"), eq("Secret123"), eq("dev"), any());
+ eq("admin"), eq("Secret123"), eq("dev"), any(), any());
}
@Test
@@ -240,7 +240,7 @@ class RedshiftServiceTest {
verify(proxyManager).startProxy(eq("111111111111:c1"), eq(cluster.getProxyPort()),
eq("172.17.0.9"), eq(32800), eq("localhost"),
- eq("admin"), eq("Secret123"), eq("dev"), any());
+ eq("admin"), eq("Secret123"), eq("dev"), any(), any());
}
@Test
@@ -456,7 +456,7 @@ class RedshiftServiceTest {
verify(proxyManager).stopProxy("111111111111:c1");
verify(proxyManager).startProxy(eq("111111111111:c1"), eq(7107),
eq("172.17.0.11"), eq(32820), eq("localhost"),
- eq("admin"), eq("Secret123"), eq("dev"), any());
+ eq("admin"), eq("Secret123"), eq("dev"), any(), any());
}
@Test
@@ -487,7 +487,7 @@ class RedshiftServiceTest {
// replacement container must be stopped too — once before the restart, once in
// rollback — so it is not left running behind a "failed" cluster.
verify(proxyManager).startProxy(eq("111111111111:c1"), eq(7107), any(), anyInt(),
- any(), any(), any(), any(), any());
+ any(), any(), any(), any(), any(), any());
verify(proxyManager, times(2)).stopProxy("111111111111:c1");
verify(cm, times(2)).stop("111111111111", "c1");
}
@@ -860,7 +860,7 @@ class RedshiftServiceTest {
assertEquals(restored.getEndpoint().getPort(), restored.getProxyPort());
verify(proxyManager).startProxy(eq("111111111111:restored"), anyInt(),
eq("172.17.0.10"), eq(32810), eq("localhost"),
- eq("admin"), eq("Secret123"), eq("dev"), any());
+ eq("admin"), eq("Secret123"), eq("dev"), any(), any());
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java
index a8d02b3020..ae88d7d8b5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java
@@ -78,6 +78,50 @@ class CopyStatementParserTest {
assertEquals("t", c.targetTable());
}
+ @Test
+ void parsesIamRoleOnCopy() {
+ CopyStatementParser.S3CopyFrom c = copyFrom(
+ "COPY t FROM 's3://b/k' IAM_ROLE 'arn:aws:iam::000000000000:role/CopyRole'");
+ assertEquals("arn:aws:iam::000000000000:role/CopyRole", c.iamRoleArn());
+ }
+
+ @Test
+ void parsesIamRoleCombinedWithOtherOptions() {
+ CopyStatementParser.S3CopyFrom c = copyFrom(
+ "COPY t FROM 's3://b/k' IAM_ROLE 'arn:aws:iam::000000000000:role/CopyRole' "
+ + "GZIP DELIMITER ','");
+ assertEquals("arn:aws:iam::000000000000:role/CopyRole", c.iamRoleArn());
+ assertTrue(c.gzip());
+ assertEquals(",", c.delimiter());
+ }
+
+ @Test
+ void copyWithoutIamRoleLeavesItNull() {
+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY t FROM 's3://b/k'");
+ assertNull(c.iamRoleArn());
+ }
+
+ @Test
+ void iamRoleDefaultKeywordIsStillUnsupported() {
+ // Bare `default` (no quotes) does not match the quoted-ARN clause, so it falls through
+ // to the existing catch-all and the whole statement is rejected (fail-open), same as today.
+ assertNull(CopyStatementParser.parse("COPY t FROM 's3://b/k' IAM_ROLE default"));
+ }
+
+ @Test
+ void parsesIamRoleOnUnload() {
+ CopyStatementParser.S3Unload u = (CopyStatementParser.S3Unload) CopyStatementParser.parse(
+ "UNLOAD ('select 1') TO 's3://b/p' IAM_ROLE 'arn:aws:iam::000000000000:role/UnloadRole'");
+ assertEquals("arn:aws:iam::000000000000:role/UnloadRole", u.iamRoleArn());
+ }
+
+ @Test
+ void unloadWithoutIamRoleLeavesItNull() {
+ CopyStatementParser.S3Unload u = (CopyStatementParser.S3Unload) CopyStatementParser.parse(
+ "UNLOAD ('select 1') TO 's3://b/p'");
+ assertNull(u.iamRoleArn());
+ }
+
@Test
void rejectsInjectionInTableName() {
assertNull(CopyStatementParser.parse("COPY t; DROP TABLE u; FROM 's3://b/k'"));
@@ -122,7 +166,6 @@ class CopyStatementParserTest {
assertNull(CopyStatementParser.parse("COPY t FROM 's3://b/k' JSON 'auto'"));
assertNull(CopyStatementParser.parse("COPY t FROM 's3://b/k' FIXEDWIDTH 'a:1,b:2'"));
assertNull(CopyStatementParser.parse("COPY t FROM 's3://b/k' GZIP MAXERROR 10"));
- assertNull(CopyStatementParser.parse("COPY t FROM 's3://b/k' IAM_ROLE 'arn:aws:iam::0:role/r'"));
assertNull(CopyStatementParser.parse("COPY t FROM 's3://b/k' DATEFORMAT 'YYYY-MM-DD'"));
}
@@ -263,7 +306,6 @@ class CopyStatementParserTest {
void unloadRejectsUnsupportedOptions() {
assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' PARQUET"));
assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' ENCRYPTED"));
- assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' IAM_ROLE 'arn:aws:iam::0:role/r'"));
assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' REGION 'us-west-2'"));
assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' EXTENSION 'csv'"));
assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' ZSTD"));
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.java
index 468ab11255..389a7747bf 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.redshift.proxy;
+import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.junit.jupiter.api.AfterEach;
@@ -34,6 +35,7 @@ class ExtendedS3ExchangeTest {
private Socket testClient;
private Socket testBackend;
private S3Service s3;
+ private IamService iamService;
private final AtomicReference<Throwable> backendFailure = new AtomicReference<>();
@BeforeEach
@@ -48,6 +50,7 @@ class ExtendedS3ExchangeTest {
testBackend = backendListener.accept();
backendListener.close();
s3 = mock(S3Service.class);
+ iamService = mock(IamService.class);
}
@AfterEach
@@ -65,7 +68,7 @@ class ExtendedS3ExchangeTest {
when(s3.getObject("b", "in/data")).thenReturn(
new S3Object("b", "in/data", "1|a\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));
CopyStatementParser.S3CopyFrom copy = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "b", "in/data", "|", 0, false, false, null);
+ "t", List.of(), "b", "in/data", "|", 0, false, false, null, null);
BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());
BackendResponseCoordinator.Ticket ticket = coordinator.register(
BackendResponseCoordinator.Operation.EXECUTE, null);
@@ -73,7 +76,7 @@ class ExtendedS3ExchangeTest {
writeSync();
Thread backend = backendThread(() -> playCopyIn(copied));
- ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), copy, s3, coordinator, ticket);
+ ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), copy, s3, iamService, coordinator, ticket);
joinBackend(backend);
PostgresWireDecoder clientDecoder = new PostgresWireDecoder(testClient.getInputStream());
@@ -94,14 +97,14 @@ class ExtendedS3ExchangeTest {
});
CopyStatementParser.S3Unload unload = new CopyStatementParser.S3Unload(
"select a from t", "b", "out/", "|", false, false, false,
- false, null, false, true, false, 0);
+ false, null, false, true, false, 0, null);
BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());
BackendResponseCoordinator.Ticket ticket = coordinator.register(
BackendResponseCoordinator.Operation.EXECUTE, null);
writeSync();
Thread backend = backendThread(this::playCopyOut);
- ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), unload, s3, coordinator, ticket);
+ ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), unload, s3, iamService, coordinator, ticket);
joinBackend(backend);
assertEquals("1\n2\n", new String(written.get("out/000"), StandardCharsets.US_ASCII));
@@ -116,7 +119,7 @@ class ExtendedS3ExchangeTest {
@Test
void missingCopyInputSendsCopyFailAndWaitsForSyncRecovery() throws Exception {
CopyStatementParser.S3CopyFrom copy = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "b", "missing", "|", 0, false, false, null);
+ "t", List.of(), "b", "missing", "|", 0, false, false, null, null);
BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());
BackendResponseCoordinator.Ticket ticket = coordinator.register(
BackendResponseCoordinator.Operation.EXECUTE, null);
@@ -134,7 +137,7 @@ class ExtendedS3ExchangeTest {
testBackend.getOutputStream().flush();
});
- ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), copy, s3, coordinator, ticket);
+ ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), copy, s3, iamService, coordinator, ticket);
joinBackend(backend);
testClient.setSoTimeout(1_000);
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java
index da27d39748..9998be6fc9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.redshift.proxy;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.services.acm.CertificateGenerator;
+import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;
import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;
@@ -74,7 +75,7 @@ class RedshiftAuthProxyTest {
proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),
"admin", "Secret123", "dev",
mock(RdsSigV4Validator.class), realTls(), (user, pw) -> PasswordValidator.AuthResult.MASTER_EQUIVALENT,
- mock(S3Service.class), 5000, 5000, 100);
+ mock(S3Service.class), mock(IamService.class), 5000, 5000, 100);
proxy.start(proxyPort);
try (Socket client = new Socket("localhost", proxyPort)) {
@@ -117,7 +118,7 @@ class RedshiftAuthProxyTest {
proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),
"admin", "Secret123", "dev",
mock(RdsSigV4Validator.class), realTls(), (user, pw) -> PasswordValidator.AuthResult.MASTER_EQUIVALENT,
- mock(S3Service.class), 5000, 5000, 100);
+ mock(S3Service.class), mock(IamService.class), 5000, 5000, 100);
proxy.start(proxyPort);
// Connect, then drop without ever sending a startup packet.
@@ -134,7 +135,7 @@ class RedshiftAuthProxyTest {
proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),
"admin", "Secret123", "dev",
mock(RdsSigV4Validator.class), realTls(), (user, pw) -> PasswordValidator.AuthResult.MASTER_EQUIVALENT,
- mock(S3Service.class), 200, 5000, 100);
+ mock(S3Service.class), mock(IamService.class), 200, 5000, 100);
proxy.start(proxyPort);
try (Socket client = new Socket("localhost", proxyPort)) {
@@ -153,7 +154,7 @@ class RedshiftAuthProxyTest {
proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),
"admin", "Secret123", "dev",
mock(RdsSigV4Validator.class), realTls(), (user, pw) -> PasswordValidator.AuthResult.MASTER_EQUIVALENT,
- mock(S3Service.class), 5000, 5000, 1);
+ mock(S3Service.class), mock(IamService.class), 5000, 5000, 1);
proxy.start(proxyPort);
// Hold the proxy's single connection permit from the test thread itself, so the
@@ -192,7 +193,7 @@ class RedshiftAuthProxyTest {
proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),
"admin", "Secret123", "dev",
mock(RdsSigV4Validator.class), realTls(), (user, pw) -> PasswordValidator.AuthResult.MASTER_EQUIVALENT,
- mock(S3Service.class), 5000, 5000, 100);
+ mock(S3Service.class), mock(IamService.class), 5000, 5000, 100);
proxy.start(proxyPort); // must not throw despite the port being busy at first
assertTrue(portAccepts(proxyPort), "proxy never bound the port after the squatter released it");
@@ -205,7 +206,7 @@ class RedshiftAuthProxyTest {
proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),
"admin", "old", "dev",
mock(RdsSigV4Validator.class), realTls(), (user, pw) -> PasswordValidator.AuthResult.MASTER_EQUIVALENT,
- mock(S3Service.class), 5000, 5000, 100);
+ mock(S3Service.class), mock(IamService.class), 5000, 5000, 100);
proxy.start(proxyPort);
proxy.updateMasterPassword("rotated");
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
index 2214d9476b..4cb26e418c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.redshift.proxy;
+import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.jboss.logging.Logger;
@@ -47,7 +48,8 @@ class RedshiftInterceptingBridgeTest {
testBackendEnd = backendListener.accept();
s3Stub = Mockito.mock(S3Service.class);
- RedshiftInterceptingBridge bridge = new RedshiftInterceptingBridge(bridgeClientEnd, bridgeBackendEnd, s3Stub);
+ RedshiftInterceptingBridge bridge = new RedshiftInterceptingBridge(
+ bridgeClientEnd, bridgeBackendEnd, s3Stub, Mockito.mock(IamService.class));
bridgeThread = Thread.ofVirtual().name("bridge-under-test").start(bridge::run);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java
index 6830b41dd0..7bee189f13 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.redshift.proxy;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;
import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;
@@ -29,10 +30,13 @@ import static org.mockito.Mockito.when;
class RedshiftProxyManagerTest {
+ private static final String KEY = "111111111111:c1";
+ private static final String SECOND_KEY = "111111111111:c2";
+
private RedshiftProxyManager newManager() {
return new RedshiftProxyManager(
mock(RdsSigV4Validator.class), mock(RdsProxyTlsCertificates.class),
- mock(S3Service.class), testConfig());
+ mock(S3Service.class), mock(IamService.class), testConfig());
}
private static EmulatorConfig testConfig() {
@@ -57,8 +61,8 @@ class RedshiftProxyManagerTest {
RedshiftProxyManager manager = newManager();
int port = availablePort();
try {
- start(manager, "111111111111:c1", port);
- assertTrue(registry(manager).containsKey("111111111111:c1"));
+ start(manager, KEY, port);
+ assertTrue(registry(manager).containsKey(KEY));
assertPortUnavailable(port);
} finally {
manager.stopAll();
@@ -69,11 +73,11 @@ class RedshiftProxyManagerTest {
void stopProxyRemovesTheKeyAndReleasesThePort() throws Exception {
RedshiftProxyManager manager = newManager();
int port = availablePort();
- start(manager, "k", port);
+ start(manager, KEY, port);
- manager.stopProxy("k");
+ manager.stopProxy(KEY);
- assertFalse(registry(manager).containsKey("k"));
+ assertFalse(registry(manager).containsKey(KEY));
assertPortAvailable(port);
}
@@ -82,10 +86,10 @@ class RedshiftProxyManagerTest {
RedshiftProxyManager manager = newManager();
int firstPort = availablePort();
try {
- start(manager, "k", firstPort);
+ start(manager, KEY, firstPort);
int replacementPort = availablePort();
- start(manager, "k", replacementPort);
+ start(manager, KEY, replacementPort);
assertPortAvailable(firstPort);
assertPortUnavailable(replacementPort);
@@ -106,9 +110,9 @@ class RedshiftProxyManagerTest {
RedshiftProxyManager manager = newManager();
int port = availablePort();
try {
- start(manager, "k", port);
- manager.updateMasterPassword("k", "rotated");
- assertEquals("rotated", masterPassword(registry(manager).get("k")));
+ start(manager, KEY, port);
+ manager.updateMasterPassword(KEY, "rotated");
+ assertEquals("rotated", masterPassword(registry(manager).get(KEY)));
} finally {
manager.stopAll();
}
@@ -118,9 +122,9 @@ class RedshiftProxyManagerTest {
void stopAllReleasesEveryListenerAndIsIdempotent() throws IOException {
RedshiftProxyManager manager = newManager();
int a = availablePort();
- start(manager, "a", a);
+ start(manager, KEY, a);
int b = availablePort();
- start(manager, "b", b);
+ start(manager, SECOND_KEY, b);
manager.stopAll();
@@ -134,8 +138,8 @@ class RedshiftProxyManagerTest {
RedshiftProxyManager manager = newManager();
try (ServerSocket occupied = new ServerSocket(0)) {
assertThrows(RuntimeException.class,
- () -> start(manager, "k", occupied.getLocalPort()));
- assertFalse(registry(manager).containsKey("k"));
+ () -> start(manager, KEY, occupied.getLocalPort()));
+ assertFalse(registry(manager).containsKey(KEY));
} finally {
manager.stopAll();
}
@@ -146,12 +150,12 @@ class RedshiftProxyManagerTest {
RedshiftProxyManager manager = newManager();
RedshiftAuthProxy badProxy = mock(RedshiftAuthProxy.class);
doThrow(new RuntimeException("close failed")).when(badProxy).stop();
- registry(manager).put("k", badProxy);
+ registry(manager).put(KEY, badProxy);
- assertThrows(RuntimeException.class, () -> manager.stopProxy("k"));
+ assertThrows(RuntimeException.class, () -> manager.stopProxy(KEY));
// Proxy stays registered so a later cleanup attempt can still reach it and retry.
- assertSame(badProxy, registry(manager).get("k"));
+ assertSame(badProxy, registry(manager).get(KEY));
}
@Test
@@ -159,12 +163,12 @@ class RedshiftProxyManagerTest {
RedshiftProxyManager manager = newManager();
RedshiftAuthProxy proxy = mock(RedshiftAuthProxy.class);
doThrow(new RuntimeException("close failed")).doNothing().when(proxy).stop();
- registry(manager).put("k", proxy);
+ registry(manager).put(KEY, proxy);
- assertThrows(RuntimeException.class, () -> manager.stopProxy("k"));
- assertDoesNotThrow(() -> manager.stopProxy("k"));
+ assertThrows(RuntimeException.class, () -> manager.stopProxy(KEY));
+ assertDoesNotThrow(() -> manager.stopProxy(KEY));
- assertFalse(registry(manager).containsKey("k"));
+ assertFalse(registry(manager).containsKey(KEY));
verify(proxy, times(2)).stop();
}
@@ -173,12 +177,12 @@ class RedshiftProxyManagerTest {
RedshiftProxyManager manager = newManager();
RedshiftAuthProxy stuck = mock(RedshiftAuthProxy.class);
doThrow(new RuntimeException("close failed")).when(stuck).stop();
- unclosable(manager).put("k", stuck);
+ unclosable(manager).put(KEY, stuck);
- assertThrows(RuntimeException.class, () -> manager.stopProxy("k"));
+ assertThrows(RuntimeException.class, () -> manager.stopProxy(KEY));
// Reference retained so the next attempt can retry the close.
- assertSame(stuck, unclosable(manager).get("k"));
+ assertSame(stuck, unclosable(manager).get(KEY));
}
@Test
@@ -186,13 +190,13 @@ class RedshiftProxyManagerTest {
RedshiftProxyManager manager = newManager();
RedshiftAuthProxy stuck = mock(RedshiftAuthProxy.class);
doThrow(new RuntimeException("close failed")).when(stuck).stop();
- unclosable(manager).put("k", stuck);
+ unclosable(manager).put(KEY, stuck);
- assertThrows(RuntimeException.class, () -> manager.stopProxy("k"));
- assertThrows(RuntimeException.class, () -> manager.stopProxy("k"));
+ assertThrows(RuntimeException.class, () -> manager.stopProxy(KEY));
+ assertThrows(RuntimeException.class, () -> manager.stopProxy(KEY));
verify(stuck, times(2)).stop();
- assertTrue(unclosable(manager).containsKey("k"));
+ assertTrue(unclosable(manager).containsKey(KEY));
}
@Test
@@ -200,12 +204,12 @@ class RedshiftProxyManagerTest {
RedshiftProxyManager manager = newManager();
RedshiftAuthProxy recovering = mock(RedshiftAuthProxy.class);
doThrow(new RuntimeException("close failed")).doNothing().when(recovering).stop();
- unclosable(manager).put("k", recovering);
+ unclosable(manager).put(KEY, recovering);
- assertThrows(RuntimeException.class, () -> manager.stopProxy("k"));
- assertDoesNotThrow(() -> manager.stopProxy("k"));
+ assertThrows(RuntimeException.class, () -> manager.stopProxy(KEY));
+ assertDoesNotThrow(() -> manager.stopProxy(KEY));
- assertFalse(unclosable(manager).containsKey("k"));
+ assertFalse(unclosable(manager).containsKey(KEY));
}
@Test
@@ -213,12 +217,12 @@ class RedshiftProxyManagerTest {
RedshiftProxyManager manager = newManager();
RedshiftAuthProxy stuck = mock(RedshiftAuthProxy.class);
doThrow(new RuntimeException("close failed")).when(stuck).stop();
- unclosable(manager).put("k", stuck);
+ unclosable(manager).put(KEY, stuck);
int port = availablePort();
try {
// A fresh start for the same key must not silently drop the leaked listener.
- start(manager, "k", port);
- assertSame(stuck, unclosable(manager).get("k"));
+ start(manager, KEY, port);
+ assertSame(stuck, unclosable(manager).get(KEY));
} finally {
manager.stopAll();
}
@@ -230,13 +234,13 @@ class RedshiftProxyManagerTest {
RedshiftAuthProxy recovered = mock(RedshiftAuthProxy.class);
RedshiftAuthProxy stillStuck = mock(RedshiftAuthProxy.class);
doThrow(new RuntimeException("close failed")).when(stillStuck).stop();
- unclosable(manager).put("a", recovered);
- unclosable(manager).put("b", stillStuck);
+ unclosable(manager).put(KEY, recovered);
+ unclosable(manager).put(SECOND_KEY, stillStuck);
manager.stopAll();
- assertFalse(unclosable(manager).containsKey("a"));
- assertTrue(unclosable(manager).containsKey("b"));
+ assertFalse(unclosable(manager).containsKey(KEY));
+ assertTrue(unclosable(manager).containsKey(SECOND_KEY));
}
// --- reflection + port helpers copied from RdsProxyManagerTest ---
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java
index b80d175b94..9aad9aca8f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java
@@ -1,6 +1,8 @@
package io.github.hectorvent.floci.services.redshift.proxy;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.iam.model.IamRole;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.junit.jupiter.api.AfterEach;
@@ -35,6 +37,10 @@ import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
class S3CopySimulatorTest {
@@ -140,7 +146,7 @@ class S3CopySimulatorTest {
private CopyStatementParser.S3Unload unloadSpec(String bucket, String prefix,
boolean gzip, boolean manifest, boolean allowOverwrite, boolean parallel, long maxFileSize) {
return new CopyStatementParser.S3Unload("select a,b from t", bucket, prefix,
- "|", false, gzip, false, false, null, manifest, allowOverwrite, parallel, maxFileSize);
+ "|", false, gzip, false, false, null, manifest, allowOverwrite, parallel, maxFileSize, null);
}
@Test
@@ -154,9 +160,9 @@ class S3CopySimulatorTest {
new S3Object("b", "prefix/a", new byte[0], "text/plain")),
List.of(), false, null));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "b", "prefix/", null, 0, false, true, null);
+ "t", List.of(), "b", "prefix/", null, 0, false, true, null, null);
- S3CopySimulator.CopyInput input = S3CopySimulator.prepareCopy(spec, s3);
+ S3CopySimulator.CopyInput input = S3CopySimulator.prepareCopy(spec, s3, null);
assertEquals(List.of("prefix/a", "prefix/z"), input.keys());
assertEquals("COPY t FROM STDIN WITH (FORMAT csv, DELIMITER ',')",
@@ -166,16 +172,140 @@ class S3CopySimulatorTest {
@Test
void preparedCopyReportsMissingObject() {
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "b", "missing", "|", 0, false, false, null);
+ "t", List.of(), "b", "missing", "|", 0, false, false, null, null);
S3CopySimulator.S3TransferException error = assertThrows(
S3CopySimulator.S3TransferException.class,
- () -> S3CopySimulator.prepareCopy(spec, s3));
+ () -> S3CopySimulator.prepareCopy(spec, s3, null));
assertEquals("XX000", error.sqlState());
assertEquals("S3 object s3://b/missing not found", error.getMessage());
}
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/CopyRole";
+
+ @Test
+ void prepareCopyWithMalformedIamRoleArnFailsBeforeTouchingS3() {
+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
+ "t", List.of(), "b", "k", "|", 0, false, false, null, "not-an-arn");
+ IamService iamService = mock(IamService.class);
+
+ S3CopySimulator.S3TransferException error = assertThrows(
+ S3CopySimulator.S3TransferException.class,
+ () -> S3CopySimulator.prepareCopy(spec, s3, iamService));
+
+ assertEquals("42501", error.sqlState());
+ verifyNoInteractions(s3);
+ }
+
+ @Test
+ void prepareCopyWithUnknownRoleFailsBeforeTouchingS3() {
+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
+ "t", List.of(), "b", "k", "|", 0, false, false, null, ROLE_ARN);
+ IamService iamService = mock(IamService.class);
+ when(iamService.findRole(any(), any())).thenReturn(java.util.Optional.empty());
+
+ S3CopySimulator.S3TransferException error = assertThrows(
+ S3CopySimulator.S3TransferException.class,
+ () -> S3CopySimulator.prepareCopy(spec, s3, iamService));
+
+ assertEquals("42501", error.sqlState());
+ verifyNoInteractions(s3);
+ }
+
+ @Test
+ void prepareCopyRejectsCrossAccountRole() {
+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
+ "t", List.of(), "b", "k", "|", 0, false, false, null, ROLE_ARN);
+ IamService iamService = mock(IamService.class);
+
+ S3CopySimulator.S3TransferException error = assertThrows(
+ S3CopySimulator.S3TransferException.class,
+ () -> S3CopySimulator.prepareCopy(spec, s3, iamService, "111111111111"));
+
+ assertEquals("42501", error.sqlState());
+ assertTrue(error.getMessage().contains("cross-account"));
+ verifyNoInteractions(s3);
+ verifyNoInteractions(iamService);
+ }
+
+ @Test
+ void prepareCopyRejectsRoleNotAssociatedWithCluster() {
+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
+ "t", List.of(), "b", "k", "|", 0, false, false, null, ROLE_ARN);
+ IamService iamService = mock(IamService.class);
+
+ S3CopySimulator.S3TransferException error = assertThrows(
+ S3CopySimulator.S3TransferException.class,
+ () -> S3CopySimulator.prepareCopy(spec, s3, iamService, "000000000000", List.of()));
+
+ assertEquals("42501", error.sqlState());
+ assertTrue(error.getMessage().contains("not associated"));
+ verifyNoInteractions(s3);
+ verifyNoInteractions(iamService);
+ }
+
+ @Test
+ void prepareCopyRejectsRoleWithoutRedshiftTrust() {
+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
+ "t", List.of(), "b", "k", "|", 0, false, false, null, ROLE_ARN);
+ IamService iamService = mock(IamService.class);
+ IamRole role = mock(IamRole.class);
+ when(iamService.findRole("000000000000", "CopyRole"))
+ .thenReturn(java.util.Optional.of(role));
+ when(role.getAssumeRolePolicyDocument()).thenReturn(""
+ + "{\"Statement\":[{\"Effect\":\"Allow\","
+ + "\"Principal\":{\"Service\":\"lambda.amazonaws.com\"},"
+ + "\"Action\":\"sts:AssumeRole\"}]}" );
+
+ S3CopySimulator.S3TransferException error = assertThrows(
+ S3CopySimulator.S3TransferException.class,
+ () -> S3CopySimulator.prepareCopy(spec, s3, iamService, "000000000000"));
+
+ assertEquals("42501", error.sqlState());
+ assertTrue(error.getMessage().contains("trust policy"));
+ verifyNoInteractions(s3);
+ }
+
+ @Test
+ void prepareCopyWithValidRoleCallsSignedAuthorization() {
+ when(s3.objectExists("b", "k")).thenReturn(true);
+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
+ "t", List.of(), "b", "k", "|", 0, false, false, null, ROLE_ARN);
+ IamService iamService = mock(IamService.class);
+ when(iamService.findRole(any(), any()))
+ .thenReturn(java.util.Optional.of(mock(io.github.hectorvent.floci.services.iam.model.IamRole.class)));
+
+ S3CopySimulator.CopyInput input = S3CopySimulator.prepareCopy(spec, s3, iamService);
+
+ verify(s3, never()).authorizeAnonymousGetObject(any(), any());
+ verify(s3, never()).authorizeAnonymousListBucket(any());
+ verify(s3).authorizeSignedListBucket(any(), any(), eq("b"));
+ verify(s3).authorizeSignedGetObject(any(), any(), eq("b"), eq("k"));
+ verify(iamService).registerSessionForAccount(
+ eq("000000000000"), any(), any(), any(), eq(ROLE_ARN), any(), isNull());
+ verify(iamService, never()).unregisterSession(any(), any());
+ }
+
+ @Test
+ void copyStreamingUsesRoleSessionForObjectAuthorization() throws Exception {
+ when(s3.objectExists("b", "k")).thenReturn(true);
+ when(s3.getObject("b", "k"))
+ .thenReturn(new S3Object("b", "k", "1|alice\n".getBytes(StandardCharsets.UTF_8), "text/plain"));
+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
+ "t", List.of(), "b", "k", "|", 0, false, false, null, ROLE_ARN);
+ IamService iamService = mock(IamService.class);
+ when(iamService.findRole(any(), any()))
+ .thenReturn(java.util.Optional.of(mock(io.github.hectorvent.floci.services.iam.model.IamRole.class)));
+
+ S3CopySimulator.CopyInput input = S3CopySimulator.prepareCopy(spec, s3, iamService);
+ S3CopySimulator.streamCopyInput(input, new ByteArrayOutputStream());
+
+ verify(s3, times(2))
+ .authorizeSignedGetObject(any(), any(), eq("b"), eq("k"));
+ verify(s3, never()).authorizeAnonymousGetObject(any(), any());
+ }
+
@Test
void unloadCollectorWritesOneEmptyObjectForZeroRows() throws Exception {
Map<String, byte[]> written = new ConcurrentHashMap<>();
@@ -185,7 +315,7 @@ class S3CopySimulatorTest {
});
CopyStatementParser.S3Unload spec = unloadSpec("b", "out/", false, false, true, false, 0);
- try (S3CopySimulator.UnloadCollector collector = S3CopySimulator.prepareUnload(spec, s3)) {
+ try (S3CopySimulator.UnloadCollector collector = S3CopySimulator.prepareUnload(spec, s3, null)) {
collector.complete();
}
@@ -208,7 +338,7 @@ class S3CopySimulatorTest {
CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
Thread backend = backendThread(() -> playUnloadBackend("1|alice\n", "2|bob\n"));
- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertTrue(handled);
@@ -236,10 +366,10 @@ class S3CopySimulatorTest {
CopyStatementParser.S3Unload spec = new CopyStatementParser.S3Unload(
"select a,b from t", "wh", "out/", "|", true, false, true, false, null,
- false, false, true, 0);
+ false, false, true, 0, null);
// The fake backend stands in for PostgreSQL: the first CopyData frame is the header row.
Thread backend = backendThread(() -> playUnloadBackend("h1|h2\n", "1|a\n", "2|b\n", "3|c\n"));
- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertEquals(3, written.size(), written.keySet().toString());
@@ -267,7 +397,7 @@ class S3CopySimulatorTest {
CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
Thread backend = backendThread(() -> playUnloadBackend("1|alice\n", "2|bob\n", "3|carol\n"));
- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertTrue(written.containsKey("out/0000_part_00"));
@@ -295,7 +425,7 @@ class S3CopySimulatorTest {
CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, false, 0);
Thread backend = backendThread(() -> playUnloadBackend("1|a\n"));
- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertTrue(written.containsKey("out/000"), written.keySet().toString());
@@ -313,7 +443,7 @@ class S3CopySimulatorTest {
CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", true, false, false, true, 0);
Thread backend = backendThread(() -> playUnloadBackend("1|a\n", "2|b\n"));
- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertTrue(written.containsKey("out/0000_part_00.gz"), written.keySet().toString());
@@ -340,7 +470,7 @@ class S3CopySimulatorTest {
CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, false, true, 0);
Thread backend = backendThread(() -> playUnloadBackend("1|alice\n", "2|bob\n", "3|carol\n"));
- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
String manifest = new String(written.get("out/manifest"), StandardCharsets.UTF_8);
@@ -360,7 +490,7 @@ class S3CopySimulatorTest {
CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
// No backend thread: the select must never run.
- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
assertTrue(handled);
PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
@@ -386,7 +516,7 @@ class S3CopySimulatorTest {
out.flush();
});
- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
@@ -423,7 +553,7 @@ class S3CopySimulatorTest {
}
});
- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
@@ -458,7 +588,7 @@ class S3CopySimulatorTest {
out.flush();
});
- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'T');
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'T');
joinBackend(backend);
PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
@@ -503,7 +633,7 @@ class S3CopySimulatorTest {
out.flush();
});
- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertTrue(handled);
@@ -527,7 +657,7 @@ class S3CopySimulatorTest {
S3CopySimulator.UNLOAD_HEAP_MIB.acquire(192);
try {
- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
assertTrue(handled);
PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
@@ -547,7 +677,7 @@ class S3CopySimulatorTest {
when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
.thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
CopyStatementParser.S3Unload spec = new CopyStatementParser.S3Unload(
- "select a from t", "wh", "out/", "\t", true, false, false, true, "\\N", false, true, true, 0);
+ "select a from t", "wh", "out/", "\t", true, false, false, true, "\\N", false, true, true, 0, null);
AtomicReference<String> seenSql = new AtomicReference<>();
Thread backend = backendThread(() -> {
@@ -565,7 +695,7 @@ class S3CopySimulatorTest {
out.flush();
});
- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
String sql = seenSql.get();
@@ -602,7 +732,7 @@ class S3CopySimulatorTest {
out.flush();
});
- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertTrue(handled);
@@ -617,7 +747,7 @@ class S3CopySimulatorTest {
.thenThrow(new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));
CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
assertTrue(handled);
PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
@@ -648,7 +778,7 @@ class S3CopySimulatorTest {
CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, false, true, 0);
Thread backend = backendThread(() -> playUnloadBackend("1|a\n"));
- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertEquals(1, deleted.size());
@@ -680,7 +810,7 @@ class S3CopySimulatorTest {
CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, true, true, 0);
Thread backend = backendThread(() -> playUnloadBackend("1|a\n", "2|b\n"));
- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
// Under ALLOWOVERWRITE a written key may have replaced pre-existing (or another
@@ -701,7 +831,7 @@ class S3CopySimulatorTest {
.when(s3).authorizeAnonymousListBucket(eq("wh"));
CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
- boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
assertTrue(handled);
PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
@@ -741,7 +871,7 @@ class S3CopySimulatorTest {
testBackend.close();
});
- S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertEquals(1, deleted.size());
@@ -809,12 +939,12 @@ class S3CopySimulatorTest {
new S3Object("wh", "d/a.txt", "1|alice\n2|bob\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "people", List.of(), "wh", "d/a.txt", "|", 0, false, false, null);
+ "people", List.of(), "wh", "d/a.txt", "|", 0, false, false, null, null);
ByteArrayOutputStream copyData = new ByteArrayOutputStream();
Thread backend = backendThread(() -> playHappyBackend(copyData));
- boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertTrue(handled);
@@ -830,12 +960,12 @@ class S3CopySimulatorTest {
when(s3.getObject("wh", "k")).thenReturn(
new S3Object("wh", "k", "x\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of("a", "b"), "wh", "k", "|", 0, false, false, null);
+ "t", List.of("a", "b"), "wh", "k", "|", 0, false, false, null, null);
AtomicReference<String> fabricated = new AtomicReference<>();
Thread backend = backendThread(() -> fabricated.set(captureFabricatedThenComplete()));
- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
String sql = fabricated.get();
@@ -851,12 +981,12 @@ class S3CopySimulatorTest {
when(s3.getObject("wh", "k")).thenReturn(
new S3Object("wh", "k", "x\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of("a", "b"), "wh", "k", null, 0, false, true, null);
+ "t", List.of("a", "b"), "wh", "k", null, 0, false, true, null, null);
AtomicReference<String> fabricated = new AtomicReference<>();
Thread backend = backendThread(() -> fabricated.set(captureFabricatedThenComplete()));
- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
String sql = fabricated.get();
@@ -871,12 +1001,12 @@ class S3CopySimulatorTest {
when(s3.getObject("wh", "k")).thenReturn(
new S3Object("wh", "k", "x\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "wh", "k", "|", 0, false, false, "\\N");
+ "t", List.of(), "wh", "k", "|", 0, false, false, "\\N", null);
AtomicReference<String> fabricated = new AtomicReference<>();
Thread backend = backendThread(() -> fabricated.set(captureFabricatedThenComplete()));
- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
String sql = fabricated.get();
@@ -898,11 +1028,11 @@ class S3CopySimulatorTest {
new S3Object("wh", "p/2", "2|b\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "wh", "p/", "|", 1, false, false, null);
+ "t", List.of(), "wh", "p/", "|", 1, false, false, null, null);
ByteArrayOutputStream copyData = new ByteArrayOutputStream();
Thread backend = backendThread(() -> playHappyBackend(copyData));
- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertEquals("1|a\n2|b\n", copyData.toString(StandardCharsets.US_ASCII));
@@ -914,11 +1044,11 @@ class S3CopySimulatorTest {
when(s3.getObject("wh", "k.gz")).thenReturn(
new S3Object("wh", "k.gz", gzip("1|a\n2|b\n"), "application/gzip"));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "wh", "k.gz", "|", 0, true, false, null);
+ "t", List.of(), "wh", "k.gz", "|", 0, true, false, null, null);
ByteArrayOutputStream copyData = new ByteArrayOutputStream();
Thread backend = backendThread(() -> playHappyBackend(copyData));
- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertEquals("1|a\n2|b\n", copyData.toString(StandardCharsets.US_ASCII));
@@ -929,9 +1059,9 @@ class S3CopySimulatorTest {
doThrow(new AwsException("AccessDenied", "no", 403))
.when(s3).authorizeAnonymousListBucket("wh");
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "wh", "k", "|", 0, false, false, null);
+ "t", List.of(), "wh", "k", "|", 0, false, false, null, null);
- boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
assertTrue(handled);
PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
@@ -947,9 +1077,9 @@ class S3CopySimulatorTest {
when(s3.listObjectsWithPrefixes(eq("wh"), eq("missing"), isNull(), anyInt(), any(), any())).thenReturn(
new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "wh", "missing", "|", 0, false, false, null);
+ "t", List.of(), "wh", "missing", "|", 0, false, false, null, null);
- boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
assertTrue(handled);
PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
@@ -965,7 +1095,7 @@ class S3CopySimulatorTest {
when(s3.getObject("wh", "k")).thenReturn(
new S3Object("wh", "k", "1\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "nosuch", List.of(), "wh", "k", "|", 0, false, false, null);
+ "nosuch", List.of(), "wh", "k", "|", 0, false, false, null, null);
Thread backend = backendThread(() -> {
new PostgresWireDecoder(testBackend.getInputStream()).nextMessage();
@@ -978,7 +1108,7 @@ class S3CopySimulatorTest {
out.flush();
});
- boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertTrue(handled);
@@ -992,7 +1122,7 @@ class S3CopySimulatorTest {
when(s3.objectExists("wh", "k")).thenReturn(true);
when(s3.getObject("wh", "k")).thenThrow(new AwsException("InternalError", "boom", 500));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "wh", "k", "|", 0, false, false, null);
+ "t", List.of(), "wh", "k", "|", 0, false, false, null, null);
Thread backend = backendThread(() -> {
OutputStream out = testBackend.getOutputStream();
@@ -1014,7 +1144,7 @@ class S3CopySimulatorTest {
out.flush();
});
- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
@@ -1031,14 +1161,14 @@ class S3CopySimulatorTest {
when(s3.getObject("wh", "k")).thenReturn(
new S3Object("wh", "k", "1\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "wh", "k", "|", 0, false, false, null);
+ "t", List.of(), "wh", "k", "|", 0, false, false, null, null);
Thread backend = backendThread(() -> {
new PostgresWireDecoder(testBackend.getInputStream()).nextMessage();
testBackend.close();
});
- boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertTrue(handled);
@@ -1053,7 +1183,7 @@ class S3CopySimulatorTest {
when(s3.listObjectsWithPrefixes(eq("wh"), eq("missing"), isNull(), anyInt(), any(), any())).thenReturn(
new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "wh", "missing", "|", 0, false, false, null);
+ "t", List.of(), "wh", "missing", "|", 0, false, false, null, null);
Thread backend = backendThread(() -> {
PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());
@@ -1069,7 +1199,7 @@ class S3CopySimulatorTest {
out.flush();
});
- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'T');
+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'T');
joinBackend(backend);
PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
@@ -1097,11 +1227,11 @@ class S3CopySimulatorTest {
new S3Object("wh", "multi/2", "2|b\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "wh", "multi/", "|", 0, false, false, null);
+ "t", List.of(), "wh", "multi/", "|", 0, false, false, null, null);
ByteArrayOutputStream copyData = new ByteArrayOutputStream();
Thread backend = backendThread(() -> playHappyBackend(copyData));
- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertEquals("1|a\n2|b\n", copyData.toString(StandardCharsets.US_ASCII));
@@ -1113,7 +1243,7 @@ class S3CopySimulatorTest {
when(s3.getObject("wh", "k")).thenReturn(
new S3Object("wh", "k", "1\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "wh", "k", "|", 0, false, false, null);
+ "t", List.of(), "wh", "k", "|", 0, false, false, null, null);
simBackend.setSoTimeout(100);
@@ -1126,7 +1256,7 @@ class S3CopySimulatorTest {
}
});
- boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'I');
+ boolean handled = S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'I');
joinBackend(backend);
assertTrue(handled);
@@ -1143,7 +1273,7 @@ class S3CopySimulatorTest {
when(s3.listObjectsWithPrefixes(eq("wh"), eq("missing"), isNull(), anyInt(), any(), any())).thenReturn(
new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "wh", "missing", "|", 0, false, false, null);
+ "t", List.of(), "wh", "missing", "|", 0, false, false, null, null);
// Backend closes socket when abort query arrives, simulating dropped connection
Thread backend = backendThread(() -> {
@@ -1154,7 +1284,7 @@ class S3CopySimulatorTest {
testBackend.close();
});
- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'T');
+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'T');
joinBackend(backend);
assertTrue(simBackend.isClosed(), "simBackend must be closed upon synchronization failure");
@@ -1172,7 +1302,7 @@ class S3CopySimulatorTest {
when(s3.listObjectsWithPrefixes(eq("wh"), eq("missing"), isNull(), anyInt(), any(), any())).thenReturn(
new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
- "t", List.of(), "wh", "missing", "|", 0, false, false, null);
+ "t", List.of(), "wh", "missing", "|", 0, false, false, null, null);
// Fake backend returns ReadyForQuery with 'T' (e.g. abort query unexpectedly succeeded)
Thread backend = backendThread(() -> {
@@ -1185,7 +1315,7 @@ class S3CopySimulatorTest {
out.flush();
});
- S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, 'T');
+ S3CopySimulator.runCopyFrom(simClient, simBackend, spec, s3, null, 'T');
joinBackend(backend);
assertTrue(simBackend.isClosed(), "simBackend must be closed when abort returns non-E status");
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
index a974d6f7a0..8f48075ca0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
@@ -1222,5 +1222,25 @@ class S3ServiceTest {
s3Service.createBucket("anon-del-bucket", "us-east-1");
assertDoesNotThrow(() -> s3Service.authorizeAnonymousDeleteObject("anon-del-bucket", "some/key"));
}
+
+ @Test
+ void authorizeSignedGetObjectIsANoOpWhenEnforceAuthIsOff() {
+ assertDoesNotThrow(() -> s3Service.authorizeSignedGetObject("ASIAFAKEKEY00000001", "sessiontoken", "signed-get-bucket", "some/key"));
+ }
+
+ @Test
+ void authorizeSignedPutObjectIsANoOpWhenEnforceAuthIsOff() {
+ assertDoesNotThrow(() -> s3Service.authorizeSignedPutObject("ASIAFAKEKEY00000001", "sessiontoken", "signed-put-bucket", "some/key"));
+ }
+
+ @Test
+ void authorizeSignedListBucketIsANoOpWhenEnforceAuthIsOff() {
+ assertDoesNotThrow(() -> s3Service.authorizeSignedListBucket("ASIAFAKEKEY00000001", "sessiontoken", "signed-list-bucket"));
+ }
+
+ @Test
+ void authorizeSignedDeleteObjectIsANoOpWhenEnforceAuthIsOff() {
+ assertDoesNotThrow(() -> s3Service.authorizeSignedDeleteObject("ASIAFAKEKEY00000001", "sessiontoken", "signed-del-bucket", "some/key"));
+ }
}
diff --git a/tools/docs/services.yaml b/tools/docs/services.yaml
index 9b98d0dc89..bc46c62091 100644
--- a/tools/docs/services.yaml
+++ b/tools/docs/services.yaml
@@ -281,10 +281,10 @@ services:
doc: docs/services/redshift.md
sources:
- { path: src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java, mode: switch }
- # SubnetIds / VpcSecurityGroupIds / TagKeys / DbGroups are labels of an inner
+ # SubnetIds / VpcSecurityGroupIds / IamRoles / TagKeys / DbGroups are labels of an inner
# member-name switch (memberKeyRegex), not actions, same shape as the rds
# exclusions above.
- exclude_actions: [SubnetIds, VpcSecurityGroupIds, TagKeys, DbGroups]
+ exclude_actions: [SubnetIds, VpcSecurityGroupIds, IamRoles, TagKeys, DbGroups]
- service: redshiftdata
doc: docs/services/redshift-data.md
sources: