Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(redshift): materialize Glue external tables into PostgreSQL (#4996)

Vũ Thế Huy06ceec2aeefd
diff --git a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftSpectrumTest.java b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftSpectrumTest.java
index 2f2b97cf5d..d57b0f09b8 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftSpectrumTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftSpectrumTest.java
@@ -4,6 +4,11 @@ import com.floci.test.TestFixtures;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.iam.IamClient;
+import software.amazon.awssdk.services.iam.model.CreateRoleRequest;
+import software.amazon.awssdk.services.iam.model.DeleteRolePolicyRequest;
+import software.amazon.awssdk.services.iam.model.DeleteRoleRequest;
+import software.amazon.awssdk.services.iam.model.PutRolePolicyRequest;
import software.amazon.awssdk.services.redshift.RedshiftClient;
import software.amazon.awssdk.services.redshift.model.Cluster;
import software.amazon.awssdk.services.redshift.model.CreateClusterRequest;
@@ -29,18 +34,33 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
class RedshiftSpectrumTest {
+ private static final String ROLE_PREFIX = "SpectrumCompatRole";
+
@Test
@DisplayName("queries an S3 CSV external table through the Redshift JDBC wire")
void queriesCsvExternalTable() throws Exception {
String suffix = UUID.randomUUID().toString().replace("-", "").substring(0, 12);
String clusterId = "spectrum-" + suffix;
String bucket = "spectrum-" + suffix;
+ String roleName = ROLE_PREFIX + suffix;
+ String roleArn = "arn:aws:iam::000000000000:role/" + roleName;
RedshiftClient redshift = TestFixtures.redshiftClient();
S3Client s3 = TestFixtures.s3Client();
+ IamClient iam = TestFixtures.iamClient();
boolean bucketCreated = false;
boolean clusterCreated = false;
try {
+ iam.createRole(CreateRoleRequest.builder().roleName(roleName).path("/")
+ .assumeRolePolicyDocument("""
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}""")
+ .build());
+ iam.putRolePolicy(PutRolePolicyRequest.builder().roleName(roleName).policyName("SpectrumS3")
+ .policyDocument("""
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Action":["s3:GetObject","s3:ListBucket"],"Resource":"*"}]}""")
+ .build());
s3.createBucket(CreateBucketRequest.builder().bucket(bucket).build());
bucketCreated = true;
s3.putObject(PutObjectRequest.builder().bucket(bucket).key("events/part-1.csv")
@@ -48,7 +68,7 @@ class RedshiftSpectrumTest {
RequestBody.fromString("id,name\n1,Alice\n2,Bob\n"));
redshift.createCluster(CreateClusterRequest.builder().clusterIdentifier(clusterId)
.nodeType("dc2.large").masterUsername("admin").masterUserPassword("Password123")
- .build());
+ .iamRoles(roleArn).build());
clusterCreated = true;
Cluster cluster = waitForCluster(redshift, clusterId);
@@ -56,7 +76,8 @@ class RedshiftSpectrumTest {
+ cluster.endpoint().port() + "/dev";
try (Connection connection = waitForConnection(jdbcUrl);
Statement statement = connection.createStatement()) {
- statement.execute("CREATE EXTERNAL SCHEMA analytics FROM DATA CATALOG DATABASE 'dev' IAM_ROLE 'role'");
+ statement.execute("CREATE EXTERNAL SCHEMA analytics FROM DATA CATALOG DATABASE 'dev' IAM_ROLE '"
+ + roleArn + "'");
statement.execute("CREATE EXTERNAL TABLE analytics.events (id INTEGER, name VARCHAR) "
+ "STORED AS TEXTFILE LOCATION 's3://" + bucket + "/events/' "
+ "TBLPROPERTIES ('skip.header.line.count'='1')");
@@ -81,8 +102,16 @@ class RedshiftSpectrumTest {
s3.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key("events/part-1.csv").build());
s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucket).build());
}
+ try {
+ iam.deleteRolePolicy(DeleteRolePolicyRequest.builder().roleName(roleName)
+ .policyName("SpectrumS3").build());
+ iam.deleteRole(DeleteRoleRequest.builder().roleName(roleName).build());
+ } catch (RuntimeException ignored) {
+ // Cleanup is best effort if role setup failed partway through.
+ }
redshift.close();
s3.close();
+ iam.close();
}
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index ea7827b44d..3cb90ccc28 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1602,6 +1602,9 @@ public interface EmulatorConfig {
int proxyBackendConnectTimeoutMillis();
@WithDefault("100")
int proxyMaxConnections();
+
+ @WithDefault("1000000")
+ long spectrumMaxRows();
}
interface RdsServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueTableResolver.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueTableResolver.java
index 492018b45f..ed7744f1a7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueTableResolver.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueTableResolver.java
@@ -1,13 +1,17 @@
package io.github.hectorvent.floci.services.glue;
import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
import io.github.hectorvent.floci.services.glue.model.Table;
import java.util.ArrayList;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* Resolves a Glue {@link Table} into the pieces of a DuckDB read plan: which {@code read_*}
@@ -27,9 +31,152 @@ public final class GlueTableResolver {
private static final String ICEBERG_TABLE_TYPE = "ICEBERG";
+ /** Hive table property naming how many header lines a delimited file carries. */
+ private static final String PARAM_SKIP_HEADER = "skip.header.line.count";
+
private GlueTableResolver() {
}
+ public record ReadPlan(String fromClause, boolean iceberg, List<Column> columns) {
+ }
+
+ public static ReadPlan readPlan(Table table) {
+ if (table == null || table.getStorageDescriptor() == null
+ || table.getStorageDescriptor().getLocation() == null
+ || table.getStorageDescriptor().getLocation().isBlank()) {
+ throw new IllegalArgumentException("Glue table has no storage location");
+ }
+ List<Column> columns = declaredColumns(table);
+ String metadata = isIcebergTable(table) ? icebergMetadataLocation(table) : null;
+ if (metadata != null && !metadata.isBlank()) {
+ return new ReadPlan(icebergReadExpression(metadata), true, columns);
+ }
+ String location = table.getStorageDescriptor().getLocation();
+ String normalized = location.endsWith("/") ? location.substring(0, location.length() - 1) : location;
+ String readFunction = inferReadFunction(table);
+ String path = readPath(table, normalized);
+ if ("read_csv_auto".equals(readFunction) && hasCsvOptions(table)) {
+ return new ReadPlan(csvReadExpression(table, path), false, columns);
+ }
+ return new ReadPlan(readExpression(readFunction, path), false, columns);
+ }
+
+ /**
+ * Read plan constrained to the exact objects that the caller listed and authorized. This is
+ * used by Redshift Spectrum, where a location-derived glob could match keys outside the
+ * literal S3 prefix that was checked against the bound role.
+ */
+ public static ReadPlan readPlan(Table table, List<String> objectUris) {
+ if (isIcebergTable(table)) {
+ return readPlan(table);
+ }
+ if (objectUris == null || objectUris.isEmpty()) {
+ throw new IllegalArgumentException("An exact-object read plan requires at least one object");
+ }
+ List<Column> columns = declaredColumns(table);
+ String fileList = "[" + objectUris.stream().map(GlueTableResolver::sqlLiteral)
+ .collect(Collectors.joining(", ")) + "]";
+ String readFunction = inferReadFunction(table);
+ String fromClause;
+ if ("read_csv_auto".equals(readFunction) && hasCsvOptions(table)) {
+ fromClause = csvReadExpressionForInput(table, fileList);
+ } else if ("read_parquet".equals(readFunction)) {
+ fromClause = "read_parquet(" + fileList + ", union_by_name = true)";
+ } else {
+ fromClause = readFunction + "(" + fileList + ")";
+ }
+ return new ReadPlan(fromClause, false, columns);
+ }
+
+ /**
+ * A table that declares CSV options (header lines to skip, a delimiter) is read with exactly
+ * those options and its declared columns rather than sniffed: a headerless file otherwise gets
+ * its first row taken as the header and the declared column names bind to nothing.
+ */
+ static boolean hasCsvOptions(Table table) {
+ Map<String, String> options = csvParameters(table);
+ return options.containsKey(PARAM_SKIP_HEADER) || options.containsKey("field.delim")
+ || options.containsKey("separatorChar");
+ }
+
+ static String csvReadExpression(Table table, String readPath) {
+ return csvReadExpressionForInput(table, "'" + escape(readPath) + "/**'");
+ }
+
+ private static String csvReadExpressionForInput(Table table, String inputExpression) {
+ Map<String, String> options = csvParameters(table);
+ int skip = skipCount(options.get(PARAM_SKIP_HEADER));
+ StringBuilder sql = new StringBuilder("read_csv(").append(inputExpression);
+ sql.append(", header = ").append(skip == 1);
+ if (skip > 1) {
+ sql.append(", skip = ").append(skip);
+ }
+ String delimiter = options.containsKey("field.delim") ? options.get("field.delim") : options.get("separatorChar");
+ sql.append(", delim = '").append(escape(delimiter == null || delimiter.isEmpty() ? "," : delimiter)).append("'");
+ appendOption(sql, "quote", options.get("quoteChar"));
+ appendOption(sql, "escape", options.get("escapeChar"));
+ appendOption(sql, "nullstr", options.get("serialization.null.format"));
+ List<Column> dataColumns = table.getStorageDescriptor().getColumns();
+ if (dataColumns != null && !dataColumns.isEmpty()) {
+ // Every column is read as VARCHAR: DuckDB's own type sniffing would disagree with the Glue types,
+ // and the caller casts each column to its declared type afterwards.
+ sql.append(", columns = {");
+ for (int i = 0; i < dataColumns.size(); i++) {
+ if (i > 0) {
+ sql.append(", ");
+ }
+ sql.append('\'').append(escape(dataColumns.get(i).getName())).append("': 'VARCHAR'");
+ }
+ sql.append('}');
+ }
+ return sql.append(')').toString();
+ }
+
+ private static void appendOption(StringBuilder sql, String name, String value) {
+ if (value != null && !value.isEmpty()) {
+ sql.append(", ").append(name).append(" = '").append(escape(value)).append('\'');
+ }
+ }
+
+ private static int skipCount(String value) {
+ if (value == null) {
+ return 0;
+ }
+ try {
+ return Math.max(0, Integer.parseInt(value.trim()));
+ } catch (NumberFormatException expected) {
+ // an unparsable skip count is treated as absent, the same as a table that declares none
+ return 0;
+ }
+ }
+
+ /** Table, storage and SerDe parameters merged; the narrower scope wins. */
+ private static Map<String, String> csvParameters(Table table) {
+ Map<String, String> merged = new LinkedHashMap<>();
+ if (table.getParameters() != null) {
+ merged.putAll(table.getParameters());
+ }
+ StorageDescriptor descriptor = table.getStorageDescriptor();
+ if (descriptor != null) {
+ if (descriptor.getParameters() != null) {
+ merged.putAll(descriptor.getParameters());
+ }
+ if (descriptor.getSerdeInfo() != null && descriptor.getSerdeInfo().getParameters() != null) {
+ merged.putAll(descriptor.getSerdeInfo().getParameters());
+ }
+ }
+ return merged;
+ }
+
+ private static String escape(String value) {
+ return value.replace("'", "''");
+ }
+
+ private static String sqlLiteral(String value) {
+ String globLiteral = value.replace("[", "[[]").replace("*", "[*]").replace("?", "[?]");
+ return "'" + globLiteral.replace("'", "''") + "'";
+ }
+
/**
* Projects the columns the catalog declares, rather than whatever the underlying files spell.
*
@@ -45,16 +192,7 @@ public final class GlueTableResolver {
* in charge.
*/
public static String buildProjection(Table table) {
- List<Column> declared = new ArrayList<>();
- if (table != null) {
- if (table.getStorageDescriptor() != null && table.getStorageDescriptor().getColumns() != null) {
- declared.addAll(table.getStorageDescriptor().getColumns());
- }
- if (table.getPartitionKeys() != null) {
- declared.addAll(table.getPartitionKeys());
- }
- }
-
+ List<Column> declared = declaredColumns(table);
List<String> names = new ArrayList<>();
Set<String> seen = new HashSet<>();
for (Column c : declared) {
@@ -82,6 +220,27 @@ public final class GlueTableResolver {
return sb.toString();
}
+ private static List<Column> declaredColumns(Table table) {
+ List<Column> declared = new ArrayList<>();
+ if (table != null) {
+ if (table.getStorageDescriptor() != null && table.getStorageDescriptor().getColumns() != null) {
+ declared.addAll(table.getStorageDescriptor().getColumns());
+ }
+ if (table.getPartitionKeys() != null) {
+ declared.addAll(table.getPartitionKeys());
+ }
+ }
+ List<Column> result = new ArrayList<>();
+ Set<String> seen = new HashSet<>();
+ for (Column column : declared) {
+ if (column != null && column.getName() != null && !column.getName().isBlank()
+ && seen.add(column.getName().toLowerCase(Locale.ROOT))) {
+ result.add(column);
+ }
+ }
+ return result;
+ }
+
public static String inferReadFunction(Table table) {
if (table == null || table.getStorageDescriptor() == null) {
return "read_csv_auto";
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftRoleAccess.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftRoleAccess.java
index 94fc92e2c5..a341ae4980 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftRoleAccess.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftRoleAccess.java
@@ -23,12 +23,12 @@ import java.util.Optional;
* Spectrum: validates an {@code IAM_ROLE} ARN against the cluster and Redshift's trust policy,
* mints a short-lived session, and evaluates the role's identity policy for one S3 action.
*
- * <p>Extracted from {@link S3CopySimulator}, whose {@code S3TransferException} this still throws
- * (same package, matching {@link ManifestReader}'s existing precedent for this kind of extraction).
+ * <p>Extracted from {@link S3CopySimulator} so the Spectrum package shares one implementation.
*/
-final class RedshiftRoleAccess {
+public final class RedshiftRoleAccess {
private static final String SQLSTATE_INSUFFICIENT_PRIVILEGE = "42501";
+ private static final String SQLSTATE_INTERNAL = "XX000";
private static final SecureRandom SECURE_RANDOM = new SecureRandom();
private static final String UPPER_ALPHANUMERIC = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
@@ -36,6 +36,8 @@ final class RedshiftRoleAccess {
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
/** COPY/UNLOAD is synchronous end-to-end; this only needs to outlive one statement. */
private static final Duration ROLE_SESSION_TTL = Duration.ofMinutes(5);
+ /** Spectrum streams rows lazily and authorizes each object as it is read, so one query can outlive a COPY. */
+ public static final Duration STREAMING_ROLE_SESSION_TTL = Duration.ofMinutes(30);
// S3Service's signed-request authorization only checks the bucket's resource policy: for a
// genuine HTTP request the identity-policy gate already happened upstream in
@@ -50,14 +52,20 @@ final class RedshiftRoleAccess {
private RedshiftRoleAccess() {
}
- record RoleSession(String accessKeyId, String sessionToken) {
+ public record RoleSession(String accessKeyId, String sessionToken) {
}
/**
* Validates the role ARN, its account and Redshift trust policy, then mints a short-lived session.
*/
- static RoleSession resolveRoleSession(String iamRoleArn, IamService iamService,
+ public static RoleSession resolveRoleSession(String iamRoleArn, IamService iamService,
String clusterAccountId, List<String> associatedRoleArns) {
+ return resolveRoleSession(iamRoleArn, iamService, clusterAccountId, associatedRoleArns, ROLE_SESSION_TTL);
+ }
+
+ public static RoleSession resolveRoleSession(String iamRoleArn, IamService iamService,
+ String clusterAccountId, List<String> associatedRoleArns,
+ Duration sessionTtl) {
AwsArnUtils.Arn parsed;
try {
parsed = AwsArnUtils.parse(iamRoleArn);
@@ -100,11 +108,11 @@ final class RedshiftRoleAccess {
String sessionToken = randomString(SECRET_CHARACTERS, 200);
iamService.registerSessionForAccount(parsed.accountId(), accessKeyId,
randomString(SECRET_CHARACTERS, 40), sessionToken, iamRoleArn,
- Instant.now().plus(ROLE_SESSION_TTL), null);
+ Instant.now().plus(sessionTtl), null);
return new RoleSession(accessKeyId, sessionToken);
}
- static void releaseRoleSession(RoleSession roleSession, String iamRoleArn, IamService iamService) {
+ public static void releaseRoleSession(RoleSession roleSession, String iamRoleArn, IamService iamService) {
if (roleSession == null) {
return;
}
@@ -116,8 +124,13 @@ final class RedshiftRoleAccess {
* Evaluates the role's identity-based policy for one S3 action, skipped entirely when
* {@code FLOCI_SERVICES_S3_ENFORCE_AUTH} is off (matching the anonymous path's behavior).
*/
- static void authorizeRoleAction(S3Service s3, IamService iamService, String roleArn,
+ public static void authorizeRoleAction(S3Service s3, IamService iamService, String roleArn,
String action, String resourceArn) {
+ authorizeRoleAction(s3, iamService, roleArn, action, resourceArn, Map.of());
+ }
+
+ public static void authorizeRoleAction(S3Service s3, IamService iamService, String roleArn,
+ String action, String resourceArn, Map<String, List<String>> conditionContext) {
if (!s3.isAuthEnforced()) {
return;
}
@@ -129,19 +142,57 @@ final class RedshiftRoleAccess {
"IAM Role '" + roleArn + "' could not be assumed: " + e.getMessage(), e);
}
IamPolicyEvaluator.SimulationDecision decision =
- ROLE_POLICY_EVALUATOR.simulatePrincipalPolicy(caller, action, resourceArn, Map.of());
+ ROLE_POLICY_EVALUATOR.simulatePrincipalPolicy(caller, action, resourceArn, conditionContext);
if (decision != IamPolicyEvaluator.SimulationDecision.ALLOWED) {
throw new S3CopySimulator.S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
"S3 access denied for IAM Role '" + roleArn + "': " + action + " on " + resourceArn, null);
}
}
+ /**
+ * Authorizes {@code s3:ListBucket} as the role's signed session: the identity policy first, then
+ * the bucket policy through the signed-request path COPY uses, so a bucket-policy deny applies
+ * even when the identity policy allows the action. {@code prefix} feeds the {@code s3:prefix}
+ * condition key an identity policy may use to constrain the listing.
+ */
+ public static void authorizeRoleList(S3Service s3, IamService iamService, RoleSession roleSession,
+ String roleArn, String bucket, String prefix) {
+ authorizeRoleAction(s3, iamService, roleArn, "s3:ListBucket", bucketArn(roleArn, bucket),
+ Map.of("s3:prefix", List.of(prefix == null ? "" : prefix)));
+ try {
+ s3.authorizeSignedListBucket(roleSession.accessKeyId(), roleSession.sessionToken(), bucket);
+ } catch (AwsException e) {
+ throw signedAccessFailure(e, "s3://" + bucket);
+ }
+ }
+
+ /** {@code s3:GetObject} counterpart of {@link #authorizeRoleList}. */
+ public static void authorizeRoleRead(S3Service s3, IamService iamService, RoleSession roleSession,
+ String roleArn, String bucket, String key) {
+ authorizeRoleAction(s3, iamService, roleArn, "s3:GetObject", objectArn(roleArn, bucket, key));
+ try {
+ s3.authorizeSignedGetObject(roleSession.accessKeyId(), roleSession.sessionToken(), bucket, key);
+ } catch (AwsException e) {
+ throw signedAccessFailure(e, "s3://" + bucket + "/" + key);
+ }
+ }
+
+ /** A bucket that does not exist is not an access denial, and reporting it as one hides the real cause. */
+ private static S3CopySimulator.S3TransferException signedAccessFailure(AwsException e, String target) {
+ if ("NoSuchBucket".equals(e.getErrorCode())) {
+ return new S3CopySimulator.S3TransferException(SQLSTATE_INTERNAL,
+ "S3 bucket does not exist for " + target, e);
+ }
+ return new S3CopySimulator.S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "S3 access denied for " + target, e);
+ }
+
/** The bucket and object ARNs live in the role's partition: a China role authorizes China buckets. */
- static String bucketArn(String roleArn, String bucket) {
+ public static String bucketArn(String roleArn, String bucket) {
return AwsArnUtils.Arn.global(AwsArnUtils.parse(roleArn).partition(), "s3", "", bucket).toString();
}
- static String objectArn(String roleArn, String bucket, String key) {
+ public static String objectArn(String roleArn, String bucket, String key) {
return bucketArn(roleArn, bucket) + "/" + key;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java
index 315521bd0f..5b2c00c3f8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java
@@ -75,15 +75,15 @@ public final class S3CopySimulator {
void close();
}
- static final class S3TransferException extends RuntimeException {
+ public static final class S3TransferException extends RuntimeException {
private final String sqlState;
- S3TransferException(String sqlState, String message, Throwable cause) {
+ public S3TransferException(String sqlState, String message, Throwable cause) {
super(message, cause);
this.sqlState = sqlState;
}
- String sqlState() {
+ public String sqlState() {
return sqlState;
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java
new file mode 100644
index 0000000000..66d6c10f63
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java
@@ -0,0 +1,756 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.services.floci.duck.FlociDuckClient;
+import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.services.glue.GlueTableResolver;
+import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.redshift.proxy.RedshiftRoleAccess;
+import io.github.hectorvent.floci.services.redshift.proxy.S3CopySimulator;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import jakarta.enterprise.context.ApplicationScoped;
+import org.jboss.logging.Logger;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+@ApplicationScoped
+public class ExternalTableMaterializer {
+ private static final Logger LOG = Logger.getLogger(ExternalTableMaterializer.class);
+ private static final ObjectMapper JSON = new ObjectMapper();
+ public static final String SCRATCH_BUCKET = S3Service.REDSHIFT_SPECTRUM_SCRATCH_BUCKET;
+ private static final String SQLSTATE_LOAD_FAILED = "58030";
+ private static final String SQLSTATE_INSUFFICIENT_PRIVILEGE = "42501";
+ private static final long LOCK_TIMEOUT_SECONDS = 30;
+ private static final String SCRATCH_KEY_PREFIX = "spectrum-";
+ private static final Duration STALE_SCRATCH_AGE = Duration.ofHours(1);
+ private static final String ICEBERG_SETUP = "INSTALL iceberg; LOAD iceberg;\n";
+ private static final String ICEBERG_MANIFESTS = "SELECT DISTINCT manifest_path FROM read_avro(%s)";
+ private static final String ICEBERG_DATA_FILES = "SELECT DISTINCT file_path FROM iceberg_metadata(%s) "
+ + "WHERE status IN ('ADDED', 'EXISTING')";
+ private static final String COPY_OPTIONS = "WITH (FORMAT csv, HEADER true, NULL '\\N')";
+ public enum Outcome { NOT_EXTERNAL, CURRENT, LOADED }
+
+ private final FlociDuckClient duckClient;
+ private final GlueService glueService;
+ private final S3Service s3Service;
+ private final IamService iamService;
+ private final EmulatorConfig config;
+ private final ConcurrentHashMap<String, String> fingerprints = new ConcurrentHashMap<>();
+ /** Column definitions each loaded table was created with, to tell a data reload from a schema change. */
+ private final ConcurrentHashMap<String, String> schemaSignatures = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, ReentrantLock> locks = new ConcurrentHashMap<>();
+ private final Set<String> sweptAccounts = ConcurrentHashMap.newKeySet();
+
+ public ExternalTableMaterializer(FlociDuckClient duckClient, GlueService glueService, S3Service s3Service,
+ IamService iamService, EmulatorConfig config) {
+ this.duckClient = duckClient;
+ this.glueService = glueService;
+ this.s3Service = s3Service;
+ this.iamService = iamService;
+ this.config = config;
+ }
+
+ public Outcome ensureCurrent(BackendSql backend, SpectrumSession session, ExternalSchemaBinding binding,
+ String tableName) {
+ Table table;
+ try {
+ table = RequestScopes.callAs(session.accountId(), () -> glueService.getTable(binding.glueDatabase(), tableName));
+ } catch (AwsException exception) {
+ if ("EntityNotFoundException".equals(exception.getErrorCode())) {
+ return Outcome.NOT_EXTERNAL;
+ }
+ throw exception;
+ }
+ String label = binding.schemaName() + "." + tableName;
+ // DuckDB reads as the account, so the bound role's access is enforced here, ahead of the read,
+ // with the same signed authorization COPY uses: identity policy and bucket policy.
+ RedshiftRoleAccess.RoleSession roleSession = openRoleSession(binding, session);
+ try {
+ Location location = Location.parse(table);
+ authorizeIcebergMetadata(session.accountId(), binding, roleSession, table, location);
+ authorizeIcebergManifests(session.accountId(), binding, roleSession, table, location);
+ List<Partition> partitions = partitions(session.accountId(), binding, table);
+ List<ReadSource> sources = readSources(session.accountId(), binding, roleSession, table, location, partitions);
+ List<S3Object> objects = sources.stream().flatMap(source -> source.objects().stream()).toList();
+ authorizeObjects(session.accountId(), binding, roleSession, sources);
+ authorizeIcebergDataFiles(session.accountId(), binding, roleSession, table, location);
+ String cacheKey = cacheKey(session, binding, tableName);
+ String fingerprint = fingerprint(table, partitions, objects);
+ // An Iceberg snapshot can change through its metadata or manifests while the Glue table and the
+ // listed objects stay the same, so no fingerprint can vouch for it: it is read on every query.
+ boolean cacheable = !session.inTransaction() && !GlueTableResolver.isIcebergTable(table);
+ if (cacheable && fingerprint.equals(fingerprints.get(cacheKey))) {
+ return Outcome.CURRENT;
+ }
+ ReentrantLock lock = locks.computeIfAbsent(cacheKey, ignored -> new ReentrantLock());
+ acquire(lock, label, session.inTransaction());
+ try {
+ if (cacheable && fingerprint.equals(fingerprints.get(cacheKey))) {
+ return Outcome.CURRENT;
+ }
+ String previousSignature = schemaSignatures.get(cacheKey);
+ String definitions = load(backend, session, binding, table, sources, previousSignature);
+ if (session.inTransaction()) {
+ // Transactional DDL may roll back, so the rows cannot be trusted afterwards. The schema
+ // signature stays when this load kept the committed columns, so a view that depends on the
+ // table can still be refilled in place; a replaced table may roll back to the old columns.
+ fingerprints.remove(cacheKey);
+ if (!definitions.equals(previousSignature)) {
+ schemaSignatures.remove(cacheKey);
+ }
+ } else {
+ if (cacheable) {
+ fingerprints.put(cacheKey, fingerprint);
+ } else {
+ fingerprints.remove(cacheKey);
+ }
+ schemaSignatures.put(cacheKey, definitions);
+ }
+ return Outcome.LOADED;
+ } finally {
+ lock.unlock();
+ }
+ } catch (SpectrumSqlException | SpectrumReadException exception) {
+ throw exception;
+ } catch (RuntimeException exception) {
+ // the cause can carry DuckDB request text, so it goes to the log and not to the client
+ LOG.warnv(exception, "Unable to load external table {0}", label);
+ throw new SpectrumReadException(SQLSTATE_LOAD_FAILED, "Unable to load external table \"" + label + "\"", exception);
+ } finally {
+ RedshiftRoleAccess.releaseRoleSession(roleSession, binding.iamRoleArn(), iamService);
+ }
+ }
+
+ /**
+ * A session inside a transaction can hold a PostgreSQL lock on the table until it commits, so
+ * waiting forever on the Java lock here could deadlock in a way PostgreSQL's detector cannot see.
+ * A session outside a transaction holds no PostgreSQL lock, so it can wait out a slow load.
+ */
+ private static void acquire(ReentrantLock lock, String label, boolean inTransaction) {
+ try {
+ if (!inTransaction) {
+ lock.lockInterruptibly();
+ } else if (!lock.tryLock(LOCK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
+ throw new SpectrumReadException(SQLSTATE_LOAD_FAILED,
+ "Timed out waiting to load external table \"" + label + "\"");
+ }
+ } catch (InterruptedException exception) {
+ Thread.currentThread().interrupt();
+ throw new SpectrumReadException(SQLSTATE_LOAD_FAILED,
+ "Interrupted while waiting to load external table \"" + label + "\"", exception);
+ }
+ }
+
+ public void forget(String clusterKey, String databaseName, String schemaName, String tableName) {
+ forgetMatching(key -> key.equals(clusterKey + "|" + databaseName + "|" + schemaName + "|" + tableName));
+ }
+
+ /** Forgets every table of one external schema, so a dropped and recreated schema is reloaded. */
+ public void forgetSchema(String clusterKey, String databaseName, String schemaName) {
+ String prefix = schemaPrefix(clusterKey, databaseName, schemaName);
+ forgetMatching(key -> key.startsWith(prefix));
+ }
+
+ private void forgetMatching(Predicate<String> matches) {
+ fingerprints.keySet().removeIf(matches);
+ schemaSignatures.keySet().removeIf(matches);
+ // Keep locks: removing a held lock lets another thread create a second lock for the same table.
+ }
+
+ /** Tables of one external schema this materializer has loaded into PostgreSQL and still tracks. */
+ public Set<String> loadedTables(String clusterKey, String databaseName, String schemaName) {
+ String prefix = schemaPrefix(clusterKey, databaseName, schemaName);
+ return fingerprints.keySet().stream().filter(key -> key.startsWith(prefix))
+ .map(key -> key.substring(prefix.length())).collect(Collectors.toSet());
+ }
+
+ public void forgetCluster(String clusterKey) {
+ forgetMatching(key -> key.startsWith(clusterKey + "|"));
+ }
+
+ private static String schemaPrefix(String clusterKey, String databaseName, String schemaName) {
+ return clusterKey + "|" + databaseName + "|" + schemaName + "|";
+ }
+
+ private RedshiftRoleAccess.RoleSession openRoleSession(ExternalSchemaBinding binding, SpectrumSession session) {
+ try {
+ return RedshiftRoleAccess.resolveRoleSession(binding.iamRoleArn(), iamService, session.accountId(),
+ session.iamRoleArns());
+ } catch (S3CopySimulator.S3TransferException exception) {
+ throw new SpectrumSqlException(exception.sqlState(), exception.getMessage());
+ }
+ }
+
+ private void authorizeList(String accountId, ExternalSchemaBinding binding,
+ RedshiftRoleAccess.RoleSession roleSession, Location location) {
+ try {
+ RequestScopes.runAs(accountId, () -> RedshiftRoleAccess.authorizeRoleList(
+ s3Service, iamService, roleSession, binding.iamRoleArn(), location.bucket(), location.prefix()));
+ } catch (S3CopySimulator.S3TransferException exception) {
+ throw new SpectrumSqlException(exception.sqlState(), exception.getMessage());
+ }
+ }
+
+ /**
+ * DuckDB follows an Iceberg table's {@code metadata_location}, a free-form Glue parameter, so it
+ * must stay inside the location the role was authorized for and be readable by the role itself.
+ */
+ private void authorizeIcebergMetadata(String accountId, ExternalSchemaBinding binding,
+ RedshiftRoleAccess.RoleSession roleSession, Table table, Location location) {
+ if (!GlueTableResolver.isIcebergTable(table)) {
+ return;
+ }
+ String metadata = GlueTableResolver.icebergMetadataLocation(table);
+ if (metadata == null || metadata.isBlank()) {
+ return;
+ }
+ if (!metadata.startsWith("s3://")) {
+ throw new SpectrumSqlException("0A000", "Iceberg table \"" + table.getName() + "\" has an unsupported metadata_location");
+ }
+ String rest = metadata.substring(5);
+ int slash = rest.indexOf('/');
+ String bucket = slash < 0 ? rest : rest.substring(0, slash);
+ String key = slash < 0 ? "" : rest.substring(slash + 1);
+ if (!bucket.equals(location.bucket()) || !key.startsWith(location.prefix())) {
+ throw new SpectrumSqlException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "Iceberg metadata_location of \"" + table.getName() + "\" is outside the table location");
+ }
+ try {
+ RequestScopes.runAs(accountId, () -> RedshiftRoleAccess.authorizeRoleRead(
+ s3Service, iamService, roleSession, binding.iamRoleArn(), bucket, key));
+ } catch (S3CopySimulator.S3TransferException exception) {
+ throw new SpectrumSqlException(exception.sqlState(), exception.getMessage());
+ }
+ }
+
+ private void authorizeObjects(String accountId, ExternalSchemaBinding binding,
+ RedshiftRoleAccess.RoleSession roleSession, List<ReadSource> sources) {
+ try {
+ for (ReadSource source : sources) {
+ for (S3Object object : source.objects()) {
+ RequestScopes.runAs(accountId, () -> RedshiftRoleAccess.authorizeRoleRead(
+ s3Service, iamService, roleSession, binding.iamRoleArn(),
+ source.location().bucket(), object.getKey()));
+ }
+ }
+ } catch (S3CopySimulator.S3TransferException exception) {
+ throw new SpectrumSqlException(exception.sqlState(), exception.getMessage());
+ }
+ }
+
+ /**
+ * Iceberg manifests can name data files outside the table location. DuckDB performs the scan
+ * with account credentials, so authorize each live data-file path as the bound role first.
+ * The scan re-reads the metadata afterwards, so a metadata object overwritten between the two
+ * steps is not covered. Before routing user queries here, pin the authorized snapshot and its
+ * file list for the scan, or enforce the role during the scan in the sidecar.
+ */
+ private void authorizeIcebergDataFiles(String accountId, ExternalSchemaBinding binding,
+ RedshiftRoleAccess.RoleSession roleSession, Table table, Location location) {
+ if (!GlueTableResolver.isIcebergTable(table)) {
+ return;
+ }
+ String metadata = GlueTableResolver.icebergMetadataLocation(table);
+ if (metadata == null || metadata.isBlank()) {
+ throw new SpectrumSqlException("0A000", "Iceberg table \"" + table.getName()
+ + "\" has no supported metadata_location");
+ }
+ String sql = ICEBERG_DATA_FILES.formatted(sqlPathLiteral(metadata));
+ List<Map<String, Object>> rows = duckClient.query(sql, ICEBERG_SETUP, accountId);
+ for (Map<String, Object> row : rows) {
+ Object value = row.get("file_path");
+ if (!(value instanceof String filePath) || filePath.isBlank()) {
+ throw new SpectrumSqlException("XX000", "Iceberg manifest contains an invalid data-file path");
+ }
+ S3Location file = icebergFileLocation(filePath, location);
+ try {
+ RequestScopes.runAs(accountId, () -> RedshiftRoleAccess.authorizeRoleRead(
+ s3Service, iamService, roleSession, binding.iamRoleArn(), file.bucket(), file.key()));
+ } catch (S3CopySimulator.S3TransferException exception) {
+ throw new SpectrumSqlException(exception.sqlState(), exception.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Iceberg's metadata scan reads manifest lists and manifests before it can return data-file paths.
+ * Resolve and authorize both levels first, so the sidecar cannot read them using the emulator's
+ * account credentials when the bound Redshift role is denied access.
+ */
+ private void authorizeIcebergManifests(String accountId, ExternalSchemaBinding binding,
+ RedshiftRoleAccess.RoleSession roleSession, Table table, Location location) {
+ if (!GlueTableResolver.isIcebergTable(table)) {
+ return;
+ }
+ String metadata = GlueTableResolver.icebergMetadataLocation(table);
+ if (metadata == null || metadata.isBlank()) {
+ return;
+ }
+ S3Location metadataObjectLocation = icebergFileLocation(metadata, location);
+ S3Object metadataObject = RequestScopes.callAs(accountId,
+ () -> s3Service.getObject(metadataObjectLocation.bucket(), metadataObjectLocation.key()));
+ if (metadataObject == null || metadataObject.getData() == null) {
+ throw new SpectrumSqlException("XX000", "Unable to read Iceberg table metadata");
+ }
+ JsonNode metadataJson;
+ try {
+ metadataJson = JSON.readTree(metadataObject.getData());
+ } catch (IOException exception) {
+ throw new SpectrumSqlException("XX000", "Iceberg table metadata is not valid JSON");
+ }
+ String currentSnapshotId = metadataJson.path("current-snapshot-id").asText();
+ if (currentSnapshotId.isBlank() || "-1".equals(currentSnapshotId)) {
+ if (metadataJson.path("snapshots").size() > 0) {
+ // the scan could pick a snapshot whose manifests were never authorized
+ throw new SpectrumSqlException("XX000", "Iceberg table metadata has snapshots but no current snapshot");
+ }
+ return;
+ }
+ String manifestListPath = null;
+ for (JsonNode snapshot : metadataJson.path("snapshots")) {
+ if (currentSnapshotId.equals(snapshot.path("snapshot-id").asText())) {
+ manifestListPath = snapshot.path("manifest-list").asText();
+ break;
+ }
+ }
+ if (manifestListPath == null || manifestListPath.isBlank()) {
+ throw new SpectrumSqlException("XX000", "Iceberg current snapshot has no manifest-list path");
+ }
+ S3Location manifestList = icebergFileLocation(manifestListPath, location);
+ authorizeIcebergObject(accountId, binding, roleSession, manifestList);
+ String manifestListUri = "s3://" + manifestList.bucket() + "/" + manifestList.key();
+ List<Map<String, Object>> manifests = duckClient.query(
+ ICEBERG_MANIFESTS.formatted(sqlPathLiteral(manifestListUri)), ICEBERG_SETUP, accountId);
+ for (Map<String, Object> manifest : manifests) {
+ Object manifestValue = manifest.get("manifest_path");
+ if (!(manifestValue instanceof String manifestPath) || manifestPath.isBlank()) {
+ throw new SpectrumSqlException("XX000", "Iceberg manifest list contains an invalid manifest path");
+ }
+ authorizeIcebergObject(accountId, binding, roleSession, icebergFileLocation(manifestPath, location));
+ }
+ }
+
+ private void authorizeIcebergObject(String accountId, ExternalSchemaBinding binding,
+ RedshiftRoleAccess.RoleSession roleSession, S3Location object) {
+ try {
+ RequestScopes.runAs(accountId, () -> RedshiftRoleAccess.authorizeRoleRead(
+ s3Service, iamService, roleSession, binding.iamRoleArn(), object.bucket(), object.key()));
+ } catch (S3CopySimulator.S3TransferException exception) {
+ throw new SpectrumSqlException(exception.sqlState(), exception.getMessage());
+ }
+ }
+
+ private static S3Location icebergFileLocation(String filePath, Location tableLocation) {
+ if (filePath.startsWith("s3://")) {
+ String rest = filePath.substring(5);
+ int slash = rest.indexOf('/');
+ if (slash <= 0 || slash == rest.length() - 1) {
+ throw new SpectrumSqlException("XX000", "Iceberg manifest contains an invalid S3 data-file path");
+ }
+ return new S3Location(rest.substring(0, slash), rest.substring(slash + 1));
+ }
+ if (filePath.contains("://") || filePath.isBlank()) {
+ throw new SpectrumSqlException("0A000", "Iceberg data-file path uses an unsupported location");
+ }
+ String key = filePath.startsWith("/") ? filePath.substring(1) : filePath;
+ if (!tableLocation.prefix().isEmpty() && !key.startsWith(tableLocation.prefix())) {
+ key = tableLocation.prefix() + key;
+ }
+ return new S3Location(tableLocation.bucket(), key);
+ }
+
+ private static String sqlPathLiteral(String path) {
+ String globLiteral = path.replace("[", "[[]").replace("*", "[*]").replace("?", "[?]");
+ return "'" + globLiteral.replace("'", "''") + "'";
+ }
+
+ private String cacheKey(SpectrumSession session, ExternalSchemaBinding binding, String tableName) {
+ return session.clusterKey() + "|" + session.databaseName() + "|" + binding.schemaName() + "|" + tableName;
+ }
+
+ private List<Partition> partitions(String accountId, ExternalSchemaBinding binding, Table table) {
+ if (GlueTableResolver.isIcebergTable(table)) {
+ return List.of();
+ }
+ List<Partition> partitions = RequestScopes.callAs(accountId,
+ () -> glueService.getPartitions(binding.glueDatabase(), table.getName()));
+ return partitions == null ? List.of() : partitions;
+ }
+
+ private List<ReadSource> readSources(String accountId, ExternalSchemaBinding binding,
+ RedshiftRoleAccess.RoleSession roleSession, Table table,
+ Location tableLocation, List<Partition> partitions) {
+ if (partitions.isEmpty() && hasPartitionKeys(table)) {
+ // Spectrum reads only registered partitions, so a partitioned table with none returns no rows.
+ return List.of();
+ }
+ if (partitions.isEmpty()) {
+ authorizeList(accountId, binding, roleSession, tableLocation);
+ return List.of(new ReadSource(table, null, tableLocation, listObjects(accountId, tableLocation)));
+ }
+ List<ReadSource> sources = new ArrayList<>();
+ for (Partition partition : partitions) {
+ StorageDescriptor descriptor = partitionDescriptor(table, partition);
+ Table partitionTable = new Table();
+ partitionTable.setName(table.getName());
+ partitionTable.setParameters(table.getParameters());
+ partitionTable.setPartitionKeys(table.getPartitionKeys());
+ partitionTable.setStorageDescriptor(descriptor);
+ Location location = Location.parse(descriptor.getLocation(), table.getName());
+ authorizeList(accountId, binding, roleSession, location);
+ sources.add(new ReadSource(partitionTable, partition, location, listObjects(accountId, location)));
+ }
+ return List.copyOf(sources);
+ }
+
+ private static boolean hasPartitionKeys(Table table) {
+ return !GlueTableResolver.isIcebergTable(table)
+ && table.getPartitionKeys() != null && !table.getPartitionKeys().isEmpty();
+ }
+
+ /**
+ * A partition's descriptor with whatever it leaves unset filled in from the table, so the
+ * columns, format and CSV options the read depends on are never lost for a sparse partition.
+ */
+ private static StorageDescriptor partitionDescriptor(Table table, Partition partition) {
+ StorageDescriptor tableDescriptor = table.getStorageDescriptor();
+ StorageDescriptor own = partition.getStorageDescriptor();
+ if (own == null) {
+ return tableDescriptor;
+ }
+ StorageDescriptor merged = new StorageDescriptor();
+ merged.setLocation(own.getLocation() == null || own.getLocation().isBlank()
+ ? tableDescriptor.getLocation() : own.getLocation());
+ merged.setColumns(own.getColumns() == null || own.getColumns().isEmpty()
+ ? tableDescriptor.getColumns() : own.getColumns());
+ merged.setInputFormat(own.getInputFormat() != null ? own.getInputFormat() : tableDescriptor.getInputFormat());
+ merged.setOutputFormat(own.getOutputFormat() != null ? own.getOutputFormat() : tableDescriptor.getOutputFormat());
+ merged.setSerdeInfo(own.getSerdeInfo() != null ? own.getSerdeInfo() : tableDescriptor.getSerdeInfo());
+ merged.setParameters(own.getParameters() != null ? own.getParameters() : tableDescriptor.getParameters());
+ merged.setCompressed(own.getCompressed() != null ? own.getCompressed() : tableDescriptor.getCompressed());
+ return merged;
+ }
+
+ private List<S3Object> listObjects(String accountId, Location location) {
+ return RequestScopes.callAs(accountId, () -> {
+ List<S3Object> objects = new ArrayList<>();
+ String token = null;
+ do {
+ S3Service.ListObjectsResult page = s3Service.listObjectsWithPrefixes(location.bucket(), location.prefix(), "", 1000, token, null);
+ objects.addAll(page.objects());
+ token = page.isTruncated() ? page.nextContinuationToken() : null;
+ } while (token != null);
+ objects.removeIf(object -> isIgnoredBySpectrum(object.getKey(), location.prefix()));
+ objects.sort(Comparator.comparing(S3Object::getKey));
+ return objects;
+ });
+ }
+
+ /**
+ * Spectrum ignores hidden files (names starting with a period, underscore or hash mark, or ending with
+ * a tilde), which job output such as {@code _SUCCESS} relies on, and a folder marker has no data.
+ * A file under a hidden folder such as {@code _temporary/} is intermediate output, so every path
+ * component below the table or partition prefix is checked, not only the file name.
+ */
+ private static boolean isIgnoredBySpectrum(String key, String prefix) {
+ String relative = key.startsWith(prefix) ? key.substring(prefix.length()) : key;
+ if (relative.isEmpty() || relative.endsWith("/")) {
+ return true;
+ }
+ for (String component : relative.split("/")) {
+ if (component.startsWith(".") || component.startsWith("_") || component.startsWith("#")
+ || component.endsWith("~")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * A partition's values and everything in its descriptor that the read depends on (location, columns,
+ * formats, SerDe and its parameters), so that any change to how its files are decoded reloads the table.
+ */
+ private static String partitionEntry(Table table, Partition partition) {
+ StorageDescriptor descriptor = partitionDescriptor(table, partition);
+ StringBuilder entry = new StringBuilder().append(partition.getValues()).append('=');
+ if (descriptor == null) {
+ return entry.toString();
+ }
+ entry.append(descriptor.getLocation()).append('|')
+ .append(descriptor.getInputFormat()).append('|')
+ .append(descriptor.getOutputFormat()).append('|')
+ .append(descriptor.getCompressed()).append('|');
+ if (descriptor.getColumns() != null) {
+ descriptor.getColumns().forEach(column -> entry.append(column.getName()).append(':')
+ .append(column.getType()).append(','));
+ }
+ entry.append('|').append(new TreeMap<>(descriptor.getParameters() == null
+ ? Map.of() : descriptor.getParameters())).append('|');
+ if (descriptor.getSerdeInfo() != null) {
+ entry.append(descriptor.getSerdeInfo().getSerializationLibrary()).append(new TreeMap<>(
+ descriptor.getSerdeInfo().getParameters() == null
+ ? Map.of() : descriptor.getSerdeInfo().getParameters()));
+ }
+ return entry.toString();
+ }
+
+ private String fingerprint(Table table, List<Partition> partitions, List<S3Object> objects) {
+ StringBuilder value = new StringBuilder().append(table.getVersionId()).append('|').append(table.getUpdateTime()).append('|');
+ if (!partitions.isEmpty()) {
+ partitions.stream().map(partition -> partitionEntry(table, partition)).sorted()
+ .forEach(entry -> value.append(entry).append(';'));
+ }
+ for (S3Object object : objects) {
+ value.append(object.getKey()).append(':').append(object.getSize()).append(':').append(object.getETag()).append(';');
+ }
+ try {
+ return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.toString().getBytes(StandardCharsets.UTF_8)));
+ } catch (NoSuchAlgorithmException exception) {
+ throw new IllegalStateException("SHA-256 is not available", exception);
+ }
+ }
+
+ /**
+ * Loads the table into PostgreSQL and returns the column definitions it was created with. When they
+ * match {@code loadedDefinitions} the target table is refilled in place, so views and grants that
+ * depend on it survive; a schema change replaces the table instead.
+ */
+ private String load(BackendSql backend, SpectrumSession session, ExternalSchemaBinding binding, Table table,
+ List<ReadSource> sources, String loadedDefinitions) {
+ GlueTableResolver.ReadPlan plan = GlueTableResolver.readPlan(table);
+ if (plan.columns().isEmpty()) {
+ throw new SpectrumSqlException("0A000", "Glue table \"" + binding.schemaName() + "." + table.getName() + "\" declares no columns");
+ }
+ String schema = quote(binding.schemaName());
+ String target = schema + "." + quote(table.getName());
+ String stagingName = "floci_spectrum_stg_" + UUID.randomUUID().toString().replace("-", "");
+ String staging = "pg_temp." + quote(stagingName);
+ String definitions = plan.columns().stream().map(column -> quote(column.getName()) + " " + GlueTypeMapper.toPostgres(column.getType())).collect(Collectors.joining(", "));
+ boolean stagingCreated = false;
+ String scratchKey = null;
+ try {
+ backend.execute("CREATE TEMP TABLE " + quote(stagingName) + " (" + definitions + ")");
+ stagingCreated = true;
+ boolean hasObjects = sources.stream().anyMatch(source -> !source.objects().isEmpty());
+ if (plan.iceberg() || hasObjects) {
+ // before a key exists: a scratch bucket that is not ours must see no write and no delete
+ prepareScratchBucket(session.accountId());
+ scratchKey = SCRATCH_KEY_PREFIX + UUID.randomUUID() + ".csv";
+ readWithDuckDb(session.accountId(), table, sources,
+ config.services().redshift().spectrumMaxRows(), scratchKey);
+ long rows;
+ String resultKey = scratchKey;
+ try (InputStream csv = RequestScopes.callAs(session.accountId(),
+ () -> s3Service.openObjectStream(SCRATCH_BUCKET, resultKey, null))) {
+ rows = backend.copyIn("COPY " + staging + " FROM STDIN " + COPY_OPTIONS, csv);
+ } catch (IOException exception) {
+ throw new SpectrumReadException(SQLSTATE_LOAD_FAILED,
+ "Unable to stream external table data into PostgreSQL", exception);
+ }
+ if (rows > config.services().redshift().spectrumMaxRows()) {
+ throw new SpectrumReadException(SQLSTATE_LOAD_FAILED, "External table \"" + binding.schemaName() + "." + table.getName() + "\" exceeds configured row limit");
+ }
+ }
+ if (definitions.equals(loadedDefinitions)) {
+ backend.execute("CREATE TABLE IF NOT EXISTS " + target + " (" + definitions + "); TRUNCATE " + target
+ + "; INSERT INTO " + target + " SELECT * FROM " + staging + "; DROP TABLE " + staging);
+ } else {
+ backend.execute("DROP TABLE IF EXISTS " + target + "; CREATE TABLE " + target + " (" + definitions
+ + "); INSERT INTO " + target + " SELECT * FROM " + staging + "; DROP TABLE " + staging);
+ }
+ stagingCreated = false;
+ return definitions;
+ } catch (RuntimeException exception) {
+ if (stagingCreated) {
+ try {
+ backend.execute("DROP TABLE IF EXISTS " + staging);
+ } catch (RuntimeException cleanupFailure) {
+ LOG.warnv(cleanupFailure, "Could not drop Spectrum staging table {0}", staging);
+ }
+ }
+ throw exception;
+ } finally {
+ if (scratchKey != null) {
+ String cleanupKey = scratchKey;
+ try {
+ RequestScopes.runAs(session.accountId(), () -> s3Service.deleteObject(SCRATCH_BUCKET, cleanupKey));
+ } catch (RuntimeException exception) {
+ LOG.warnv(exception, "Could not delete Spectrum scratch object {0}", cleanupKey);
+ }
+ }
+ }
+ }
+
+ private void prepareScratchBucket(String accountId) {
+ RequestScopes.runAs(accountId, () -> {
+ ensureScratchBucket();
+ if (sweptAccounts.add(accountId)) {
+ sweepStaleScratchObjects();
+ }
+ });
+ }
+
+ private void readWithDuckDb(String accountId, Table table, List<ReadSource> sources, long maxRows,
+ String scratchKey) {
+ List<String> selects = sources.stream().filter(source -> source.partition() == null || !source.objects().isEmpty())
+ .map(this::readSelect).toList();
+ if (selects.isEmpty() && GlueTableResolver.isIcebergTable(table)) {
+ GlueTableResolver.ReadPlan plan = GlueTableResolver.readPlan(table);
+ selects = List.of("SELECT " + projection(plan.columns()) + " FROM " + fromClause(plan, table));
+ }
+ String query = selects.size() == 1 ? selects.getFirst()
+ : "SELECT * FROM (" + String.join(" UNION ALL ", selects) + ") AS spectrum_partitions";
+ String setup = GlueTableResolver.isIcebergTable(table) ? ICEBERG_SETUP : null;
+ duckClient.execute(query + " LIMIT " + (maxRows + 1), setup,
+ "s3://" + SCRATCH_BUCKET + "/" + scratchKey, accountId);
+ }
+
+ /**
+ * Creates the scratch bucket and tags it as ours. A bucket of that name that carries no tag belongs to
+ * the user: it is neither written to nor swept, and the load fails instead of mixing with it.
+ */
+ private void ensureScratchBucket() {
+ boolean created = true;
+ try {
+ s3Service.createBucket(SCRATCH_BUCKET, config.defaultRegion());
+ } catch (AwsException exception) {
+ if (!"BucketAlreadyOwnedByYou".equals(exception.getErrorCode())) {
+ throw exception;
+ }
+ created = false;
+ }
+ if (created) {
+ s3Service.putBucketTagging(SCRATCH_BUCKET,
+ Map.of(S3Service.INTERNAL_BUCKET_TAG_KEY, S3Service.REDSHIFT_SPECTRUM_SCRATCH_TAG_VALUE));
+ } else if (!S3Service.REDSHIFT_SPECTRUM_SCRATCH_TAG_VALUE.equals(
+ s3Service.getBucketTagging(SCRATCH_BUCKET).get(S3Service.INTERNAL_BUCKET_TAG_KEY))) {
+ throw new SpectrumReadException(SQLSTATE_LOAD_FAILED, "Bucket \"" + SCRATCH_BUCKET
+ + "\" already exists and is not the Spectrum scratch bucket");
+ }
+ }
+
+ /**
+ * A JVM that dies mid-load leaves its scratch object behind. The first load of an account removes the
+ * old ones; a recent object may belong to a load still running, so only stale ones go.
+ */
+ private void sweepStaleScratchObjects() {
+ Instant cutoff = Instant.now().minus(STALE_SCRATCH_AGE);
+ try {
+ String token = null;
+ do {
+ S3Service.ListObjectsResult page = s3Service.listObjectsWithPrefixes(
+ SCRATCH_BUCKET, SCRATCH_KEY_PREFIX, "", 1000, token, null);
+ for (S3Object object : page.objects()) {
+ if (object.getLastModified().isBefore(cutoff)) {
+ s3Service.deleteObject(SCRATCH_BUCKET, object.getKey());
+ }
+ }
+ token = page.isTruncated() ? page.nextContinuationToken() : null;
+ } while (token != null);
+ } catch (RuntimeException exception) {
+ LOG.warnv(exception, "Could not sweep stale Spectrum scratch objects from {0}", SCRATCH_BUCKET);
+ }
+ }
+
+ private String readSelect(ReadSource source) {
+ List<String> objectUris = source.objects().stream()
+ .map(object -> "s3://" + source.location().bucket() + "/" + object.getKey()).toList();
+ GlueTableResolver.ReadPlan plan = GlueTableResolver.readPlan(source.table(), objectUris);
+ String projection;
+ if (source.partition() == null) {
+ projection = projection(plan.columns());
+ } else {
+ List<Column> dataColumns = source.table().getStorageDescriptor().getColumns();
+ List<String> parts = new ArrayList<>(dataColumns.stream()
+ .map(column -> GlueTypeMapper.duckProjection(column.getName(), column.getType())).toList());
+ List<Column> partitionKeys = source.table().getPartitionKeys();
+ List<String> values = source.partition().getValues();
+ for (int index = 0; index < partitionKeys.size(); index++) {
+ Column key = partitionKeys.get(index);
+ String value = index < values.size() ? values.get(index) : null;
+ if (value != null && GlueTypeMapper.exceedsWidth(key.getType(), value)) {
+ // a data column wider than its width is NULL, so a partition value is as well
+ value = null;
+ }
+ String expression = value == null || "__HIVE_DEFAULT_PARTITION__".equals(value)
+ ? "CAST(NULL AS VARCHAR)"
+ : "CAST('" + value.replace("'", "''") + "' AS VARCHAR)";
+ parts.add("COALESCE(" + expression + ", '\\N') AS " + quote(key.getName()));
+ }
+ projection = String.join(", ", parts);
+ }
+ return "SELECT " + projection + " FROM " + fromClause(plan, source.table());
+ }
+
+ /**
+ * An Iceberg scan reads the same glob-escaped metadata literal that the authorization queries used,
+ * so that every step sees one path and none can match a different object.
+ */
+ private static String fromClause(GlueTableResolver.ReadPlan plan, Table table) {
+ if (!plan.iceberg()) {
+ return plan.fromClause();
+ }
+ return "iceberg_scan(" + sqlPathLiteral(GlueTableResolver.icebergMetadataLocation(table)) + ")";
+ }
+
+ private static String projection(List<Column> columns) {
+ return columns.stream().map(column -> GlueTypeMapper.duckProjection(column.getName(), column.getType()))
+ .collect(Collectors.joining(", "));
+ }
+
+ static String quote(String identifier) {
+ return "\"" + identifier.replace("\"", "\"\"") + "\"";
+ }
+
+ private record Location(String bucket, String prefix) {
+ static Location parse(Table table) {
+ String location = table.getStorageDescriptor() == null ? null : table.getStorageDescriptor().getLocation();
+ return parse(location, table.getName());
+ }
+
+ static Location parse(String location, String tableName) {
+ if (location == null || location.isBlank() || !location.startsWith("s3://")) {
+ throw new SpectrumSqlException("0A000", "Glue table \"" + tableName + "\" has no supported storage location");
+ }
+ String rest = location.substring(5);
+ int slash = rest.indexOf('/');
+ String bucket = slash < 0 ? rest : rest.substring(0, slash);
+ String prefix = slash < 0 ? "" : rest.substring(slash + 1);
+ return new Location(bucket, prefix.isEmpty() || prefix.endsWith("/") ? prefix : prefix + "/");
+ }
+ }
+
+ private record ReadSource(Table table, Partition partition, Location location, List<S3Object> objects) {
+ }
+
+ private record S3Location(String bucket, String key) {
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapper.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapper.java
index 92a2305162..65bca1d47f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapper.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapper.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.redshift.spectrum;
+import java.nio.charset.StandardCharsets;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -74,6 +75,12 @@ public final class GlueTypeMapper {
};
}
+ /** True when {@code glueType} is a sized string whose width, in bytes, the value does not fit in. */
+ public static boolean exceedsWidth(String glueType, String value) {
+ Matcher sized = SIZED_STRING.matcher(canonicalGlueType(glueType));
+ return sized.matches() && value.getBytes(StandardCharsets.UTF_8).length > Integer.parseInt(sized.group(1));
+ }
+
public static boolean isNested(String glueType) {
String type = normalize(glueType);
return type.startsWith("array<") || type.startsWith("map<") || type.startsWith("struct<");
@@ -82,8 +89,20 @@ public final class GlueTypeMapper {
public static String duckProjection(String columnName, String glueType) {
String column = quote(columnName);
String type = canonicalGlueType(glueType);
- String expression = isNested(type) ? "to_json(" + column + ")"
- : "binary".equals(type) ? "'\\x' || hex(" + column + ")" : column;
+ Matcher sized = SIZED_STRING.matcher(type);
+ String expression;
+ if (isNested(type)) {
+ expression = "to_json(" + column + ")";
+ } else if ("binary".equals(type)) {
+ expression = "'\\x' || hex(" + column + ")";
+ } else if (sized.matches()) {
+ // Spectrum nulls a value wider than its column (surplus_char_handling defaults to SET_TO_NULL),
+ // where COPY into varchar(n) would fail the whole query. strlen counts bytes, as VARCHAR(n) does.
+ expression = "CASE WHEN strlen(CAST(" + column + " AS VARCHAR)) > " + sized.group(1)
+ + " THEN NULL ELSE " + column + " END";
+ } else {
+ expression = column;
+ }
return "COALESCE(CAST(" + expression + " AS VARCHAR), " + NULL_MARKER + ") AS " + column;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumCatalog.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumCatalog.java
index 69b0f38e5c..288c3ea6d4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumCatalog.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumCatalog.java
@@ -45,6 +45,50 @@ public class SpectrumCatalog implements Resettable {
return schemas.getForAccount(accountId, schemaKey(accountId, databaseName, schemaName));
}
+ /**
+ * Phase one stored a schema under the Glue database name, while its tables carry the Redshift
+ * database, so the schema's own {@code databaseName} cannot be matched against the session's.
+ * The Redshift database scope comes from {@link #legacySchemaNames}, which requires tables in it.
+ */
+ public Optional<SpectrumExternalSchema> findLegacySchema(String accountId, String databaseName, String schemaName) {
+ if (!legacySchemaNames(accountId, databaseName).contains(schemaName)) {
+ return Optional.empty();
+ }
+ String prefix = accountId + ":";
+ List<SpectrumExternalSchema> candidates = schemas.scanForAccount(accountId, key -> key.startsWith(prefix))
+ .stream()
+ .filter(schema -> accountId.equals(schema.accountId()) && schema.schemaName().equals(schemaName))
+ .sorted(Comparator.comparing(SpectrumExternalSchema::databaseName))
+ .toList();
+ if (candidates.size() > 1) {
+ // same-named schemas of other Glue databases must not lend their IAM role to this one: keep
+ // those whose location covers a table of this Redshift database
+ List<SpectrumExternalTable> tables = tablesForSchema(accountId, databaseName, schemaName);
+ List<SpectrumExternalSchema> owning = candidates.stream()
+ .filter(schema -> tables.stream().anyMatch(table -> table.location().startsWith(schema.location())))
+ .toList();
+ if (!owning.isEmpty()) {
+ return Optional.of(owning.getFirst());
+ }
+ }
+ return candidates.stream().findFirst();
+ }
+
+ public List<String> legacySchemaNames(String accountId, String databaseName) {
+ String prefix = accountId + ":";
+ String tablePrefix = accountId + ":" + databaseName + ":";
+ List<String> knownSchemas = schemas.scanForAccount(accountId, key -> key.startsWith(prefix)).stream()
+ .filter(schema -> accountId.equals(schema.accountId()))
+ .map(SpectrumExternalSchema::schemaName)
+ .toList();
+ return tables.scanForAccount(accountId, key -> key.startsWith(tablePrefix)).stream()
+ .filter(table -> accountId.equals(table.accountId()) && knownSchemas.contains(table.schemaName()))
+ .map(SpectrumExternalTable::schemaName)
+ .distinct()
+ .sorted()
+ .toList();
+ }
+
public Optional<SpectrumExternalTable> table(String accountId, String databaseName,
String schemaName, String tableName) {
return tables.getForAccount(accountId, tableKey(accountId, databaseName, schemaName, tableName));
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumCatalogResolver.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumCatalogResolver.java
new file mode 100644
index 0000000000..ac88a84f97
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumCatalogResolver.java
@@ -0,0 +1,57 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import jakarta.enterprise.context.ApplicationScoped;
+
+import java.util.List;
+import java.util.Optional;
+
+@ApplicationScoped
+public class SpectrumCatalogResolver {
+ private final ExternalCatalogRegistry registry;
+ private final SpectrumCatalog catalog;
+
+ public SpectrumCatalogResolver(ExternalCatalogRegistry registry, SpectrumCatalog catalog) {
+ this.registry = registry;
+ this.catalog = catalog;
+ }
+
+ public Optional<Resolution> resolve(String accountId, String clusterKey, String databaseName, String schemaName) {
+ Optional<ExternalSchemaBinding> glue = registry.find(accountId, clusterKey, databaseName, schemaName);
+ if (glue.isPresent()) {
+ return Optional.of(new Resolution.Glue(glue.get()));
+ }
+ return catalog.findLegacySchema(accountId, databaseName, schemaName).map(Resolution.PhaseOne::new);
+ }
+
+ public Optional<SpectrumExternalTable> legacyTable(String accountId, String databaseName,
+ String schemaName, String tableName) {
+ return catalog.table(accountId, databaseName, schemaName, tableName);
+ }
+
+ public List<String> legacySchemaNames(String accountId, String databaseName) {
+ return catalog.legacySchemaNames(accountId, databaseName);
+ }
+
+ public sealed interface Resolution permits Resolution.Glue, Resolution.PhaseOne {
+ Kind kind();
+
+ enum Kind {
+ GLUE,
+ PHASE_ONE
+ }
+
+ record Glue(ExternalSchemaBinding binding) implements Resolution {
+ @Override
+ public Kind kind() {
+ return Kind.GLUE;
+ }
+ }
+
+ record PhaseOne(SpectrumExternalSchema schema) implements Resolution {
+ @Override
+ public Kind kind() {
+ return Kind.PHASE_ONE;
+ }
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumGlueCsvAdapter.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumGlueCsvAdapter.java
new file mode 100644
index 0000000000..7c496cc1c3
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumGlueCsvAdapter.java
@@ -0,0 +1,189 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import jakarta.enterprise.context.ApplicationScoped;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/** Converts a strictly supported subset of Glue CSV metadata into the Phase 1 streaming descriptors. */
+@ApplicationScoped
+public final class SpectrumGlueCsvAdapter {
+
+ private static final String TEXT_INPUT_FORMAT = "org.apache.hadoop.mapred.TextInputFormat";
+ private static final String LAZY_SIMPLE_SERDE = "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
+ private static final String OPEN_CSV_SERDE = "org.apache.hadoop.hive.serde2.OpenCSVSerde";
+ private static final Set<String> TABLE_PROPERTIES = Set.of("EXTERNAL", "skip.header.line.count", "field.delim",
+ "serialization.format", "quoteChar", "escapeChar", "serialization.null.format");
+ private static final Set<String> CSV_PROPERTIES = Set.of("field.delim", "serialization.format", "separatorChar",
+ "quoteChar", "escapeChar", "serialization.null.format");
+
+ public Optional<CsvTable> adapt(ExternalSchemaBinding binding, Table table,
+ String accountId, String databaseName) {
+ if (binding == null || table == null || accountId == null || databaseName == null
+ || !"EXTERNAL_TABLE".equalsIgnoreCase(table.getTableType())
+ || table.getPartitionKeys() != null && !table.getPartitionKeys().isEmpty()) {
+ return Optional.empty();
+ }
+ StorageDescriptor descriptor = table.getStorageDescriptor();
+ if (descriptor == null || descriptor.getLocation() == null || descriptor.getLocation().isBlank()
+ || !isS3Location(descriptor.getLocation())
+ || !TEXT_INPUT_FORMAT.equals(descriptor.getInputFormat())
+ || Boolean.TRUE.equals(descriptor.getCompressed())
+ || descriptor.getNumberOfBuckets() != null && descriptor.getNumberOfBuckets() > 0
+ || descriptor.getSchemaReference() != null || !empty(descriptor.getParameters())) {
+ return Optional.empty();
+ }
+ StorageDescriptor.SerDeInfo serde = descriptor.getSerdeInfo();
+ if (serde == null || !isSupportedSerde(serde.getSerializationLibrary())) {
+ return Optional.empty();
+ }
+ try {
+ if (table.getParameters() != null) {
+ for (String property : table.getParameters().keySet()) {
+ if (!TABLE_PROPERTIES.contains(property)) {
+ return Optional.empty();
+ }
+ }
+ String external = table.getParameters().get("EXTERNAL");
+ if (external != null && !"TRUE".equalsIgnoreCase(external)) {
+ return Optional.empty();
+ }
+ }
+ Map<String, String> parameters = supportedParameters(table, serde);
+ CsvProperties properties = csvProperties(parameters);
+ List<SpectrumColumn> columns = columns(descriptor.getColumns());
+ SpectrumExternalSchema schema = new SpectrumExternalSchema(accountId, databaseName,
+ binding.schemaName(), descriptor.getLocation(), binding.iamRoleArn());
+ SpectrumExternalTable externalTable = new SpectrumExternalTable(accountId, databaseName,
+ binding.schemaName(), table.getName(), columns, descriptor.getLocation(), properties.delimiter(),
+ properties.quote(), properties.escape(), properties.nullValue(), properties.headerLines());
+ return Optional.of(new CsvTable(schema, externalTable));
+ } catch (IllegalArgumentException | SpectrumSqlException expected) {
+ // an unsupported CSV shape or type is not an error: the table is read through the DuckDB path
+ return Optional.empty();
+ }
+ }
+
+ private static boolean isS3Location(String location) {
+ try {
+ SpectrumExternalSchema.validateS3Location(location);
+ return true;
+ } catch (IllegalArgumentException expected) {
+ // validateS3Location signals an unusable location by throwing
+ return false;
+ }
+ }
+
+ private static boolean isSupportedSerde(String serializationLibrary) {
+ return LAZY_SIMPLE_SERDE.equals(serializationLibrary) || OPEN_CSV_SERDE.equals(serializationLibrary);
+ }
+
+ private static boolean empty(Map<String, String> values) {
+ return values == null || values.isEmpty();
+ }
+
+ private static Map<String, String> supportedParameters(Table table, StorageDescriptor.SerDeInfo serde) {
+ Map<String, String> parameters = new LinkedHashMap<>();
+ merge(parameters, serde.getParameters());
+ if (table.getParameters() != null) {
+ Map<String, String> tableCsv = new LinkedHashMap<>();
+ for (Map.Entry<String, String> entry : table.getParameters().entrySet()) {
+ if (!"EXTERNAL".equals(entry.getKey())) {
+ tableCsv.put(entry.getKey(), entry.getValue());
+ }
+ }
+ merge(tableCsv, parameters);
+ parameters = tableCsv;
+ }
+ return parameters;
+ }
+
+ private static void merge(Map<String, String> destination, Map<String, String> source) {
+ if (source == null) {
+ return;
+ }
+ for (Map.Entry<String, String> entry : source.entrySet()) {
+ if (!CSV_PROPERTIES.contains(entry.getKey()) || entry.getValue() == null) {
+ throw new IllegalArgumentException("Unsupported Glue CSV property");
+ }
+ String existing = destination.putIfAbsent(entry.getKey(), entry.getValue());
+ if (existing != null && !existing.equals(entry.getValue())) {
+ throw new IllegalArgumentException("Conflicting Glue CSV properties");
+ }
+ }
+ }
+
+ private static CsvProperties csvProperties(Map<String, String> values) {
+ char delimiter = singleCharacter(values, "field.delim", ',');
+ delimiter = singleCharacter(values, "serialization.format", delimiter);
+ delimiter = singleCharacter(values, "separatorChar", delimiter);
+ char quote = singleCharacter(values, "quoteChar", '"');
+ char escape = singleCharacter(values, "escapeChar", '\\');
+ String nullValue = values.getOrDefault("serialization.null.format", "\\N");
+ String header = values.getOrDefault("skip.header.line.count", "0");
+ int headerLines;
+ try {
+ headerLines = Integer.parseInt(header);
+ } catch (NumberFormatException exception) {
+ throw new IllegalArgumentException("Invalid CSV header count", exception);
+ }
+ if (headerLines < 0 || nullValue.isEmpty()) {
+ throw new IllegalArgumentException("Invalid Glue CSV properties");
+ }
+ return new CsvProperties(delimiter, quote, escape, nullValue, headerLines);
+ }
+
+ private static char singleCharacter(Map<String, String> values, String key, char defaultValue) {
+ String value = values.get(key);
+ if (value == null) {
+ return defaultValue;
+ }
+ if (value.length() != 1) {
+ throw new IllegalArgumentException("Glue CSV property must contain exactly one character");
+ }
+ return value.charAt(0);
+ }
+
+ private static List<SpectrumColumn> columns(List<Column> source) {
+ if (source == null || source.isEmpty()) {
+ throw new IllegalArgumentException("Glue table has no declared columns");
+ }
+ List<SpectrumColumn> columns = new ArrayList<>(source.size());
+ for (Column column : source) {
+ if (column == null || column.getName() == null || column.getType() == null) {
+ throw new IllegalArgumentException("Glue table has an incomplete column");
+ }
+ String normalizedGlueType = column.getType().trim().toLowerCase(Locale.ROOT).replace(" ", "");
+ String postgresType = GlueTypeMapper.toPostgres(column.getType());
+ SpectrumColumn.Type legacyType = switch (normalizedGlueType.equals("char") ? "char"
+ : postgresType.toLowerCase(Locale.ROOT)) {
+ // Phase 1 models exactly VARCHAR(256); any other width needs the DuckDB path
+ case "text", "varchar", "varchar(256)" -> SpectrumColumn.Type.VARCHAR;
+ case "char" -> SpectrumColumn.Type.CHAR;
+ case "integer" -> SpectrumColumn.Type.INTEGER;
+ case "bigint" -> SpectrumColumn.Type.BIGINT;
+ case "numeric" -> SpectrumColumn.Type.DECIMAL;
+ case "boolean" -> SpectrumColumn.Type.BOOLEAN;
+ case "date" -> SpectrumColumn.Type.DATE;
+ case "timestamp" -> SpectrumColumn.Type.TIMESTAMP;
+ default -> throw new IllegalArgumentException("Glue type is not supported by Phase 1 CSV");
+ };
+ columns.add(new SpectrumColumn(column.getName(), legacyType));
+ }
+ return List.copyOf(columns);
+ }
+
+ private record CsvProperties(char delimiter, char quote, char escape, String nullValue, int headerLines) {
+ }
+
+ public record CsvTable(SpectrumExternalSchema schema, SpectrumExternalTable table) {
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializer.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializer.java
index a6a0384882..fb6d264f48 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializer.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializer.java
@@ -1,11 +1,10 @@
package io.github.hectorvent.floci.services.redshift.spectrum;
-import jakarta.enterprise.context.ApplicationScoped;
-
import io.github.hectorvent.floci.services.redshift.proxy.PostgresWireDecoder;
+import jakarta.enterprise.context.ApplicationScoped;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
@@ -13,6 +12,7 @@ import java.security.SecureRandom;
import java.util.HexFormat;
import java.util.Iterator;
import java.util.List;
+import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -63,6 +63,33 @@ public final class SpectrumMaterializer {
}
}
+ public Materialization materialize(BackendSql backend, SpectrumSession session, SpectrumExternalSchema schema,
+ SpectrumExternalTable table, SpectrumS3Reader reader, String identifier) {
+ Materialization materialization = new Materialization(identifier, table.columns());
+ boolean tableCreated = false;
+ try {
+ backend.execute(createTableSql(identifier, table));
+ tableCreated = true;
+ try (Stream<SpectrumRow> rows = reader.read(session, schema, table);
+ SpectrumRowInputStream input = new SpectrumRowInputStream(rows.iterator(), table.columns())) {
+ backend.copyIn(copySql(identifier, table), input);
+ }
+ return materialization;
+ } catch (SpectrumReadException failure) {
+ SpectrumReadException original = findRowFailure(failure);
+ if (original == null) {
+ original = failure;
+ }
+ cleanupAfterFailure(backend, materialization, tableCreated, original);
+ throw original;
+ } catch (RuntimeException failure) {
+ SpectrumReadException readFailure = new SpectrumReadException(
+ SQLSTATE_DATA, "Unable to materialize Spectrum rows", failure);
+ cleanupAfterFailure(backend, materialization, tableCreated, readFailure);
+ throw readFailure;
+ }
+ }
+
public String nextIdentifier() {
return "spectrum_tmp_" + HexFormat.of().formatHex(randomBytes(12));
}
@@ -77,9 +104,40 @@ public final class SpectrumMaterializer {
}
}
+ public void cleanup(BackendSql backend, Materialization materialization) {
+ backend.execute(dropTableSql(materialization.identifier()));
+ }
+
+ private static String dropTableSql(String identifier) {
+ return "DROP TABLE IF EXISTS \"" + quoteIdentifier(identifier) + "\"";
+ }
+
+ private void cleanupAfterFailure(BackendSql backend, Materialization materialization, boolean tableCreated,
+ SpectrumReadException original) {
+ if (tableCreated) {
+ try {
+ cleanup(backend, materialization);
+ } catch (RuntimeException cleanupFailure) {
+ original.addSuppressed(cleanupFailure);
+ }
+ }
+ }
+
+ private static SpectrumReadException findRowFailure(Throwable failure) {
+ Throwable current = failure;
+ while (current != null) {
+ if (current instanceof SpectrumReadException readException) {
+ if (readException != failure) {
+ return readException;
+ }
+ }
+ current = current.getCause();
+ }
+ return null;
+ }
+
private static String createTableSql(String identifier, SpectrumExternalTable table) {
- StringBuilder sql = new StringBuilder("CREATE TEMP TABLE \"")
- .append(quoteIdentifier(identifier)).append("\" (");
+ StringBuilder sql = new StringBuilder("CREATE TEMP TABLE \"").append(quoteIdentifier(identifier)).append("\" (");
for (int i = 0; i < table.columns().size(); i++) {
if (i > 0) {
sql.append(", ");
@@ -93,8 +151,7 @@ public final class SpectrumMaterializer {
private static String copySql(String identifier, SpectrumExternalTable table) {
return "COPY \"" + quoteIdentifier(identifier) + "\" ("
+ table.columns().stream().map(column -> "\"" + quoteIdentifier(column.name()) + "\"")
- .collect(Collectors.joining(", "))
- + ") FROM STDIN";
+ .collect(Collectors.joining(", ")) + ") FROM STDIN";
}
private static byte[] encodeRow(SpectrumRow row, List<SpectrumColumn> columns) {
@@ -120,6 +177,52 @@ public final class SpectrumMaterializer {
return line.toString().getBytes(StandardCharsets.UTF_8);
}
+ private static final class SpectrumRowInputStream extends InputStream {
+ private final Iterator<SpectrumRow> rows;
+ private final List<SpectrumColumn> columns;
+ private byte[] currentRow = new byte[0];
+ private int offset;
+
+ private SpectrumRowInputStream(Iterator<SpectrumRow> rows, List<SpectrumColumn> columns) {
+ this.rows = rows;
+ this.columns = columns;
+ }
+
+ @Override
+ public int read() throws IOException {
+ byte[] value = new byte[1];
+ int count = read(value, 0, 1);
+ return count < 0 ? -1 : value[0] & 0xff;
+ }
+
+ @Override
+ public int read(byte[] bytes, int start, int length) throws IOException {
+ Objects.checkFromIndexSize(start, length, bytes.length);
+ if (length == 0) {
+ return 0;
+ }
+ if (offset >= currentRow.length) {
+ try {
+ if (!rows.hasNext()) {
+ return -1;
+ }
+ currentRow = encodeRow(rows.next(), columns);
+ offset = 0;
+ } catch (RuntimeException exception) {
+ throw new IOException("Unable to read Spectrum row", exception);
+ }
+ }
+ int count = Math.min(length, currentRow.length - offset);
+ System.arraycopy(currentRow, offset, bytes, start, count);
+ offset += count;
+ return count;
+ }
+
+ @Override
+ public void close() {
+ }
+ }
+
private static void sendQuery(OutputStream output, String sql) throws IOException {
output.write(PostgresWireDecoder.encodeQuery(sql));
output.flush();
@@ -190,8 +293,7 @@ public final class SpectrumMaterializer {
}
private void recoverFromFailure(Socket backend, Materialization materialization, OutputStream output,
- boolean tableCreated, boolean copyInProgress,
- SpectrumReadException original) {
+ boolean tableCreated, boolean copyInProgress, SpectrumReadException original) {
if (copyInProgress && output != null) {
try {
writeCopyFail(output, original.getMessage());
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumS3Reader.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumS3Reader.java
index aff800e436..89d590485a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumS3Reader.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumS3Reader.java
@@ -1,7 +1,10 @@
package io.github.hectorvent.floci.services.redshift.spectrum;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.redshift.proxy.RedshiftRoleAccess;
+import io.github.hectorvent.floci.services.redshift.proxy.S3CopySimulator;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import jakarta.enterprise.context.ApplicationScoped;
@@ -33,8 +36,6 @@ public final class SpectrumS3Reader {
private static final String SQLSTATE_INTERNAL = "XX000";
private final S3Service s3Service;
- // Stored in the schema now so role-aware Spectrum authorization can be added without changing metadata.
- @SuppressWarnings("unused")
private final IamService iamService;
public SpectrumS3Reader(S3Service s3Service, IamService iamService) {
@@ -43,28 +44,77 @@ public final class SpectrumS3Reader {
}
public Stream<SpectrumRow> read(SpectrumExternalSchema schema, SpectrumExternalTable table) {
+ // The phase-one interceptor knows only the account and database, not the cluster's roles, so the
+ // role is enforced here without the cluster-association check. The session overload has both.
+ return read(schema, table, schema.iamRoleArn(), null);
+ }
+
+ public Stream<SpectrumRow> read(SpectrumSession session, SpectrumExternalSchema schema,
+ SpectrumExternalTable table) {
+ if (session == null || !session.accountId().equals(schema.accountId())
+ || !session.databaseName().equals(table.databaseName())) {
+ throw new SpectrumReadException(SQLSTATE_AUTHORIZATION, "Spectrum session does not match external table");
+ }
+ String roleArn = schema.iamRoleArn();
+ if (roleArn != null && !session.iamRoleArns().contains(roleArn)) {
+ throw new SpectrumReadException(SQLSTATE_AUTHORIZATION,
+ "Spectrum IAM role is not associated with the Redshift cluster");
+ }
+ return read(schema, table, roleArn, session.iamRoleArns());
+ }
+
+ /**
+ * The returned stream owns the role session minted here and releases it on close, so callers
+ * must close it (try-with-resources) even when they stop reading early.
+ */
+ private Stream<SpectrumRow> read(SpectrumExternalSchema schema, SpectrumExternalTable table, String roleArn,
+ List<String> associatedRoleArns) {
+ String accountId = schema.accountId();
Location location = Location.parse(table.location());
+ RedshiftRoleAccess.RoleSession roleSession = null;
try {
- s3Service.authorizeAnonymousListBucket(location.bucket());
- List<S3Object> objects = new ArrayList<>();
- String continuationToken = null;
- do {
- S3Service.ListObjectsResult result = s3Service.listObjectsWithPrefixes(
- location.bucket(), location.key(), "", 1000, continuationToken, null);
- objects.addAll(result.objects());
- continuationToken = result.isTruncated() ? result.nextContinuationToken() : null;
- } while (continuationToken != null);
- List<String> keys = objects.stream()
- .map(S3Object::getKey)
- .sorted(Comparator.naturalOrder())
- .toList();
+ if (roleArn != null) {
+ roleSession = RedshiftRoleAccess.resolveRoleSession(roleArn, iamService, accountId, associatedRoleArns,
+ RedshiftRoleAccess.STREAMING_ROLE_SESSION_TTL);
+ }
+ RedshiftRoleAccess.RoleSession session = roleSession;
+ // Account-aware S3 storage resolves the account from the request context, which the
+ // proxy thread does not have; without it the read could hit the default account.
+ List<String> keys = RequestScopes.callAs(accountId, () -> {
+ if (session == null) {
+ s3Service.authorizeAnonymousListBucket(location.bucket());
+ } else {
+ RedshiftRoleAccess.authorizeRoleList(s3Service, iamService, session, roleArn, location.bucket(), location.key());
+ }
+ List<S3Object> objects = new ArrayList<>();
+ String continuationToken = null;
+ do {
+ S3Service.ListObjectsResult result = s3Service.listObjectsWithPrefixes(
+ location.bucket(), location.key(), "", 1000, continuationToken, null);
+ objects.addAll(result.objects());
+ continuationToken = result.isTruncated() ? result.nextContinuationToken() : null;
+ } while (continuationToken != null);
+ return objects.stream().map(S3Object::getKey).sorted(Comparator.naturalOrder()).toList();
+ });
return StreamSupport.stream(Spliterators.spliteratorUnknownSize(
- new RowIterator(location.bucket(), keys.iterator(), table), 0), false);
+ new RowIterator(accountId, location.bucket(), keys.iterator(), table, roleArn, session), 0), false)
+ .onClose(() -> RedshiftRoleAccess.releaseRoleSession(session, roleArn, iamService));
+ } catch (S3CopySimulator.S3TransferException exception) {
+ RedshiftRoleAccess.releaseRoleSession(roleSession, roleArn, iamService);
+ throw mapRoleException(exception);
} catch (AwsException exception) {
+ RedshiftRoleAccess.releaseRoleSession(roleSession, roleArn, iamService);
throw mapAwsException(location, exception);
+ } catch (RuntimeException exception) {
+ RedshiftRoleAccess.releaseRoleSession(roleSession, roleArn, iamService);
+ throw exception;
}
}
+ private static SpectrumReadException mapRoleException(S3CopySimulator.S3TransferException exception) {
+ return new SpectrumReadException(exception.sqlState(), "Unable to read Spectrum data with IAM role", exception);
+ }
+
private SpectrumReadException mapAwsException(Location location, AwsException exception) {
String state = switch (exception.getErrorCode()) {
case "AccessDenied", "Unauthorized", "Forbidden" -> SQLSTATE_AUTHORIZATION;
@@ -75,18 +125,25 @@ public final class SpectrumS3Reader {
}
private final class RowIterator implements Iterator<SpectrumRow> {
+ private final String accountId;
private final String bucket;
private final Iterator<String> keys;
private final SpectrumExternalTable table;
+ private final String roleArn;
+ private final RedshiftRoleAccess.RoleSession roleSession;
private BufferedReader reader;
private SpectrumRow next;
private boolean finished;
private String currentKey;
- private RowIterator(String bucket, Iterator<String> keys, SpectrumExternalTable table) {
+ private RowIterator(String accountId, String bucket, Iterator<String> keys, SpectrumExternalTable table,
+ String roleArn, RedshiftRoleAccess.RoleSession roleSession) {
+ this.accountId = accountId;
this.bucket = bucket;
this.keys = keys;
this.table = table;
+ this.roleArn = roleArn;
+ this.roleSession = roleSession;
}
@Override
@@ -116,8 +173,16 @@ public final class SpectrumS3Reader {
return;
}
currentKey = keys.next();
- s3Service.authorizeAnonymousGetObject(bucket, currentKey);
- S3Object object = s3Service.getObject(bucket, currentKey);
+ String key = currentKey;
+ S3Object object = RequestScopes.callAs(accountId, () -> {
+ if (roleSession == null) {
+ s3Service.authorizeAnonymousGetObject(bucket, key);
+ } else {
+ RedshiftRoleAccess.authorizeRoleRead(s3Service, iamService, roleSession, roleArn,
+ bucket, key);
+ }
+ return s3Service.getObject(bucket, key);
+ });
reader = new BufferedReader(new InputStreamReader(
new ByteArrayInputStream(object.getData()), StandardCharsets.UTF_8));
skipHeaders();
@@ -134,6 +199,8 @@ public final class SpectrumS3Reader {
next = convert(record, table);
return;
}
+ } catch (S3CopySimulator.S3TransferException exception) {
+ throw mapRoleException(exception);
} catch (AwsException exception) {
throw mapAwsException(new Location(bucket, currentKey), exception);
} catch (IOException exception) {
@@ -233,7 +300,7 @@ public final class SpectrumS3Reader {
int value;
while ((value = reader.read()) != -1) {
char current = (char) value;
- if (current == escape && !quoted) {
+ if (current == escape) {
record.append(current);
int next = reader.read();
if (next == -1) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 97c401eb16..3bc0ca79b2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -99,6 +99,11 @@ public class S3Service implements Resettable, ResourceProvider {
return "000000000000";
}
static final String DEFAULT_OWNER_DISPLAY_NAME = "floci";
+ public static final String INTERNAL_BUCKET_PREFIX = "floci-internal-";
+ public static final String REDSHIFT_SPECTRUM_SCRATCH_BUCKET =
+ INTERNAL_BUCKET_PREFIX + "redshift-spectrum-scratch";
+ public static final String INTERNAL_BUCKET_TAG_KEY = "floci:internal";
+ public static final String REDSHIFT_SPECTRUM_SCRATCH_TAG_VALUE = "redshift-spectrum-scratch";
private static final String AUTHENTICATED_USERS_GROUP_URI = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers";
private static final String LOG_DELIVERY_GROUP_URI = "http://acs.amazonaws.com/groups/s3/LogDelivery";
private static final String LEGACY_ACCESS_KEY_ID = "test";
@@ -510,7 +515,19 @@ public class S3Service implements Resettable, ResourceProvider {
}
public List<Bucket> listBuckets() {
- return bucketStore.scan(key -> true);
+ return bucketStore.scan(key -> true).stream()
+ .filter(bucket -> !isSpectrumScratchBucket(bucket))
+ .toList();
+ }
+
+ /**
+ * The Spectrum scratch bucket is told apart from a user bucket of the same name by the tag the
+ * materializer puts on it, so a user's own bucket is never hidden or treated as scratch space.
+ */
+ private static boolean isSpectrumScratchBucket(Bucket bucket) {
+ return REDSHIFT_SPECTRUM_SCRATCH_BUCKET.equals(bucket.getName())
+ && bucket.getTags() != null
+ && REDSHIFT_SPECTRUM_SCRATCH_TAG_VALUE.equals(bucket.getTags().get(INTERNAL_BUCKET_TAG_KEY));
}
public String getBucketOwnerAccountId(String bucketName) {
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index ed918f59dd..3b5950f494 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -451,6 +451,7 @@ floci:
proxy-handshake-timeout-millis: 10000 # Max time a client has to complete the startup/auth handshake
proxy-backend-connect-timeout-millis: 5000 # Max time to wait for the backend TCP connect
proxy-max-connections: 100 # Max concurrent proxy connections before new ones are refused
+ spectrum-max-rows: 1000000 # FLOCI_SERVICES_REDSHIFT_SPECTRUM_MAX_ROWS; rows loaded per external table per query
rds:
enabled: true
mock: false # FLOCI_SERVICES_RDS_MOCK — true = metadata only, no Docker (useful for CI)
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueTableResolverTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueTableResolverTest.java
index e9f46f51d9..d05833ed54 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueTableResolverTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueTableResolverTest.java
@@ -6,13 +6,64 @@ import io.github.hectorvent.floci.services.glue.model.Table;
import org.junit.jupiter.api.Test;
import java.util.List;
+import java.util.Map;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
+import static org.junit.jupiter.api.Assertions.assertThrows;
class GlueTableResolverTest {
+ private static Column column(String name, String type) {
+ Column column = new Column();
+ column.setName(name);
+ column.setType(type);
+ return column;
+ }
+
+ private static Table table(String format, String location, List<Column> columns) {
+ StorageDescriptor descriptor = new StorageDescriptor();
+ descriptor.setInputFormat(format);
+ descriptor.setLocation(location);
+ descriptor.setColumns(columns);
+ Table table = new Table();
+ table.setName("events");
+ table.setStorageDescriptor(descriptor);
+ return table;
+ }
+
+ @Test
+ void readPlanForParquetUsesLocationGlob() {
+ GlueTableResolver.ReadPlan plan = GlueTableResolver.readPlan(table(
+ "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "s3://bucket/events/",
+ List.of(column("id", "int"))));
+ assertThat(plan.fromClause(), equalTo("read_parquet('s3://bucket/events/**', union_by_name = true)"));
+ assertThat(plan.iceberg(), is(false));
+ }
+
+ @Test
+ void readPlanForIcebergUsesMetadataLocation() {
+ Table table = table(null, "s3://bucket/iceberg/", List.of(column("id", "int")));
+ table.setParameters(Map.of("table_type", "ICEBERG", "metadata_location", "s3://bucket/iceberg/metadata/v1.json"));
+ assertThat(GlueTableResolver.readPlan(table).fromClause(), equalTo("iceberg_scan('s3://bucket/iceberg/metadata/v1.json')"));
+ }
+
+ @Test
+ void readPlanAppendsPartitionKeysWithoutDuplicates() {
+ Table table = table("org.apache.hadoop.mapred.TextInputFormat", "s3://bucket/events",
+ List.of(column("id", "int"), column("dt", "string")));
+ table.setPartitionKeys(List.of(column("DT", "string"), column("region", "string")));
+ assertThat(GlueTableResolver.readPlan(table).columns().stream().map(Column::getName).toList(), contains("id", "dt", "region"));
+ }
+
+ @Test
+ void readPlanRejectsMissingLocation() {
+ assertThrows(IllegalArgumentException.class, () -> GlueTableResolver.readPlan(table("input", " ", List.of())));
+ }
+
@Test
void inferReadFunctionDetectsParquetFromInputFormat() {
StorageDescriptor sd = new StorageDescriptor();
@@ -64,4 +115,52 @@ class GlueTableResolverTest {
void isIcebergTableFalseWhenParametersMissing() {
assertThat(GlueTableResolver.isIcebergTable(new Table()), is(false));
}
+
+ @Test
+ void readPlanUsesExplicitCsvOptionsInsteadOfSniffingWhenTheTableDeclaresThem() {
+ Table table = table("org.apache.hadoop.mapred.TextInputFormat", "s3://bucket/events/",
+ List.of(column("id", "int"), column("name", "string")));
+ table.setParameters(Map.of("skip.header.line.count", "0", "serialization.null.format", "NA"));
+ StorageDescriptor.SerDeInfo serde = new StorageDescriptor.SerDeInfo();
+ serde.setParameters(Map.of("field.delim", "|", "quoteChar", "'", "escapeChar", "\\"));
+ table.getStorageDescriptor().setSerdeInfo(serde);
+
+ String from = GlueTableResolver.readPlan(table).fromClause();
+
+ assertThat(from, equalTo("read_csv('s3://bucket/events/**', header = false, delim = '|', quote = '''', "
+ + "escape = '\\', nullstr = 'NA', columns = {'id': 'VARCHAR', 'name': 'VARCHAR'})"));
+ }
+
+ @Test
+ void readPlanDefaultsTheDelimiterToACommaWhenOnlyTheHeaderCountIsDeclared() {
+ Table table = table("org.apache.hadoop.mapred.TextInputFormat", "s3://bucket/events/",
+ List.of(column("id", "int")));
+ table.setParameters(Map.of("skip.header.line.count", "0"));
+
+ String from = GlueTableResolver.readPlan(table).fromClause();
+
+ assertThat(from, equalTo("read_csv('s3://bucket/events/**', header = false, delim = ',', "
+ + "columns = {'id': 'VARCHAR'})"));
+ }
+
+ @Test
+ void readPlanTreatsOneSkippedLineAsAHeaderAndMoreAsSkippedRows() {
+ Table oneLine = table("org.apache.hadoop.mapred.TextInputFormat", "s3://bucket/events/",
+ List.of(column("id", "int")));
+ oneLine.setParameters(Map.of("skip.header.line.count", "1"));
+ Table twoLines = table("org.apache.hadoop.mapred.TextInputFormat", "s3://bucket/events/",
+ List.of(column("id", "int")));
+ twoLines.setParameters(Map.of("skip.header.line.count", "2"));
+
+ assertThat(GlueTableResolver.readPlan(oneLine).fromClause(), containsString("header = true"));
+ assertThat(GlueTableResolver.readPlan(twoLines).fromClause(), containsString("header = false, skip = 2"));
+ }
+
+ @Test
+ void readPlanKeepsSniffingWhenTheTableDeclaresNoCsvOptions() {
+ Table table = table("org.apache.hadoop.mapred.TextInputFormat", "s3://bucket/events/",
+ List.of(column("id", "int")));
+
+ assertThat(GlueTableResolver.readPlan(table).fromClause(), equalTo("read_csv_auto('s3://bucket/events/**')"));
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java
index 4a82ef3a37..988ddc863e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.redshift;
+import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.redshift.model.Cluster;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.quarkus.test.junit.QuarkusTest;
@@ -35,6 +36,15 @@ class RedshiftSpectrumIntegrationTest {
@Inject
S3Service s3Service;
+ @Inject
+ IamService iamService;
+
+ private static final String ROLE_NAME = "SpectrumItRole";
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/" + ROLE_NAME;
+ private static final String TRUST_POLICY = """
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}""";
+
private String clusterId;
private String bucket;
@@ -55,6 +65,17 @@ class RedshiftSpectrumIntegrationTest {
}
}
+ private Cluster createClusterWithRole(String identifier) {
+ if (iamService.findRole("000000000000", ROLE_NAME).isEmpty()) {
+ iamService.createRole(ROLE_NAME, "/", TRUST_POLICY, null, 0, null);
+ iamService.putRolePolicy(ROLE_NAME, "AllowS3", """
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Action":["s3:GetObject","s3:ListBucket"],"Resource":"*"}]}""");
+ }
+ return redshiftService.createCluster(identifier, "dc2.large", "admin", "Secret123",
+ null, List.of(), List.of(ROLE_ARN));
+ }
+
@AfterEach
void cleanUp() {
if (clusterId != null) {
@@ -75,13 +96,13 @@ class RedshiftSpectrumIntegrationTest {
s3Service.putObject(bucket, "events/part-1.csv",
"id,name\n1,Alice\n2,Bob\n".getBytes(StandardCharsets.UTF_8), "text/csv", null);
clusterId = "it-spectrum-" + System.nanoTime();
- Cluster cluster = redshiftService.createCluster(clusterId, "dc2.large", "admin", "Secret123");
+ Cluster cluster = createClusterWithRole(clusterId);
try (Connection connection = waitForConnection(cluster, "admin", "Secret123");
Statement statement = connection.createStatement()) {
- statement.execute("CREATE EXTERNAL SCHEMA analytics FROM DATA CATALOG DATABASE 'dev' IAM_ROLE 'role'");
+ statement.execute("CREATE EXTERNAL SCHEMA analytics FROM DATA CATALOG DATABASE 'dev' IAM_ROLE '" + ROLE_ARN + "'");
SQLException duplicateSchema = assertThrows(SQLException.class, () -> statement.execute(
- "CREATE EXTERNAL SCHEMA analytics FROM DATA CATALOG DATABASE 'dev' IAM_ROLE 'role'"));
+ "CREATE EXTERNAL SCHEMA analytics FROM DATA CATALOG DATABASE 'dev' IAM_ROLE '" + ROLE_ARN + "'"));
assertEquals("42P06", duplicateSchema.getSQLState());
statement.execute("CREATE EXTERNAL TABLE analytics.events (id INTEGER, name VARCHAR) "
+ "STORED AS TEXTFILE LOCATION 's3://" + bucket + "/events/' "
@@ -117,7 +138,7 @@ class RedshiftSpectrumIntegrationTest {
s3Service.putObject(bucket, "events/part-1.csv",
"id,name\n1,Alice\n2,Bob\n".getBytes(StandardCharsets.UTF_8), "text/csv", null);
clusterId = "it-spectrum-ext-" + System.nanoTime();
- Cluster cluster = redshiftService.createCluster(clusterId, "dc2.large", "admin", "Secret123");
+ Cluster cluster = createClusterWithRole(clusterId);
String url = "jdbc:postgresql://127.0.0.1:" + cluster.getEndpoint().getPort()
+ "/dev?socketTimeout=20&loginTimeout=20";
@@ -125,7 +146,7 @@ class RedshiftSpectrumIntegrationTest {
.pollInterval(Duration.ofMillis(500)).ignoreExceptions()
.until(() -> DriverManager.getConnection(url, "admin", "Secret123"), Objects::nonNull);
Statement setup = connection.createStatement()) {
- setup.execute("CREATE EXTERNAL SCHEMA analytics_ext FROM DATA CATALOG DATABASE 'dev' IAM_ROLE 'role'");
+ setup.execute("CREATE EXTERNAL SCHEMA analytics_ext FROM DATA CATALOG DATABASE 'dev' IAM_ROLE '" + ROLE_ARN + "'");
setup.execute("CREATE EXTERNAL TABLE analytics_ext.events (id INTEGER, name VARCHAR) "
+ "STORED AS TEXTFILE LOCATION 's3://" + bucket + "/events/' "
+ "TBLPROPERTIES ('skip.header.line.count'='1')");
@@ -156,7 +177,7 @@ class RedshiftSpectrumIntegrationTest {
"id,name\n1,Alice\n2,Bob\n3,Carol\n4,Dave\n5,Eve\n".getBytes(StandardCharsets.UTF_8),
"text/csv", null);
clusterId = "it-spectrum-fetch-" + System.nanoTime();
- Cluster cluster = redshiftService.createCluster(clusterId, "dc2.large", "admin", "Secret123");
+ Cluster cluster = createClusterWithRole(clusterId);
String url = "jdbc:postgresql://127.0.0.1:" + cluster.getEndpoint().getPort()
+ "/dev?socketTimeout=20&loginTimeout=20";
@@ -164,7 +185,7 @@ class RedshiftSpectrumIntegrationTest {
.pollInterval(Duration.ofMillis(500)).ignoreExceptions()
.until(() -> DriverManager.getConnection(url, "admin", "Secret123"), Objects::nonNull);
Statement setup = connection.createStatement()) {
- setup.execute("CREATE EXTERNAL SCHEMA analytics_fetch FROM DATA CATALOG DATABASE 'dev' IAM_ROLE 'role'");
+ setup.execute("CREATE EXTERNAL SCHEMA analytics_fetch FROM DATA CATALOG DATABASE 'dev' IAM_ROLE '" + ROLE_ARN + "'");
setup.execute("CREATE EXTERNAL TABLE analytics_fetch.events (id INTEGER, name VARCHAR) "
+ "STORED AS TEXTFILE LOCATION 's3://" + bucket + "/events/' "
+ "TBLPROPERTIES ('skip.header.line.count'='1')");
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftRoleAccessTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftRoleAccessTest.java
index 4cf4415df2..4a15cc2bf4 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftRoleAccessTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftRoleAccessTest.java
@@ -1,22 +1,42 @@
package io.github.hectorvent.floci.services.redshift.proxy;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.iam.model.CallerContext;
+import io.github.hectorvent.floci.services.iam.model.IamRole;
import io.github.hectorvent.floci.services.s3.S3Service;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import java.time.Duration;
+import java.time.Instant;
import java.util.List;
+import java.util.Optional;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
class RedshiftRoleAccessTest {
+ @Test
+ void s3ResourcesInheritTheRoleArnPartition() {
+ String roleArn = "arn:aws-cn:iam::111111111111:role/spectrum-role";
+
+ assertEquals("arn:aws-cn:s3:::bucket", RedshiftRoleAccess.bucketArn(roleArn, "bucket"));
+ assertEquals("arn:aws-cn:s3:::bucket/key", RedshiftRoleAccess.objectArn(roleArn, "bucket", "key"));
+ }
+
@Test
void authorizeRoleActionThrowsWhenEnforcedPolicyDeniesTheAction() {
S3Service s3 = mock(S3Service.class);
@@ -35,6 +55,134 @@ class RedshiftRoleAccessTest {
assertThat(exception.getMessage(), containsString("S3 access denied"));
}
+ private static final String ACCOUNT = "111111111111";
+ private static final String ROLE_ARN = "arn:aws:iam::111111111111:role/spectrum-role";
+ private static final String TRUST_POLICY = """
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}""";
+ private static final String EC2_TRUST_POLICY = """
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}""";
+
+ private static IamService iamServiceWithRole(String trustPolicy) {
+ IamService iamService = mock(IamService.class);
+ IamRole role = mock(IamRole.class);
+ when(role.getAssumeRolePolicyDocument()).thenReturn(trustPolicy);
+ when(iamService.findRole(ACCOUNT, "spectrum-role")).thenReturn(Optional.of(role));
+ return iamService;
+ }
+
+ private static void assertRefused(String message, String iamRoleArn, IamService iamService,
+ String clusterAccount, List<String> associated) {
+ S3CopySimulator.S3TransferException exception = assertThrows(S3CopySimulator.S3TransferException.class,
+ () -> RedshiftRoleAccess.resolveRoleSession(iamRoleArn, iamService, clusterAccount, associated));
+ assertThat(exception.sqlState(), equalTo("42501"));
+ assertThat(exception.getMessage(), containsString(message));
+ }
+
+ @Test
+ void resolveRoleSessionRefusesMalformedNonRoleAndCrossAccountArns() {
+ IamService iamService = iamServiceWithRole(TRUST_POLICY);
+
+ assertRefused("malformed ARN", "not-an-arn", iamService, ACCOUNT, List.of());
+ assertRefused("not an IAM role ARN", "arn:aws:iam::111111111111:user/someone", iamService, ACCOUNT, List.of());
+ assertRefused("cross-account", ROLE_ARN, iamService, "222222222222", List.of(ROLE_ARN));
+ }
+
+ @Test
+ void resolveRoleSessionRefusesARoleTheClusterDoesNotHaveOrThatDoesNotExist() {
+ assertRefused("not associated", ROLE_ARN, iamServiceWithRole(TRUST_POLICY), ACCOUNT, List.of());
+
+ IamService missing = mock(IamService.class);
+ when(missing.findRole(ACCOUNT, "spectrum-role")).thenReturn(Optional.empty());
+ assertRefused("does not exist", ROLE_ARN, missing, ACCOUNT, List.of(ROLE_ARN));
+ }
+
+ @Test
+ void resolveRoleSessionRefusesARoleWhoseTrustPolicyExcludesRedshift() {
+ assertRefused("trust policy", ROLE_ARN, iamServiceWithRole(EC2_TRUST_POLICY), ACCOUNT, List.of(ROLE_ARN));
+ }
+
+ @Test
+ void resolveRoleSessionMintsASessionThatExpiresAfterTheRequestedTtl() {
+ IamService iamService = iamServiceWithRole(TRUST_POLICY);
+ Instant before = Instant.now();
+
+ RedshiftRoleAccess.RoleSession session = RedshiftRoleAccess.resolveRoleSession(
+ ROLE_ARN, iamService, ACCOUNT, List.of(ROLE_ARN), RedshiftRoleAccess.STREAMING_ROLE_SESSION_TTL);
+
+ ArgumentCaptor<Instant> expiry = ArgumentCaptor.forClass(Instant.class);
+ verify(iamService).registerSessionForAccount(eq(ACCOUNT), eq(session.accessKeyId()), anyString(),
+ eq(session.sessionToken()), eq(ROLE_ARN), expiry.capture(), isNull());
+ assertThat(expiry.getValue().isAfter(before.plus(Duration.ofMinutes(29))), equalTo(true));
+ assertThat(expiry.getValue().isBefore(before.plus(Duration.ofMinutes(31))), equalTo(true));
+ }
+
+ @Test
+ void authorizeRoleActionAllowsWhatTheIdentityPolicyGrants() {
+ S3Service s3 = mock(S3Service.class);
+ when(s3.isAuthEnforced()).thenReturn(true);
+ IamService iamService = mock(IamService.class);
+ when(iamService.resolvePrincipalContext(ROLE_ARN)).thenReturn(CallerContext.of(List.of("""
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}""")));
+
+ RedshiftRoleAccess.authorizeRoleAction(s3, iamService, ROLE_ARN, "s3:GetObject", "arn:aws:s3:::bucket/key");
+ }
+
+ @Test
+ void authorizeRoleListHonoursAnS3PrefixCondition() {
+ S3Service s3 = mock(S3Service.class);
+ when(s3.isAuthEnforced()).thenReturn(true);
+ IamService iamService = mock(IamService.class);
+ when(iamService.resolvePrincipalContext(ROLE_ARN)).thenReturn(CallerContext.of(List.of("""
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket",
+ "Resource":"arn:aws:s3:::bucket","Condition":{"StringLike":{"s3:prefix":["events/*"]}}}]}""")));
+ RedshiftRoleAccess.RoleSession session = new RedshiftRoleAccess.RoleSession("ASIAX", "token");
+
+ RedshiftRoleAccess.authorizeRoleList(s3, iamService, session, ROLE_ARN, "bucket", "events/");
+ S3CopySimulator.S3TransferException denied = assertThrows(S3CopySimulator.S3TransferException.class,
+ () -> RedshiftRoleAccess.authorizeRoleList(s3, iamService, session, ROLE_ARN, "bucket", "other/"));
+
+ assertThat(denied.sqlState(), equalTo("42501"));
+ }
+
+ @Test
+ void aMissingBucketIsNotReportedAsAccessDenied() {
+ S3Service s3 = mock(S3Service.class);
+ when(s3.isAuthEnforced()).thenReturn(false);
+ IamService iamService = mock(IamService.class);
+ RedshiftRoleAccess.RoleSession session = new RedshiftRoleAccess.RoleSession("ASIAX", "token");
+ doThrow(new AwsException("NoSuchBucket", "The specified bucket does not exist", 404)).when(s3)
+ .authorizeSignedListBucket("ASIAX", "token", "bucket");
+
+ S3CopySimulator.S3TransferException missing = assertThrows(S3CopySimulator.S3TransferException.class,
+ () -> RedshiftRoleAccess.authorizeRoleList(s3, iamService, session, ROLE_ARN, "bucket", ""));
+
+ assertThat(missing.sqlState(), equalTo("XX000"));
+ assertThat(missing.getMessage(), containsString("does not exist"));
+ }
+
+ @Test
+ void aBucketPolicyDenyIsReportedAsAccessDeniedForListAndRead() {
+ S3Service s3 = mock(S3Service.class);
+ when(s3.isAuthEnforced()).thenReturn(false);
+ IamService iamService = mock(IamService.class);
+ RedshiftRoleAccess.RoleSession session = new RedshiftRoleAccess.RoleSession("ASIAX", "token");
+ doThrow(new AwsException("AccessDenied", "Access Denied", 403)).when(s3)
+ .authorizeSignedListBucket("ASIAX", "token", "bucket");
+ doThrow(new AwsException("AccessDenied", "Access Denied", 403)).when(s3)
+ .authorizeSignedGetObject("ASIAX", "token", "bucket", "key");
+
+ S3CopySimulator.S3TransferException list = assertThrows(S3CopySimulator.S3TransferException.class,
+ () -> RedshiftRoleAccess.authorizeRoleList(s3, iamService, session, ROLE_ARN, "bucket", ""));
+ S3CopySimulator.S3TransferException read = assertThrows(S3CopySimulator.S3TransferException.class,
+ () -> RedshiftRoleAccess.authorizeRoleRead(s3, iamService, session, ROLE_ARN, "bucket", "key"));
+
+ assertThat(list.sqlState(), equalTo("42501"));
+ assertThat(read.sqlState(), equalTo("42501"));
+ assertThat(read.getMessage(), containsString("s3://bucket/key"));
+ }
+
@Test
void authorizeRoleActionSkipsCheckWhenEnforcementDisabled() {
S3Service s3 = mock(S3Service.class);
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java
new file mode 100644
index 0000000000..e2662a87b8
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java
@@ -0,0 +1,896 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.floci.duck.FlociDuckClient;
+import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.iam.model.CallerContext;
+import io.github.hectorvent.floci.services.iam.model.IamRole;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Answers;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class ExternalTableMaterializerTest {
+ private static final String ACCOUNT = "000000000000";
+ private static final ExternalSchemaBinding BINDING = new ExternalSchemaBinding(ACCOUNT, ACCOUNT + ":c",
+ "dev", "analytics", "lake", "arn:aws:iam::000000000000:role/R");
+ private static final String TRUST_POLICY = """
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}""";
+ private FlociDuckClient duck;
+ private GlueService glue;
+ private S3Service s3;
+ private IamService iam;
+ private RecordingBackend backend;
+ private ExternalTableMaterializer materializer;
+
+ private static final class RecordingBackend implements BackendSql {
+ private final List<String> statements = new ArrayList<>();
+
+ @Override
+ public void execute(String sql) {
+ statements.add(sql);
+ }
+
+ @Override
+ public long copyIn(String copySql, InputStream data) {
+ statements.add(copySql);
+ return 2;
+ }
+ }
+
+ @BeforeEach
+ void setUp() {
+ duck = mock(FlociDuckClient.class);
+ glue = mock(GlueService.class);
+ s3 = mock(S3Service.class);
+ iam = mock(IamService.class);
+ IamRole role = mock(IamRole.class);
+ when(role.getAssumeRolePolicyDocument()).thenReturn(TRUST_POLICY);
+ when(iam.findRole(ACCOUNT, "R")).thenReturn(Optional.of(role));
+ EmulatorConfig config = mock(EmulatorConfig.class, Answers.RETURNS_DEEP_STUBS);
+ when(config.services().redshift().spectrumMaxRows()).thenReturn(1000L);
+ when(config.defaultRegion()).thenReturn("us-east-1");
+ when(duck.query(anyString(), any(), anyString())).thenReturn(List.of());
+ backend = new RecordingBackend();
+ materializer = new ExternalTableMaterializer(duck, glue, s3, iam, config);
+ when(s3.openObjectStream(eq(ExternalTableMaterializer.SCRATCH_BUCKET), anyString(), isNull()))
+ .thenReturn(new ByteArrayInputStream("id,name\n1,Alice\n2,Bob\n".getBytes(StandardCharsets.UTF_8)));
+ S3Object icebergMetadata = new S3Object("bucket", "events/metadata/v1.json",
+ "{\"current-snapshot-id\":\"1\",\"snapshots\":[{\"snapshot-id\":\"1\","
+ .concat("\"manifest-list\":\"s3://bucket/events/metadata/snap*.avro\"}]}")
+ .getBytes(StandardCharsets.UTF_8), "application/json");
+ when(s3.getObject(eq("bucket"), argThat(key -> key.startsWith("events/metadata/"))))
+ .thenReturn(icebergMetadata);
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(
+ List.of(new S3Object("bucket", "events/p1.csv", new byte[]{1}, "text/csv", "etag1")),
+ List.of(), false, null));
+ when(s3.isAuthEnforced()).thenReturn(false);
+ when(s3.listObjectsWithPrefixes(eq(ExternalTableMaterializer.SCRATCH_BUCKET), eq("spectrum-"), eq(""),
+ eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+ }
+
+ @Test
+ void loadsThroughAStagingTableAndCachesUnchangedData() {
+ Table table = csvTable();
+ when(glue.getTable("lake", "events")).thenReturn(table);
+
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ assertThat(backend.statements.get(0), containsString("CREATE TEMP TABLE \"floci_spectrum_stg_"));
+ assertThat(backend.statements.get(0), containsString("\"id\" integer, \"name\" text"));
+ assertThat(backend.statements.get(1), containsString("COPY pg_temp.\"floci_spectrum_stg_"));
+ assertThat(backend.statements.get(2), containsString("CREATE TABLE \"analytics\".\"events\""));
+ assertThat(backend.statements.get(2), containsString("FROM pg_temp.\"floci_spectrum_stg_"));
+ assertThat(backend.statements.stream().noneMatch(sql -> sql.contains("events__stg")), equalTo(true));
+ verify(s3).openObjectStream(eq(ExternalTableMaterializer.SCRATCH_BUCKET), anyString(), isNull());
+ verify(s3, never()).getObject(eq(ExternalTableMaterializer.SCRATCH_BUCKET), anyString());
+ verify(duck).execute(argThat(sql -> sql.contains("LIMIT 1001")), isNull(), anyString(), eq(ACCOUNT));
+ int size = backend.statements.size();
+
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.CURRENT));
+ assertThat(backend.statements.size(), equalTo(size));
+ }
+
+ @Test
+ void readsOnlyTheExactS3ObjectsThatWereListedAndAuthorized() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(
+ new S3Object("bucket", "events/part*'one'.csv", new byte[]{1}, "text/csv", "etag1")),
+ List.of(), false, null));
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(duck).execute(argThat(sql -> sql.contains("read_csv_auto(['s3://bucket/events/part[*]''one''.csv'])")
+ && !sql.contains("/**")), isNull(), anyString(), eq(ACCOUNT));
+ }
+
+ @Test
+ void deniesIcebergDataFilesReferencedByManifestBeforeScanningThem() {
+ Table table = icebergTable("s3://bucket/events/metadata/v[1].json");
+ when(glue.getTable("lake", "events")).thenReturn(table);
+ when(s3.isAuthEnforced()).thenReturn(true);
+ when(duck.query(anyString(), any(), eq(ACCOUNT))).thenAnswer(invocation -> {
+ String sql = invocation.getArgument(0);
+ if (!sql.contains("status IN ('ADDED', 'EXISTING')") || sql.contains("content = 'DATA'")) {
+ return List.of();
+ }
+ return List.of(
+ Map.of("file_path", "s3://bucket/events/data.parquet", "manifest_content", "DATA",
+ "content", "EXISTING", "status", "ADDED"),
+ Map.of("file_path", "s3://bucket/private/deletes.parquet", "manifest_content", "DELETES",
+ "content", "POSITION_DELETES", "status", "ADDED"));
+ });
+ when(iam.resolvePrincipalContext(BINDING.iamRoleArn())).thenReturn(CallerContext.of(List.of("""
+ {"Version":"2012-10-17","Statement":[
+ {"Effect":"Allow","Action":["s3:ListBucket","s3:GetObject"],"Resource":"*"},
+ {"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::bucket/private/deletes.parquet"}
+ ]}""")));
+
+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class,
+ () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+
+ assertThat(exception.sqlState(), equalTo("42501"));
+ verify(duck).query(argThat(sql -> sql.contains("iceberg_metadata('s3://bucket/events/metadata/v[[]1].json')")),
+ any(), eq(ACCOUNT));
+ verify(duck, never()).execute(anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void deniesIcebergManifestBeforeTheMetadataScanReadsIt() {
+ Table table = icebergTable("s3://bucket/events/metadata/v1.json");
+ when(glue.getTable("lake", "events")).thenReturn(table);
+ when(s3.isAuthEnforced()).thenReturn(true);
+ when(duck.query(anyString(), any(), eq(ACCOUNT))).thenAnswer(invocation -> {
+ String sql = invocation.getArgument(0);
+ if (sql.contains("read_avro")) {
+ return List.of(Map.of("manifest_path", "s3://bucket/private/manifest.avro"));
+ }
+ return List.of();
+ });
+ when(iam.resolvePrincipalContext(BINDING.iamRoleArn())).thenReturn(CallerContext.of(List.of("""
+ {"Version":"2012-10-17","Statement":[
+ {"Effect":"Allow","Action":["s3:ListBucket","s3:GetObject"],"Resource":"*"},
+ {"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::bucket/private/manifest.avro"}
+ ]}""")));
+
+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class,
+ () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+
+ assertThat(exception.sqlState(), equalTo("42501"));
+ verify(duck).query(argThat(sql -> sql.contains("read_avro('s3://bucket/events/metadata/snap[*].avro')")),
+ any(), eq(ACCOUNT));
+ verify(duck, never()).query(argThat(sql -> sql.contains("iceberg_metadata")), any(), eq(ACCOUNT));
+ verify(duck, never()).execute(anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void emptyS3LocationCreatesAnEmptyTableWithoutCallingDuckDb() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ verify(duck, never()).execute(anyString(), any(), anyString(), anyString());
+ assertThat(backend.statements.size(), equalTo(2));
+ }
+
+ @Test
+ void doesNotCacheLoadsThatMayBeRolledBack() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+
+ materializer.ensureCurrent(backend, session(true), BINDING, "events");
+ assertThat(materializer.ensureCurrent(backend, session(true), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ }
+
+ @Test
+ void forgettingOneClusterInvalidatesOnlyItsFingerprints() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ SpectrumSession first = session(ACCOUNT + ":first", false);
+ SpectrumSession second = session(ACCOUNT + ":second", false);
+ ExternalSchemaBinding firstBinding = new ExternalSchemaBinding(ACCOUNT, first.clusterKey(), "dev",
+ "analytics", "lake", BINDING.iamRoleArn());
+ ExternalSchemaBinding secondBinding = new ExternalSchemaBinding(ACCOUNT, second.clusterKey(), "dev",
+ "analytics", "lake", BINDING.iamRoleArn());
+
+ materializer.ensureCurrent(backend, first, firstBinding, "events");
+ materializer.ensureCurrent(backend, second, secondBinding, "events");
+ int sizeBeforeForget = backend.statements.size();
+ materializer.forgetCluster(first.clusterKey());
+
+ assertThat(materializer.ensureCurrent(backend, first, firstBinding, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ assertThat(materializer.ensureCurrent(backend, second, secondBinding, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.CURRENT));
+ assertThat(backend.statements.size(), equalTo(sizeBeforeForget + 3));
+ }
+
+ @Test
+ void missingGlueTableIsNotTreatedAsAnExternalTable() {
+ when(glue.getTable("lake", "native")).thenThrow(new AwsException("EntityNotFoundException", "missing", 400));
+
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "native"),
+ equalTo(ExternalTableMaterializer.Outcome.NOT_EXTERNAL));
+ assertThat(backend.statements.size(), equalTo(0));
+ }
+
+ @Test
+ void deniesAListedObjectUsingItsExactKeyBeforeDuckDbReadsIt() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ when(s3.isAuthEnforced()).thenReturn(true);
+ S3Object publicObject = new S3Object("bucket", "events/public.csv", new byte[]{1}, "text/csv", "etag1");
+ S3Object privateObject = new S3Object("bucket", "events/private.csv", new byte[]{2}, "text/csv", "etag2");
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(publicObject, privateObject), List.of(), false, null));
+ when(iam.resolvePrincipalContext(BINDING.iamRoleArn())).thenReturn(CallerContext.of(List.of("""
+ {"Version":"2012-10-17","Statement":[
+ {"Effect":"Allow","Action":["s3:ListBucket","s3:GetObject"],"Resource":"*"},
+ {"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::bucket/events/private.csv"}
+ ]}""")));
+
+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class,
+ () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+
+ assertThat(exception.sqlState(), equalTo("42501"));
+ verify(duck, never()).execute(anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void doesNotReuseAStagingTableCacheEntryAcrossRedshiftDatabases() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ SpectrumSession otherDatabase = new SpectrumSession(ACCOUNT, BINDING.clusterKey(), "reporting",
+ List.of(BINDING.iamRoleArn()), false);
+ ExternalSchemaBinding otherBinding = new ExternalSchemaBinding(ACCOUNT, BINDING.clusterKey(),
+ "reporting", "analytics", "lake", BINDING.iamRoleArn());
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ int statementCount = backend.statements.size();
+
+ assertThat(materializer.ensureCurrent(backend, otherDatabase, otherBinding, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ assertThat(backend.statements.size(), equalTo(statementCount + 3));
+ }
+
+ @Test
+ void partitionedTableWithoutRegisteredPartitionsLoadsNoRows() {
+ Table table = csvTable();
+ table.setPartitionKeys(List.of(new Column("day", "string")));
+ when(glue.getTable("lake", "events")).thenReturn(table);
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(duck, never()).execute(anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void readsObjectsFromRegisteredPartitionLocationsOutsideTheTableRoot() {
+ Table table = csvTable();
+ table.setPartitionKeys(List.of(new Column("day", "string")));
+ when(glue.getTable("lake", "events")).thenReturn(table);
+ StorageDescriptor partitionDescriptor = new StorageDescriptor();
+ partitionDescriptor.setLocation("s3://bucket/archived/day=2026-09-25/");
+ partitionDescriptor.setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
+ partitionDescriptor.setColumns(table.getStorageDescriptor().getColumns());
+ Partition partition = new Partition();
+ partition.setValues(List.of("2026-09-25"));
+ partition.setStorageDescriptor(partitionDescriptor);
+ when(glue.getPartitions("lake", "events")).thenReturn(List.of(partition));
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("archived/day=2026-09-25/"), eq(""), eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(
+ new S3Object("bucket", "archived/day=2026-09-25/part.csv", new byte[]{1}, "text/csv", "partition-etag")),
+ List.of(), false, null));
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(duck).execute(argThat(sql -> sql.contains("s3://bucket/archived/day=2026-09-25/part.csv")
+ && sql.contains("CAST('2026-09-25' AS VARCHAR)") && sql.contains("AS \"day\"")),
+ any(), anyString(), eq(ACCOUNT));
+ }
+
+ @Test
+ void reloadRefillsTheExistingTableInPlaceSoDependentViewsSurvive() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(
+ List.of(new S3Object("bucket", "events/p1.csv", new byte[]{1, 2}, "text/csv", "etag2")),
+ List.of(), false, null));
+ backend.statements.clear();
+
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+
+ String swap = backend.statements.getLast();
+ assertThat(swap, containsString("TRUNCATE \"analytics\".\"events\""));
+ assertThat(swap, containsString("INSERT INTO \"analytics\".\"events\" SELECT * FROM pg_temp.\"floci_spectrum_stg_"));
+ assertThat(swap.contains("RENAME TO"), equalTo(false));
+ assertThat(swap.contains("DROP TABLE IF EXISTS \"analytics\".\"events\";"), equalTo(false));
+ }
+
+ @Test
+ void reloadRunsTheInPlaceStepsInOrderAndDropsTheStagingTable() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ forceReload();
+ backend.statements.clear();
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ String swap = backend.statements.getLast();
+ int create = swap.indexOf("CREATE TABLE IF NOT EXISTS \"analytics\".\"events\"");
+ int truncate = swap.indexOf("TRUNCATE");
+ int insert = swap.indexOf("INSERT INTO");
+ int dropStaging = swap.indexOf("DROP TABLE pg_temp.\"floci_spectrum_stg_");
+ assertThat(create >= 0 && create < truncate && truncate < insert && insert < dropStaging, equalTo(true));
+ }
+
+ @Test
+ void aLoadInsideATransactionThatKeepsTheColumnsStillRefillsInPlaceAfterwards() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ materializer.ensureCurrent(backend, session(true), BINDING, "events");
+ backend.statements.clear();
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ assertThat(backend.statements.getLast(), containsString("CREATE TABLE IF NOT EXISTS \"analytics\".\"events\""));
+ assertThat(backend.statements.getLast(), not(containsString("DROP TABLE IF EXISTS \"analytics\"")));
+ }
+
+ @Test
+ void aTransactionalLoadThatReplacesTheTableIsNotTrustedAsItsCommittedColumns() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ Table changed = csvTable();
+ changed.setVersionId("2");
+ changed.getStorageDescriptor().setColumns(List.of(new Column("id", "bigint")));
+ when(glue.getTable("lake", "events")).thenReturn(changed);
+ materializer.ensureCurrent(backend, session(true), BINDING, "events");
+ backend.statements.clear();
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ assertThat(backend.statements.getLast(), containsString("CREATE TABLE \"analytics\".\"events\""));
+ }
+
+ @Test
+ void forgettingATableDropsItsSchemaSignatureSoItIsRecreated() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ materializer.forget(BINDING.clusterKey(), "dev", "analytics", "events");
+ backend.statements.clear();
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ assertThat(backend.statements.getLast(), containsString("CREATE TABLE \"analytics\".\"events\""));
+ }
+
+ private void forceReload() {
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(
+ List.of(new S3Object("bucket", "events/p1.csv", new byte[]{9}, "text/csv", "etag-changed")),
+ List.of(), false, null));
+ }
+
+ @Test
+ void schemaChangeReplacesTheTableInsteadOfRefillingIt() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ Table changed = csvTable();
+ changed.setVersionId("2");
+ changed.getStorageDescriptor().setColumns(List.of(new Column("id", "bigint")));
+ when(glue.getTable("lake", "events")).thenReturn(changed);
+ backend.statements.clear();
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ assertThat(backend.statements.getLast(), containsString("CREATE TABLE \"analytics\".\"events\""));
+ }
+
+ @Test
+ void deniesAnIcebergMetadataLocationOutsideTheTableLocation() {
+ Table table = icebergTable("s3://other-bucket/events/metadata/v1.metadata.json");
+ when(glue.getTable("lake", "events")).thenReturn(table);
+
+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class,
+ () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+
+ assertThat(exception.sqlState(), equalTo("42501"));
+ verify(duck, never()).execute(anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void scansIcebergThroughTheSameGlobEscapedMetadataLiteralThatWasAuthorized() {
+ when(glue.getTable("lake", "events")).thenReturn(icebergTable("s3://bucket/events/metadata/v[1].json"));
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(duck).execute(argThat(sql -> sql.contains("iceberg_scan('s3://bucket/events/metadata/v[[]1].json')")),
+ any(), anyString(), eq(ACCOUNT));
+ }
+
+ @Test
+ void deniesAnIcebergTableWhoseSnapshotsHaveNoCurrentSnapshot() {
+ when(s3.getObject(eq("bucket"), eq("events/metadata/v2.json"))).thenReturn(new S3Object("bucket",
+ "events/metadata/v2.json",
+ "{\"current-snapshot-id\":-1,\"snapshots\":[{\"snapshot-id\":\"1\",\"manifest-list\":\"s3://bucket/events/metadata/snap.avro\"}]}"
+ .getBytes(StandardCharsets.UTF_8), "application/json"));
+ when(glue.getTable("lake", "events")).thenReturn(icebergTable("s3://bucket/events/metadata/v2.json"));
+
+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class,
+ () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+
+ assertThat(exception.sqlState(), equalTo("XX000"));
+ verify(duck, never()).execute(anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void ignoresHiddenFilesAndFolderMarkersLikeSpectrum() {
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(
+ new S3Object("bucket", "events/p1.csv", new byte[]{1}, "text/csv", "etag1"),
+ new S3Object("bucket", "events/_SUCCESS", new byte[0], "text/plain", "etag2"),
+ new S3Object("bucket", "events/.hidden.csv", new byte[]{1}, "text/csv", "etag3"),
+ new S3Object("bucket", "events/#draft.csv", new byte[]{1}, "text/csv", "etag4"),
+ new S3Object("bucket", "events/old.csv~", new byte[]{1}, "text/csv", "etag5"),
+ new S3Object("bucket", "events/nested/", new byte[0], "binary/octet-stream", "etag6")),
+ List.of(), false, null));
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(duck).execute(argThat(sql -> sql.contains("s3://bucket/events/p1.csv")
+ && !sql.contains("_SUCCESS") && !sql.contains(".hidden") && !sql.contains("#draft")
+ && !sql.contains("old.csv~") && !sql.contains("nested/")), any(), anyString(), eq(ACCOUNT));
+ }
+
+ @Test
+ void ignoresFilesUnderAHiddenFolderBelowThePrefix() {
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(
+ new S3Object("bucket", "events/p1.csv", new byte[]{1}, "text/csv", "etag1"),
+ new S3Object("bucket", "events/_temporary/0/part.csv", new byte[]{1}, "text/csv", "etag2"),
+ new S3Object("bucket", "events/.staging/part.csv", new byte[]{1}, "text/csv", "etag3"),
+ new S3Object("bucket", "events/day=1/part.csv", new byte[]{1}, "text/csv", "etag4")),
+ List.of(), false, null));
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(duck).execute(argThat(sql -> sql.contains("s3://bucket/events/p1.csv")
+ && sql.contains("s3://bucket/events/day=1/part.csv")
+ && !sql.contains("_temporary") && !sql.contains(".staging")), any(), anyString(), eq(ACCOUNT));
+ }
+
+ @Test
+ void reloadsWhenAPartitionChangesItsDelimiterOrColumnsAtTheSameLocationAndFormat() {
+ Table table = csvTable();
+ table.setPartitionKeys(List.of(new Column("day", "string")));
+ when(glue.getTable("lake", "events")).thenReturn(table);
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(
+ List.of(new S3Object("bucket", "events/p1.csv", new byte[]{1}, "text/csv", "etag1")),
+ List.of(), false, null));
+ Partition partition = partitionAt("org.apache.hadoop.mapred.TextInputFormat", table);
+ partition.getStorageDescriptor().setSerdeInfo(serdeWith(Map.of("field.delim", ",")));
+ when(glue.getPartitions("lake", "events")).thenReturn(List.of(partition));
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.CURRENT));
+
+ Partition delimiterChanged = partitionAt("org.apache.hadoop.mapred.TextInputFormat", table);
+ delimiterChanged.getStorageDescriptor().setSerdeInfo(serdeWith(Map.of("field.delim", "|")));
+ when(glue.getPartitions("lake", "events")).thenReturn(List.of(delimiterChanged));
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+
+ Partition columnsChanged = partitionAt("org.apache.hadoop.mapred.TextInputFormat", table);
+ columnsChanged.getStorageDescriptor().setSerdeInfo(serdeWith(Map.of("field.delim", "|")));
+ columnsChanged.getStorageDescriptor().setColumns(List.of(new Column("id", "bigint")));
+ when(glue.getPartitions("lake", "events")).thenReturn(List.of(columnsChanged));
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ }
+
+ private static StorageDescriptor.SerDeInfo serdeWith(Map<String, String> parameters) {
+ StorageDescriptor.SerDeInfo serde = new StorageDescriptor.SerDeInfo();
+ serde.setSerializationLibrary("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
+ serde.setParameters(parameters);
+ return serde;
+ }
+
+ @Test
+ void readsAnIcebergTableOnEveryQueryBecauseItsSnapshotCanChangeUnseen() {
+ when(glue.getTable("lake", "events")).thenReturn(icebergTable("s3://bucket/events/metadata/v1.metadata.json"));
+
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ }
+
+ @Test
+ void refusesToUseAScratchBucketNameThatBelongsToAUserBucket() {
+ when(s3.createBucket(eq(ExternalTableMaterializer.SCRATCH_BUCKET), anyString()))
+ .thenThrow(new AwsException("BucketAlreadyOwnedByYou", "owned", 409));
+ when(s3.getBucketTagging(ExternalTableMaterializer.SCRATCH_BUCKET)).thenReturn(Map.of());
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+
+ SpectrumReadException exception = assertThrows(SpectrumReadException.class,
+ () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+
+ assertThat(exception.getMessage(), containsString("not the Spectrum scratch bucket"));
+ verify(duck, never()).execute(anyString(), any(), anyString(), anyString());
+ verify(s3, never()).deleteObject(eq(ExternalTableMaterializer.SCRATCH_BUCKET), anyString());
+ }
+
+ @Test
+ void tagsTheScratchBucketItCreatesAndReusesATaggedOne() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ verify(s3).putBucketTagging(ExternalTableMaterializer.SCRATCH_BUCKET, Map.of(
+ S3Service.INTERNAL_BUCKET_TAG_KEY, S3Service.REDSHIFT_SPECTRUM_SCRATCH_TAG_VALUE));
+
+ when(s3.createBucket(eq(ExternalTableMaterializer.SCRATCH_BUCKET), anyString()))
+ .thenThrow(new AwsException("BucketAlreadyOwnedByYou", "owned", 409));
+ when(s3.getBucketTagging(ExternalTableMaterializer.SCRATCH_BUCKET)).thenReturn(Map.of(
+ S3Service.INTERNAL_BUCKET_TAG_KEY, S3Service.REDSHIFT_SPECTRUM_SCRATCH_TAG_VALUE));
+ forceReload();
+
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ }
+
+ @Test
+ void authorizesTheIcebergMetadataObjectAsTheRole() {
+ when(glue.getTable("lake", "events")).thenReturn(icebergTable("s3://bucket/events/metadata/v1.metadata.json"));
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(s3).authorizeSignedGetObject(anyString(), anyString(), eq("bucket"), eq("events/metadata/v1.metadata.json"));
+ }
+
+ @Test
+ void listsWithTheTableLocationPrefixSoPrefixConditionsApply() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ when(s3.isAuthEnforced()).thenReturn(true);
+ when(iam.resolvePrincipalContext(BINDING.iamRoleArn())).thenReturn(CallerContext.of(List.of("""
+ {"Version":"2012-10-17","Statement":[
+ {"Effect":"Allow","Action":"s3:ListBucket","Resource":"arn:aws:s3:::bucket",
+ "Condition":{"StringLike":{"s3:prefix":["events/*"]}}},
+ {"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}
+ ]}""")));
+
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ }
+
+ @Test
+ void deniesListingWhenThePrefixConditionDoesNotMatchTheTableLocation() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ when(s3.isAuthEnforced()).thenReturn(true);
+ when(iam.resolvePrincipalContext(BINDING.iamRoleArn())).thenReturn(CallerContext.of(List.of("""
+ {"Version":"2012-10-17","Statement":[
+ {"Effect":"Allow","Action":"s3:ListBucket","Resource":"arn:aws:s3:::bucket",
+ "Condition":{"StringLike":{"s3:prefix":["other/*"]}}},
+ {"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}
+ ]}""")));
+
+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class,
+ () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+
+ assertThat(exception.sqlState(), equalTo("42501"));
+ }
+
+
+ @Test
+ void projectingTableWithoutRegisteredPartitionsIgnoresItsLocationTemplate() {
+ Table table = csvTable();
+ table.setPartitionKeys(List.of(new Column("dt", "string")));
+ table.setParameters(Map.of("projection.enabled", "true",
+ "storage.location.template", "s3://other-bucket/events/${dt}/"));
+ when(glue.getTable("lake", "events")).thenReturn(table);
+
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+
+ verify(duck, never()).execute(anyString(), any(), anyString(), anyString());
+ verify(s3, never()).listObjectsWithPrefixes(eq("other-bucket"), anyString(), anyString(), anyInt(), any(), any());
+ }
+
+ @Test
+ void sweepsStaleScratchObjectsOnceAndKeepsRecentOnes() {
+ S3Object stale = new S3Object(ExternalTableMaterializer.SCRATCH_BUCKET, "spectrum-old.csv",
+ new byte[]{1}, "text/csv", "e1");
+ stale.setLastModified(Instant.now().minus(Duration.ofHours(3)));
+ S3Object recent = new S3Object(ExternalTableMaterializer.SCRATCH_BUCKET, "spectrum-new.csv",
+ new byte[]{1}, "text/csv", "e2");
+ when(s3.listObjectsWithPrefixes(eq(ExternalTableMaterializer.SCRATCH_BUCKET), eq("spectrum-"), eq(""),
+ eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(stale, recent), List.of(), false, null));
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ forceReload();
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(s3).deleteObject(ExternalTableMaterializer.SCRATCH_BUCKET, "spectrum-old.csv");
+ verify(s3, never()).deleteObject(ExternalTableMaterializer.SCRATCH_BUCKET, "spectrum-new.csv");
+ verify(s3, times(1)).listObjectsWithPrefixes(eq(ExternalTableMaterializer.SCRATCH_BUCKET),
+ eq("spectrum-"), eq(""), eq(1000), any(), any());
+ }
+
+ @Test
+ void nullsAPartitionValueWiderThanItsVarcharKey() {
+ Table table = csvTable();
+ table.setPartitionKeys(List.of(new Column("day", "varchar(3)")));
+ when(glue.getTable("lake", "events")).thenReturn(table);
+ StorageDescriptor descriptor = new StorageDescriptor();
+ descriptor.setLocation("s3://bucket/events/");
+ descriptor.setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
+ descriptor.setColumns(table.getStorageDescriptor().getColumns());
+ Partition partition = new Partition();
+ partition.setValues(List.of("2026-09-25"));
+ partition.setStorageDescriptor(descriptor);
+ when(glue.getPartitions("lake", "events")).thenReturn(List.of(partition));
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(duck).execute(argThat(sql -> sql.contains("CAST(NULL AS VARCHAR)")
+ && !sql.contains("2026-09-25") && sql.contains("AS \"day\"")), any(), anyString(), eq(ACCOUNT));
+ }
+
+ @Test
+ void reloadsWhenAPartitionChangesFormatAtTheSameLocation() {
+ Table table = csvTable();
+ table.setPartitionKeys(List.of(new Column("day", "string")));
+ when(glue.getTable("lake", "events")).thenReturn(table);
+ Partition partition = partitionAt("org.apache.hadoop.mapred.TextInputFormat", table);
+ when(glue.getPartitions("lake", "events")).thenReturn(List.of(partition));
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(
+ List.of(new S3Object("bucket", "events/p1.csv", new byte[]{1}, "text/csv", "etag1")),
+ List.of(), false, null));
+
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.CURRENT));
+
+ when(glue.getPartitions("lake", "events")).thenReturn(List.of(
+ partitionAt("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", table)));
+
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ }
+
+ @Test
+ void aSessionOutsideATransactionWaitsOutASlowLoadInsteadOfTimingOut() throws Exception {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ CountDownLatch loading = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ loading.countDown();
+ release.await(10, TimeUnit.SECONDS);
+ return null;
+ }).when(duck).execute(anyString(), any(), anyString(), anyString());
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ Future<ExternalTableMaterializer.Outcome> first = executor.submit(
+ () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+ assertThat(loading.await(10, TimeUnit.SECONDS), equalTo(true));
+ Future<ExternalTableMaterializer.Outcome> second = executor.submit(
+ () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+ assertThrows(TimeoutException.class, () -> second.get(300, TimeUnit.MILLISECONDS));
+
+ release.countDown();
+
+ assertThat(first.get(10, TimeUnit.SECONDS), equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ assertThat(second.get(10, TimeUnit.SECONDS), equalTo(ExternalTableMaterializer.Outcome.CURRENT));
+ } finally {
+ release.countDown();
+ executor.shutdownNow();
+ }
+ }
+
+ private Partition partitionAt(String inputFormat, Table table) {
+ StorageDescriptor descriptor = new StorageDescriptor();
+ descriptor.setLocation("s3://bucket/events/");
+ descriptor.setInputFormat(inputFormat);
+ descriptor.setColumns(table.getStorageDescriptor().getColumns());
+ Partition partition = new Partition();
+ partition.setValues(List.of("2026-09-25"));
+ partition.setStorageDescriptor(descriptor);
+ return partition;
+ }
+
+ @Test
+ void listsEveryPageOfTheTableLocation() {
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), isNull(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(
+ List.of(new S3Object("bucket", "events/p1.csv", new byte[]{1}, "text/csv", "etag1")),
+ List.of(), true, "page-2"));
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), eq("page-2"), any()))
+ .thenReturn(new S3Service.ListObjectsResult(
+ List.of(new S3Object("bucket", "events/p2.csv", new byte[]{1}, "text/csv", "etag2")),
+ List.of(), false, null));
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(duck).execute(argThat(sql -> sql.contains("s3://bucket/events/p1.csv")
+ && sql.contains("s3://bucket/events/p2.csv")), any(), anyString(), eq(ACCOUNT));
+ }
+
+ private Table icebergTable(String metadataLocation) {
+ Table table = csvTable();
+ table.setParameters(Map.of("table_type", "ICEBERG", "metadata_location", metadataLocation));
+ return table;
+ }
+
+ private static SpectrumSession session(boolean inTransaction) {
+ return session(BINDING.clusterKey(), inTransaction);
+ }
+
+ private static SpectrumSession session(String clusterKey, boolean inTransaction) {
+ return new SpectrumSession(ACCOUNT, clusterKey, "dev", List.of(BINDING.iamRoleArn()), inTransaction);
+ }
+
+ private Table csvTable() {
+ Column id = new Column();
+ id.setName("id");
+ id.setType("int");
+ Column name = new Column();
+ name.setName("name");
+ name.setType("string");
+ StorageDescriptor descriptor = new StorageDescriptor();
+ descriptor.setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
+ descriptor.setLocation("s3://bucket/events/");
+ descriptor.setColumns(List.of(id, name));
+ Table table = new Table();
+ table.setName("events");
+ table.setVersionId("1");
+ table.setStorageDescriptor(descriptor);
+ return table;
+ }
+
+ @Test
+ void deniesReadsThatOnlyTheBucketPolicyForbids() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ doThrow(new AwsException("AccessDenied", "Access Denied", 403)).when(s3)
+ .authorizeSignedGetObject(anyString(), anyString(), eq("bucket"), eq("events/p1.csv"));
+
+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class,
+ () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+
+ assertThat(exception.sqlState(), equalTo("42501"));
+ verify(duck, never()).execute(anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void authorizesListingAndEveryObjectAsTheRoleSessionAndReleasesIt() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(s3).authorizeSignedListBucket(anyString(), anyString(), eq("bucket"));
+ verify(s3).authorizeSignedGetObject(anyString(), anyString(), eq("bucket"), eq("events/p1.csv"));
+ verify(iam).unregisterSession(eq(ACCOUNT), anyString());
+ }
+
+ @Test
+ void releasesTheRoleSessionWhenAuthorizationFails() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ doThrow(new AwsException("AccessDenied", "Access Denied", 403)).when(s3)
+ .authorizeSignedListBucket(anyString(), anyString(), eq("bucket"));
+
+ assertThrows(SpectrumSqlException.class,
+ () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+
+ verify(iam).unregisterSession(eq(ACCOUNT), anyString());
+ }
+
+ @Test
+ void reloadsAfterTheSchemaWasForgottenEvenWhenNothingChanged() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ assertThat(materializer.loadedTables(BINDING.clusterKey(), "dev", "analytics"),
+ equalTo(Set.of("events")));
+
+ materializer.forgetSchema(BINDING.clusterKey(), "dev", "analytics");
+
+ assertThat(materializer.loadedTables(BINDING.clusterKey(), "dev", "analytics"), equalTo(Set.of()));
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ }
+
+ @Test
+ void readsHeaderlessDelimitedCsvWithTheTablesDeclaredOptions() {
+ Table table = csvTable();
+ table.setParameters(Map.of("skip.header.line.count", "0"));
+ StorageDescriptor.SerDeInfo serde = new StorageDescriptor.SerDeInfo();
+ serde.setParameters(Map.of("field.delim", "|"));
+ table.getStorageDescriptor().setSerdeInfo(serde);
+ when(glue.getTable("lake", "events")).thenReturn(table);
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(duck).execute(argThat(sql -> sql.contains("read_csv(['s3://bucket/events/p1.csv']")
+ && sql.contains("header = false") && sql.contains("delim = '|'")
+ && sql.contains("columns = {'id': 'VARCHAR', 'name': 'VARCHAR'}")),
+ isNull(), anyString(), eq(ACCOUNT));
+ }
+
+ @Test
+ void sparsePartitionKeepsTheTablesColumnsAndCsvOptions() {
+ Table table = csvTable();
+ table.setPartitionKeys(List.of(new Column("day", "string")));
+ table.setParameters(Map.of("skip.header.line.count", "0"));
+ when(glue.getTable("lake", "events")).thenReturn(table);
+ StorageDescriptor sparse = new StorageDescriptor();
+ sparse.setLocation("s3://bucket/archived/day=2026-09-25/");
+ Partition partition = new Partition();
+ partition.setValues(List.of("2026-09-25"));
+ partition.setStorageDescriptor(sparse);
+ when(glue.getPartitions("lake", "events")).thenReturn(List.of(partition));
+ when(s3.listObjectsWithPrefixes(eq("bucket"), eq("archived/day=2026-09-25/"), eq(""), eq(1000), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(
+ new S3Object("bucket", "archived/day=2026-09-25/part.csv", new byte[]{1}, "text/csv", "e")),
+ List.of(), false, null));
+
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+
+ verify(duck).execute(argThat(sql -> sql.contains("header = false")
+ && sql.contains("columns = {'id': 'VARCHAR', 'name': 'VARCHAR'}")
+ && sql.contains("CAST('2026-09-25' AS VARCHAR)")), any(), anyString(), eq(ACCOUNT));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapperTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapperTest.java
index 916b07d884..1c2aac6cfa 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapperTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapperTest.java
@@ -38,6 +38,16 @@ class GlueTypeMapperTest {
equalTo("COALESCE(CAST('\\x' || hex(\"payload\") AS VARCHAR), '\\N') AS \"payload\""));
}
+ @Test
+ void nullsSizedStringValuesWiderThanTheColumnInBytes() {
+ String expected = "COALESCE(CAST(CASE WHEN strlen(CAST(\"name\" AS VARCHAR)) > %d THEN NULL ELSE \"name\" END"
+ + " AS VARCHAR), '\\N') AS \"name\"";
+
+ assertThat(GlueTypeMapper.duckProjection("name", "varchar(10)"), equalTo(expected.formatted(10)));
+ assertThat(GlueTypeMapper.duckProjection("name", "varchar"), equalTo(expected.formatted(256)));
+ assertThat(GlueTypeMapper.duckProjection("name", "char"), equalTo(expected.formatted(1)));
+ }
+
@Test
void canonicalisesRedshiftAliases() {
assertThat(GlueTypeMapper.canonicalGlueType("DOUBLE PRECISION"), equalTo("double"));
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumCatalogTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumCatalogTest.java
index ddd2ff831a..96751b68d8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumCatalogTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumCatalogTest.java
@@ -41,6 +41,19 @@ class SpectrumCatalogTest {
assertEquals(java.util.Optional.of(schema), catalog.schema(ACCOUNT, "dev", "analytics"));
}
+ @Test
+ void legacySchemaLookupKeepsTheSchemaWhoseLocationCoversTheTablesOfThisDatabase() {
+ String roleA = "arn:aws:iam::000000000000:role/A";
+ String roleB = "arn:aws:iam::000000000000:role/B";
+ catalog.createSchema(new SpectrumExternalSchema(ACCOUNT, "glue_a", "analytics", "s3://other/root/", roleA));
+ SpectrumExternalSchema owning = new SpectrumExternalSchema(
+ ACCOUNT, "glue_b", "analytics", "s3://warehouse/events/", roleB);
+ catalog.createSchema(owning);
+ catalog.createTable(table("events"));
+
+ assertEquals(java.util.Optional.of(owning), catalog.findLegacySchema(ACCOUNT, "dev", "analytics"));
+ }
+
@Test
void createsAndListsExternalTablesInStableNameOrder() {
SpectrumExternalTable second = table("zebra");
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumGlueCsvAdapterTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumGlueCsvAdapterTest.java
new file mode 100644
index 0000000000..faac7a261d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumGlueCsvAdapterTest.java
@@ -0,0 +1,101 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class SpectrumGlueCsvAdapterTest {
+
+ private static final String ACCOUNT = "000000000000";
+ private static final String ROLE = "arn:aws:iam::000000000000:role/Reader";
+ private static final ExternalSchemaBinding BINDING = new ExternalSchemaBinding(
+ ACCOUNT, "cluster-1", "dev", "analytics", "warehouse", ROLE);
+
+ private final SpectrumGlueCsvAdapter adapter = new SpectrumGlueCsvAdapter();
+
+ @Test
+ void adaptsDeclaredTextfileColumnsAndHeaderCount() {
+ Table table = csvTable("s3://events-bucket/events/");
+ table.getStorageDescriptor().setColumns(List.of(
+ new Column("id", "int"), new Column("name", "string"), new Column("code", "char")));
+ table.setParameters(Map.of("EXTERNAL", "TRUE", "skip.header.line.count", "1"));
+
+ SpectrumGlueCsvAdapter.CsvTable adapted = adapter.adapt(BINDING, table, ACCOUNT, "dev").orElseThrow();
+
+ assertEquals("s3://events-bucket/events/", adapted.table().location());
+ assertEquals(List.of(
+ new SpectrumColumn("id", SpectrumColumn.Type.INTEGER),
+ new SpectrumColumn("name", SpectrumColumn.Type.VARCHAR),
+ new SpectrumColumn("code", SpectrumColumn.Type.CHAR)), adapted.table().columns());
+ assertEquals(',', adapted.table().delimiter());
+ assertEquals(1, adapted.table().headerLines());
+ assertEquals(ACCOUNT, adapted.schema().accountId());
+ assertEquals("dev", adapted.schema().databaseName());
+ assertEquals("analytics", adapted.schema().schemaName());
+ assertEquals(ROLE, adapted.schema().iamRoleArn());
+ }
+
+ @Test
+ void adaptsUnsizedVarcharButLeavesOtherWidthsToTheDuckDbPath() {
+ Table unsized = csvTable("s3://events-bucket/events/");
+ unsized.getStorageDescriptor().setColumns(List.of(new Column("name", "varchar")));
+ assertEquals(List.of(new SpectrumColumn("name", SpectrumColumn.Type.VARCHAR)),
+ adapter.adapt(BINDING, unsized, ACCOUNT, "dev").orElseThrow().table().columns());
+
+ Table sized = csvTable("s3://events-bucket/events/");
+ sized.getStorageDescriptor().setColumns(List.of(new Column("name", "varchar(10)")));
+ assertTrue(adapter.adapt(BINDING, sized, ACCOUNT, "dev").isEmpty());
+ }
+
+ @Test
+ void adaptsExplicitQuoteAndEscapeProperties() {
+ Table table = csvTable("s3://events-bucket/events/");
+ StorageDescriptor.SerDeInfo serde = table.getStorageDescriptor().getSerdeInfo();
+ serde.setSerializationLibrary("org.apache.hadoop.hive.serde2.OpenCSVSerde");
+ serde.setParameters(Map.of("separatorChar", "|", "quoteChar", "~", "escapeChar", "!"));
+
+ SpectrumExternalTable adapted = adapter.adapt(BINDING, table, ACCOUNT, "dev").orElseThrow().table();
+
+ assertEquals('|', adapted.delimiter());
+ assertEquals('~', adapted.quote());
+ assertEquals('!', adapted.escape());
+ }
+
+ @Test
+ void refusesMissingLocationUnsupportedSerdeAndPartitionedTables() {
+ assertTrue(adapter.adapt(BINDING, csvTable(null), ACCOUNT, "dev").isEmpty());
+
+ Table unsupportedSerde = csvTable("s3://events-bucket/events/");
+ unsupportedSerde.getStorageDescriptor().getSerdeInfo()
+ .setSerializationLibrary("org.openx.data.jsonserde.JsonSerDe");
+ assertTrue(adapter.adapt(BINDING, unsupportedSerde, ACCOUNT, "dev").isEmpty());
+
+ Table partitioned = csvTable("s3://events-bucket/events/");
+ partitioned.setPartitionKeys(List.of(new Column("day", "date")));
+ assertTrue(adapter.adapt(BINDING, partitioned, ACCOUNT, "dev").isEmpty());
+ }
+
+ private static Table csvTable(String location) {
+ StorageDescriptor descriptor = new StorageDescriptor();
+ descriptor.setLocation(location);
+ descriptor.setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
+ descriptor.setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
+ descriptor.setColumns(List.of(new Column("id", "int"), new Column("name", "string")));
+ StorageDescriptor.SerDeInfo serde = new StorageDescriptor.SerDeInfo();
+ serde.setSerializationLibrary("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
+ serde.setParameters(Map.of("field.delim", ",", "serialization.format", ","));
+ descriptor.setSerdeInfo(serde);
+ Table table = new Table();
+ table.setName("events");
+ table.setTableType("EXTERNAL_TABLE");
+ table.setStorageDescriptor(descriptor);
+ return table;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializerTest.java
index b2a690d6c0..20c7fc835a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializerTest.java
@@ -9,6 +9,7 @@ import java.io.OutputStream;
import java.net.ServerSocket;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutorService;
@@ -16,20 +17,94 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Stream;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
class SpectrumMaterializerTest {
@Test
- void materializesRowsAndSetsSurplusVarcharValuesToNull() throws Exception {
+ void materializesThroughBackendSqlWithoutBufferingRowsInTheCaller() throws Exception {
+ SpectrumS3Reader reader = mock(SpectrumS3Reader.class);
+ when(reader.read(any(SpectrumSession.class),
+ any(SpectrumExternalSchema.class),
+ any(SpectrumExternalTable.class)))
+ .thenReturn(Stream.of(new SpectrumRow(List.of("1", "Alice\nSmith")),
+ new SpectrumRow(Arrays.asList("2", null))));
+ SpectrumExternalTable table = new SpectrumExternalTable("000000000000", "dev", "analytics", "events",
+ List.of(new SpectrumColumn("id", SpectrumColumn.Type.INTEGER),
+ new SpectrumColumn("name", SpectrumColumn.Type.VARCHAR)),
+ "s3://warehouse/events/", ',', '"', '\\', "\\N", 0);
+ CapturingBackendSql backend = new CapturingBackendSql();
+
+ SpectrumMaterializer.Materialization materialized = new SpectrumMaterializer().materialize(
+ backend, new SpectrumSession("000000000000", "cluster-1", "dev", List.of(), false),
+ schema(), table, reader, "spectrum_tmp_test");
+
+ assertEquals("spectrum_tmp_test", materialized.identifier());
+ assertEquals(List.of(
+ "CREATE TEMP TABLE \"spectrum_tmp_test\" (\"id\" INTEGER, \"name\" VARCHAR(256))",
+ "COPY \"spectrum_tmp_test\" (\"id\", \"name\") FROM STDIN"), backend.commands);
+ assertEquals("1\tAlice\\nSmith\n2\t\\N\n",
+ new String(backend.copiedInput, StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void copyFailureDropsTemporaryTableAndPreservesBackendSqlState() {
+ SpectrumS3Reader reader = mock(SpectrumS3Reader.class);
+ when(reader.read(any(SpectrumSession.class),
+ any(SpectrumExternalSchema.class),
+ any(SpectrumExternalTable.class)))
+ .thenReturn(Stream.of(new SpectrumRow(List.of("1"))));
+ SpectrumReadException backendFailure = new SpectrumReadException("42501", "COPY denied");
+ CapturingBackendSql backend = new CapturingBackendSql(backendFailure);
+
+ SpectrumReadException thrown = assertThrows(SpectrumReadException.class,
+ () -> new SpectrumMaterializer().materialize(backend, session(), schema(), oneColumnTable(), reader,
+ "spectrum_tmp_test"));
+
+ assertSame(backendFailure, thrown);
+ assertEquals(List.of(
+ "CREATE TEMP TABLE \"spectrum_tmp_test\" (\"id\" INTEGER)",
+ "COPY \"spectrum_tmp_test\" (\"id\") FROM STDIN",
+ "DROP TABLE IF EXISTS \"spectrum_tmp_test\""), backend.commands);
+ }
+
+ @Test
+ void rowReadFailurePreservesSqlStateAndDropsTemporaryTable() {
+ SpectrumS3Reader reader = mock(SpectrumS3Reader.class);
+ SpectrumReadException rowFailure = new SpectrumReadException("22000", "bad CSV row");
+ when(reader.read(any(SpectrumSession.class),
+ any(SpectrumExternalSchema.class),
+ any(SpectrumExternalTable.class)))
+ .thenReturn(Stream.generate(() -> {
+ throw rowFailure;
+ }));
+ CapturingBackendSql backend = new CapturingBackendSql();
+
+ SpectrumReadException thrown = assertThrows(SpectrumReadException.class,
+ () -> new SpectrumMaterializer().materialize(backend, session(), schema(), oneColumnTable(), reader,
+ "spectrum_tmp_test"));
+
+ assertSame(rowFailure, thrown);
+ assertEquals(List.of(
+ "CREATE TEMP TABLE \"spectrum_tmp_test\" (\"id\" INTEGER)",
+ "COPY \"spectrum_tmp_test\" (\"id\") FROM STDIN",
+ "DROP TABLE IF EXISTS \"spectrum_tmp_test\""), backend.commands);
+ }
+
+ @Test
+ void materializesRowsWithGeneratedIdentifierAndCopyFramingAndNullsSurplusVarcharValues() throws Exception {
try (ServerSocket listener = new ServerSocket(0); Socket backend = new Socket("localhost", listener.getLocalPort())) {
ExecutorService executor = Executors.newSingleThreadExecutor();
- Future<BackendCapture> captured = executor.submit(() -> serve(listener));
+ Future<byte[]> captured = executor.submit(() -> serve(listener));
SpectrumS3Reader reader = mock(SpectrumS3Reader.class);
- when(reader.read(org.mockito.ArgumentMatchers.any(), org.mockito.ArgumentMatchers.any()))
+ when(reader.read(any(), any()))
.thenReturn(Stream.of(new SpectrumRow(Arrays.asList("1", null)),
new SpectrumRow(List.of("two", "x")),
new SpectrumRow(List.of("3", "v".repeat(256))),
@@ -44,23 +119,22 @@ class SpectrumMaterializerTest {
.materialize(backend, table, null, reader);
new SpectrumMaterializer().cleanup(backend, materialization);
- BackendCapture capture = captured.get();
+ byte[] copy = captured.get();
assertTrue(materialization.identifier().startsWith("spectrum_tmp_"));
assertFalse(materialization.identifier().contains("events"));
- assertTrue(capture.createSql().contains("\"value\" VARCHAR(256)"));
- String copyRows = new String(capture.copyRows(), StandardCharsets.UTF_8);
+ String copyRows = new String(copy, StandardCharsets.UTF_8);
assertTrue(copyRows.contains("1\t\\N\ntwo\tx\n"));
assertTrue(copyRows.contains("3\t" + "v".repeat(256) + "\n4\t\\N\n5\t\\N\n"));
executor.shutdownNow();
}
}
- private static BackendCapture serve(ServerSocket listener) throws IOException {
+ private static byte[] serve(ServerSocket listener) throws IOException {
try (Socket socket = listener.accept()) {
InputStream input = socket.getInputStream();
OutputStream output = socket.getOutputStream();
ByteArrayOutputStream copyRows = new ByteArrayOutputStream();
- String createSql = readQuery(input);
+ readQuery(input);
send(output, 'C', "CREATE 0\0".getBytes(StandardCharsets.UTF_8));
send(output, 'Z', new byte[]{'I'});
readQuery(input);
@@ -79,16 +153,15 @@ class SpectrumMaterializerTest {
readQuery(input);
send(output, 'C', "DROP TABLE\0".getBytes(StandardCharsets.UTF_8));
send(output, 'Z', new byte[]{'I'});
- return new BackendCapture(createSql, copyRows.toByteArray());
+ return copyRows.toByteArray();
}
}
- private static String readQuery(InputStream input) throws IOException {
+ private static void readQuery(InputStream input) throws IOException {
Frame frame = readFrame(input);
if (frame.type() != 'Q') {
throw new IOException("expected query");
}
- return new String(frame.body(), 0, frame.body().length - 1, StandardCharsets.UTF_8);
}
private static Frame readFrame(InputStream input) throws IOException {
@@ -116,6 +189,50 @@ class SpectrumMaterializerTest {
private record Frame(char type, byte[] body) {
}
- private record BackendCapture(String createSql, byte[] copyRows) {
+ private static SpectrumExternalSchema schema() {
+ return new SpectrumExternalSchema("000000000000", "dev", "analytics", "s3://warehouse/root/", null);
+ }
+
+ private static SpectrumSession session() {
+ return new SpectrumSession("000000000000", "cluster-1", "dev", List.of(), false);
+ }
+
+ private static SpectrumExternalTable oneColumnTable() {
+ return new SpectrumExternalTable("000000000000", "dev", "analytics", "events",
+ List.of(new SpectrumColumn("id", SpectrumColumn.Type.INTEGER)),
+ "s3://warehouse/events/", ',', '"', '\\', "\\N", 0);
+ }
+
+ private static final class CapturingBackendSql implements BackendSql {
+ private final ArrayList<String> commands = new ArrayList<>();
+ private final SpectrumReadException copyFailure;
+ private byte[] copiedInput;
+
+ private CapturingBackendSql() {
+ this(null);
+ }
+
+ private CapturingBackendSql(SpectrumReadException copyFailure) {
+ this.copyFailure = copyFailure;
+ }
+
+ @Override
+ public void execute(String sql) {
+ commands.add(sql);
+ }
+
+ @Override
+ public long copyIn(String copySql, InputStream data) {
+ commands.add(copySql);
+ if (copyFailure != null) {
+ throw copyFailure;
+ }
+ try {
+ copiedInput = data.readAllBytes();
+ } catch (IOException exception) {
+ throw new SpectrumReadException("22000", "Unable to capture Spectrum rows", exception);
+ }
+ return 2;
+ }
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumS3ReaderTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumS3ReaderTest.java
index a9e6034c52..e964a1cdfa 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumS3ReaderTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumS3ReaderTest.java
@@ -1,6 +1,10 @@
package io.github.hectorvent.floci.services.redshift.spectrum;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.iam.model.CallerContext;
+import io.github.hectorvent.floci.services.iam.model.IamRole;
+import io.github.hectorvent.floci.services.redshift.proxy.RedshiftRoleAccess;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.junit.jupiter.api.Test;
@@ -8,11 +12,17 @@ import org.junit.jupiter.api.Test;
import java.util.List;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import java.util.Optional;
+import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class SpectrumS3ReaderTest {
@@ -43,6 +53,24 @@ class SpectrumS3ReaderTest {
new SpectrumRow(Arrays.asList("2", null, "false", "3.00", "2026-01-03"))), rows);
}
+ @Test
+ void keepsEscapedQuotesInsideQuotedFieldsWithoutJoiningTheNextRecord() {
+ S3Service s3 = mock(S3Service.class);
+ S3Object data = object("events/data.csv", "\"a\\\"b\",c\nnext,row\n");
+ when(s3.listObjectsWithPrefixes("warehouse", "events/", "", 1000, null, null))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(data), List.of(), false, null));
+ when(s3.getObject("warehouse", "events/data.csv")).thenReturn(data);
+ SpectrumExternalTable table = new SpectrumExternalTable("000000000000", "dev", "analytics", "events",
+ List.of(new SpectrumColumn("first", SpectrumColumn.Type.VARCHAR),
+ new SpectrumColumn("second", SpectrumColumn.Type.VARCHAR)),
+ "s3://warehouse/events/", ',', '"', '\\', "\\N", 0);
+
+ List<SpectrumRow> rows = new SpectrumS3Reader(s3, null).read(schema(), table).toList();
+
+ assertEquals(List.of(new SpectrumRow(List.of("a\"b", "c")),
+ new SpectrumRow(List.of("next", "row"))), rows);
+ }
+
@Test
void readsPrefixObjectsInSortedKeyOrder() {
S3Service s3 = mock(S3Service.class);
@@ -107,10 +135,83 @@ class SpectrumS3ReaderTest {
assertEquals("42501", exception.sqlState());
}
+ @Test
+ void deniesRoleGetObjectBeforeFetchingTheObject() {
+ S3Service s3 = mock(S3Service.class);
+ when(s3.isAuthEnforced()).thenReturn(true);
+ S3Object data = object("events/data.csv", "value\n");
+ when(s3.listObjectsWithPrefixes("warehouse", "events/", "", 1000, null, null))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(data), List.of(), false, null));
+ IamService iam = iamWithRole();
+ String role = "arn:aws:iam::000000000000:role/Reader";
+ when(iam.resolvePrincipalContext(role)).thenReturn(CallerContext.of(List.of(policy(
+ "s3:ListBucket", RedshiftRoleAccess.bucketArn(role, "warehouse")))));
+
+ SpectrumReadException exception = assertThrows(SpectrumReadException.class,
+ () -> new SpectrumS3Reader(s3, iam).read(session(role), schema(role), table(
+ SpectrumColumn.Type.VARCHAR)).toList());
+
+ assertEquals("42501", exception.sqlState());
+ verify(s3, never()).getObject("warehouse", "events/data.csv");
+ verify(s3, never()).authorizeAnonymousGetObject("warehouse", "events/data.csv");
+ }
+
+ @Test
+ void readsRowsWhenTheBoundRoleAllowsListingAndObjectReads() {
+ S3Service s3 = mock(S3Service.class);
+ when(s3.isAuthEnforced()).thenReturn(true);
+ S3Object data = object("events/data.csv", "hello\n");
+ when(s3.listObjectsWithPrefixes("warehouse", "events/", "", 1000, null, null))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(data), List.of(), false, null));
+ when(s3.getObject("warehouse", "events/data.csv")).thenReturn(data);
+ IamService iam = iamWithRole();
+ String role = "arn:aws:iam::000000000000:role/Reader";
+ when(iam.resolvePrincipalContext(role)).thenReturn(CallerContext.of(List.of(
+ policy("s3:ListBucket", RedshiftRoleAccess.bucketArn(role, "warehouse")),
+ policy("s3:GetObject", RedshiftRoleAccess.objectArn(role, "warehouse", "events/data.csv")))));
+
+ List<SpectrumRow> rows = new SpectrumS3Reader(s3, iam)
+ .read(session(role), schema(role), table(SpectrumColumn.Type.VARCHAR)).toList();
+
+ assertEquals(List.of(new SpectrumRow(List.of("hello"))), rows);
+ verify(s3, never()).authorizeAnonymousListBucket("warehouse");
+ verify(s3, never()).authorizeAnonymousGetObject("warehouse", "events/data.csv");
+ }
+
+ @Test
+ void deniesRoleListBucketBeforeListingObjects() {
+ S3Service s3 = mock(S3Service.class);
+ when(s3.isAuthEnforced()).thenReturn(true);
+ IamService iam = iamWithRole();
+ String role = "arn:aws:iam::000000000000:role/Reader";
+ when(iam.resolvePrincipalContext(role)).thenReturn(CallerContext.of(List.of(policy(
+ "s3:GetObject", RedshiftRoleAccess.objectArn(role, "warehouse", "events/data.csv")))));
+
+ SpectrumReadException exception = assertThrows(SpectrumReadException.class,
+ () -> new SpectrumS3Reader(s3, iam).read(session(role), schema(role), table(
+ SpectrumColumn.Type.VARCHAR)).toList());
+
+ assertEquals("42501", exception.sqlState());
+ verify(s3, never()).listObjectsWithPrefixes("warehouse", "events/", "", 1000, null, null);
+ }
+
private static SpectrumExternalSchema schema() {
return new SpectrumExternalSchema("000000000000", "dev", "analytics", "s3://warehouse/root/", null);
}
+ private static SpectrumExternalSchema schema(String role) {
+ return new SpectrumExternalSchema("000000000000", "dev", "analytics", "s3://warehouse/root/", role);
+ }
+
+ private static SpectrumSession session(String role) {
+ return new SpectrumSession("000000000000", "cluster-1", "dev", List.of(role), false);
+ }
+
+ private static String policy(String action, String resource) {
+ return "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\""
+ + action + "\",\"Resource\":\"" + resource + "\"}]}";
+ }
+
private static SpectrumExternalTable table(SpectrumColumn.Type type) {
return new SpectrumExternalTable("000000000000", "dev", "analytics", "events",
List.of(new SpectrumColumn("value", type)), "s3://warehouse/events/", ',', '"', '\\', "\\N", 0);
@@ -119,4 +220,71 @@ class SpectrumS3ReaderTest {
private static S3Object object(String key, String content) {
return new S3Object("warehouse", key, content.getBytes(StandardCharsets.UTF_8), "text/csv");
}
+
+ @Test
+ void deniesReadsThatOnlyTheBucketPolicyForbids() {
+ S3Service s3 = mock(S3Service.class);
+ when(s3.isAuthEnforced()).thenReturn(true);
+ S3Object data = object("events/data.csv", "value\n");
+ when(s3.listObjectsWithPrefixes("warehouse", "events/", "", 1000, null, null))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(data), List.of(), false, null));
+ IamService iam = iamWithRole();
+ String role = "arn:aws:iam::000000000000:role/Reader";
+ when(iam.resolvePrincipalContext(role)).thenReturn(CallerContext.of(List.of(
+ policy("s3:ListBucket", RedshiftRoleAccess.bucketArn(role, "warehouse")),
+ policy("s3:GetObject", RedshiftRoleAccess.objectArn(role, "warehouse", "events/data.csv")))));
+ doThrow(new AwsException("AccessDenied", "Access Denied", 403)).when(s3)
+ .authorizeSignedGetObject(anyString(), anyString(), eq("warehouse"), eq("events/data.csv"));
+
+ SpectrumReadException exception = assertThrows(SpectrumReadException.class,
+ () -> new SpectrumS3Reader(s3, iam).read(session(role), schema(role), table(
+ SpectrumColumn.Type.VARCHAR)).toList());
+
+ assertEquals("42501", exception.sqlState());
+ verify(s3, never()).getObject("warehouse", "events/data.csv");
+ }
+
+ @Test
+ void closingTheStreamReleasesTheRoleSession() {
+ S3Service s3 = mock(S3Service.class);
+ S3Object data = object("events/data.csv", "hello\n");
+ when(s3.listObjectsWithPrefixes("warehouse", "events/", "", 1000, null, null))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(data), List.of(), false, null));
+ when(s3.getObject("warehouse", "events/data.csv")).thenReturn(data);
+ IamService iam = iamWithRole();
+ String role = "arn:aws:iam::000000000000:role/Reader";
+
+ try (Stream<SpectrumRow> rows = new SpectrumS3Reader(s3, iam)
+ .read(session(role), schema(role), table(SpectrumColumn.Type.VARCHAR))) {
+ assertEquals(1, rows.count());
+ }
+
+ verify(s3).authorizeSignedListBucket(anyString(), anyString(), eq("warehouse"));
+ verify(s3).authorizeSignedGetObject(anyString(), anyString(), eq("warehouse"), eq("events/data.csv"));
+ verify(iam).unregisterSession(eq("000000000000"), anyString());
+ }
+
+ @Test
+ void releasesTheRoleSessionWhenListingIsDenied() {
+ S3Service s3 = mock(S3Service.class);
+ IamService iam = iamWithRole();
+ String role = "arn:aws:iam::000000000000:role/Reader";
+ doThrow(new AwsException("AccessDenied", "Access Denied", 403)).when(s3)
+ .authorizeSignedListBucket(anyString(), anyString(), eq("warehouse"));
+
+ assertThrows(SpectrumReadException.class, () -> new SpectrumS3Reader(s3, iam)
+ .read(session(role), schema(role), table(SpectrumColumn.Type.VARCHAR)).toList());
+
+ verify(iam).unregisterSession(eq("000000000000"), anyString());
+ }
+
+ private static IamService iamWithRole() {
+ IamService iam = mock(IamService.class);
+ IamRole role = mock(IamRole.class);
+ when(role.getAssumeRolePolicyDocument()).thenReturn("""
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}""");
+ when(iam.findRole("000000000000", "Reader")).thenReturn(Optional.of(role));
+ return iam;
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
index 7a1b25aff6..589a94667a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
@@ -144,6 +144,34 @@ class S3ServiceTest {
assertEquals(2, buckets.size());
}
+ @Test
+ void listBucketsHidesSpectrumScratchBucketButKeepsOtherInternalPrefixBuckets() {
+ s3Service.createBucket("bucket-a", "us-east-1");
+ s3Service.createBucket(S3Service.REDSHIFT_SPECTRUM_SCRATCH_BUCKET, "us-east-1");
+ s3Service.putBucketTagging(S3Service.REDSHIFT_SPECTRUM_SCRATCH_BUCKET, Map.of(
+ S3Service.INTERNAL_BUCKET_TAG_KEY, S3Service.REDSHIFT_SPECTRUM_SCRATCH_TAG_VALUE));
+ String userBucket = S3Service.INTERNAL_BUCKET_PREFIX + "customer-data";
+ s3Service.createBucket(userBucket, "us-east-1");
+
+ List<Bucket> buckets = s3Service.listBuckets();
+
+ assertEquals(2, buckets.size());
+ assertTrue(buckets.stream().anyMatch(bucket -> "bucket-a".equals(bucket.getName())));
+ assertTrue(buckets.stream().anyMatch(bucket -> userBucket.equals(bucket.getName())));
+ assertFalse(buckets.stream().anyMatch(bucket ->
+ S3Service.REDSHIFT_SPECTRUM_SCRATCH_BUCKET.equals(bucket.getName())));
+ }
+
+ @Test
+ void listBucketsKeepsAUserBucketThatOnlySharesTheScratchBucketName() {
+ s3Service.createBucket(S3Service.REDSHIFT_SPECTRUM_SCRATCH_BUCKET, "us-east-1");
+
+ List<Bucket> buckets = s3Service.listBuckets();
+
+ assertEquals(1, buckets.size());
+ assertEquals(S3Service.REDSHIFT_SPECTRUM_SCRATCH_BUCKET, buckets.get(0).getName());
+ }
+
@Test
void putObjectLastModifiedHasMillisecondPrecision() {
s3Service.createBucket("test-bucket", null);
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index aae5bcd737..df4d899918 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -202,6 +202,7 @@ floci:
proxy-handshake-timeout-millis: 10000
proxy-backend-connect-timeout-millis: 5000
proxy-max-connections: 100
+ spectrum-max-rows: 1000000
rds:
enabled: true
mock: false