Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(glue): catalog resource policy and Data Catalog encryption settings (#4027)

David628952ecf06b
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
index ee5bebef7f..3afd35859c 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
@@ -7,12 +7,21 @@ import org.junit.jupiter.api.Test;
import software.amazon.awssdk.services.glue.GlueClient;
import software.amazon.awssdk.services.glue.model.BatchCreatePartitionRequest;
import software.amazon.awssdk.services.glue.model.BatchDeleteConnectionRequest;
+import software.amazon.awssdk.services.glue.model.ConditionCheckFailureException;
+import software.amazon.awssdk.services.glue.model.DeleteResourcePolicyRequest;
+import software.amazon.awssdk.services.glue.model.ExistCondition;
+import software.amazon.awssdk.services.glue.model.GetResourcePoliciesRequest;
+import software.amazon.awssdk.services.glue.model.GetResourcePolicyRequest;
+import software.amazon.awssdk.services.glue.model.GetResourcePolicyResponse;
+import software.amazon.awssdk.services.glue.model.GetDataCatalogEncryptionSettingsRequest;
+import software.amazon.awssdk.services.glue.model.PutResourcePolicyRequest;
import software.amazon.awssdk.services.glue.model.BatchDeleteConnectionResponse;
import software.amazon.awssdk.services.glue.model.Connection;
import software.amazon.awssdk.services.glue.model.ConnectionInput;
import software.amazon.awssdk.services.glue.model.ConnectionPropertyKey;
import software.amazon.awssdk.services.glue.model.ConnectionType;
import software.amazon.awssdk.services.glue.model.CreateConnectionRequest;
+import software.amazon.awssdk.services.glue.model.DataCatalogEncryptionSettings;
import software.amazon.awssdk.services.glue.model.CreateConnectionResponse;
import software.amazon.awssdk.services.glue.model.DeleteConnectionRequest;
import software.amazon.awssdk.services.glue.model.GetConnectionRequest;
@@ -640,6 +649,55 @@ class GlueCatalogTest {
assertThat(batch.errors()).containsOnlyKeys(CONNECTION_NAME);
}
+ @Test
+ @DisplayName("The catalog resource policy follows the create, update and delete conditions Terraform sends")
+ void catalogResourcePolicyLifecycle() {
+ String policy = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\","
+ + "\"Principal\":{\"AWS\":\"arn:aws:iam::111122223333:root\"},\"Action\":\"glue:GetTable\",\"Resource\":\"*\"}]}";
+ String policyV2 = policy.replace("glue:GetTable", "glue:GetTables");
+ // Start from no policy; the catalog is shared with other tests.
+ try {
+ glue.deleteResourcePolicy(DeleteResourcePolicyRequest.builder().build());
+ }
+ catch (EntityNotFoundException ignored) {}
+
+ String hash = glue.putResourcePolicy(PutResourcePolicyRequest.builder()
+ .policyInJson(policy)
+ .policyExistsCondition(ExistCondition.NOT_EXIST)
+ .build()).policyHash();
+ GetResourcePolicyResponse read = glue.getResourcePolicy(GetResourcePolicyRequest.builder().build());
+ assertThat(read.policyInJson()).isEqualTo(policy);
+ assertThat(read.policyHash()).isEqualTo(hash);
+ assertThat(read.createTime()).isNotNull();
+ assertThat(read.updateTime()).isNotNull();
+
+ assertThatThrownBy(() -> glue.putResourcePolicy(PutResourcePolicyRequest.builder()
+ .policyInJson(policyV2)
+ .policyExistsCondition(ExistCondition.NOT_EXIST)
+ .build()))
+ .isInstanceOf(ConditionCheckFailureException.class);
+
+ String hash2 = glue.putResourcePolicy(PutResourcePolicyRequest.builder()
+ .policyInJson(policyV2)
+ .policyExistsCondition(ExistCondition.MUST_EXIST)
+ .policyHashCondition(hash)
+ .build()).policyHash();
+ assertThat(glue.getResourcePolicy(GetResourcePolicyRequest.builder().build()).policyInJson()).isEqualTo(policyV2);
+ assertThat(glue.getResourcePolicies(GetResourcePoliciesRequest.builder().build()).getResourcePoliciesResponseList()).hasSize(1);
+
+ assertThatThrownBy(() -> glue.deleteResourcePolicy(DeleteResourcePolicyRequest.builder()
+ .policyHashCondition(hash).build()))
+ .isInstanceOf(ConditionCheckFailureException.class);
+ glue.deleteResourcePolicy(DeleteResourcePolicyRequest.builder().policyHashCondition(hash2).build());
+ assertThatThrownBy(() -> glue.getResourcePolicy(GetResourcePolicyRequest.builder().build()))
+ .isInstanceOf(EntityNotFoundException.class);
+
+ DataCatalogEncryptionSettings settings = glue.getDataCatalogEncryptionSettings(GetDataCatalogEncryptionSettingsRequest.builder().build())
+ .dataCatalogEncryptionSettings();
+ assertThat(settings.encryptionAtRest().catalogEncryptionModeAsString()).isEqualTo("DISABLED");
+ assertThat(settings.connectionPasswordEncryption().returnConnectionPasswordEncrypted()).isFalse();
+ }
+
private static TableInput tableInput(String description) {
return tableInput(TABLE_NAME, description);
}
diff --git a/docs/services/glue.md b/docs/services/glue.md
index 9e0f41c26b..770c038361 100644
--- a/docs/services/glue.md
+++ b/docs/services/glue.md
@@ -86,11 +86,44 @@ under `AuthenticationConfiguration` are accepted and never returned by a read. A
`HidePassword` removes `PASSWORD` and `ENCRYPTED_PASSWORD`, the two members the API reference defines as the
connection's password. The Kafka credentials (`KAFKA_CLIENT_KEYSTORE_PASSWORD`, `KAFKA_CLIENT_KEY_PASSWORD`,
`KAFKA_SASL_PLAIN_PASSWORD`, `KAFKA_SASL_SCRAM_PASSWORD` and their `ENCRYPTED_` forms) are returned as stored: the
-reference does not say whether the flag covers them, so Floci does not guess.
+reference does not say whether the flag covers them, so Floci does not guess. The `ENCRYPTED_` forms are what the
+catalog's `ConnectionPasswordEncryption` setting produces (see Encryption settings below).
`TestConnection` is asynchronous on AWS and returns nothing; Floci checks the request's shape and accepts it
without opening a socket to the data store, since no job or crawler runs against a connection yet.
+#### Resource policy
+
+| Action | Description |
+|--------|-------------|
+| PutResourcePolicy | Sets the catalog's resource policy from `PolicyInJson` and returns its `PolicyHash`. `PolicyExistsCondition` (`NOT_EXIST`, `MUST_EXIST`, `NONE`) and `PolicyHashCondition` are checked against the stored policy and fail with `ConditionCheckFailureException`. |
+| GetResourcePolicy | Returns the catalog policy with its hash and timestamps, or `EntityNotFoundException` when none is set. |
+| GetResourcePolicies | Lists the catalog policy (zero or one entry) under `GetResourcePoliciesResponseList`, paged. |
+| DeleteResourcePolicy | Deletes the catalog policy; honours `PolicyHashCondition`; `EntityNotFoundException` when none is set. |
+
+The catalog holds one policy, as on AWS. `PolicyInJson` must be a JSON object. AWS documents `PolicyHash` only as
+an opaque value to echo back; Floci derives it from the document, so the same policy always has the same hash.
+`ResourceArn` ("for internal use only" in the reference) and `EnableHybrid` are accepted. Per-resource policies
+that Resource Access Manager creates on AWS are not emulated, so `GetResourcePolicies` never has more than one entry.
+
+#### Encryption settings
+
+| Action | Description |
+|--------|-------------|
+| GetDataCatalogEncryptionSettings | Returns the catalog's security configuration: `EncryptionAtRest` (`CatalogEncryptionMode` `DISABLED` by default) and `ConnectionPasswordEncryption` (`ReturnConnectionPasswordEncrypted` false by default). |
+| PutDataCatalogEncryptionSettings | Replaces the configuration. `CatalogEncryptionMode` is `DISABLED`, `SSE-KMS` or `SSE-KMS-WITH-SERVICE-ROLE`; `ReturnConnectionPasswordEncrypted` is required inside its block. A block left out keeps its default. |
+
+`EncryptionAtRest` is stored and reported; catalog metadata is not encrypted on disk, which nothing observes through
+the API. `ConnectionPasswordEncryption` is applied: while `ReturnConnectionPasswordEncrypted` is true, a connection
+created or updated has each password property encrypted with `AwsKmsKeyId` through Floci's KMS and stored under the
+name the `Connection` structure documents for it (`PASSWORD` as `ENCRYPTED_PASSWORD`, and the four Kafka passwords as
+their `ENCRYPTED_KAFKA_*` forms). Reads return the ciphertext, base64-encoded, which `KMS.Decrypt` turns back into
+the password. A missing key, or none configured, fails the create or update with `GlueEncryptionException`, the error
+`CreateConnection` and `UpdateConnection` list for a failed encryption operation. Connections created before the setting
+was switched on keep their plaintext: the developer guide ("Encrypting connection passwords") states that whether a
+password is encrypted "was determined when the connection was created or updated", so the setting is not applied
+retroactively.
+
#### Jobs
| Action | Description |
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
index 4179ddfa76..83cf2b8832 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
@@ -289,7 +289,7 @@ public class GlueJsonHandler {
case "CreateConnection" -> handleCreateConnection(request, region);
case "GetConnection" -> handleGetConnection(request);
case "GetConnections" -> handleGetConnections(request);
- case "UpdateConnection" -> handleUpdateConnection(request);
+ case "UpdateConnection" -> handleUpdateConnection(request, region);
case "DeleteConnection" -> {
glueService.deleteConnection(request.path("ConnectionName").asText(null), region);
yield Response.ok(Map.of()).build();
@@ -301,6 +301,33 @@ public class GlueJsonHandler {
yield Response.ok(glueService.batchDeleteConnections(names, region)).build();
}
case "TestConnection" -> handleTestConnection(request);
+ case "PutResourcePolicy" -> {
+ String hash = glueService.putResourcePolicy(
+ request.path("PolicyInJson").asText(null),
+ request.path("PolicyHashCondition").asText(null),
+ request.path("PolicyExistsCondition").asText(null),
+ request.path("EnableHybrid").asText(null));
+ yield Response.ok(Map.of("PolicyHash", hash)).build();
+ }
+ case "GetResourcePolicy" -> Response.ok(glueService.getResourcePolicy()).build();
+ case "GetResourcePolicies" -> {
+ GlueService.Page<GluePolicy> page =
+ glueService.getResourcePolicies(readMaxResults(request), readNextToken(request));
+ yield Response.ok(pageResponse("GetResourcePoliciesResponseList", page.items(), page.nextToken())).build();
+ }
+ case "DeleteResourcePolicy" -> {
+ glueService.deleteResourcePolicy(request.path("PolicyHashCondition").asText(null));
+ yield Response.ok(Map.of()).build();
+ }
+ case "GetDataCatalogEncryptionSettings" -> Response.ok(Map.of(
+ "DataCatalogEncryptionSettings", glueService.getDataCatalogEncryptionSettings())).build();
+ case "PutDataCatalogEncryptionSettings" -> {
+ DataCatalogEncryptionSettings settings = request.hasNonNull("DataCatalogEncryptionSettings")
+ ? mapper.treeToValue(request.get("DataCatalogEncryptionSettings"), DataCatalogEncryptionSettings.class)
+ : null;
+ glueService.putDataCatalogEncryptionSettings(settings);
+ yield Response.ok(Map.of()).build();
+ }
// Read-only Glue actions for resources the emulator does not model. The AWS SDK
// expects each to return a 200 with its result key present (empty), so we emit the
// documented empty shape rather than an InvalidAction 400 that callers can't read.
@@ -907,11 +934,11 @@ public class GlueJsonHandler {
return Response.ok(pageResponse("ConnectionList", page.items(), page.nextToken())).build();
}
- private Response handleUpdateConnection(JsonNode request) throws Exception {
+ private Response handleUpdateConnection(JsonNode request, String region) throws Exception {
ConnectionInput input = request.hasNonNull("ConnectionInput")
? mapper.treeToValue(request.get("ConnectionInput"), ConnectionInput.class)
: null;
- glueService.updateConnection(request.path("Name").asText(null), input);
+ glueService.updateConnection(request.path("Name").asText(null), input, region);
return Response.ok(Map.of()).build();
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index 4c357ee34d..16f52d8d8c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -1,7 +1,10 @@
package io.github.hectorvent.floci.services.glue;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.StorageBackend;
@@ -10,6 +13,10 @@ import io.github.hectorvent.floci.services.glue.model.Classifier;
import io.github.hectorvent.floci.services.glue.model.Column;
import io.github.hectorvent.floci.services.glue.model.Connection;
import io.github.hectorvent.floci.services.glue.model.ConnectionInput;
+import io.github.hectorvent.floci.services.glue.model.ConnectionPasswordEncryption;
+import io.github.hectorvent.floci.services.glue.model.DataCatalogEncryptionSettings;
+import io.github.hectorvent.floci.services.glue.model.EncryptionAtRest;
+import io.github.hectorvent.floci.services.glue.model.GluePolicy;
import io.github.hectorvent.floci.services.glue.model.Crawler;
import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;
import io.github.hectorvent.floci.services.glue.model.Database;
@@ -27,6 +34,7 @@ import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistr
import io.github.hectorvent.floci.services.glue.schemaregistry.SchemaToColumnsConverter;
import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;
import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaVersion;
+import io.github.hectorvent.floci.services.kms.KmsService;
import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;
import io.quarkus.runtime.annotations.RegisterForReflection;
import jakarta.enterprise.context.ApplicationScoped;
@@ -35,6 +43,8 @@ import org.jboss.logging.Logger;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Base64;
@@ -113,6 +123,23 @@ public class GlueService {
"ENDPOINT_TYPE", "ROLE_ARN", "REGION", "WORKGROUP_NAME", "CLUSTER_IDENTIFIER", "DATABASE");
private static final String CONNECTION_STATUS_READY = "READY";
+ // The catalog holds one resource policy and one security configuration; both stores are
+ // account-scoped by StorageFactory, so a fixed key is the whole address.
+ private static final String CATALOG_KEY = "catalog";
+ private static final ObjectMapper POLICY_JSON = new ObjectMapper();
+ private static final Set<String> POLICY_EXISTS_CONDITIONS = Set.of("MUST_EXIST", "NOT_EXIST", "NONE");
+ private static final Set<String> ENABLE_HYBRID_VALUES = Set.of("TRUE", "FALSE");
+ private static final Set<String> CATALOG_ENCRYPTION_MODES = Set.of(
+ "DISABLED", "SSE-KMS", "SSE-KMS-WITH-SERVICE-ROLE");
+ // The Connection structure names the stored form of each password when the catalog's
+ // ConnectionPasswordEncryption is on: the plaintext key is replaced by its ENCRYPTED_ twin.
+ private static final Map<String, String> ENCRYPTED_CONNECTION_PROPERTY_KEYS = Map.of(
+ "PASSWORD", "ENCRYPTED_PASSWORD",
+ "KAFKA_CLIENT_KEYSTORE_PASSWORD", "ENCRYPTED_KAFKA_CLIENT_KEYSTORE_PASSWORD",
+ "KAFKA_CLIENT_KEY_PASSWORD", "ENCRYPTED_KAFKA_CLIENT_KEY_PASSWORD",
+ "KAFKA_SASL_PLAIN_PASSWORD", "ENCRYPTED_KAFKA_SASL_PLAIN_PASSWORD",
+ "KAFKA_SASL_SCRAM_PASSWORD", "ENCRYPTED_KAFKA_SASL_SCRAM_PASSWORD");
+
// Glue's partition index states. FAILED also exists but is only reachable through a backfill
// failure, which is not emulated.
private static final String INDEX_STATUS_CREATING = "CREATING";
@@ -131,15 +158,19 @@ public class GlueService {
private final StorageBackend<String, Crawler> crawlerStore;
private final StorageBackend<String, Classifier> classifierStore;
private final StorageBackend<String, Connection> connectionStore;
+ private final StorageBackend<String, GluePolicy> resourcePolicyStore;
+ private final StorageBackend<String, DataCatalogEncryptionSettings> encryptionSettingsStore;
private final GlueSchemaRegistryService schemaRegistryService;
private final RegionResolver regionResolver;
private final ResourceGroupsTaggingService resourceGroupsTaggingService;
+ private final KmsService kmsService;
@Inject
public GlueService(StorageFactory storageFactory,
GlueSchemaRegistryService schemaRegistryService,
RegionResolver regionResolver,
- ResourceGroupsTaggingService resourceGroupsTaggingService) {
+ ResourceGroupsTaggingService resourceGroupsTaggingService,
+ KmsService kmsService) {
this.databaseStore = storageFactory.create("glue", "databases.json", new TypeReference<>() {});
this.tableStore = storageFactory.create("glue", "tables.json", new TypeReference<>() {});
this.tableVersionStore = storageFactory.create("glue", "table_versions.json", new TypeReference<>() {});
@@ -154,9 +185,13 @@ public class GlueService {
this.crawlerStore = storageFactory.create("glue", "crawlers.json", new TypeReference<>() {});
this.classifierStore = storageFactory.create("glue", "classifiers.json", new TypeReference<>() {});
this.connectionStore = storageFactory.create("glue", "connections.json", new TypeReference<>() {});
+ this.resourcePolicyStore = storageFactory.create("glue", "resource_policy.json", new TypeReference<>() {});
+ this.encryptionSettingsStore = storageFactory.create(
+ "glue", "catalog_encryption_settings.json", new TypeReference<>() {});
this.schemaRegistryService = schemaRegistryService;
this.regionResolver = regionResolver;
this.resourceGroupsTaggingService = resourceGroupsTaggingService;
+ this.kmsService = kmsService;
}
GlueService(StorageBackend<String, Database> databaseStore,
@@ -171,9 +206,12 @@ public class GlueService {
StorageBackend<String, Crawler> crawlerStore,
StorageBackend<String, Classifier> classifierStore,
StorageBackend<String, Connection> connectionStore,
+ StorageBackend<String, GluePolicy> resourcePolicyStore,
+ StorageBackend<String, DataCatalogEncryptionSettings> encryptionSettingsStore,
GlueSchemaRegistryService schemaRegistryService,
RegionResolver regionResolver,
- ResourceGroupsTaggingService resourceGroupsTaggingService) {
+ ResourceGroupsTaggingService resourceGroupsTaggingService,
+ KmsService kmsService) {
this.databaseStore = databaseStore;
this.tableStore = tableStore;
this.tableVersionStore = tableVersionStore;
@@ -186,9 +224,12 @@ public class GlueService {
this.crawlerStore = crawlerStore;
this.classifierStore = classifierStore;
this.connectionStore = connectionStore;
+ this.resourcePolicyStore = resourcePolicyStore;
+ this.encryptionSettingsStore = encryptionSettingsStore;
this.schemaRegistryService = schemaRegistryService;
this.regionResolver = regionResolver;
this.resourceGroupsTaggingService = resourceGroupsTaggingService;
+ this.kmsService = kmsService;
}
public void createDatabase(Database database) {
@@ -1951,6 +1992,7 @@ public class GlueService {
}
Instant now = Instant.now();
Connection connection = toConnection(input);
+ encryptConnectionPasswords(connection.getConnectionProperties(), region);
connection.setCreationTime(now);
connection.setLastUpdatedTime(now);
connectionStore.put(name, connection);
@@ -2002,13 +2044,14 @@ public class GlueService {
* (API reference), so the stored definition is replaced rather than merged: a member left
* out of the input is gone afterwards. The connection keeps its name and creation time.
*/
- public void updateConnection(String name, ConnectionInput input) {
+ public void updateConnection(String name, ConnectionInput input, String region) {
validateRequired(name, "Name");
validateRequired(input, "ConnectionInput");
Connection existing = connectionStore.get(name)
.orElseThrow(() -> new AwsException("EntityNotFoundException", "Connection " + name + " not found.", 400));
validateConnectionInput(input);
Connection updated = toConnection(input);
+ encryptConnectionPasswords(updated.getConnectionProperties(), region);
updated.setName(existing.getName());
updated.setCreationTime(existing.getCreationTime());
updated.setLastUpdatedTime(Instant.now());
@@ -2136,6 +2179,173 @@ public class GlueService {
@JsonProperty("Succeeded") List<String> succeeded,
@JsonProperty("Errors") Map<String, ErrorDetail> errors) {}
+ // ---- Catalog resource policy and encryption settings -----------------------------------
+
+ /**
+ * Sets the catalog's one resource policy. {@code PolicyExistsCondition} and
+ * {@code PolicyHashCondition} are checked against the stored policy first and fail with
+ * ConditionCheckFailureException, which is how Terraform's create (NOT_EXIST) and update
+ * (MUST_EXIST) tell each other apart. Returns the new policy's hash.
+ */
+ public String putResourcePolicy(String policyInJson, String policyHashCondition,
+ String policyExistsCondition, String enableHybrid) {
+ validateRequired(policyInJson, "PolicyInJson");
+ validatePolicyDocument(policyInJson);
+ if (policyExistsCondition != null && !POLICY_EXISTS_CONDITIONS.contains(policyExistsCondition)) {
+ throw new AwsException("InvalidInputException",
+ "Unsupported PolicyExistsCondition: " + policyExistsCondition, 400);
+ }
+ if (enableHybrid != null && !ENABLE_HYBRID_VALUES.contains(enableHybrid)) {
+ throw new AwsException("InvalidInputException", "Unsupported EnableHybrid value: " + enableHybrid, 400);
+ }
+ Optional<GluePolicy> existing = resourcePolicyStore.get(CATALOG_KEY);
+ if ("NOT_EXIST".equals(policyExistsCondition) && existing.isPresent()) {
+ throw new AwsException("ConditionCheckFailureException",
+ "A resource policy already exists and PolicyExistsCondition is NOT_EXIST.", 400);
+ }
+ if ("MUST_EXIST".equals(policyExistsCondition) && existing.isEmpty()) {
+ throw new AwsException("ConditionCheckFailureException",
+ "No resource policy exists and PolicyExistsCondition is MUST_EXIST.", 400);
+ }
+ checkPolicyHashCondition(policyHashCondition, existing);
+
+ Instant now = Instant.now();
+ GluePolicy policy = new GluePolicy();
+ policy.setPolicyInJson(policyInJson);
+ policy.setPolicyHash(policyHash(policyInJson));
+ policy.setCreateTime(existing.map(GluePolicy::getCreateTime).orElse(now));
+ policy.setUpdateTime(now);
+ resourcePolicyStore.put(CATALOG_KEY, policy);
+ LOG.infov("Set Glue catalog resource policy (hash {0})", policy.getPolicyHash());
+ return policy.getPolicyHash();
+ }
+
+ public GluePolicy getResourcePolicy() {
+ return resourcePolicyStore.get(CATALOG_KEY)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Policy not found", 400));
+ }
+
+ /** The catalog policy is the only entry; RAM-granted per-resource policies are not emulated. */
+ public Page<GluePolicy> getResourcePolicies(Integer maxResults, String nextToken) {
+ List<GluePolicy> policies = new ArrayList<>(resourcePolicyStore.get(CATALOG_KEY).stream().toList());
+ return paginate(policies, maxResults, nextToken);
+ }
+
+ public void deleteResourcePolicy(String policyHashCondition) {
+ Optional<GluePolicy> existing = resourcePolicyStore.get(CATALOG_KEY);
+ if (existing.isEmpty()) {
+ throw new AwsException("EntityNotFoundException", "Policy not found", 400);
+ }
+ checkPolicyHashCondition(policyHashCondition, existing);
+ resourcePolicyStore.delete(CATALOG_KEY);
+ LOG.info("Deleted Glue catalog resource policy");
+ }
+
+ /** Before anything is put, a catalog reports both blocks with encryption off. */
+ public DataCatalogEncryptionSettings getDataCatalogEncryptionSettings() {
+ return encryptionSettingsStore.get(CATALOG_KEY).orElseGet(DataCatalogEncryptionSettings::defaults);
+ }
+
+ /**
+ * Replaces the catalog's security configuration. A block left out of the request keeps the
+ * default (off), so a later read always carries both blocks, as AWS's does.
+ */
+ public void putDataCatalogEncryptionSettings(DataCatalogEncryptionSettings settings) {
+ validateRequired(settings, "DataCatalogEncryptionSettings");
+ DataCatalogEncryptionSettings stored = DataCatalogEncryptionSettings.defaults();
+ EncryptionAtRest atRest = settings.getEncryptionAtRest();
+ if (atRest != null) {
+ validateRequired(atRest.getCatalogEncryptionMode(), "EncryptionAtRest.CatalogEncryptionMode");
+ if (!CATALOG_ENCRYPTION_MODES.contains(atRest.getCatalogEncryptionMode())) {
+ throw new AwsException("InvalidInputException",
+ "Unsupported CatalogEncryptionMode: " + atRest.getCatalogEncryptionMode(), 400);
+ }
+ stored.setEncryptionAtRest(atRest);
+ }
+ ConnectionPasswordEncryption passwords = settings.getConnectionPasswordEncryption();
+ if (passwords != null) {
+ validateRequired(passwords.getReturnConnectionPasswordEncrypted(),
+ "ConnectionPasswordEncryption.ReturnConnectionPasswordEncrypted");
+ stored.setConnectionPasswordEncryption(passwords);
+ }
+ encryptionSettingsStore.put(CATALOG_KEY, stored);
+ LOG.infov("Updated Glue Data Catalog encryption settings: at rest {0}, password encryption {1}",
+ stored.getEncryptionAtRest().getCatalogEncryptionMode(),
+ stored.getConnectionPasswordEncryption().getReturnConnectionPasswordEncrypted());
+ }
+
+ /**
+ * Applies the catalog's ConnectionPasswordEncryption to a connection being created or
+ * updated: each plaintext password property is encrypted with the configured KMS key and
+ * stored under its ENCRYPTED_ name (the Connection structure documents each pair), so every
+ * later read returns it encrypted. A KMS failure is the GlueEncryptionException both
+ * operations list.
+ */
+ private void encryptConnectionPasswords(Map<String, String> properties, String region) {
+ ConnectionPasswordEncryption setting = getDataCatalogEncryptionSettings().getConnectionPasswordEncryption();
+ if (setting == null || !Boolean.TRUE.equals(setting.getReturnConnectionPasswordEncrypted())) {
+ return;
+ }
+ String keyId = setting.getAwsKmsKeyId();
+ for (Map.Entry<String, String> pair : ENCRYPTED_CONNECTION_PROPERTY_KEYS.entrySet()) {
+ String plaintext = properties.get(pair.getKey());
+ if (plaintext == null) {
+ continue;
+ }
+ if (keyId == null || keyId.isBlank()) {
+ throw new AwsException("GlueEncryptionException",
+ "Connection password encryption is enabled but the catalog settings name no AwsKmsKeyId.", 400);
+ }
+ byte[] ciphertext;
+ try {
+ ciphertext = kmsService.encrypt(keyId, plaintext.getBytes(StandardCharsets.UTF_8), region);
+ } catch (AwsException e) {
+ throw new AwsException("GlueEncryptionException",
+ "An encryption operation failed: " + e.getMessage(), 400);
+ }
+ properties.remove(pair.getKey());
+ properties.put(pair.getValue(), Base64.getEncoder().encodeToString(ciphertext));
+ }
+ }
+
+ private static void validatePolicyDocument(String policyInJson) {
+ if (policyInJson.length() < 2) {
+ throw new AwsException("InvalidInputException", "PolicyInJson must be at least 2 characters.", 400);
+ }
+ try {
+ JsonNode document = POLICY_JSON.readTree(policyInJson);
+ if (document == null || !document.isObject()) {
+ throw new AwsException("InvalidInputException", "PolicyInJson must be a JSON policy document.", 400);
+ }
+ } catch (JsonProcessingException e) {
+ throw new AwsException("InvalidInputException", "PolicyInJson is not valid JSON: " + e.getOriginalMessage(), 400);
+ }
+ }
+
+ private static void checkPolicyHashCondition(String policyHashCondition, Optional<GluePolicy> existing) {
+ if (policyHashCondition == null || policyHashCondition.isBlank()) {
+ return;
+ }
+ String current = existing.map(GluePolicy::getPolicyHash).orElse(null);
+ if (!policyHashCondition.equals(current)) {
+ throw new AwsException("ConditionCheckFailureException",
+ "PolicyHashCondition does not match the current policy hash.", 400);
+ }
+ }
+
+ /**
+ * AWS documents PolicyHash only as an opaque value to echo back in PolicyHashCondition; this
+ * derives it from the document so the same policy always yields the same hash.
+ */
+ private static String policyHash(String policyInJson) {
+ try {
+ byte[] digest = MessageDigest.getInstance("MD5").digest(policyInJson.getBytes(StandardCharsets.UTF_8));
+ return Base64.getEncoder().encodeToString(digest);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("MD5 is a required JDK algorithm", e);
+ }
+ }
+
public void tagResource(String arn, Map<String, String> tags, String region) {
validateArn(arn);
if (arn.contains(":registry/") || arn.contains(":schema/")) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionPasswordEncryption.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionPasswordEncryption.java
new file mode 100644
index 0000000000..2d21d92e71
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionPasswordEncryption.java
@@ -0,0 +1,28 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+/**
+ * The {@code ConnectionPasswordEncryption} block of the Data Catalog encryption settings. With
+ * {@code ReturnConnectionPasswordEncrypted} true, connection passwords are encrypted with
+ * {@code AwsKmsKeyId} when a connection is created or updated and stay encrypted in every read.
+ */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ConnectionPasswordEncryption {
+ @JsonProperty("ReturnConnectionPasswordEncrypted")
+ private Boolean returnConnectionPasswordEncrypted;
+
+ @JsonProperty("AwsKmsKeyId")
+ private String awsKmsKeyId;
+
+ public Boolean getReturnConnectionPasswordEncrypted() { return returnConnectionPasswordEncrypted; }
+ public void setReturnConnectionPasswordEncrypted(Boolean returnConnectionPasswordEncrypted) {
+ this.returnConnectionPasswordEncrypted = returnConnectionPasswordEncrypted;
+ }
+
+ public String getAwsKmsKeyId() { return awsKmsKeyId; }
+ public void setAwsKmsKeyId(String awsKmsKeyId) { this.awsKmsKeyId = awsKmsKeyId; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/DataCatalogEncryptionSettings.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/DataCatalogEncryptionSettings.java
new file mode 100644
index 0000000000..53038e937d
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/DataCatalogEncryptionSettings.java
@@ -0,0 +1,36 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+/** The Data Catalog's security configuration, one per catalog. */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class DataCatalogEncryptionSettings {
+ @JsonProperty("EncryptionAtRest")
+ private EncryptionAtRest encryptionAtRest;
+
+ @JsonProperty("ConnectionPasswordEncryption")
+ private ConnectionPasswordEncryption connectionPasswordEncryption;
+
+ /** What a catalog reports before anything was ever put: both blocks present, both off. */
+ public static DataCatalogEncryptionSettings defaults() {
+ DataCatalogEncryptionSettings settings = new DataCatalogEncryptionSettings();
+ EncryptionAtRest atRest = new EncryptionAtRest();
+ atRest.setCatalogEncryptionMode("DISABLED");
+ settings.encryptionAtRest = atRest;
+ ConnectionPasswordEncryption passwords = new ConnectionPasswordEncryption();
+ passwords.setReturnConnectionPasswordEncrypted(false);
+ settings.connectionPasswordEncryption = passwords;
+ return settings;
+ }
+
+ public EncryptionAtRest getEncryptionAtRest() { return encryptionAtRest; }
+ public void setEncryptionAtRest(EncryptionAtRest encryptionAtRest) { this.encryptionAtRest = encryptionAtRest; }
+
+ public ConnectionPasswordEncryption getConnectionPasswordEncryption() { return connectionPasswordEncryption; }
+ public void setConnectionPasswordEncryption(ConnectionPasswordEncryption connectionPasswordEncryption) {
+ this.connectionPasswordEncryption = connectionPasswordEncryption;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/EncryptionAtRest.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/EncryptionAtRest.java
new file mode 100644
index 0000000000..52c09a9148
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/EncryptionAtRest.java
@@ -0,0 +1,30 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+/** The {@code EncryptionAtRest} block of the Data Catalog encryption settings. */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class EncryptionAtRest {
+ @JsonProperty("CatalogEncryptionMode")
+ private String catalogEncryptionMode;
+
+ @JsonProperty("SseAwsKmsKeyId")
+ private String sseAwsKmsKeyId;
+
+ @JsonProperty("CatalogEncryptionServiceRole")
+ private String catalogEncryptionServiceRole;
+
+ public String getCatalogEncryptionMode() { return catalogEncryptionMode; }
+ public void setCatalogEncryptionMode(String catalogEncryptionMode) { this.catalogEncryptionMode = catalogEncryptionMode; }
+
+ public String getSseAwsKmsKeyId() { return sseAwsKmsKeyId; }
+ public void setSseAwsKmsKeyId(String sseAwsKmsKeyId) { this.sseAwsKmsKeyId = sseAwsKmsKeyId; }
+
+ public String getCatalogEncryptionServiceRole() { return catalogEncryptionServiceRole; }
+ public void setCatalogEncryptionServiceRole(String catalogEncryptionServiceRole) {
+ this.catalogEncryptionServiceRole = catalogEncryptionServiceRole;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java
new file mode 100644
index 0000000000..c4d2a4aa4a
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java
@@ -0,0 +1,42 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+
+/**
+ * The Data Catalog resource policy as GetResourcePolicy returns it, and as each entry of
+ * GetResourcePolicies (the {@code GluePolicy} structure of the API reference).
+ */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class GluePolicy {
+ @JsonProperty("PolicyInJson")
+ private String policyInJson;
+
+ @JsonProperty("PolicyHash")
+ private String policyHash;
+
+ @JsonProperty("CreateTime")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant createTime;
+
+ @JsonProperty("UpdateTime")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant updateTime;
+
+ public String getPolicyInJson() { return policyInJson; }
+ public void setPolicyInJson(String policyInJson) { this.policyInJson = policyInJson; }
+
+ public String getPolicyHash() { return policyHash; }
+ public void setPolicyHash(String policyHash) { this.policyHash = policyHash; }
+
+ public Instant getCreateTime() { return createTime; }
+ public void setCreateTime(Instant createTime) { this.createTime = createTime; }
+
+ public Instant getUpdateTime() { return updateTime; }
+ public void setUpdateTime(Instant updateTime) { this.updateTime = updateTime; }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueCatalogPolicyAndEncryptionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCatalogPolicyAndEncryptionIntegrationTest.java
new file mode 100644
index 0000000000..ff758349a1
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCatalogPolicyAndEncryptionIntegrationTest.java
@@ -0,0 +1,182 @@
+package io.github.hectorvent.floci.services.glue;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.UUID;
+
+import static io.github.hectorvent.floci.testing.RestAssuredJsonUtils.awsAction;
+import static io.github.hectorvent.floci.testing.RestAssuredJsonUtils.awsActionJson;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * The catalog's resource policy and encryption settings over JSON 1.1, in the order Terraform
+ * drives them, plus the observable effect of ConnectionPasswordEncryption: a connection created
+ * while it is on stores its password KMS-encrypted and GetConnection returns it that way.
+ * Both settings are one per catalog, so the class runs in order and puts the catalog back at
+ * the end.
+ */
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class GlueCatalogPolicyAndEncryptionIntegrationTest {
+
+ private static final String GLUE = "AWSGlue";
+ private static final String KMS = "TrentService";
+ private static final String POLICY = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\","
+ + "\"Principal\":{\"AWS\":\"arn:aws:iam::111122223333:root\"},\"Action\":\"glue:GetTable\",\"Resource\":\"*\"}]}";
+ private static final String POLICY_V2 = POLICY.replace("glue:GetTable", "glue:GetTables");
+
+ private static String policyHash;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void resourcePolicy_absentAtFirst_isEntityNotFound() {
+ // The catalog may carry a policy from another class; start from a known state.
+ awsAction(GLUE, "DeleteResourcePolicy", "{}");
+ awsAction(GLUE, "GetResourcePolicy", "{}")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("EntityNotFoundException"));
+ awsAction(GLUE, "GetResourcePolicies", "{}")
+ .then()
+ .statusCode(200)
+ .body("GetResourcePoliciesResponseList.size()", equalTo(0));
+ }
+
+ @Test
+ @Order(2)
+ void putResourcePolicy_notExistThenMustExist_createsThenUpdates() throws Exception {
+ JsonNode created = awsActionJson(GLUE, "PutResourcePolicy",
+ "{\"PolicyInJson\": " + quoted(POLICY) + ", \"PolicyExistsCondition\": \"NOT_EXIST\"}");
+ policyHash = created.get("PolicyHash").asText();
+
+ awsAction(GLUE, "GetResourcePolicy", "{}")
+ .then()
+ .statusCode(200)
+ .body("PolicyInJson", equalTo(POLICY))
+ .body("PolicyHash", equalTo(policyHash))
+ .body("CreateTime", notNullValue())
+ .body("UpdateTime", notNullValue());
+
+ // Terraform's second create against an existing policy must be refused.
+ awsAction(GLUE, "PutResourcePolicy",
+ "{\"PolicyInJson\": " + quoted(POLICY_V2) + ", \"PolicyExistsCondition\": \"NOT_EXIST\"}")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ConditionCheckFailureException"));
+
+ JsonNode updated = awsActionJson(GLUE, "PutResourcePolicy",
+ "{\"PolicyInJson\": " + quoted(POLICY_V2) + ", \"PolicyExistsCondition\": \"MUST_EXIST\","
+ + " \"PolicyHashCondition\": " + quoted(policyHash) + ", \"EnableHybrid\": \"TRUE\"}");
+ policyHash = updated.get("PolicyHash").asText();
+ awsAction(GLUE, "GetResourcePolicies", "{}")
+ .then()
+ .statusCode(200)
+ .body("GetResourcePoliciesResponseList.size()", equalTo(1))
+ .body("GetResourcePoliciesResponseList[0].PolicyInJson", equalTo(POLICY_V2));
+ }
+
+ @Test
+ @Order(3)
+ void deleteResourcePolicy_wrongHashRefused_rightHashDeletes() {
+ awsAction(GLUE, "DeleteResourcePolicy", "{\"PolicyHashCondition\": \"stale\"}")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ConditionCheckFailureException"));
+ awsAction(GLUE, "DeleteResourcePolicy", "{\"PolicyHashCondition\": " + quoted(policyHash) + "}")
+ .then()
+ .statusCode(200);
+ awsAction(GLUE, "DeleteResourcePolicy", "{}")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("EntityNotFoundException"));
+ }
+
+ @Test
+ @Order(4)
+ void encryptionSettings_defaultOff_thenEncryptConnectionPasswordsWithARealKmsKey() throws Exception {
+ awsAction(GLUE, "GetDataCatalogEncryptionSettings", "{}")
+ .then()
+ .statusCode(200)
+ .body("DataCatalogEncryptionSettings.EncryptionAtRest.CatalogEncryptionMode", equalTo("DISABLED"))
+ .body("DataCatalogEncryptionSettings.ConnectionPasswordEncryption.ReturnConnectionPasswordEncrypted", equalTo(false));
+
+ String keyId = awsActionJson(KMS, "CreateKey", "{\"Description\": \"glue connection passwords\"}")
+ .path("KeyMetadata").path("KeyId").asText();
+ awsAction(GLUE, "PutDataCatalogEncryptionSettings", """
+ {"DataCatalogEncryptionSettings": {
+ "EncryptionAtRest": {"CatalogEncryptionMode": "DISABLED"},
+ "ConnectionPasswordEncryption": {"ReturnConnectionPasswordEncrypted": true, "AwsKmsKeyId": "%s"}
+ }}
+ """.formatted(keyId))
+ .then().statusCode(200);
+ awsAction(GLUE, "GetDataCatalogEncryptionSettings", "{}")
+ .then()
+ .statusCode(200)
+ .body("DataCatalogEncryptionSettings.ConnectionPasswordEncryption.ReturnConnectionPasswordEncrypted", equalTo(true))
+ .body("DataCatalogEncryptionSettings.ConnectionPasswordEncryption.AwsKmsKeyId", equalTo(keyId));
+
+ String name = "enc-" + UUID.randomUUID().toString().substring(0, 8);
+ awsAction(GLUE, "CreateConnection", """
+ {"ConnectionInput": {"Name": "%s", "ConnectionType": "JDBC",
+ "ConnectionProperties": {"JDBC_CONNECTION_URL": "jdbc:postgresql://db:5432/orders", "USERNAME": "app", "PASSWORD": "s3cret"}}}
+ """.formatted(name))
+ .then().statusCode(200);
+
+ JsonNode connection = awsActionJson(GLUE, "GetConnection", "{\"Name\": \"" + name + "\"}").path("Connection");
+ JsonNode properties = connection.path("ConnectionProperties");
+ assertEquals("app", properties.path("USERNAME").asText());
+ assertEquals(false, properties.has("PASSWORD"));
+ String encrypted = properties.path("ENCRYPTED_PASSWORD").asText();
+ // The stored value is a real KMS ciphertext: KMS itself decrypts it back to the password.
+ JsonNode decrypted = awsActionJson(KMS, "Decrypt", "{\"CiphertextBlob\": " + quoted(encrypted) + "}");
+ assertEquals("s3cret", new String(Base64.getDecoder().decode(decrypted.path("Plaintext").asText()), StandardCharsets.UTF_8));
+
+ awsAction(GLUE, "GetConnection", "{\"Name\": \"" + name + "\", \"HidePassword\": true}")
+ .then()
+ .statusCode(200)
+ .body("Connection.ConnectionProperties", not(hasKey("ENCRYPTED_PASSWORD")))
+ .body("Connection.ConnectionProperties", not(hasKey("PASSWORD")));
+
+ awsAction(GLUE, "DeleteConnection", "{\"ConnectionName\": \"" + name + "\"}").then().statusCode(200);
+ // Put the catalog back so other classes see the default.
+ awsAction(GLUE, "PutDataCatalogEncryptionSettings", """
+ {"DataCatalogEncryptionSettings": {
+ "EncryptionAtRest": {"CatalogEncryptionMode": "DISABLED"},
+ "ConnectionPasswordEncryption": {"ReturnConnectionPasswordEncrypted": false}
+ }}
+ """)
+ .then().statusCode(200);
+ }
+
+ @Test
+ @Order(5)
+ void putDataCatalogEncryptionSettings_unknownMode_isInvalidInput() {
+ awsAction(GLUE, "PutDataCatalogEncryptionSettings",
+ "{\"DataCatalogEncryptionSettings\": {\"EncryptionAtRest\": {\"CatalogEncryptionMode\": \"AES\"}}}")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidInputException"));
+ }
+
+ private static String quoted(String value) {
+ return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"") + "\"";
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java
index 6188bff544..a60827decd 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java
@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;
+import io.github.hectorvent.floci.services.kms.KmsService;
import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;
import jakarta.ws.rs.core.Response;
import org.junit.jupiter.api.BeforeEach;
@@ -41,7 +42,8 @@ class GlueJsonHandlerEmptyListTest {
GlueSchemaRegistryService schemaRegistryService =
new GlueSchemaRegistryService(storageFactory, regionResolver);
GlueService glueService = new GlueService(
- storageFactory, schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(storageFactory));
+ storageFactory, schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(storageFactory),
+ new KmsService(storageFactory, regionResolver));
handler = new GlueJsonHandler(glueService, schemaRegistryService, mapper);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
index 0843b32551..7326862cfa 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;
+import io.github.hectorvent.floci.services.kms.KmsService;
import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;
import jakarta.ws.rs.core.Response;
import org.junit.jupiter.api.BeforeEach;
@@ -41,7 +42,8 @@ class GlueJsonHandlerTest {
GlueSchemaRegistryService schemaRegistryService =
new GlueSchemaRegistryService(storageFactory, regionResolver);
GlueService glueService = new GlueService(
- storageFactory, schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(storageFactory));
+ storageFactory, schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(storageFactory),
+ new KmsService(storageFactory, regionResolver));
handler = new GlueJsonHandler(glueService, schemaRegistryService, mapper);
}
@@ -737,4 +739,58 @@ class GlueJsonHandlerTest {
inline.putObject("ConnectionProperties").put("JDBC_CONNECTION_URL", "jdbc:mysql://h:3306/d");
assertEquals(200, handler.handle("TestConnection", test, REGION).getStatus());
}
+
+ @Test
+ void resourcePolicyOperationsAnswerWithTheDocumentedBodies() throws Exception {
+ String policy = "{\"Version\":\"2012-10-17\",\"Statement\":[]}";
+ ObjectNode put = mapper.createObjectNode();
+ put.put("PolicyInJson", policy);
+ put.put("PolicyExistsCondition", "NOT_EXIST");
+ JsonNode putBody = mapper.valueToTree(handler.handle("PutResourcePolicy", put, REGION).getEntity());
+ String hash = putBody.get("PolicyHash").asText();
+ assertFalse(hash.isBlank());
+
+ JsonNode got = mapper.valueToTree(handler.handle("GetResourcePolicy", mapper.createObjectNode(), REGION).getEntity());
+ assertEquals(policy, got.get("PolicyInJson").asText());
+ assertEquals(hash, got.get("PolicyHash").asText());
+ assertTrue(got.get("CreateTime").isNumber());
+ assertTrue(got.get("UpdateTime").isNumber());
+
+ JsonNode list = mapper.valueToTree(handler.handle("GetResourcePolicies", mapper.createObjectNode(), REGION).getEntity());
+ assertEquals(1, list.get("GetResourcePoliciesResponseList").size());
+ assertEquals(hash, list.get("GetResourcePoliciesResponseList").get(0).get("PolicyHash").asText());
+ assertFalse(list.has("NextToken"));
+
+ AwsException conflict = assertThrows(AwsException.class, () -> handler.handle("PutResourcePolicy", put, REGION));
+ assertEquals("ConditionCheckFailureException", conflict.getErrorCode());
+
+ Response deleted = handler.handle("DeleteResourcePolicy", mapper.createObjectNode(), REGION);
+ assertEquals(0, mapper.valueToTree(deleted.getEntity()).size());
+ AwsException gone = assertThrows(AwsException.class,
+ () -> handler.handle("GetResourcePolicy", mapper.createObjectNode(), REGION));
+ assertEquals("EntityNotFoundException", gone.getErrorCode());
+ }
+
+ @Test
+ void encryptionSettingsReportBothBlocksBeforeAndAfterAPut() throws Exception {
+ JsonNode defaults = mapper.valueToTree(
+ handler.handle("GetDataCatalogEncryptionSettings", mapper.createObjectNode(), REGION).getEntity());
+ JsonNode settings = defaults.get("DataCatalogEncryptionSettings");
+ assertEquals("DISABLED", settings.get("EncryptionAtRest").get("CatalogEncryptionMode").asText());
+ assertFalse(settings.get("EncryptionAtRest").has("SseAwsKmsKeyId"));
+ assertFalse(settings.get("ConnectionPasswordEncryption").get("ReturnConnectionPasswordEncrypted").asBoolean());
+
+ ObjectNode put = mapper.createObjectNode();
+ ObjectNode block = put.putObject("DataCatalogEncryptionSettings").putObject("ConnectionPasswordEncryption");
+ block.put("ReturnConnectionPasswordEncrypted", true);
+ block.put("AwsKmsKeyId", "alias/glue");
+ assertEquals(0, mapper.valueToTree(handler.handle("PutDataCatalogEncryptionSettings", put, REGION).getEntity()).size());
+
+ JsonNode after = mapper.valueToTree(
+ handler.handle("GetDataCatalogEncryptionSettings", mapper.createObjectNode(), REGION).getEntity())
+ .get("DataCatalogEncryptionSettings");
+ assertTrue(after.get("ConnectionPasswordEncryption").get("ReturnConnectionPasswordEncrypted").asBoolean());
+ assertEquals("alias/glue", after.get("ConnectionPasswordEncryption").get("AwsKmsKeyId").asText());
+ assertEquals("DISABLED", after.get("EncryptionAtRest").get("CatalogEncryptionMode").asText());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
index d4e0794c77..261686a1b8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -12,6 +12,10 @@ import io.github.hectorvent.floci.services.glue.model.Column;
import io.github.hectorvent.floci.services.glue.model.Database;
import io.github.hectorvent.floci.services.glue.model.Connection;
import io.github.hectorvent.floci.services.glue.model.ConnectionInput;
+import io.github.hectorvent.floci.services.glue.model.ConnectionPasswordEncryption;
+import io.github.hectorvent.floci.services.glue.model.DataCatalogEncryptionSettings;
+import io.github.hectorvent.floci.services.glue.model.EncryptionAtRest;
+import io.github.hectorvent.floci.services.glue.model.GluePolicy;
import io.github.hectorvent.floci.services.glue.model.Crawler;
import io.github.hectorvent.floci.services.glue.model.AuthenticationConfiguration;
import io.github.hectorvent.floci.services.glue.model.PhysicalConnectionRequirements;
@@ -28,6 +32,7 @@ import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
import io.github.hectorvent.floci.services.glue.model.Table;
import io.github.hectorvent.floci.services.glue.model.UserDefinedFunction;
import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;
+import io.github.hectorvent.floci.services.kms.KmsService;
import io.github.hectorvent.floci.services.glue.schemaregistry.model.RegistryId;
import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;
import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;
@@ -35,6 +40,8 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.time.Instant;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -44,6 +51,7 @@ import java.util.concurrent.CountDownLatch;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -65,6 +73,7 @@ class GlueServiceTest {
private GlueService glueService;
private GlueSchemaRegistryService schemaRegistryService;
+ private KmsService kmsService;
private StorageBackend<String, Database> databaseStore;
private StorageBackend<String, Table> tableStore;
private StorageBackend<String, Table> tableVersionStore;
@@ -76,6 +85,7 @@ class GlueServiceTest {
RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT_ID);
StorageFactory storageFactory = new InMemoryStorageFactory();
schemaRegistryService = new GlueSchemaRegistryService(storageFactory, regionResolver);
+ kmsService = new KmsService(storageFactory, regionResolver);
databaseStore = new InMemoryStorage<>();
tableStore = new InMemoryStorage<>();
tableVersionStore = new InMemoryStorage<>();
@@ -94,7 +104,10 @@ class GlueServiceTest {
new InMemoryStorage<String, Crawler>(),
new InMemoryStorage<String, Classifier>(),
new InMemoryStorage<String, Connection>(),
- schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(null));
+ new InMemoryStorage<String, GluePolicy>(),
+ new InMemoryStorage<String, DataCatalogEncryptionSettings>(),
+ schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(null),
+ kmsService);
glueService.createDatabase(new Database("db1"));
}
@@ -1528,7 +1541,7 @@ class GlueServiceTest {
redefinition.setName("orders");
redefinition.setConnectionType("JDBC");
redefinition.setConnectionProperties(Map.of("JDBC_CONNECTION_URL", "jdbc:postgresql://db2.internal:5432/orders"));
- glueService.updateConnection("orders", redefinition);
+ glueService.updateConnection("orders", redefinition, REGION);
Connection updated = glueService.getConnection("orders", false);
assertEquals("jdbc:postgresql://db2.internal:5432/orders", updated.getConnectionProperties().get("JDBC_CONNECTION_URL"));
@@ -1553,7 +1566,7 @@ class GlueServiceTest {
assertEquals("AlreadyExistsException", duplicate.getErrorCode());
AwsException missing = assertThrows(AwsException.class,
- () -> glueService.updateConnection("absent", jdbcConnection("absent")));
+ () -> glueService.updateConnection("absent", jdbcConnection("absent"), REGION));
assertEquals("EntityNotFoundException", missing.getErrorCode());
AwsException missingDelete = assertThrows(AwsException.class,
@@ -1720,6 +1733,176 @@ class GlueServiceTest {
assertThrows(AwsException.class, () -> glueService.getTags(arn, REGION)).getErrorCode());
}
+ // ---- Catalog resource policy and encryption settings -----------------------------------
+
+ private static final String POLICY_V1 = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::111122223333:root\"},\"Action\":\"glue:GetTable\",\"Resource\":\"*\"}]}";
+ private static final String POLICY_V2 = POLICY_V1.replace("glue:GetTable", "glue:GetTables");
+
+ @Test
+ void resourcePolicyFollowsTheTerraformSequenceOfConditions() {
+ assertEquals("EntityNotFoundException",
+ assertThrows(AwsException.class, () -> glueService.getResourcePolicy()).getErrorCode());
+ assertTrue(glueService.getResourcePolicies(null, null).items().isEmpty());
+
+ // Create: NOT_EXIST must pass on an empty catalog.
+ String hash = glueService.putResourcePolicy(POLICY_V1, null, "NOT_EXIST", null);
+ GluePolicy stored = glueService.getResourcePolicy();
+ assertEquals(POLICY_V1, stored.getPolicyInJson());
+ assertEquals(hash, stored.getPolicyHash());
+ assertNotNull(stored.getCreateTime());
+ assertEquals(stored.getCreateTime(), stored.getUpdateTime());
+ assertEquals(1, glueService.getResourcePolicies(null, null).items().size());
+
+ // A second create must fail, an update must pass and keep CreateTime.
+ assertEquals("ConditionCheckFailureException",
+ assertThrows(AwsException.class,
+ () -> glueService.putResourcePolicy(POLICY_V2, null, "NOT_EXIST", null)).getErrorCode());
+ String hash2 = glueService.putResourcePolicy(POLICY_V2, hash, "MUST_EXIST", "TRUE");
+ GluePolicy updated = glueService.getResourcePolicy();
+ assertEquals(POLICY_V2, updated.getPolicyInJson());
+ assertNotEquals(hash, hash2);
+ assertEquals(stored.getCreateTime(), updated.getCreateTime());
+ assertFalse(updated.getUpdateTime().isBefore(stored.getUpdateTime()));
+
+ // A stale hash is refused on put and on delete; the right one is accepted.
+ assertEquals("ConditionCheckFailureException",
+ assertThrows(AwsException.class,
+ () -> glueService.putResourcePolicy(POLICY_V1, hash, "NONE", null)).getErrorCode());
+ assertEquals("ConditionCheckFailureException",
+ assertThrows(AwsException.class, () -> glueService.deleteResourcePolicy(hash)).getErrorCode());
+ glueService.deleteResourcePolicy(hash2);
+ assertEquals("EntityNotFoundException",
+ assertThrows(AwsException.class, () -> glueService.deleteResourcePolicy(null)).getErrorCode());
+ assertEquals("ConditionCheckFailureException",
+ assertThrows(AwsException.class,
+ () -> glueService.putResourcePolicy(POLICY_V1, null, "MUST_EXIST", null)).getErrorCode());
+ }
+
+ @Test
+ void resourcePolicyInputIsValidated() {
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.putResourcePolicy(null, null, null, null)).getErrorCode());
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.putResourcePolicy("{", null, null, null)).getErrorCode());
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.putResourcePolicy("[]", null, null, null)).getErrorCode());
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.putResourcePolicy(POLICY_V1, null, "MAYBE", null)).getErrorCode());
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.putResourcePolicy(POLICY_V1, null, null, "yes")).getErrorCode());
+ // The same document always hashes the same, so a client can compare hashes across reads.
+ assertEquals(glueService.putResourcePolicy(POLICY_V1, null, null, null),
+ glueService.putResourcePolicy(POLICY_V1, null, null, null));
+ }
+
+ @Test
+ void encryptionSettingsDefaultToOffAndAreReplacedByPut() {
+ DataCatalogEncryptionSettings defaults = glueService.getDataCatalogEncryptionSettings();
+ assertEquals("DISABLED", defaults.getEncryptionAtRest().getCatalogEncryptionMode());
+ assertEquals(false, defaults.getConnectionPasswordEncryption().getReturnConnectionPasswordEncrypted());
+ assertNull(defaults.getConnectionPasswordEncryption().getAwsKmsKeyId());
+
+ DataCatalogEncryptionSettings put = new DataCatalogEncryptionSettings();
+ EncryptionAtRest atRest = new EncryptionAtRest();
+ atRest.setCatalogEncryptionMode("SSE-KMS");
+ atRest.setSseAwsKmsKeyId("alias/catalog");
+ put.setEncryptionAtRest(atRest);
+ glueService.putDataCatalogEncryptionSettings(put);
+
+ DataCatalogEncryptionSettings read = glueService.getDataCatalogEncryptionSettings();
+ assertEquals("SSE-KMS", read.getEncryptionAtRest().getCatalogEncryptionMode());
+ assertEquals("alias/catalog", read.getEncryptionAtRest().getSseAwsKmsKeyId());
+ // The block left out of the put is reported with its default, not dropped.
+ assertEquals(false, read.getConnectionPasswordEncryption().getReturnConnectionPasswordEncrypted());
+
+ EncryptionAtRest badMode = new EncryptionAtRest();
+ badMode.setCatalogEncryptionMode("AES");
+ DataCatalogEncryptionSettings invalid = new DataCatalogEncryptionSettings();
+ invalid.setEncryptionAtRest(badMode);
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.putDataCatalogEncryptionSettings(invalid)).getErrorCode());
+ DataCatalogEncryptionSettings missingFlag = new DataCatalogEncryptionSettings();
+ missingFlag.setConnectionPasswordEncryption(new ConnectionPasswordEncryption());
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.putDataCatalogEncryptionSettings(missingFlag)).getErrorCode());
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.putDataCatalogEncryptionSettings(null)).getErrorCode());
+ }
+
+ private void enableConnectionPasswordEncryption(String keyId) {
+ ConnectionPasswordEncryption passwords = new ConnectionPasswordEncryption();
+ passwords.setReturnConnectionPasswordEncrypted(true);
+ passwords.setAwsKmsKeyId(keyId);
+ DataCatalogEncryptionSettings settings = new DataCatalogEncryptionSettings();
+ settings.setConnectionPasswordEncryption(passwords);
+ glueService.putDataCatalogEncryptionSettings(settings);
+ }
+
+ @Test
+ void connectionPasswordsAreStoredEncryptedOnceTheCatalogSettingIsOn() {
+ String keyId = kmsService.createKey("glue connection passwords", REGION).getKeyId();
+ enableConnectionPasswordEncryption(keyId);
+
+ ConnectionInput jdbc = jdbcConnection("enc-jdbc");
+ glueService.createConnection(jdbc, null, REGION);
+ Connection stored = glueService.getConnection("enc-jdbc", false);
+ assertNull(stored.getConnectionProperties().get("PASSWORD"));
+ String encrypted = stored.getConnectionProperties().get("ENCRYPTED_PASSWORD");
+ assertNotNull(encrypted);
+ assertEquals("s3cret", new String(
+ kmsService.decrypt(Base64.getDecoder().decode(encrypted), REGION), StandardCharsets.UTF_8));
+ assertEquals("app", stored.getConnectionProperties().get("USERNAME"));
+ // HidePassword removes the encrypted form as it removes the plaintext one.
+ assertNull(glueService.getConnection("enc-jdbc", true).getConnectionProperties().get("ENCRYPTED_PASSWORD"));
+
+ ConnectionInput kafka = jdbcConnection("enc-kafka");
+ kafka.setConnectionType("KAFKA");
+ Map<String, String> properties = new LinkedHashMap<>();
+ properties.put("KAFKA_BOOTSTRAP_SERVERS", "broker:9092");
+ properties.put("KAFKA_SASL_MECHANISM", "SCRAM-SHA-512");
+ properties.put("KAFKA_SASL_SCRAM_USERNAME", "svc");
+ properties.put("KAFKA_SASL_SCRAM_PASSWORD", "scram-pw");
+ properties.put("KAFKA_CLIENT_KEYSTORE_PASSWORD", "ks-pw");
+ kafka.setConnectionProperties(properties);
+ glueService.createConnection(kafka, null, REGION);
+ Map<String, String> kafkaStored = glueService.getConnection("enc-kafka", false).getConnectionProperties();
+ assertNull(kafkaStored.get("KAFKA_SASL_SCRAM_PASSWORD"));
+ assertNull(kafkaStored.get("KAFKA_CLIENT_KEYSTORE_PASSWORD"));
+ assertNotNull(kafkaStored.get("ENCRYPTED_KAFKA_SASL_SCRAM_PASSWORD"));
+ assertNotNull(kafkaStored.get("ENCRYPTED_KAFKA_CLIENT_KEYSTORE_PASSWORD"));
+ assertEquals("svc", kafkaStored.get("KAFKA_SASL_SCRAM_USERNAME"));
+
+ // An update goes through the same path.
+ ConnectionInput redefinition = jdbcConnection("enc-jdbc");
+ redefinition.setConnectionProperties(Map.of("JDBC_CONNECTION_URL", "jdbc:x://h/d", "PASSWORD", "changed"));
+ glueService.updateConnection("enc-jdbc", redefinition, REGION);
+ Map<String, String> afterUpdate = glueService.getConnection("enc-jdbc", false).getConnectionProperties();
+ assertNull(afterUpdate.get("PASSWORD"));
+ assertEquals("changed", new String(kmsService.decrypt(
+ Base64.getDecoder().decode(afterUpdate.get("ENCRYPTED_PASSWORD")), REGION), StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void connectionPasswordEncryptionFailuresAreGlueEncryptionExceptions() {
+ enableConnectionPasswordEncryption("arn:aws:kms:us-east-1:000000000000:key/00000000-0000-0000-0000-000000000000");
+ AwsException unknownKey = assertThrows(AwsException.class,
+ () -> glueService.createConnection(jdbcConnection("enc-bad-key"), null, REGION));
+ assertEquals("GlueEncryptionException", unknownKey.getErrorCode());
+ assertEquals("EntityNotFoundException",
+ assertThrows(AwsException.class, () -> glueService.getConnection("enc-bad-key", false)).getErrorCode());
+
+ enableConnectionPasswordEncryption(null);
+ assertEquals("GlueEncryptionException",
+ assertThrows(AwsException.class,
+ () -> glueService.createConnection(jdbcConnection("enc-no-key"), null, REGION)).getErrorCode());
+
+ // A connection without any password property is unaffected by the setting.
+ ConnectionInput network = jdbcConnection("enc-network");
+ network.setConnectionType("NETWORK");
+ network.setConnectionProperties(Map.of());
+ assertEquals("READY", glueService.createConnection(network, null, REGION));
+ }
+
private static List<String> names(GlueService.Page<Table> page) {
return page.items().stream().map(Table::getName).toList();
}