Light, fluffy, and always free - The AWS Local Emulator alternative
feat(glue): catalog resource policy and Data Catalog encryption settings (#4027)
David628952ecf06b
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.javaindex ee5bebef7f..3afd35859c 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java@@ -7,12 +7,21 @@ import org.junit.jupiter.api.Test;import software.amazon.awssdk.services.glue.GlueClient;import software.amazon.awssdk.services.glue.model.BatchCreatePartitionRequest;import software.amazon.awssdk.services.glue.model.BatchDeleteConnectionRequest;+import software.amazon.awssdk.services.glue.model.ConditionCheckFailureException;+import software.amazon.awssdk.services.glue.model.DeleteResourcePolicyRequest;+import software.amazon.awssdk.services.glue.model.ExistCondition;+import software.amazon.awssdk.services.glue.model.GetResourcePoliciesRequest;+import software.amazon.awssdk.services.glue.model.GetResourcePolicyRequest;+import software.amazon.awssdk.services.glue.model.GetResourcePolicyResponse;+import software.amazon.awssdk.services.glue.model.GetDataCatalogEncryptionSettingsRequest;+import software.amazon.awssdk.services.glue.model.PutResourcePolicyRequest;import software.amazon.awssdk.services.glue.model.BatchDeleteConnectionResponse;import software.amazon.awssdk.services.glue.model.Connection;import software.amazon.awssdk.services.glue.model.ConnectionInput;import software.amazon.awssdk.services.glue.model.ConnectionPropertyKey;import software.amazon.awssdk.services.glue.model.ConnectionType;import software.amazon.awssdk.services.glue.model.CreateConnectionRequest;+import software.amazon.awssdk.services.glue.model.DataCatalogEncryptionSettings;import software.amazon.awssdk.services.glue.model.CreateConnectionResponse;import software.amazon.awssdk.services.glue.model.DeleteConnectionRequest;import software.amazon.awssdk.services.glue.model.GetConnectionRequest;@@ -640,6 +649,55 @@ class GlueCatalogTest {assertThat(batch.errors()).containsOnlyKeys(CONNECTION_NAME);}+ @Test+ @DisplayName("The catalog resource policy follows the create, update and delete conditions Terraform sends")+ void catalogResourcePolicyLifecycle() {+ String policy = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\","+ + "\"Principal\":{\"AWS\":\"arn:aws:iam::111122223333:root\"},\"Action\":\"glue:GetTable\",\"Resource\":\"*\"}]}";+ String policyV2 = policy.replace("glue:GetTable", "glue:GetTables");+ // Start from no policy; the catalog is shared with other tests.+ try {+ glue.deleteResourcePolicy(DeleteResourcePolicyRequest.builder().build());+ }+ catch (EntityNotFoundException ignored) {}++ String hash = glue.putResourcePolicy(PutResourcePolicyRequest.builder()+ .policyInJson(policy)+ .policyExistsCondition(ExistCondition.NOT_EXIST)+ .build()).policyHash();+ GetResourcePolicyResponse read = glue.getResourcePolicy(GetResourcePolicyRequest.builder().build());+ assertThat(read.policyInJson()).isEqualTo(policy);+ assertThat(read.policyHash()).isEqualTo(hash);+ assertThat(read.createTime()).isNotNull();+ assertThat(read.updateTime()).isNotNull();++ assertThatThrownBy(() -> glue.putResourcePolicy(PutResourcePolicyRequest.builder()+ .policyInJson(policyV2)+ .policyExistsCondition(ExistCondition.NOT_EXIST)+ .build()))+ .isInstanceOf(ConditionCheckFailureException.class);++ String hash2 = glue.putResourcePolicy(PutResourcePolicyRequest.builder()+ .policyInJson(policyV2)+ .policyExistsCondition(ExistCondition.MUST_EXIST)+ .policyHashCondition(hash)+ .build()).policyHash();+ assertThat(glue.getResourcePolicy(GetResourcePolicyRequest.builder().build()).policyInJson()).isEqualTo(policyV2);+ assertThat(glue.getResourcePolicies(GetResourcePoliciesRequest.builder().build()).getResourcePoliciesResponseList()).hasSize(1);++ assertThatThrownBy(() -> glue.deleteResourcePolicy(DeleteResourcePolicyRequest.builder()+ .policyHashCondition(hash).build()))+ .isInstanceOf(ConditionCheckFailureException.class);+ glue.deleteResourcePolicy(DeleteResourcePolicyRequest.builder().policyHashCondition(hash2).build());+ assertThatThrownBy(() -> glue.getResourcePolicy(GetResourcePolicyRequest.builder().build()))+ .isInstanceOf(EntityNotFoundException.class);++ DataCatalogEncryptionSettings settings = glue.getDataCatalogEncryptionSettings(GetDataCatalogEncryptionSettingsRequest.builder().build())+ .dataCatalogEncryptionSettings();+ assertThat(settings.encryptionAtRest().catalogEncryptionModeAsString()).isEqualTo("DISABLED");+ assertThat(settings.connectionPasswordEncryption().returnConnectionPasswordEncrypted()).isFalse();+ }+private static TableInput tableInput(String description) {return tableInput(TABLE_NAME, description);}diff --git a/docs/services/glue.md b/docs/services/glue.mdindex 9e0f41c26b..770c038361 100644--- a/docs/services/glue.md+++ b/docs/services/glue.md@@ -86,11 +86,44 @@ under `AuthenticationConfiguration` are accepted and never returned by a read. A`HidePassword` removes `PASSWORD` and `ENCRYPTED_PASSWORD`, the two members the API reference defines as theconnection's password. The Kafka credentials (`KAFKA_CLIENT_KEYSTORE_PASSWORD`, `KAFKA_CLIENT_KEY_PASSWORD`,`KAFKA_SASL_PLAIN_PASSWORD`, `KAFKA_SASL_SCRAM_PASSWORD` and their `ENCRYPTED_` forms) are returned as stored: the-reference does not say whether the flag covers them, so Floci does not guess.+reference does not say whether the flag covers them, so Floci does not guess. The `ENCRYPTED_` forms are what the+catalog's `ConnectionPasswordEncryption` setting produces (see Encryption settings below).`TestConnection` is asynchronous on AWS and returns nothing; Floci checks the request's shape and accepts itwithout opening a socket to the data store, since no job or crawler runs against a connection yet.+#### Resource policy++| Action | Description |+|--------|-------------|+| PutResourcePolicy | Sets the catalog's resource policy from `PolicyInJson` and returns its `PolicyHash`. `PolicyExistsCondition` (`NOT_EXIST`, `MUST_EXIST`, `NONE`) and `PolicyHashCondition` are checked against the stored policy and fail with `ConditionCheckFailureException`. |+| GetResourcePolicy | Returns the catalog policy with its hash and timestamps, or `EntityNotFoundException` when none is set. |+| GetResourcePolicies | Lists the catalog policy (zero or one entry) under `GetResourcePoliciesResponseList`, paged. |+| DeleteResourcePolicy | Deletes the catalog policy; honours `PolicyHashCondition`; `EntityNotFoundException` when none is set. |++The catalog holds one policy, as on AWS. `PolicyInJson` must be a JSON object. AWS documents `PolicyHash` only as+an opaque value to echo back; Floci derives it from the document, so the same policy always has the same hash.+`ResourceArn` ("for internal use only" in the reference) and `EnableHybrid` are accepted. Per-resource policies+that Resource Access Manager creates on AWS are not emulated, so `GetResourcePolicies` never has more than one entry.++#### Encryption settings++| Action | Description |+|--------|-------------|+| GetDataCatalogEncryptionSettings | Returns the catalog's security configuration: `EncryptionAtRest` (`CatalogEncryptionMode` `DISABLED` by default) and `ConnectionPasswordEncryption` (`ReturnConnectionPasswordEncrypted` false by default). |+| PutDataCatalogEncryptionSettings | Replaces the configuration. `CatalogEncryptionMode` is `DISABLED`, `SSE-KMS` or `SSE-KMS-WITH-SERVICE-ROLE`; `ReturnConnectionPasswordEncrypted` is required inside its block. A block left out keeps its default. |++`EncryptionAtRest` is stored and reported; catalog metadata is not encrypted on disk, which nothing observes through+the API. `ConnectionPasswordEncryption` is applied: while `ReturnConnectionPasswordEncrypted` is true, a connection+created or updated has each password property encrypted with `AwsKmsKeyId` through Floci's KMS and stored under the+name the `Connection` structure documents for it (`PASSWORD` as `ENCRYPTED_PASSWORD`, and the four Kafka passwords as+their `ENCRYPTED_KAFKA_*` forms). Reads return the ciphertext, base64-encoded, which `KMS.Decrypt` turns back into+the password. A missing key, or none configured, fails the create or update with `GlueEncryptionException`, the error+`CreateConnection` and `UpdateConnection` list for a failed encryption operation. Connections created before the setting+was switched on keep their plaintext: the developer guide ("Encrypting connection passwords") states that whether a+password is encrypted "was determined when the connection was created or updated", so the setting is not applied+retroactively.+#### Jobs| Action | Description |diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.javaindex 4179ddfa76..83cf2b8832 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java@@ -289,7 +289,7 @@ public class GlueJsonHandler {case "CreateConnection" -> handleCreateConnection(request, region);case "GetConnection" -> handleGetConnection(request);case "GetConnections" -> handleGetConnections(request);- case "UpdateConnection" -> handleUpdateConnection(request);+ case "UpdateConnection" -> handleUpdateConnection(request, region);case "DeleteConnection" -> {glueService.deleteConnection(request.path("ConnectionName").asText(null), region);yield Response.ok(Map.of()).build();@@ -301,6 +301,33 @@ public class GlueJsonHandler {yield Response.ok(glueService.batchDeleteConnections(names, region)).build();}case "TestConnection" -> handleTestConnection(request);+ case "PutResourcePolicy" -> {+ String hash = glueService.putResourcePolicy(+ request.path("PolicyInJson").asText(null),+ request.path("PolicyHashCondition").asText(null),+ request.path("PolicyExistsCondition").asText(null),+ request.path("EnableHybrid").asText(null));+ yield Response.ok(Map.of("PolicyHash", hash)).build();+ }+ case "GetResourcePolicy" -> Response.ok(glueService.getResourcePolicy()).build();+ case "GetResourcePolicies" -> {+ GlueService.Page<GluePolicy> page =+ glueService.getResourcePolicies(readMaxResults(request), readNextToken(request));+ yield Response.ok(pageResponse("GetResourcePoliciesResponseList", page.items(), page.nextToken())).build();+ }+ case "DeleteResourcePolicy" -> {+ glueService.deleteResourcePolicy(request.path("PolicyHashCondition").asText(null));+ yield Response.ok(Map.of()).build();+ }+ case "GetDataCatalogEncryptionSettings" -> Response.ok(Map.of(+ "DataCatalogEncryptionSettings", glueService.getDataCatalogEncryptionSettings())).build();+ case "PutDataCatalogEncryptionSettings" -> {+ DataCatalogEncryptionSettings settings = request.hasNonNull("DataCatalogEncryptionSettings")+ ? mapper.treeToValue(request.get("DataCatalogEncryptionSettings"), DataCatalogEncryptionSettings.class)+ : null;+ glueService.putDataCatalogEncryptionSettings(settings);+ yield Response.ok(Map.of()).build();+ }// Read-only Glue actions for resources the emulator does not model. The AWS SDK// expects each to return a 200 with its result key present (empty), so we emit the// documented empty shape rather than an InvalidAction 400 that callers can't read.@@ -907,11 +934,11 @@ public class GlueJsonHandler {return Response.ok(pageResponse("ConnectionList", page.items(), page.nextToken())).build();}- private Response handleUpdateConnection(JsonNode request) throws Exception {+ private Response handleUpdateConnection(JsonNode request, String region) throws Exception {ConnectionInput input = request.hasNonNull("ConnectionInput")? mapper.treeToValue(request.get("ConnectionInput"), ConnectionInput.class): null;- glueService.updateConnection(request.path("Name").asText(null), input);+ glueService.updateConnection(request.path("Name").asText(null), input, region);return Response.ok(Map.of()).build();}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.javaindex 4c357ee34d..16f52d8d8c 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java@@ -1,7 +1,10 @@package io.github.hectorvent.floci.services.glue;import com.fasterxml.jackson.annotation.JsonProperty;+import com.fasterxml.jackson.core.JsonProcessingException;import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.StorageBackend;@@ -10,6 +13,10 @@ import io.github.hectorvent.floci.services.glue.model.Classifier;import io.github.hectorvent.floci.services.glue.model.Column;import io.github.hectorvent.floci.services.glue.model.Connection;import io.github.hectorvent.floci.services.glue.model.ConnectionInput;+import io.github.hectorvent.floci.services.glue.model.ConnectionPasswordEncryption;+import io.github.hectorvent.floci.services.glue.model.DataCatalogEncryptionSettings;+import io.github.hectorvent.floci.services.glue.model.EncryptionAtRest;+import io.github.hectorvent.floci.services.glue.model.GluePolicy;import io.github.hectorvent.floci.services.glue.model.Crawler;import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;import io.github.hectorvent.floci.services.glue.model.Database;@@ -27,6 +34,7 @@ import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistrimport io.github.hectorvent.floci.services.glue.schemaregistry.SchemaToColumnsConverter;import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaVersion;+import io.github.hectorvent.floci.services.kms.KmsService;import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;import io.quarkus.runtime.annotations.RegisterForReflection;import jakarta.enterprise.context.ApplicationScoped;@@ -35,6 +43,8 @@ import org.jboss.logging.Logger;import java.math.BigDecimal;import java.nio.charset.StandardCharsets;+import java.security.MessageDigest;+import java.security.NoSuchAlgorithmException;import java.time.Instant;import java.util.ArrayList;import java.util.Base64;@@ -113,6 +123,23 @@ public class GlueService {"ENDPOINT_TYPE", "ROLE_ARN", "REGION", "WORKGROUP_NAME", "CLUSTER_IDENTIFIER", "DATABASE");private static final String CONNECTION_STATUS_READY = "READY";+ // The catalog holds one resource policy and one security configuration; both stores are+ // account-scoped by StorageFactory, so a fixed key is the whole address.+ private static final String CATALOG_KEY = "catalog";+ private static final ObjectMapper POLICY_JSON = new ObjectMapper();+ private static final Set<String> POLICY_EXISTS_CONDITIONS = Set.of("MUST_EXIST", "NOT_EXIST", "NONE");+ private static final Set<String> ENABLE_HYBRID_VALUES = Set.of("TRUE", "FALSE");+ private static final Set<String> CATALOG_ENCRYPTION_MODES = Set.of(+ "DISABLED", "SSE-KMS", "SSE-KMS-WITH-SERVICE-ROLE");+ // The Connection structure names the stored form of each password when the catalog's+ // ConnectionPasswordEncryption is on: the plaintext key is replaced by its ENCRYPTED_ twin.+ private static final Map<String, String> ENCRYPTED_CONNECTION_PROPERTY_KEYS = Map.of(+ "PASSWORD", "ENCRYPTED_PASSWORD",+ "KAFKA_CLIENT_KEYSTORE_PASSWORD", "ENCRYPTED_KAFKA_CLIENT_KEYSTORE_PASSWORD",+ "KAFKA_CLIENT_KEY_PASSWORD", "ENCRYPTED_KAFKA_CLIENT_KEY_PASSWORD",+ "KAFKA_SASL_PLAIN_PASSWORD", "ENCRYPTED_KAFKA_SASL_PLAIN_PASSWORD",+ "KAFKA_SASL_SCRAM_PASSWORD", "ENCRYPTED_KAFKA_SASL_SCRAM_PASSWORD");+// Glue's partition index states. FAILED also exists but is only reachable through a backfill// failure, which is not emulated.private static final String INDEX_STATUS_CREATING = "CREATING";@@ -131,15 +158,19 @@ public class GlueService {private final StorageBackend<String, Crawler> crawlerStore;private final StorageBackend<String, Classifier> classifierStore;private final StorageBackend<String, Connection> connectionStore;+ private final StorageBackend<String, GluePolicy> resourcePolicyStore;+ private final StorageBackend<String, DataCatalogEncryptionSettings> encryptionSettingsStore;private final GlueSchemaRegistryService schemaRegistryService;private final RegionResolver regionResolver;private final ResourceGroupsTaggingService resourceGroupsTaggingService;+ private final KmsService kmsService;@Injectpublic GlueService(StorageFactory storageFactory,GlueSchemaRegistryService schemaRegistryService,RegionResolver regionResolver,- ResourceGroupsTaggingService resourceGroupsTaggingService) {+ ResourceGroupsTaggingService resourceGroupsTaggingService,+ KmsService kmsService) {this.databaseStore = storageFactory.create("glue", "databases.json", new TypeReference<>() {});this.tableStore = storageFactory.create("glue", "tables.json", new TypeReference<>() {});this.tableVersionStore = storageFactory.create("glue", "table_versions.json", new TypeReference<>() {});@@ -154,9 +185,13 @@ public class GlueService {this.crawlerStore = storageFactory.create("glue", "crawlers.json", new TypeReference<>() {});this.classifierStore = storageFactory.create("glue", "classifiers.json", new TypeReference<>() {});this.connectionStore = storageFactory.create("glue", "connections.json", new TypeReference<>() {});+ this.resourcePolicyStore = storageFactory.create("glue", "resource_policy.json", new TypeReference<>() {});+ this.encryptionSettingsStore = storageFactory.create(+ "glue", "catalog_encryption_settings.json", new TypeReference<>() {});this.schemaRegistryService = schemaRegistryService;this.regionResolver = regionResolver;this.resourceGroupsTaggingService = resourceGroupsTaggingService;+ this.kmsService = kmsService;}GlueService(StorageBackend<String, Database> databaseStore,@@ -171,9 +206,12 @@ public class GlueService {StorageBackend<String, Crawler> crawlerStore,StorageBackend<String, Classifier> classifierStore,StorageBackend<String, Connection> connectionStore,+ StorageBackend<String, GluePolicy> resourcePolicyStore,+ StorageBackend<String, DataCatalogEncryptionSettings> encryptionSettingsStore,GlueSchemaRegistryService schemaRegistryService,RegionResolver regionResolver,- ResourceGroupsTaggingService resourceGroupsTaggingService) {+ ResourceGroupsTaggingService resourceGroupsTaggingService,+ KmsService kmsService) {this.databaseStore = databaseStore;this.tableStore = tableStore;this.tableVersionStore = tableVersionStore;@@ -186,9 +224,12 @@ public class GlueService {this.crawlerStore = crawlerStore;this.classifierStore = classifierStore;this.connectionStore = connectionStore;+ this.resourcePolicyStore = resourcePolicyStore;+ this.encryptionSettingsStore = encryptionSettingsStore;this.schemaRegistryService = schemaRegistryService;this.regionResolver = regionResolver;this.resourceGroupsTaggingService = resourceGroupsTaggingService;+ this.kmsService = kmsService;}public void createDatabase(Database database) {@@ -1951,6 +1992,7 @@ public class GlueService {}Instant now = Instant.now();Connection connection = toConnection(input);+ encryptConnectionPasswords(connection.getConnectionProperties(), region);connection.setCreationTime(now);connection.setLastUpdatedTime(now);connectionStore.put(name, connection);@@ -2002,13 +2044,14 @@ public class GlueService {* (API reference), so the stored definition is replaced rather than merged: a member left* out of the input is gone afterwards. The connection keeps its name and creation time.*/- public void updateConnection(String name, ConnectionInput input) {+ public void updateConnection(String name, ConnectionInput input, String region) {validateRequired(name, "Name");validateRequired(input, "ConnectionInput");Connection existing = connectionStore.get(name).orElseThrow(() -> new AwsException("EntityNotFoundException", "Connection " + name + " not found.", 400));validateConnectionInput(input);Connection updated = toConnection(input);+ encryptConnectionPasswords(updated.getConnectionProperties(), region);updated.setName(existing.getName());updated.setCreationTime(existing.getCreationTime());updated.setLastUpdatedTime(Instant.now());@@ -2136,6 +2179,173 @@ public class GlueService {@JsonProperty("Succeeded") List<String> succeeded,@JsonProperty("Errors") Map<String, ErrorDetail> errors) {}+ // ---- Catalog resource policy and encryption settings -----------------------------------++ /**+ * Sets the catalog's one resource policy. {@code PolicyExistsCondition} and+ * {@code PolicyHashCondition} are checked against the stored policy first and fail with+ * ConditionCheckFailureException, which is how Terraform's create (NOT_EXIST) and update+ * (MUST_EXIST) tell each other apart. Returns the new policy's hash.+ */+ public String putResourcePolicy(String policyInJson, String policyHashCondition,+ String policyExistsCondition, String enableHybrid) {+ validateRequired(policyInJson, "PolicyInJson");+ validatePolicyDocument(policyInJson);+ if (policyExistsCondition != null && !POLICY_EXISTS_CONDITIONS.contains(policyExistsCondition)) {+ throw new AwsException("InvalidInputException",+ "Unsupported PolicyExistsCondition: " + policyExistsCondition, 400);+ }+ if (enableHybrid != null && !ENABLE_HYBRID_VALUES.contains(enableHybrid)) {+ throw new AwsException("InvalidInputException", "Unsupported EnableHybrid value: " + enableHybrid, 400);+ }+ Optional<GluePolicy> existing = resourcePolicyStore.get(CATALOG_KEY);+ if ("NOT_EXIST".equals(policyExistsCondition) && existing.isPresent()) {+ throw new AwsException("ConditionCheckFailureException",+ "A resource policy already exists and PolicyExistsCondition is NOT_EXIST.", 400);+ }+ if ("MUST_EXIST".equals(policyExistsCondition) && existing.isEmpty()) {+ throw new AwsException("ConditionCheckFailureException",+ "No resource policy exists and PolicyExistsCondition is MUST_EXIST.", 400);+ }+ checkPolicyHashCondition(policyHashCondition, existing);++ Instant now = Instant.now();+ GluePolicy policy = new GluePolicy();+ policy.setPolicyInJson(policyInJson);+ policy.setPolicyHash(policyHash(policyInJson));+ policy.setCreateTime(existing.map(GluePolicy::getCreateTime).orElse(now));+ policy.setUpdateTime(now);+ resourcePolicyStore.put(CATALOG_KEY, policy);+ LOG.infov("Set Glue catalog resource policy (hash {0})", policy.getPolicyHash());+ return policy.getPolicyHash();+ }++ public GluePolicy getResourcePolicy() {+ return resourcePolicyStore.get(CATALOG_KEY)+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Policy not found", 400));+ }++ /** The catalog policy is the only entry; RAM-granted per-resource policies are not emulated. */+ public Page<GluePolicy> getResourcePolicies(Integer maxResults, String nextToken) {+ List<GluePolicy> policies = new ArrayList<>(resourcePolicyStore.get(CATALOG_KEY).stream().toList());+ return paginate(policies, maxResults, nextToken);+ }++ public void deleteResourcePolicy(String policyHashCondition) {+ Optional<GluePolicy> existing = resourcePolicyStore.get(CATALOG_KEY);+ if (existing.isEmpty()) {+ throw new AwsException("EntityNotFoundException", "Policy not found", 400);+ }+ checkPolicyHashCondition(policyHashCondition, existing);+ resourcePolicyStore.delete(CATALOG_KEY);+ LOG.info("Deleted Glue catalog resource policy");+ }++ /** Before anything is put, a catalog reports both blocks with encryption off. */+ public DataCatalogEncryptionSettings getDataCatalogEncryptionSettings() {+ return encryptionSettingsStore.get(CATALOG_KEY).orElseGet(DataCatalogEncryptionSettings::defaults);+ }++ /**+ * Replaces the catalog's security configuration. A block left out of the request keeps the+ * default (off), so a later read always carries both blocks, as AWS's does.+ */+ public void putDataCatalogEncryptionSettings(DataCatalogEncryptionSettings settings) {+ validateRequired(settings, "DataCatalogEncryptionSettings");+ DataCatalogEncryptionSettings stored = DataCatalogEncryptionSettings.defaults();+ EncryptionAtRest atRest = settings.getEncryptionAtRest();+ if (atRest != null) {+ validateRequired(atRest.getCatalogEncryptionMode(), "EncryptionAtRest.CatalogEncryptionMode");+ if (!CATALOG_ENCRYPTION_MODES.contains(atRest.getCatalogEncryptionMode())) {+ throw new AwsException("InvalidInputException",+ "Unsupported CatalogEncryptionMode: " + atRest.getCatalogEncryptionMode(), 400);+ }+ stored.setEncryptionAtRest(atRest);+ }+ ConnectionPasswordEncryption passwords = settings.getConnectionPasswordEncryption();+ if (passwords != null) {+ validateRequired(passwords.getReturnConnectionPasswordEncrypted(),+ "ConnectionPasswordEncryption.ReturnConnectionPasswordEncrypted");+ stored.setConnectionPasswordEncryption(passwords);+ }+ encryptionSettingsStore.put(CATALOG_KEY, stored);+ LOG.infov("Updated Glue Data Catalog encryption settings: at rest {0}, password encryption {1}",+ stored.getEncryptionAtRest().getCatalogEncryptionMode(),+ stored.getConnectionPasswordEncryption().getReturnConnectionPasswordEncrypted());+ }++ /**+ * Applies the catalog's ConnectionPasswordEncryption to a connection being created or+ * updated: each plaintext password property is encrypted with the configured KMS key and+ * stored under its ENCRYPTED_ name (the Connection structure documents each pair), so every+ * later read returns it encrypted. A KMS failure is the GlueEncryptionException both+ * operations list.+ */+ private void encryptConnectionPasswords(Map<String, String> properties, String region) {+ ConnectionPasswordEncryption setting = getDataCatalogEncryptionSettings().getConnectionPasswordEncryption();+ if (setting == null || !Boolean.TRUE.equals(setting.getReturnConnectionPasswordEncrypted())) {+ return;+ }+ String keyId = setting.getAwsKmsKeyId();+ for (Map.Entry<String, String> pair : ENCRYPTED_CONNECTION_PROPERTY_KEYS.entrySet()) {+ String plaintext = properties.get(pair.getKey());+ if (plaintext == null) {+ continue;+ }+ if (keyId == null || keyId.isBlank()) {+ throw new AwsException("GlueEncryptionException",+ "Connection password encryption is enabled but the catalog settings name no AwsKmsKeyId.", 400);+ }+ byte[] ciphertext;+ try {+ ciphertext = kmsService.encrypt(keyId, plaintext.getBytes(StandardCharsets.UTF_8), region);+ } catch (AwsException e) {+ throw new AwsException("GlueEncryptionException",+ "An encryption operation failed: " + e.getMessage(), 400);+ }+ properties.remove(pair.getKey());+ properties.put(pair.getValue(), Base64.getEncoder().encodeToString(ciphertext));+ }+ }++ private static void validatePolicyDocument(String policyInJson) {+ if (policyInJson.length() < 2) {+ throw new AwsException("InvalidInputException", "PolicyInJson must be at least 2 characters.", 400);+ }+ try {+ JsonNode document = POLICY_JSON.readTree(policyInJson);+ if (document == null || !document.isObject()) {+ throw new AwsException("InvalidInputException", "PolicyInJson must be a JSON policy document.", 400);+ }+ } catch (JsonProcessingException e) {+ throw new AwsException("InvalidInputException", "PolicyInJson is not valid JSON: " + e.getOriginalMessage(), 400);+ }+ }++ private static void checkPolicyHashCondition(String policyHashCondition, Optional<GluePolicy> existing) {+ if (policyHashCondition == null || policyHashCondition.isBlank()) {+ return;+ }+ String current = existing.map(GluePolicy::getPolicyHash).orElse(null);+ if (!policyHashCondition.equals(current)) {+ throw new AwsException("ConditionCheckFailureException",+ "PolicyHashCondition does not match the current policy hash.", 400);+ }+ }++ /**+ * AWS documents PolicyHash only as an opaque value to echo back in PolicyHashCondition; this+ * derives it from the document so the same policy always yields the same hash.+ */+ private static String policyHash(String policyInJson) {+ try {+ byte[] digest = MessageDigest.getInstance("MD5").digest(policyInJson.getBytes(StandardCharsets.UTF_8));+ return Base64.getEncoder().encodeToString(digest);+ } catch (NoSuchAlgorithmException e) {+ throw new IllegalStateException("MD5 is a required JDK algorithm", e);+ }+ }+public void tagResource(String arn, Map<String, String> tags, String region) {validateArn(arn);if (arn.contains(":registry/") || arn.contains(":schema/")) {diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionPasswordEncryption.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionPasswordEncryption.javanew file mode 100644index 0000000000..2d21d92e71--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionPasswordEncryption.java@@ -0,0 +1,28 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++/**+ * The {@code ConnectionPasswordEncryption} block of the Data Catalog encryption settings. With+ * {@code ReturnConnectionPasswordEncrypted} true, connection passwords are encrypted with+ * {@code AwsKmsKeyId} when a connection is created or updated and stay encrypted in every read.+ */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class ConnectionPasswordEncryption {+ @JsonProperty("ReturnConnectionPasswordEncrypted")+ private Boolean returnConnectionPasswordEncrypted;++ @JsonProperty("AwsKmsKeyId")+ private String awsKmsKeyId;++ public Boolean getReturnConnectionPasswordEncrypted() { return returnConnectionPasswordEncrypted; }+ public void setReturnConnectionPasswordEncrypted(Boolean returnConnectionPasswordEncrypted) {+ this.returnConnectionPasswordEncrypted = returnConnectionPasswordEncrypted;+ }++ public String getAwsKmsKeyId() { return awsKmsKeyId; }+ public void setAwsKmsKeyId(String awsKmsKeyId) { this.awsKmsKeyId = awsKmsKeyId; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/DataCatalogEncryptionSettings.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/DataCatalogEncryptionSettings.javanew file mode 100644index 0000000000..53038e937d--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/DataCatalogEncryptionSettings.java@@ -0,0 +1,36 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++/** The Data Catalog's security configuration, one per catalog. */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class DataCatalogEncryptionSettings {+ @JsonProperty("EncryptionAtRest")+ private EncryptionAtRest encryptionAtRest;++ @JsonProperty("ConnectionPasswordEncryption")+ private ConnectionPasswordEncryption connectionPasswordEncryption;++ /** What a catalog reports before anything was ever put: both blocks present, both off. */+ public static DataCatalogEncryptionSettings defaults() {+ DataCatalogEncryptionSettings settings = new DataCatalogEncryptionSettings();+ EncryptionAtRest atRest = new EncryptionAtRest();+ atRest.setCatalogEncryptionMode("DISABLED");+ settings.encryptionAtRest = atRest;+ ConnectionPasswordEncryption passwords = new ConnectionPasswordEncryption();+ passwords.setReturnConnectionPasswordEncrypted(false);+ settings.connectionPasswordEncryption = passwords;+ return settings;+ }++ public EncryptionAtRest getEncryptionAtRest() { return encryptionAtRest; }+ public void setEncryptionAtRest(EncryptionAtRest encryptionAtRest) { this.encryptionAtRest = encryptionAtRest; }++ public ConnectionPasswordEncryption getConnectionPasswordEncryption() { return connectionPasswordEncryption; }+ public void setConnectionPasswordEncryption(ConnectionPasswordEncryption connectionPasswordEncryption) {+ this.connectionPasswordEncryption = connectionPasswordEncryption;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/EncryptionAtRest.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/EncryptionAtRest.javanew file mode 100644index 0000000000..52c09a9148--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/EncryptionAtRest.java@@ -0,0 +1,30 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++/** The {@code EncryptionAtRest} block of the Data Catalog encryption settings. */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class EncryptionAtRest {+ @JsonProperty("CatalogEncryptionMode")+ private String catalogEncryptionMode;++ @JsonProperty("SseAwsKmsKeyId")+ private String sseAwsKmsKeyId;++ @JsonProperty("CatalogEncryptionServiceRole")+ private String catalogEncryptionServiceRole;++ public String getCatalogEncryptionMode() { return catalogEncryptionMode; }+ public void setCatalogEncryptionMode(String catalogEncryptionMode) { this.catalogEncryptionMode = catalogEncryptionMode; }++ public String getSseAwsKmsKeyId() { return sseAwsKmsKeyId; }+ public void setSseAwsKmsKeyId(String sseAwsKmsKeyId) { this.sseAwsKmsKeyId = sseAwsKmsKeyId; }++ public String getCatalogEncryptionServiceRole() { return catalogEncryptionServiceRole; }+ public void setCatalogEncryptionServiceRole(String catalogEncryptionServiceRole) {+ this.catalogEncryptionServiceRole = catalogEncryptionServiceRole;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.javanew file mode 100644index 0000000000..c4d2a4aa4a--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java@@ -0,0 +1,42 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonFormat;+import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;++/**+ * The Data Catalog resource policy as GetResourcePolicy returns it, and as each entry of+ * GetResourcePolicies (the {@code GluePolicy} structure of the API reference).+ */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class GluePolicy {+ @JsonProperty("PolicyInJson")+ private String policyInJson;++ @JsonProperty("PolicyHash")+ private String policyHash;++ @JsonProperty("CreateTime")+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant createTime;++ @JsonProperty("UpdateTime")+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant updateTime;++ public String getPolicyInJson() { return policyInJson; }+ public void setPolicyInJson(String policyInJson) { this.policyInJson = policyInJson; }++ public String getPolicyHash() { return policyHash; }+ public void setPolicyHash(String policyHash) { this.policyHash = policyHash; }++ public Instant getCreateTime() { return createTime; }+ public void setCreateTime(Instant createTime) { this.createTime = createTime; }++ public Instant getUpdateTime() { return updateTime; }+ public void setUpdateTime(Instant updateTime) { this.updateTime = updateTime; }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueCatalogPolicyAndEncryptionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCatalogPolicyAndEncryptionIntegrationTest.javanew file mode 100644index 0000000000..ff758349a1--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueCatalogPolicyAndEncryptionIntegrationTest.java@@ -0,0 +1,182 @@+package io.github.hectorvent.floci.services.glue;++import com.fasterxml.jackson.databind.JsonNode;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import java.nio.charset.StandardCharsets;+import java.util.Base64;+import java.util.UUID;++import static io.github.hectorvent.floci.testing.RestAssuredJsonUtils.awsAction;+import static io.github.hectorvent.floci.testing.RestAssuredJsonUtils.awsActionJson;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasKey;+import static org.hamcrest.Matchers.not;+import static org.hamcrest.Matchers.notNullValue;+import static org.junit.jupiter.api.Assertions.assertEquals;++/**+ * The catalog's resource policy and encryption settings over JSON 1.1, in the order Terraform+ * drives them, plus the observable effect of ConnectionPasswordEncryption: a connection created+ * while it is on stores its password KMS-encrypted and GetConnection returns it that way.+ * Both settings are one per catalog, so the class runs in order and puts the catalog back at+ * the end.+ */+@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class GlueCatalogPolicyAndEncryptionIntegrationTest {++ private static final String GLUE = "AWSGlue";+ private static final String KMS = "TrentService";+ private static final String POLICY = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\","+ + "\"Principal\":{\"AWS\":\"arn:aws:iam::111122223333:root\"},\"Action\":\"glue:GetTable\",\"Resource\":\"*\"}]}";+ private static final String POLICY_V2 = POLICY.replace("glue:GetTable", "glue:GetTables");++ private static String policyHash;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ @Order(1)+ void resourcePolicy_absentAtFirst_isEntityNotFound() {+ // The catalog may carry a policy from another class; start from a known state.+ awsAction(GLUE, "DeleteResourcePolicy", "{}");+ awsAction(GLUE, "GetResourcePolicy", "{}")+ .then()+ .statusCode(400)+ .body("__type", equalTo("EntityNotFoundException"));+ awsAction(GLUE, "GetResourcePolicies", "{}")+ .then()+ .statusCode(200)+ .body("GetResourcePoliciesResponseList.size()", equalTo(0));+ }++ @Test+ @Order(2)+ void putResourcePolicy_notExistThenMustExist_createsThenUpdates() throws Exception {+ JsonNode created = awsActionJson(GLUE, "PutResourcePolicy",+ "{\"PolicyInJson\": " + quoted(POLICY) + ", \"PolicyExistsCondition\": \"NOT_EXIST\"}");+ policyHash = created.get("PolicyHash").asText();++ awsAction(GLUE, "GetResourcePolicy", "{}")+ .then()+ .statusCode(200)+ .body("PolicyInJson", equalTo(POLICY))+ .body("PolicyHash", equalTo(policyHash))+ .body("CreateTime", notNullValue())+ .body("UpdateTime", notNullValue());++ // Terraform's second create against an existing policy must be refused.+ awsAction(GLUE, "PutResourcePolicy",+ "{\"PolicyInJson\": " + quoted(POLICY_V2) + ", \"PolicyExistsCondition\": \"NOT_EXIST\"}")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ConditionCheckFailureException"));++ JsonNode updated = awsActionJson(GLUE, "PutResourcePolicy",+ "{\"PolicyInJson\": " + quoted(POLICY_V2) + ", \"PolicyExistsCondition\": \"MUST_EXIST\","+ + " \"PolicyHashCondition\": " + quoted(policyHash) + ", \"EnableHybrid\": \"TRUE\"}");+ policyHash = updated.get("PolicyHash").asText();+ awsAction(GLUE, "GetResourcePolicies", "{}")+ .then()+ .statusCode(200)+ .body("GetResourcePoliciesResponseList.size()", equalTo(1))+ .body("GetResourcePoliciesResponseList[0].PolicyInJson", equalTo(POLICY_V2));+ }++ @Test+ @Order(3)+ void deleteResourcePolicy_wrongHashRefused_rightHashDeletes() {+ awsAction(GLUE, "DeleteResourcePolicy", "{\"PolicyHashCondition\": \"stale\"}")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ConditionCheckFailureException"));+ awsAction(GLUE, "DeleteResourcePolicy", "{\"PolicyHashCondition\": " + quoted(policyHash) + "}")+ .then()+ .statusCode(200);+ awsAction(GLUE, "DeleteResourcePolicy", "{}")+ .then()+ .statusCode(400)+ .body("__type", equalTo("EntityNotFoundException"));+ }++ @Test+ @Order(4)+ void encryptionSettings_defaultOff_thenEncryptConnectionPasswordsWithARealKmsKey() throws Exception {+ awsAction(GLUE, "GetDataCatalogEncryptionSettings", "{}")+ .then()+ .statusCode(200)+ .body("DataCatalogEncryptionSettings.EncryptionAtRest.CatalogEncryptionMode", equalTo("DISABLED"))+ .body("DataCatalogEncryptionSettings.ConnectionPasswordEncryption.ReturnConnectionPasswordEncrypted", equalTo(false));++ String keyId = awsActionJson(KMS, "CreateKey", "{\"Description\": \"glue connection passwords\"}")+ .path("KeyMetadata").path("KeyId").asText();+ awsAction(GLUE, "PutDataCatalogEncryptionSettings", """+ {"DataCatalogEncryptionSettings": {+ "EncryptionAtRest": {"CatalogEncryptionMode": "DISABLED"},+ "ConnectionPasswordEncryption": {"ReturnConnectionPasswordEncrypted": true, "AwsKmsKeyId": "%s"}+ }}+ """.formatted(keyId))+ .then().statusCode(200);+ awsAction(GLUE, "GetDataCatalogEncryptionSettings", "{}")+ .then()+ .statusCode(200)+ .body("DataCatalogEncryptionSettings.ConnectionPasswordEncryption.ReturnConnectionPasswordEncrypted", equalTo(true))+ .body("DataCatalogEncryptionSettings.ConnectionPasswordEncryption.AwsKmsKeyId", equalTo(keyId));++ String name = "enc-" + UUID.randomUUID().toString().substring(0, 8);+ awsAction(GLUE, "CreateConnection", """+ {"ConnectionInput": {"Name": "%s", "ConnectionType": "JDBC",+ "ConnectionProperties": {"JDBC_CONNECTION_URL": "jdbc:postgresql://db:5432/orders", "USERNAME": "app", "PASSWORD": "s3cret"}}}+ """.formatted(name))+ .then().statusCode(200);++ JsonNode connection = awsActionJson(GLUE, "GetConnection", "{\"Name\": \"" + name + "\"}").path("Connection");+ JsonNode properties = connection.path("ConnectionProperties");+ assertEquals("app", properties.path("USERNAME").asText());+ assertEquals(false, properties.has("PASSWORD"));+ String encrypted = properties.path("ENCRYPTED_PASSWORD").asText();+ // The stored value is a real KMS ciphertext: KMS itself decrypts it back to the password.+ JsonNode decrypted = awsActionJson(KMS, "Decrypt", "{\"CiphertextBlob\": " + quoted(encrypted) + "}");+ assertEquals("s3cret", new String(Base64.getDecoder().decode(decrypted.path("Plaintext").asText()), StandardCharsets.UTF_8));++ awsAction(GLUE, "GetConnection", "{\"Name\": \"" + name + "\", \"HidePassword\": true}")+ .then()+ .statusCode(200)+ .body("Connection.ConnectionProperties", not(hasKey("ENCRYPTED_PASSWORD")))+ .body("Connection.ConnectionProperties", not(hasKey("PASSWORD")));++ awsAction(GLUE, "DeleteConnection", "{\"ConnectionName\": \"" + name + "\"}").then().statusCode(200);+ // Put the catalog back so other classes see the default.+ awsAction(GLUE, "PutDataCatalogEncryptionSettings", """+ {"DataCatalogEncryptionSettings": {+ "EncryptionAtRest": {"CatalogEncryptionMode": "DISABLED"},+ "ConnectionPasswordEncryption": {"ReturnConnectionPasswordEncrypted": false}+ }}+ """)+ .then().statusCode(200);+ }++ @Test+ @Order(5)+ void putDataCatalogEncryptionSettings_unknownMode_isInvalidInput() {+ awsAction(GLUE, "PutDataCatalogEncryptionSettings",+ "{\"DataCatalogEncryptionSettings\": {\"EncryptionAtRest\": {\"CatalogEncryptionMode\": \"AES\"}}}")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidInputException"));+ }++ private static String quoted(String value) {+ return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"") + "\"";+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.javaindex 6188bff544..a60827decd 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerEmptyListTest.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;+import io.github.hectorvent.floci.services.kms.KmsService;import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;import jakarta.ws.rs.core.Response;import org.junit.jupiter.api.BeforeEach;@@ -41,7 +42,8 @@ class GlueJsonHandlerEmptyListTest {GlueSchemaRegistryService schemaRegistryService =new GlueSchemaRegistryService(storageFactory, regionResolver);GlueService glueService = new GlueService(- storageFactory, schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(storageFactory));+ storageFactory, schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(storageFactory),+ new KmsService(storageFactory, regionResolver));handler = new GlueJsonHandler(glueService, schemaRegistryService, mapper);}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.javaindex 0843b32551..7326862cfa 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;+import io.github.hectorvent.floci.services.kms.KmsService;import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;import jakarta.ws.rs.core.Response;import org.junit.jupiter.api.BeforeEach;@@ -41,7 +42,8 @@ class GlueJsonHandlerTest {GlueSchemaRegistryService schemaRegistryService =new GlueSchemaRegistryService(storageFactory, regionResolver);GlueService glueService = new GlueService(- storageFactory, schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(storageFactory));+ storageFactory, schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(storageFactory),+ new KmsService(storageFactory, regionResolver));handler = new GlueJsonHandler(glueService, schemaRegistryService, mapper);}@@ -737,4 +739,58 @@ class GlueJsonHandlerTest {inline.putObject("ConnectionProperties").put("JDBC_CONNECTION_URL", "jdbc:mysql://h:3306/d");assertEquals(200, handler.handle("TestConnection", test, REGION).getStatus());}++ @Test+ void resourcePolicyOperationsAnswerWithTheDocumentedBodies() throws Exception {+ String policy = "{\"Version\":\"2012-10-17\",\"Statement\":[]}";+ ObjectNode put = mapper.createObjectNode();+ put.put("PolicyInJson", policy);+ put.put("PolicyExistsCondition", "NOT_EXIST");+ JsonNode putBody = mapper.valueToTree(handler.handle("PutResourcePolicy", put, REGION).getEntity());+ String hash = putBody.get("PolicyHash").asText();+ assertFalse(hash.isBlank());++ JsonNode got = mapper.valueToTree(handler.handle("GetResourcePolicy", mapper.createObjectNode(), REGION).getEntity());+ assertEquals(policy, got.get("PolicyInJson").asText());+ assertEquals(hash, got.get("PolicyHash").asText());+ assertTrue(got.get("CreateTime").isNumber());+ assertTrue(got.get("UpdateTime").isNumber());++ JsonNode list = mapper.valueToTree(handler.handle("GetResourcePolicies", mapper.createObjectNode(), REGION).getEntity());+ assertEquals(1, list.get("GetResourcePoliciesResponseList").size());+ assertEquals(hash, list.get("GetResourcePoliciesResponseList").get(0).get("PolicyHash").asText());+ assertFalse(list.has("NextToken"));++ AwsException conflict = assertThrows(AwsException.class, () -> handler.handle("PutResourcePolicy", put, REGION));+ assertEquals("ConditionCheckFailureException", conflict.getErrorCode());++ Response deleted = handler.handle("DeleteResourcePolicy", mapper.createObjectNode(), REGION);+ assertEquals(0, mapper.valueToTree(deleted.getEntity()).size());+ AwsException gone = assertThrows(AwsException.class,+ () -> handler.handle("GetResourcePolicy", mapper.createObjectNode(), REGION));+ assertEquals("EntityNotFoundException", gone.getErrorCode());+ }++ @Test+ void encryptionSettingsReportBothBlocksBeforeAndAfterAPut() throws Exception {+ JsonNode defaults = mapper.valueToTree(+ handler.handle("GetDataCatalogEncryptionSettings", mapper.createObjectNode(), REGION).getEntity());+ JsonNode settings = defaults.get("DataCatalogEncryptionSettings");+ assertEquals("DISABLED", settings.get("EncryptionAtRest").get("CatalogEncryptionMode").asText());+ assertFalse(settings.get("EncryptionAtRest").has("SseAwsKmsKeyId"));+ assertFalse(settings.get("ConnectionPasswordEncryption").get("ReturnConnectionPasswordEncrypted").asBoolean());++ ObjectNode put = mapper.createObjectNode();+ ObjectNode block = put.putObject("DataCatalogEncryptionSettings").putObject("ConnectionPasswordEncryption");+ block.put("ReturnConnectionPasswordEncrypted", true);+ block.put("AwsKmsKeyId", "alias/glue");+ assertEquals(0, mapper.valueToTree(handler.handle("PutDataCatalogEncryptionSettings", put, REGION).getEntity()).size());++ JsonNode after = mapper.valueToTree(+ handler.handle("GetDataCatalogEncryptionSettings", mapper.createObjectNode(), REGION).getEntity())+ .get("DataCatalogEncryptionSettings");+ assertTrue(after.get("ConnectionPasswordEncryption").get("ReturnConnectionPasswordEncrypted").asBoolean());+ assertEquals("alias/glue", after.get("ConnectionPasswordEncryption").get("AwsKmsKeyId").asText());+ assertEquals("DISABLED", after.get("EncryptionAtRest").get("CatalogEncryptionMode").asText());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.javaindex d4e0794c77..261686a1b8 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java@@ -12,6 +12,10 @@ import io.github.hectorvent.floci.services.glue.model.Column;import io.github.hectorvent.floci.services.glue.model.Database;import io.github.hectorvent.floci.services.glue.model.Connection;import io.github.hectorvent.floci.services.glue.model.ConnectionInput;+import io.github.hectorvent.floci.services.glue.model.ConnectionPasswordEncryption;+import io.github.hectorvent.floci.services.glue.model.DataCatalogEncryptionSettings;+import io.github.hectorvent.floci.services.glue.model.EncryptionAtRest;+import io.github.hectorvent.floci.services.glue.model.GluePolicy;import io.github.hectorvent.floci.services.glue.model.Crawler;import io.github.hectorvent.floci.services.glue.model.AuthenticationConfiguration;import io.github.hectorvent.floci.services.glue.model.PhysicalConnectionRequirements;@@ -28,6 +32,7 @@ import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;import io.github.hectorvent.floci.services.glue.model.Table;import io.github.hectorvent.floci.services.glue.model.UserDefinedFunction;import io.github.hectorvent.floci.services.glue.schemaregistry.GlueSchemaRegistryService;+import io.github.hectorvent.floci.services.kms.KmsService;import io.github.hectorvent.floci.services.glue.schemaregistry.model.RegistryId;import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;@@ -35,6 +40,8 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import java.time.Instant;+import java.nio.charset.StandardCharsets;+import java.util.Base64;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;@@ -44,6 +51,7 @@ import java.util.concurrent.CountDownLatch;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotEquals;import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;@@ -65,6 +73,7 @@ class GlueServiceTest {private GlueService glueService;private GlueSchemaRegistryService schemaRegistryService;+ private KmsService kmsService;private StorageBackend<String, Database> databaseStore;private StorageBackend<String, Table> tableStore;private StorageBackend<String, Table> tableVersionStore;@@ -76,6 +85,7 @@ class GlueServiceTest {RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT_ID);StorageFactory storageFactory = new InMemoryStorageFactory();schemaRegistryService = new GlueSchemaRegistryService(storageFactory, regionResolver);+ kmsService = new KmsService(storageFactory, regionResolver);databaseStore = new InMemoryStorage<>();tableStore = new InMemoryStorage<>();tableVersionStore = new InMemoryStorage<>();@@ -94,7 +104,10 @@ class GlueServiceTest {new InMemoryStorage<String, Crawler>(),new InMemoryStorage<String, Classifier>(),new InMemoryStorage<String, Connection>(),- schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(null));+ new InMemoryStorage<String, GluePolicy>(),+ new InMemoryStorage<String, DataCatalogEncryptionSettings>(),+ schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(null),+ kmsService);glueService.createDatabase(new Database("db1"));}@@ -1528,7 +1541,7 @@ class GlueServiceTest {redefinition.setName("orders");redefinition.setConnectionType("JDBC");redefinition.setConnectionProperties(Map.of("JDBC_CONNECTION_URL", "jdbc:postgresql://db2.internal:5432/orders"));- glueService.updateConnection("orders", redefinition);+ glueService.updateConnection("orders", redefinition, REGION);Connection updated = glueService.getConnection("orders", false);assertEquals("jdbc:postgresql://db2.internal:5432/orders", updated.getConnectionProperties().get("JDBC_CONNECTION_URL"));@@ -1553,7 +1566,7 @@ class GlueServiceTest {assertEquals("AlreadyExistsException", duplicate.getErrorCode());AwsException missing = assertThrows(AwsException.class,- () -> glueService.updateConnection("absent", jdbcConnection("absent")));+ () -> glueService.updateConnection("absent", jdbcConnection("absent"), REGION));assertEquals("EntityNotFoundException", missing.getErrorCode());AwsException missingDelete = assertThrows(AwsException.class,@@ -1720,6 +1733,176 @@ class GlueServiceTest {assertThrows(AwsException.class, () -> glueService.getTags(arn, REGION)).getErrorCode());}+ // ---- Catalog resource policy and encryption settings -----------------------------------++ private static final String POLICY_V1 = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::111122223333:root\"},\"Action\":\"glue:GetTable\",\"Resource\":\"*\"}]}";+ private static final String POLICY_V2 = POLICY_V1.replace("glue:GetTable", "glue:GetTables");++ @Test+ void resourcePolicyFollowsTheTerraformSequenceOfConditions() {+ assertEquals("EntityNotFoundException",+ assertThrows(AwsException.class, () -> glueService.getResourcePolicy()).getErrorCode());+ assertTrue(glueService.getResourcePolicies(null, null).items().isEmpty());++ // Create: NOT_EXIST must pass on an empty catalog.+ String hash = glueService.putResourcePolicy(POLICY_V1, null, "NOT_EXIST", null);+ GluePolicy stored = glueService.getResourcePolicy();+ assertEquals(POLICY_V1, stored.getPolicyInJson());+ assertEquals(hash, stored.getPolicyHash());+ assertNotNull(stored.getCreateTime());+ assertEquals(stored.getCreateTime(), stored.getUpdateTime());+ assertEquals(1, glueService.getResourcePolicies(null, null).items().size());++ // A second create must fail, an update must pass and keep CreateTime.+ assertEquals("ConditionCheckFailureException",+ assertThrows(AwsException.class,+ () -> glueService.putResourcePolicy(POLICY_V2, null, "NOT_EXIST", null)).getErrorCode());+ String hash2 = glueService.putResourcePolicy(POLICY_V2, hash, "MUST_EXIST", "TRUE");+ GluePolicy updated = glueService.getResourcePolicy();+ assertEquals(POLICY_V2, updated.getPolicyInJson());+ assertNotEquals(hash, hash2);+ assertEquals(stored.getCreateTime(), updated.getCreateTime());+ assertFalse(updated.getUpdateTime().isBefore(stored.getUpdateTime()));++ // A stale hash is refused on put and on delete; the right one is accepted.+ assertEquals("ConditionCheckFailureException",+ assertThrows(AwsException.class,+ () -> glueService.putResourcePolicy(POLICY_V1, hash, "NONE", null)).getErrorCode());+ assertEquals("ConditionCheckFailureException",+ assertThrows(AwsException.class, () -> glueService.deleteResourcePolicy(hash)).getErrorCode());+ glueService.deleteResourcePolicy(hash2);+ assertEquals("EntityNotFoundException",+ assertThrows(AwsException.class, () -> glueService.deleteResourcePolicy(null)).getErrorCode());+ assertEquals("ConditionCheckFailureException",+ assertThrows(AwsException.class,+ () -> glueService.putResourcePolicy(POLICY_V1, null, "MUST_EXIST", null)).getErrorCode());+ }++ @Test+ void resourcePolicyInputIsValidated() {+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.putResourcePolicy(null, null, null, null)).getErrorCode());+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.putResourcePolicy("{", null, null, null)).getErrorCode());+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.putResourcePolicy("[]", null, null, null)).getErrorCode());+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.putResourcePolicy(POLICY_V1, null, "MAYBE", null)).getErrorCode());+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.putResourcePolicy(POLICY_V1, null, null, "yes")).getErrorCode());+ // The same document always hashes the same, so a client can compare hashes across reads.+ assertEquals(glueService.putResourcePolicy(POLICY_V1, null, null, null),+ glueService.putResourcePolicy(POLICY_V1, null, null, null));+ }++ @Test+ void encryptionSettingsDefaultToOffAndAreReplacedByPut() {+ DataCatalogEncryptionSettings defaults = glueService.getDataCatalogEncryptionSettings();+ assertEquals("DISABLED", defaults.getEncryptionAtRest().getCatalogEncryptionMode());+ assertEquals(false, defaults.getConnectionPasswordEncryption().getReturnConnectionPasswordEncrypted());+ assertNull(defaults.getConnectionPasswordEncryption().getAwsKmsKeyId());++ DataCatalogEncryptionSettings put = new DataCatalogEncryptionSettings();+ EncryptionAtRest atRest = new EncryptionAtRest();+ atRest.setCatalogEncryptionMode("SSE-KMS");+ atRest.setSseAwsKmsKeyId("alias/catalog");+ put.setEncryptionAtRest(atRest);+ glueService.putDataCatalogEncryptionSettings(put);++ DataCatalogEncryptionSettings read = glueService.getDataCatalogEncryptionSettings();+ assertEquals("SSE-KMS", read.getEncryptionAtRest().getCatalogEncryptionMode());+ assertEquals("alias/catalog", read.getEncryptionAtRest().getSseAwsKmsKeyId());+ // The block left out of the put is reported with its default, not dropped.+ assertEquals(false, read.getConnectionPasswordEncryption().getReturnConnectionPasswordEncrypted());++ EncryptionAtRest badMode = new EncryptionAtRest();+ badMode.setCatalogEncryptionMode("AES");+ DataCatalogEncryptionSettings invalid = new DataCatalogEncryptionSettings();+ invalid.setEncryptionAtRest(badMode);+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.putDataCatalogEncryptionSettings(invalid)).getErrorCode());+ DataCatalogEncryptionSettings missingFlag = new DataCatalogEncryptionSettings();+ missingFlag.setConnectionPasswordEncryption(new ConnectionPasswordEncryption());+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.putDataCatalogEncryptionSettings(missingFlag)).getErrorCode());+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.putDataCatalogEncryptionSettings(null)).getErrorCode());+ }++ private void enableConnectionPasswordEncryption(String keyId) {+ ConnectionPasswordEncryption passwords = new ConnectionPasswordEncryption();+ passwords.setReturnConnectionPasswordEncrypted(true);+ passwords.setAwsKmsKeyId(keyId);+ DataCatalogEncryptionSettings settings = new DataCatalogEncryptionSettings();+ settings.setConnectionPasswordEncryption(passwords);+ glueService.putDataCatalogEncryptionSettings(settings);+ }++ @Test+ void connectionPasswordsAreStoredEncryptedOnceTheCatalogSettingIsOn() {+ String keyId = kmsService.createKey("glue connection passwords", REGION).getKeyId();+ enableConnectionPasswordEncryption(keyId);++ ConnectionInput jdbc = jdbcConnection("enc-jdbc");+ glueService.createConnection(jdbc, null, REGION);+ Connection stored = glueService.getConnection("enc-jdbc", false);+ assertNull(stored.getConnectionProperties().get("PASSWORD"));+ String encrypted = stored.getConnectionProperties().get("ENCRYPTED_PASSWORD");+ assertNotNull(encrypted);+ assertEquals("s3cret", new String(+ kmsService.decrypt(Base64.getDecoder().decode(encrypted), REGION), StandardCharsets.UTF_8));+ assertEquals("app", stored.getConnectionProperties().get("USERNAME"));+ // HidePassword removes the encrypted form as it removes the plaintext one.+ assertNull(glueService.getConnection("enc-jdbc", true).getConnectionProperties().get("ENCRYPTED_PASSWORD"));++ ConnectionInput kafka = jdbcConnection("enc-kafka");+ kafka.setConnectionType("KAFKA");+ Map<String, String> properties = new LinkedHashMap<>();+ properties.put("KAFKA_BOOTSTRAP_SERVERS", "broker:9092");+ properties.put("KAFKA_SASL_MECHANISM", "SCRAM-SHA-512");+ properties.put("KAFKA_SASL_SCRAM_USERNAME", "svc");+ properties.put("KAFKA_SASL_SCRAM_PASSWORD", "scram-pw");+ properties.put("KAFKA_CLIENT_KEYSTORE_PASSWORD", "ks-pw");+ kafka.setConnectionProperties(properties);+ glueService.createConnection(kafka, null, REGION);+ Map<String, String> kafkaStored = glueService.getConnection("enc-kafka", false).getConnectionProperties();+ assertNull(kafkaStored.get("KAFKA_SASL_SCRAM_PASSWORD"));+ assertNull(kafkaStored.get("KAFKA_CLIENT_KEYSTORE_PASSWORD"));+ assertNotNull(kafkaStored.get("ENCRYPTED_KAFKA_SASL_SCRAM_PASSWORD"));+ assertNotNull(kafkaStored.get("ENCRYPTED_KAFKA_CLIENT_KEYSTORE_PASSWORD"));+ assertEquals("svc", kafkaStored.get("KAFKA_SASL_SCRAM_USERNAME"));++ // An update goes through the same path.+ ConnectionInput redefinition = jdbcConnection("enc-jdbc");+ redefinition.setConnectionProperties(Map.of("JDBC_CONNECTION_URL", "jdbc:x://h/d", "PASSWORD", "changed"));+ glueService.updateConnection("enc-jdbc", redefinition, REGION);+ Map<String, String> afterUpdate = glueService.getConnection("enc-jdbc", false).getConnectionProperties();+ assertNull(afterUpdate.get("PASSWORD"));+ assertEquals("changed", new String(kmsService.decrypt(+ Base64.getDecoder().decode(afterUpdate.get("ENCRYPTED_PASSWORD")), REGION), StandardCharsets.UTF_8));+ }++ @Test+ void connectionPasswordEncryptionFailuresAreGlueEncryptionExceptions() {+ enableConnectionPasswordEncryption("arn:aws:kms:us-east-1:000000000000:key/00000000-0000-0000-0000-000000000000");+ AwsException unknownKey = assertThrows(AwsException.class,+ () -> glueService.createConnection(jdbcConnection("enc-bad-key"), null, REGION));+ assertEquals("GlueEncryptionException", unknownKey.getErrorCode());+ assertEquals("EntityNotFoundException",+ assertThrows(AwsException.class, () -> glueService.getConnection("enc-bad-key", false)).getErrorCode());++ enableConnectionPasswordEncryption(null);+ assertEquals("GlueEncryptionException",+ assertThrows(AwsException.class,+ () -> glueService.createConnection(jdbcConnection("enc-no-key"), null, REGION)).getErrorCode());++ // A connection without any password property is unaffected by the setting.+ ConnectionInput network = jdbcConnection("enc-network");+ network.setConnectionType("NETWORK");+ network.setConnectionProperties(Map.of());+ assertEquals("READY", glueService.createConnection(network, null, REGION));+ }+private static List<String> names(GlueService.Page<Table> page) {return page.items().stream().map(Table::getName).toList();}