Light, fluffy, and always free - The AWS Local Emulator alternative
feat(glue): Data Catalog connections (#3939)
Davida793776c1a99
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.javaindex 6e5ff90a3d..ee5bebef7f 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java@@ -6,6 +6,20 @@ import org.junit.jupiter.api.DisplayName;import org.junit.jupiter.api.Test;import software.amazon.awssdk.services.glue.GlueClient;import software.amazon.awssdk.services.glue.model.BatchCreatePartitionRequest;+import software.amazon.awssdk.services.glue.model.BatchDeleteConnectionRequest;+import software.amazon.awssdk.services.glue.model.BatchDeleteConnectionResponse;+import software.amazon.awssdk.services.glue.model.Connection;+import software.amazon.awssdk.services.glue.model.ConnectionInput;+import software.amazon.awssdk.services.glue.model.ConnectionPropertyKey;+import software.amazon.awssdk.services.glue.model.ConnectionType;+import software.amazon.awssdk.services.glue.model.CreateConnectionRequest;+import software.amazon.awssdk.services.glue.model.CreateConnectionResponse;+import software.amazon.awssdk.services.glue.model.DeleteConnectionRequest;+import software.amazon.awssdk.services.glue.model.GetConnectionRequest;+import software.amazon.awssdk.services.glue.model.GetConnectionsFilter;+import software.amazon.awssdk.services.glue.model.GetConnectionsRequest;+import software.amazon.awssdk.services.glue.model.PhysicalConnectionRequirements;+import software.amazon.awssdk.services.glue.model.UpdateConnectionRequest;import software.amazon.awssdk.services.glue.model.BatchDeleteTableRequest;import software.amazon.awssdk.services.glue.model.BatchGetPartitionRequest;import software.amazon.awssdk.services.glue.model.BatchUpdatePartitionRequest;@@ -54,6 +68,7 @@ import software.amazon.awssdk.services.glue.model.UserDefinedFunctionInput;import software.amazon.awssdk.services.resourcegroupstaggingapi.ResourceGroupsTaggingApiClient;import java.time.Instant;+import java.util.List;import java.util.Map;import java.util.stream.Collectors;@@ -69,6 +84,8 @@ class GlueCatalogTest {private static final String SECOND_TABLE_NAME = "catalog_table_second";private static final String FUNCTION_NAME = "catalog_function";private static final String DATABASE_TAGGED_NAME = TestFixtures.uniqueName("catalog_tagged_db");+ private static final String CONNECTION_NAME = TestFixtures.uniqueName("catalog_connection");+ private static final String KAFKA_CONNECTION_NAME = TestFixtures.uniqueName("catalog_kafka_connection");private static GlueClient glue;private static ResourceGroupsTaggingApiClient tagging;@@ -84,6 +101,12 @@ class GlueCatalogTest {if (glue == null) {return;}+ try {+ glue.batchDeleteConnection(BatchDeleteConnectionRequest.builder()+ .connectionNameList(CONNECTION_NAME, KAFKA_CONNECTION_NAME)+ .build());+ }+ catch (Exception ignored) {}try {glue.deleteUserDefinedFunction(DeleteUserDefinedFunctionRequest.builder().databaseName(DATABASE_NAME)@@ -531,6 +554,92 @@ class GlueCatalogTest {});}+ @Test+ @DisplayName("Connections are created, read with and without the password, redefined and deleted")+ void connectionLifecycle() {+ CreateConnectionResponse created = glue.createConnection(CreateConnectionRequest.builder()+ .connectionInput(ConnectionInput.builder()+ .name(CONNECTION_NAME)+ .description("orders database")+ .connectionType(ConnectionType.JDBC)+ .matchCriteria("orders")+ .connectionProperties(Map.of(+ ConnectionPropertyKey.JDBC_CONNECTION_URL, "jdbc:postgresql://db.internal:5432/orders",+ ConnectionPropertyKey.USERNAME, "app",+ ConnectionPropertyKey.PASSWORD, "s3cret"))+ .physicalConnectionRequirements(PhysicalConnectionRequirements.builder()+ .subnetId("subnet-0123456789abcdef0")+ .securityGroupIdList("sg-0123456789abcdef0")+ .availabilityZone("us-east-1a")+ .build())+ .build())+ .tags(Map.of("env", "dev"))+ .build());+ assertThat(created.createConnectionStatusAsString()).isEqualTo("READY");++ Connection connection = glue.getConnection(GetConnectionRequest.builder()+ .name(CONNECTION_NAME)+ .build()).connection();+ assertThat(connection.name()).isEqualTo(CONNECTION_NAME);+ assertThat(connection.connectionType()).isEqualTo(ConnectionType.JDBC);+ assertThat(connection.description()).isEqualTo("orders database");+ assertThat(connection.connectionProperties()).containsEntry(ConnectionPropertyKey.PASSWORD, "s3cret");+ assertThat(connection.physicalConnectionRequirements().subnetId()).isEqualTo("subnet-0123456789abcdef0");+ assertThat(connection.statusAsString()).isEqualTo("READY");+ assertThat(connection.connectionSchemaVersion()).isEqualTo(1);+ assertThat(connection.creationTime()).isNotNull();+ assertThat(connection.lastUpdatedTime()).isNotNull();++ Connection hidden = glue.getConnection(GetConnectionRequest.builder()+ .name(CONNECTION_NAME)+ .hidePassword(true)+ .build()).connection();+ assertThat(hidden.connectionProperties()).doesNotContainKey(ConnectionPropertyKey.PASSWORD);+ assertThat(hidden.connectionProperties()).containsEntry(ConnectionPropertyKey.USERNAME, "app");++ glue.createConnection(CreateConnectionRequest.builder()+ .connectionInput(ConnectionInput.builder()+ .name(KAFKA_CONNECTION_NAME)+ .connectionType(ConnectionType.KAFKA)+ .connectionProperties(Map.of(ConnectionPropertyKey.KAFKA_BOOTSTRAP_SERVERS, "broker:9092"))+ .build())+ .build());+ List<Connection> kafkaOnly = glue.getConnections(GetConnectionsRequest.builder()+ .filter(GetConnectionsFilter.builder().connectionType(ConnectionType.KAFKA).build())+ .build()).connectionList();+ assertThat(kafkaOnly).extracting(Connection::name).contains(KAFKA_CONNECTION_NAME);+ assertThat(kafkaOnly).extracting(Connection::connectionType).containsOnly(ConnectionType.KAFKA);++ // UpdateConnection redefines the connection: the description is gone afterwards.+ glue.updateConnection(UpdateConnectionRequest.builder()+ .name(CONNECTION_NAME)+ .connectionInput(ConnectionInput.builder()+ .name(CONNECTION_NAME)+ .connectionType(ConnectionType.JDBC)+ .connectionProperties(Map.of(+ ConnectionPropertyKey.JDBC_CONNECTION_URL, "jdbc:postgresql://db2.internal:5432/orders",+ ConnectionPropertyKey.SECRET_ID, "prod/orders"))+ .build())+ .build());+ Connection redefined = glue.getConnection(GetConnectionRequest.builder()+ .name(CONNECTION_NAME)+ .build()).connection();+ assertThat(redefined.description()).isNull();+ assertThat(redefined.connectionProperties()).doesNotContainKey(ConnectionPropertyKey.PASSWORD);+ assertThat(redefined.connectionProperties()).containsEntry(ConnectionPropertyKey.SECRET_ID, "prod/orders");+ assertThat(redefined.creationTime()).isEqualTo(connection.creationTime());++ glue.deleteConnection(DeleteConnectionRequest.builder().connectionName(CONNECTION_NAME).build());+ assertThatThrownBy(() -> glue.getConnection(GetConnectionRequest.builder().name(CONNECTION_NAME).build()))+ .isInstanceOf(EntityNotFoundException.class);++ BatchDeleteConnectionResponse batch = glue.batchDeleteConnection(BatchDeleteConnectionRequest.builder()+ .connectionNameList(KAFKA_CONNECTION_NAME, CONNECTION_NAME)+ .build());+ assertThat(batch.succeeded()).containsExactly(KAFKA_CONNECTION_NAME);+ assertThat(batch.errors()).containsOnlyKeys(CONNECTION_NAME);+ }+private static TableInput tableInput(String description) {return tableInput(TABLE_NAME, description);}diff --git a/docs/services/glue.md b/docs/services/glue.mdindex f323578538..9e0f41c26b 100644--- a/docs/services/glue.md+++ b/docs/services/glue.md@@ -3,7 +3,7 @@**Protocol:** JSON 1.1**Endpoint:** `http://localhost:4566/`-Floci emulates the AWS Glue Data Catalog and Glue Schema Registry, allowing you to manage local data lake metadata and schema-version workflows.+Floci emulates the AWS Glue Data Catalog (databases, tables, partitions, functions, connections) and Glue Schema Registry, allowing you to manage local data lake metadata and schema-version workflows.## Supported Actions@@ -64,6 +64,33 @@ only arises from a backfill failure.| UpdateUserDefinedFunction | Updates a stored user-defined function. || DeleteUserDefinedFunction | Deletes a user-defined function from a database. |+#### Connections++| Action | Description |+|--------|-------------|+| CreateConnection | Creates a connection definition from a `ConnectionInput`, with optional `Tags`, and answers `CreateConnectionStatus: READY`. |+| GetConnection | Returns a stored connection. `HidePassword` omits `PASSWORD` and `ENCRYPTED_PASSWORD` from `ConnectionProperties`. |+| GetConnections | Lists connections, narrowed by `Filter.MatchCriteria`, `Filter.ConnectionType` and `Filter.ConnectionSchemaVersion`, paged by `MaxResults` and `NextToken`. |+| UpdateConnection | Redefines a connection from a full `ConnectionInput`, as on AWS: members left out of the input are dropped; the name and `CreationTime` are kept. |+| DeleteConnection | Deletes a connection and its tags. |+| BatchDeleteConnection | Deletes up to 25 connections, reporting the ones not found in `Errors`. |+| TestConnection | Accepts a connection name or an inline `TestConnectionInput` and answers with an empty body. |++`ConnectionInput` is validated against the API reference: `Name` (1 to 255 characters), `ConnectionType`+(the documented enumeration) and `ConnectionProperties` (the documented key list, at most 100 entries, and+empty for a `NETWORK` connection) are required; `MatchCriteria` holds at most 10 entries. Credentials given+under `AuthenticationConfiguration` are accepted and never returned by a read. A connection that uses+`AuthenticationConfiguration` or the `SparkProperties`, `AthenaProperties` or `PythonProperties` maps reports+`ConnectionSchemaVersion` 2; the classic JDBC, Kafka and network shape reports 1.++`HidePassword` removes `PASSWORD` and `ENCRYPTED_PASSWORD`, the two members the API reference defines as the+connection's password. The Kafka credentials (`KAFKA_CLIENT_KEYSTORE_PASSWORD`, `KAFKA_CLIENT_KEY_PASSWORD`,+`KAFKA_SASL_PLAIN_PASSWORD`, `KAFKA_SASL_SCRAM_PASSWORD` and their `ENCRYPTED_` forms) are returned as stored: the+reference does not say whether the flag covers them, so Floci does not guess.++`TestConnection` is asynchronous on AWS and returns nothing; Floci checks the request's shape and accepts it+without opening a socket to the data store, since no job or crawler runs against a connection yet.+#### Jobs| Action | Description |diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.javaindex e0216f8317..4179ddfa76 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java@@ -26,6 +26,7 @@ import java.util.Objects;public class GlueJsonHandler {private static final TypeReference<List<String>> STRING_LIST = new TypeReference<>() {};+ private static final TypeReference<Map<String, String>> STRING_MAP = new TypeReference<>() {};private static final TypeReference<List<Map<String, Object>>> MAP_LIST = new TypeReference<>() {};private static final TypeReference<List<Partition>> PARTITION_LIST = new TypeReference<>() {};@@ -285,6 +286,21 @@ public class GlueJsonHandler {glueService.deleteCrawler(req.getName(), region);yield Response.ok().build();}+ case "CreateConnection" -> handleCreateConnection(request, region);+ case "GetConnection" -> handleGetConnection(request);+ case "GetConnections" -> handleGetConnections(request);+ case "UpdateConnection" -> handleUpdateConnection(request);+ case "DeleteConnection" -> {+ glueService.deleteConnection(request.path("ConnectionName").asText(null), region);+ yield Response.ok(Map.of()).build();+ }+ case "BatchDeleteConnection" -> {+ List<String> names = request.hasNonNull("ConnectionNameList")+ ? mapper.convertValue(request.get("ConnectionNameList"), STRING_LIST)+ : null;+ yield Response.ok(glueService.batchDeleteConnections(names, region)).build();+ }+ case "TestConnection" -> handleTestConnection(request);// Read-only Glue actions for resources the emulator does not model. The AWS SDK// expects each to return a 200 with its result key present (empty), so we emit the// documented empty shape rather than an InvalidAction 400 that callers can't read.@@ -858,6 +874,59 @@ public class GlueJsonHandler {return Response.ok().build();}+ private Response handleCreateConnection(JsonNode request, String region) throws Exception {+ ConnectionInput input = request.hasNonNull("ConnectionInput")+ ? mapper.treeToValue(request.get("ConnectionInput"), ConnectionInput.class)+ : null;+ Map<String, String> tags = request.hasNonNull("Tags")+ ? mapper.convertValue(request.get("Tags"), STRING_MAP)+ : null;+ String status = glueService.createConnection(input, tags, region);+ return Response.ok(Map.of("CreateConnectionStatus", status)).build();+ }++ private Response handleGetConnection(JsonNode request) {+ Connection connection = glueService.getConnection(+ request.path("Name").asText(null), request.path("HidePassword").asBoolean(false));+ return Response.ok(Map.of("Connection", connection)).build();+ }++ private Response handleGetConnections(JsonNode request) {+ JsonNode filter = request.path("Filter");+ List<String> matchCriteria = filter.hasNonNull("MatchCriteria")+ ? mapper.convertValue(filter.get("MatchCriteria"), STRING_LIST)+ : null;+ String connectionType = filter.path("ConnectionType").asText(null);+ Integer schemaVersion = filter.hasNonNull("ConnectionSchemaVersion")+ ? filter.get("ConnectionSchemaVersion").asInt()+ : null;+ GlueService.Page<Connection> page = glueService.getConnections(+ matchCriteria, connectionType, schemaVersion,+ request.path("HidePassword").asBoolean(false),+ readMaxResults(request), readNextToken(request));+ return Response.ok(pageResponse("ConnectionList", page.items(), page.nextToken())).build();+ }++ private Response handleUpdateConnection(JsonNode request) throws Exception {+ ConnectionInput input = request.hasNonNull("ConnectionInput")+ ? mapper.treeToValue(request.get("ConnectionInput"), ConnectionInput.class)+ : null;+ glueService.updateConnection(request.path("Name").asText(null), input);+ return Response.ok(Map.of()).build();+ }++ private Response handleTestConnection(JsonNode request) {+ JsonNode input = request.path("TestConnectionInput");+ Map<String, String> properties = input.hasNonNull("ConnectionProperties")+ ? mapper.convertValue(input.get("ConnectionProperties"), STRING_MAP)+ : null;+ glueService.testConnection(+ request.path("ConnectionName").asText(null),+ input.path("ConnectionType").asText(null),+ properties);+ return Response.ok(Map.of()).build();+ }+private Response handleGetTags(JsonNode request, String region) {String arn = request.path("ResourceArn").asText(null);Map<String, String> tags = glueService.getTags(arn, region);diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.javaindex 619cb1d9d4..4c357ee34d 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java@@ -8,6 +8,8 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.glue.model.Classifier;import io.github.hectorvent.floci.services.glue.model.Column;+import io.github.hectorvent.floci.services.glue.model.Connection;+import io.github.hectorvent.floci.services.glue.model.ConnectionInput;import io.github.hectorvent.floci.services.glue.model.Crawler;import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;import io.github.hectorvent.floci.services.glue.model.Database;@@ -72,6 +74,45 @@ public class GlueService {/** Measured against real Glue (us-west-2): a fourth index reports the limit. */private static final int MAX_PARTITION_INDEXES_PER_TABLE = 3;+ // Connection limits and enumerations from the Glue API reference (ConnectionInput and the+ // Connection structure). AWS rejects a value outside them with InvalidInputException.+ private static final int MAX_CONNECTION_NAME_LENGTH = 255;+ private static final int MAX_CONNECTION_DESCRIPTION_LENGTH = 2048;+ private static final int MAX_CONNECTION_MATCH_CRITERIA = 10;+ private static final int MAX_CONNECTION_PROPERTIES = 100;+ private static final int MAX_BATCH_DELETE_CONNECTIONS = 25;+ private static final Set<String> CONNECTION_TYPES = Set.of(+ "JDBC", "SFTP", "MONGODB", "KAFKA", "NETWORK", "MARKETPLACE", "CUSTOM", "SALESFORCE",+ "VIEW_VALIDATION_REDSHIFT", "VIEW_VALIDATION_ATHENA", "GOOGLEADS", "GOOGLESHEETS",+ "GOOGLEANALYTICS4", "SERVICENOW", "MARKETO", "SAPODATA", "ZENDESK", "JIRACLOUD", "NETSUITEERP",+ "HUBSPOT", "FACEBOOKADS", "INSTAGRAMADS", "ZOHOCRM", "SALESFORCEPARDOT",+ "SALESFORCEMARKETINGCLOUD", "ADOBEANALYTICS", "SLACK", "LINKEDIN", "MIXPANEL", "ASANA", "STRIPE",+ "SMARTSHEET", "DATADOG", "WOOCOMMERCE", "INTERCOM", "SNAPCHATADS", "PAYPAL", "QUICKBOOKS",+ "FACEBOOKPAGEINSIGHTS", "FRESHDESK", "TWILIO", "DOCUSIGNMONITOR", "FRESHSALES", "ZOOM",+ "GOOGLESEARCHCONSOLE", "SALESFORCECOMMERCECLOUD", "SAPCONCUR", "DYNATRACE",+ "MICROSOFTDYNAMIC365FINANCEANDOPS", "MICROSOFTTEAMS", "BLACKBAUDRAISEREDGENXT", "MAILCHIMP",+ "GITLAB", "PENDO", "PRODUCTBOARD", "CIRCLECI", "PIPEDIVE", "SENDGRID", "AZURECOSMOS", "AZURESQL",+ "BIGQUERY", "BLACKBAUD", "CLOUDERAHIVE", "CLOUDERAIMPALA", "CLOUDWATCH", "CLOUDWATCHMETRICS",+ "CMDB", "DATALAKEGEN2", "DB2", "DB2AS400", "DOCUMENTDB", "DOMO", "DYNAMODB",+ "GOOGLECLOUDSTORAGE", "HBASE", "KUSTOMER", "MICROSOFTDYNAMICS365CRM", "MONDAY", "MYSQL", "OKTA",+ "OPENSEARCH", "ORACLE", "PIPEDRIVE", "POSTGRESQL", "SAPHANA", "SQLSERVER", "SYNAPSE", "TERADATA",+ "TERADATANOS", "TIMESTREAM", "TPCDS", "VERTICA");+ private static final Set<String> CONNECTION_PROPERTY_KEYS = Set.of(+ "HOST", "PORT", "USERNAME", "PASSWORD", "ENCRYPTED_PASSWORD", "JDBC_DRIVER_JAR_URI",+ "JDBC_DRIVER_CLASS_NAME", "JDBC_ENGINE", "JDBC_ENGINE_VERSION", "CONFIG_FILES", "INSTANCE_ID",+ "JDBC_CONNECTION_URL", "JDBC_ENFORCE_SSL", "CUSTOM_JDBC_CERT", "SKIP_CUSTOM_JDBC_CERT_VALIDATION",+ "CUSTOM_JDBC_CERT_STRING", "CONNECTION_URL", "KAFKA_BOOTSTRAP_SERVERS", "KAFKA_SSL_ENABLED",+ "KAFKA_CUSTOM_CERT", "KAFKA_SKIP_CUSTOM_CERT_VALIDATION", "KAFKA_CLIENT_KEYSTORE",+ "KAFKA_CLIENT_KEYSTORE_PASSWORD", "KAFKA_CLIENT_KEY_PASSWORD",+ "ENCRYPTED_KAFKA_CLIENT_KEYSTORE_PASSWORD", "ENCRYPTED_KAFKA_CLIENT_KEY_PASSWORD",+ "KAFKA_SASL_MECHANISM", "KAFKA_SASL_PLAIN_USERNAME", "KAFKA_SASL_PLAIN_PASSWORD",+ "ENCRYPTED_KAFKA_SASL_PLAIN_PASSWORD", "KAFKA_SASL_SCRAM_USERNAME", "KAFKA_SASL_SCRAM_PASSWORD",+ "KAFKA_SASL_SCRAM_SECRETS_ARN", "ENCRYPTED_KAFKA_SASL_SCRAM_PASSWORD", "KAFKA_SASL_GSSAPI_KEYTAB",+ "KAFKA_SASL_GSSAPI_KRB5_CONF", "KAFKA_SASL_GSSAPI_SERVICE", "KAFKA_SASL_GSSAPI_PRINCIPAL",+ "SECRET_ID", "CONNECTOR_URL", "CONNECTOR_TYPE", "CONNECTOR_CLASS_NAME", "ENDPOINT",+ "ENDPOINT_TYPE", "ROLE_ARN", "REGION", "WORKGROUP_NAME", "CLUSTER_IDENTIFIER", "DATABASE");+ private static final String CONNECTION_STATUS_READY = "READY";+// Glue's partition index states. FAILED also exists but is only reachable through a backfill// failure, which is not emulated.private static final String INDEX_STATUS_CREATING = "CREATING";@@ -89,6 +130,7 @@ public class GlueService {private final StorageBackend<String, Job> jobStore;private final StorageBackend<String, Crawler> crawlerStore;private final StorageBackend<String, Classifier> classifierStore;+ private final StorageBackend<String, Connection> connectionStore;private final GlueSchemaRegistryService schemaRegistryService;private final RegionResolver regionResolver;private final ResourceGroupsTaggingService resourceGroupsTaggingService;@@ -111,6 +153,7 @@ public class GlueService {this.jobStore = storageFactory.create("glue", "jobs.json", new TypeReference<>() {});this.crawlerStore = storageFactory.create("glue", "crawlers.json", new TypeReference<>() {});this.classifierStore = storageFactory.create("glue", "classifiers.json", new TypeReference<>() {});+ this.connectionStore = storageFactory.create("glue", "connections.json", new TypeReference<>() {});this.schemaRegistryService = schemaRegistryService;this.regionResolver = regionResolver;this.resourceGroupsTaggingService = resourceGroupsTaggingService;@@ -127,6 +170,7 @@ public class GlueService {StorageBackend<String, Job> jobStore,StorageBackend<String, Crawler> crawlerStore,StorageBackend<String, Classifier> classifierStore,+ StorageBackend<String, Connection> connectionStore,GlueSchemaRegistryService schemaRegistryService,RegionResolver regionResolver,ResourceGroupsTaggingService resourceGroupsTaggingService) {@@ -141,6 +185,7 @@ public class GlueService {this.jobStore = jobStore;this.crawlerStore = crawlerStore;this.classifierStore = classifierStore;+ this.connectionStore = connectionStore;this.schemaRegistryService = schemaRegistryService;this.regionResolver = regionResolver;this.resourceGroupsTaggingService = resourceGroupsTaggingService;@@ -1337,6 +1382,10 @@ public class GlueService {return regionResolver.buildArn("glue", region, "job/" + jobName);}+ private String connectionArn(String region, String connectionName) {+ return regionResolver.buildArn("glue", region, "connection/" + connectionName);+ }+private String crawlerArn(String region, String crawlerName) {return regionResolver.buildArn("glue", region, "crawler/" + crawlerName);}@@ -1891,6 +1940,202 @@ public class GlueService {LOG.infov("Deleted Glue Crawler: {0}", name);}+ // ---- Connections -----------------------------------------------------------------------++ public String createConnection(ConnectionInput input, Map<String, String> tags, String region) {+ validateConnectionInput(input);+ // Connections keep their case, as jobs and crawlers do: AWS matches the name exactly.+ String name = input.getName();+ if (connectionStore.get(name).isPresent()) {+ throw new AwsException("AlreadyExistsException", "Connection " + name + " already exists.", 400);+ }+ Instant now = Instant.now();+ Connection connection = toConnection(input);+ connection.setCreationTime(now);+ connection.setLastUpdatedTime(now);+ connectionStore.put(name, connection);+ if (tags != null && !tags.isEmpty()) {+ resourceGroupsTaggingService.tagResources(List.of(connectionArn(region, name)), tags, region);+ }+ LOG.infov("Created Glue Connection: {0}", name);+ // Nothing is validated against the data store, so there is no IN_PROGRESS phase to report.+ return CONNECTION_STATUS_READY;+ }++ public Connection getConnection(String name, boolean hidePassword) {+ validateRequired(name, "Name");+ Connection connection = connectionStore.get(name)+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Connection " + name + " not found.", 400));+ return hidePassword ? connection.withoutPassword() : connection;+ }++ /**+ * Lists connections, narrowed by the GetConnections filter: every {@code MatchCriteria}+ * entry must appear on the connection, and {@code ConnectionType} and+ * {@code ConnectionSchemaVersion} must match exactly when given.+ */+ public Page<Connection> getConnections(List<String> matchCriteria, String connectionType,+ Integer connectionSchemaVersion, boolean hidePassword,+ Integer maxResults, String nextToken) {+ List<Connection> matching = new ArrayList<>();+ for (Connection connection : connectionStore.scan(k -> true)) {+ if (matchCriteria != null && !matchCriteria.isEmpty()+ && (connection.getMatchCriteria() == null+ || !connection.getMatchCriteria().containsAll(matchCriteria))) {+ continue;+ }+ if (connectionType != null && !connectionType.equals(connection.getConnectionType())) {+ continue;+ }+ if (connectionSchemaVersion != null+ && !connectionSchemaVersion.equals(connection.getConnectionSchemaVersion())) {+ continue;+ }+ matching.add(hidePassword ? connection.withoutPassword() : connection);+ }+ matching.sort(Comparator.comparing(Connection::getName));+ return paginate(matching, maxResults, nextToken);+ }++ /**+ * UpdateConnection takes a ConnectionInput that "redefines the connection in question"+ * (API reference), so the stored definition is replaced rather than merged: a member left+ * out of the input is gone afterwards. The connection keeps its name and creation time.+ */+ public void updateConnection(String name, ConnectionInput input) {+ validateRequired(name, "Name");+ validateRequired(input, "ConnectionInput");+ Connection existing = connectionStore.get(name)+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Connection " + name + " not found.", 400));+ validateConnectionInput(input);+ Connection updated = toConnection(input);+ updated.setName(existing.getName());+ updated.setCreationTime(existing.getCreationTime());+ updated.setLastUpdatedTime(Instant.now());+ connectionStore.put(name, updated);+ LOG.infov("Updated Glue Connection: {0}", name);+ }++ public void deleteConnection(String name, String region) {+ validateRequired(name, "ConnectionName");+ if (connectionStore.get(name).isEmpty()) {+ throw new AwsException("EntityNotFoundException", "Connection " + name + " not found.", 400);+ }+ connectionStore.delete(name);+ resourceGroupsTaggingService.deleteResources(List.of(connectionArn(region, name)), region);+ LOG.infov("Deleted Glue Connection: {0}", name);+ }++ /** Deletes what it can; the names that could not be deleted come back keyed in the errors map. */+ public BatchDeleteConnectionResult batchDeleteConnections(List<String> names, String region) {+ validateRequired(names, "ConnectionNameList");+ if (names.size() > MAX_BATCH_DELETE_CONNECTIONS) {+ throw new AwsException("InvalidInputException",+ "ConnectionNameList must have at most " + MAX_BATCH_DELETE_CONNECTIONS + " items.", 400);+ }+ List<String> succeeded = new ArrayList<>();+ Map<String, ErrorDetail> errors = new LinkedHashMap<>();+ for (String name : names) {+ try {+ deleteConnection(name, region);+ succeeded.add(name);+ } catch (AwsException e) {+ errors.put(name, new ErrorDetail(e.getErrorCode(), e.getMessage()));+ }+ }+ return new BatchDeleteConnectionResult(succeeded, errors);+ }++ /**+ * TestConnection on AWS is asynchronous and answers with an empty body: the outcome is only+ * visible later, on the connection's status. Floci never reaches the data store (no job can+ * run against it yet), so the request is checked for shape and accepted. Either an existing+ * connection is named, or an inline definition with a type and properties is given.+ */+ public void testConnection(String connectionName, String connectionType,+ Map<String, String> connectionProperties) {+ if (connectionName != null && !connectionName.isBlank()) {+ getConnection(connectionName, false);+ return;+ }+ validateRequired(connectionType, "TestConnectionInput.ConnectionType");+ validateConnectionType(connectionType);+ validateRequired(connectionProperties, "TestConnectionInput.ConnectionProperties");+ validateConnectionProperties(connectionProperties);+ }++ private Connection toConnection(ConnectionInput input) {+ Connection connection = new Connection();+ connection.setName(input.getName());+ connection.setDescription(input.getDescription());+ connection.setConnectionType(input.getConnectionType());+ connection.setMatchCriteria(input.getMatchCriteria());+ connection.setConnectionProperties(new LinkedHashMap<>(input.getConnectionProperties()));+ connection.setSparkProperties(input.getSparkProperties());+ connection.setAthenaProperties(input.getAthenaProperties());+ connection.setPythonProperties(input.getPythonProperties());+ connection.setPhysicalConnectionRequirements(input.getPhysicalConnectionRequirements());+ if (input.getAuthenticationConfiguration() != null) {+ connection.setAuthenticationConfiguration(input.getAuthenticationConfiguration().toOutput());+ }+ connection.setStatus(CONNECTION_STATUS_READY);+ // Schema version 2 "supports properties for specific compute environments" (Connection+ // structure). A definition that uses any of those members is a version 2 connection;+ // the classic JDBC/Kafka/Network shape is version 1.+ boolean usesComputeEnvironmentMembers = input.getAuthenticationConfiguration() != null+ || input.getSparkProperties() != null+ || input.getAthenaProperties() != null+ || input.getPythonProperties() != null;+ connection.setConnectionSchemaVersion(usesComputeEnvironmentMembers ? 2 : 1);+ return connection;+ }++ private void validateConnectionInput(ConnectionInput input) {+ validateRequired(input, "ConnectionInput");+ validateRequired(input.getName(), "ConnectionInput.Name");+ if (input.getName().length() > MAX_CONNECTION_NAME_LENGTH) {+ throw new AwsException("InvalidInputException",+ "ConnectionInput.Name must be between 1 and " + MAX_CONNECTION_NAME_LENGTH + " characters.", 400);+ }+ validateRequired(input.getConnectionType(), "ConnectionInput.ConnectionType");+ validateConnectionType(input.getConnectionType());+ validateRequired(input.getConnectionProperties(), "ConnectionInput.ConnectionProperties");+ validateConnectionProperties(input.getConnectionProperties());+ if (input.getDescription() != null && input.getDescription().length() > MAX_CONNECTION_DESCRIPTION_LENGTH) {+ throw new AwsException("InvalidInputException",+ "ConnectionInput.Description must be at most " + MAX_CONNECTION_DESCRIPTION_LENGTH + " characters.", 400);+ }+ if (input.getMatchCriteria() != null && input.getMatchCriteria().size() > MAX_CONNECTION_MATCH_CRITERIA) {+ throw new AwsException("InvalidInputException",+ "ConnectionInput.MatchCriteria must have at most " + MAX_CONNECTION_MATCH_CRITERIA + " items.", 400);+ }+ }++ private void validateConnectionType(String connectionType) {+ if (!CONNECTION_TYPES.contains(connectionType)) {+ throw new AwsException("InvalidInputException",+ "Unsupported connection type: " + connectionType, 400);+ }+ }++ private void validateConnectionProperties(Map<String, String> properties) {+ if (properties.size() > MAX_CONNECTION_PROPERTIES) {+ throw new AwsException("InvalidInputException",+ "ConnectionProperties must have at most " + MAX_CONNECTION_PROPERTIES + " entries.", 400);+ }+ for (String key : properties.keySet()) {+ if (!CONNECTION_PROPERTY_KEYS.contains(key)) {+ throw new AwsException("InvalidInputException",+ "Unsupported connection property key: " + key, 400);+ }+ }+ }++ @RegisterForReflection+ public record BatchDeleteConnectionResult(+ @JsonProperty("Succeeded") List<String> succeeded,+ @JsonProperty("Errors") Map<String, ErrorDetail> errors) {}+public void tagResource(String arn, Map<String, String> tags, String region) {validateArn(arn);if (arn.contains(":registry/") || arn.contains(":schema/")) {@@ -1937,6 +2182,9 @@ public class GlueService {} else if (resource.startsWith("crawler/")) {getCrawler(resource.substring(8));return;+ } else if (resource.startsWith("connection/")) {+ getConnection(resource.substring(11), false);+ return;} else if (resource.startsWith("database/")) {getDatabase(resource.substring(9));return;diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/AuthenticationConfiguration.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/AuthenticationConfiguration.javanew file mode 100644index 0000000000..16712f3b48--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/AuthenticationConfiguration.java@@ -0,0 +1,65 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.Map;++/**+ * How a connection authenticates. {@code BasicAuthenticationCredentials} and+ * {@code CustomAuthenticationCredentials} are input-only on AWS: a Get never echoes a+ * credential, so {@link #toOutput()} drops them along with the OAuth2 secrets.+ */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class AuthenticationConfiguration {+ @JsonProperty("AuthenticationType")+ private String authenticationType;++ @JsonProperty("SecretArn")+ private String secretArn;++ @JsonProperty("KmsKeyArn")+ private String kmsKeyArn;++ @JsonProperty("OAuth2Properties")+ private OAuth2Properties oAuth2Properties;++ @JsonProperty("BasicAuthenticationCredentials")+ private Map<String, String> basicAuthenticationCredentials;++ @JsonProperty("CustomAuthenticationCredentials")+ private Map<String, String> customAuthenticationCredentials;++ public AuthenticationConfiguration toOutput() {+ AuthenticationConfiguration output = new AuthenticationConfiguration();+ output.authenticationType = authenticationType;+ output.secretArn = secretArn;+ output.kmsKeyArn = kmsKeyArn;+ output.oAuth2Properties = oAuth2Properties == null ? null : oAuth2Properties.toOutput();+ return output;+ }++ public String getAuthenticationType() { return authenticationType; }+ public void setAuthenticationType(String authenticationType) { this.authenticationType = authenticationType; }++ public String getSecretArn() { return secretArn; }+ public void setSecretArn(String secretArn) { this.secretArn = secretArn; }++ public String getKmsKeyArn() { return kmsKeyArn; }+ public void setKmsKeyArn(String kmsKeyArn) { this.kmsKeyArn = kmsKeyArn; }++ public OAuth2Properties getOAuth2Properties() { return oAuth2Properties; }+ public void setOAuth2Properties(OAuth2Properties oAuth2Properties) { this.oAuth2Properties = oAuth2Properties; }++ public Map<String, String> getBasicAuthenticationCredentials() { return basicAuthenticationCredentials; }+ public void setBasicAuthenticationCredentials(Map<String, String> basicAuthenticationCredentials) {+ this.basicAuthenticationCredentials = basicAuthenticationCredentials;+ }++ public Map<String, String> getCustomAuthenticationCredentials() { return customAuthenticationCredentials; }+ public void setCustomAuthenticationCredentials(Map<String, String> customAuthenticationCredentials) {+ this.customAuthenticationCredentials = customAuthenticationCredentials;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.javanew file mode 100644index 0000000000..95105c8fe3--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java@@ -0,0 +1,183 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonFormat;+import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++/**+ * A Data Catalog connection as GetConnection returns it: the caller's {@link ConnectionInput}+ * plus the members the service owns (timestamps, status, schema version). Credentials never+ * reach this shape; see {@link AuthenticationConfiguration#toOutput()}.+ */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class Connection {+ @JsonProperty("Name")+ private String name;++ @JsonProperty("Description")+ private String description;++ @JsonProperty("ConnectionType")+ private String connectionType;++ @JsonProperty("MatchCriteria")+ private List<String> matchCriteria;++ @JsonProperty("ConnectionProperties")+ private Map<String, String> connectionProperties;++ @JsonProperty("SparkProperties")+ private Map<String, String> sparkProperties;++ @JsonProperty("AthenaProperties")+ private Map<String, String> athenaProperties;++ @JsonProperty("PythonProperties")+ private Map<String, String> pythonProperties;++ @JsonProperty("PhysicalConnectionRequirements")+ private PhysicalConnectionRequirements physicalConnectionRequirements;++ @JsonProperty("CreationTime")+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant creationTime;++ @JsonProperty("LastUpdatedTime")+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant lastUpdatedTime;++ @JsonProperty("LastUpdatedBy")+ private String lastUpdatedBy;++ @JsonProperty("Status")+ private String status;++ @JsonProperty("StatusReason")+ private String statusReason;++ @JsonProperty("LastConnectionValidationTime")+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)+ private Instant lastConnectionValidationTime;++ @JsonProperty("AuthenticationConfiguration")+ private AuthenticationConfiguration authenticationConfiguration;++ @JsonProperty("ConnectionSchemaVersion")+ private Integer connectionSchemaVersion;++ @JsonProperty("CompatibleComputeEnvironments")+ private List<String> compatibleComputeEnvironments;++ /**+ * A copy with the password members of {@code ConnectionProperties} removed: what+ * GetConnection and GetConnections return for {@code HidePassword=true}. The API reference+ * names {@code PASSWORD} and {@code ENCRYPTED_PASSWORD} as the connection's password+ * fields, the latter being the form stored under catalog password encryption.+ */+ public Connection withoutPassword() {+ Connection copy = copy();+ if (connectionProperties != null) {+ Map<String, String> visible = new LinkedHashMap<>(connectionProperties);+ visible.remove("PASSWORD");+ visible.remove("ENCRYPTED_PASSWORD");+ copy.connectionProperties = visible;+ }+ return copy;+ }++ private Connection copy() {+ Connection copy = new Connection();+ copy.name = name;+ copy.description = description;+ copy.connectionType = connectionType;+ copy.matchCriteria = matchCriteria;+ copy.connectionProperties = connectionProperties;+ copy.sparkProperties = sparkProperties;+ copy.athenaProperties = athenaProperties;+ copy.pythonProperties = pythonProperties;+ copy.physicalConnectionRequirements = physicalConnectionRequirements;+ copy.creationTime = creationTime;+ copy.lastUpdatedTime = lastUpdatedTime;+ copy.lastUpdatedBy = lastUpdatedBy;+ copy.status = status;+ copy.statusReason = statusReason;+ copy.lastConnectionValidationTime = lastConnectionValidationTime;+ copy.authenticationConfiguration = authenticationConfiguration;+ copy.connectionSchemaVersion = connectionSchemaVersion;+ copy.compatibleComputeEnvironments = compatibleComputeEnvironments;+ return copy;+ }++ public String getName() { return name; }+ public void setName(String name) { this.name = name; }++ public String getDescription() { return description; }+ public void setDescription(String description) { this.description = description; }++ public String getConnectionType() { return connectionType; }+ public void setConnectionType(String connectionType) { this.connectionType = connectionType; }++ public List<String> getMatchCriteria() { return matchCriteria; }+ public void setMatchCriteria(List<String> matchCriteria) { this.matchCriteria = matchCriteria; }++ public Map<String, String> getConnectionProperties() { return connectionProperties; }+ public void setConnectionProperties(Map<String, String> connectionProperties) {+ this.connectionProperties = connectionProperties;+ }++ public Map<String, String> getSparkProperties() { return sparkProperties; }+ public void setSparkProperties(Map<String, String> sparkProperties) { this.sparkProperties = sparkProperties; }++ public Map<String, String> getAthenaProperties() { return athenaProperties; }+ public void setAthenaProperties(Map<String, String> athenaProperties) { this.athenaProperties = athenaProperties; }++ public Map<String, String> getPythonProperties() { return pythonProperties; }+ public void setPythonProperties(Map<String, String> pythonProperties) { this.pythonProperties = pythonProperties; }++ public PhysicalConnectionRequirements getPhysicalConnectionRequirements() { return physicalConnectionRequirements; }+ public void setPhysicalConnectionRequirements(PhysicalConnectionRequirements physicalConnectionRequirements) {+ this.physicalConnectionRequirements = physicalConnectionRequirements;+ }++ public Instant getCreationTime() { return creationTime; }+ public void setCreationTime(Instant creationTime) { this.creationTime = creationTime; }++ public Instant getLastUpdatedTime() { return lastUpdatedTime; }+ public void setLastUpdatedTime(Instant lastUpdatedTime) { this.lastUpdatedTime = lastUpdatedTime; }++ public String getLastUpdatedBy() { return lastUpdatedBy; }+ public void setLastUpdatedBy(String lastUpdatedBy) { this.lastUpdatedBy = lastUpdatedBy; }++ public String getStatus() { return status; }+ public void setStatus(String status) { this.status = status; }++ public String getStatusReason() { return statusReason; }+ public void setStatusReason(String statusReason) { this.statusReason = statusReason; }++ public Instant getLastConnectionValidationTime() { return lastConnectionValidationTime; }+ public void setLastConnectionValidationTime(Instant lastConnectionValidationTime) {+ this.lastConnectionValidationTime = lastConnectionValidationTime;+ }++ public AuthenticationConfiguration getAuthenticationConfiguration() { return authenticationConfiguration; }+ public void setAuthenticationConfiguration(AuthenticationConfiguration authenticationConfiguration) {+ this.authenticationConfiguration = authenticationConfiguration;+ }++ public Integer getConnectionSchemaVersion() { return connectionSchemaVersion; }+ public void setConnectionSchemaVersion(Integer connectionSchemaVersion) {+ this.connectionSchemaVersion = connectionSchemaVersion;+ }++ public List<String> getCompatibleComputeEnvironments() { return compatibleComputeEnvironments; }+ public void setCompatibleComputeEnvironments(List<String> compatibleComputeEnvironments) {+ this.compatibleComputeEnvironments = compatibleComputeEnvironments;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionInput.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionInput.javanew file mode 100644index 0000000000..b61c572b30--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionInput.java@@ -0,0 +1,95 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.List;+import java.util.Map;++/**+ * The {@code ConnectionInput} structure of CreateConnection and UpdateConnection: the caller's+ * full definition of a connection. On Update it redefines the connection rather than patching+ * it, which is why it carries the required members again instead of an optional subset.+ */+@RegisterForReflection+public class ConnectionInput {+ @JsonProperty("Name")+ private String name;++ @JsonProperty("Description")+ private String description;++ @JsonProperty("ConnectionType")+ private String connectionType;++ @JsonProperty("MatchCriteria")+ private List<String> matchCriteria;++ @JsonProperty("ConnectionProperties")+ private Map<String, String> connectionProperties;++ @JsonProperty("SparkProperties")+ private Map<String, String> sparkProperties;++ @JsonProperty("AthenaProperties")+ private Map<String, String> athenaProperties;++ @JsonProperty("PythonProperties")+ private Map<String, String> pythonProperties;++ @JsonProperty("PhysicalConnectionRequirements")+ private PhysicalConnectionRequirements physicalConnectionRequirements;++ @JsonProperty("AuthenticationConfiguration")+ private AuthenticationConfiguration authenticationConfiguration;++ @JsonProperty("ValidateCredentials")+ private Boolean validateCredentials;++ @JsonProperty("ValidateForComputeEnvironments")+ private List<String> validateForComputeEnvironments;++ public String getName() { return name; }+ public void setName(String name) { this.name = name; }++ public String getDescription() { return description; }+ public void setDescription(String description) { this.description = description; }++ public String getConnectionType() { return connectionType; }+ public void setConnectionType(String connectionType) { this.connectionType = connectionType; }++ public List<String> getMatchCriteria() { return matchCriteria; }+ public void setMatchCriteria(List<String> matchCriteria) { this.matchCriteria = matchCriteria; }++ public Map<String, String> getConnectionProperties() { return connectionProperties; }+ public void setConnectionProperties(Map<String, String> connectionProperties) {+ this.connectionProperties = connectionProperties;+ }++ public Map<String, String> getSparkProperties() { return sparkProperties; }+ public void setSparkProperties(Map<String, String> sparkProperties) { this.sparkProperties = sparkProperties; }++ public Map<String, String> getAthenaProperties() { return athenaProperties; }+ public void setAthenaProperties(Map<String, String> athenaProperties) { this.athenaProperties = athenaProperties; }++ public Map<String, String> getPythonProperties() { return pythonProperties; }+ public void setPythonProperties(Map<String, String> pythonProperties) { this.pythonProperties = pythonProperties; }++ public PhysicalConnectionRequirements getPhysicalConnectionRequirements() { return physicalConnectionRequirements; }+ public void setPhysicalConnectionRequirements(PhysicalConnectionRequirements physicalConnectionRequirements) {+ this.physicalConnectionRequirements = physicalConnectionRequirements;+ }++ public AuthenticationConfiguration getAuthenticationConfiguration() { return authenticationConfiguration; }+ public void setAuthenticationConfiguration(AuthenticationConfiguration authenticationConfiguration) {+ this.authenticationConfiguration = authenticationConfiguration;+ }++ public Boolean getValidateCredentials() { return validateCredentials; }+ public void setValidateCredentials(Boolean validateCredentials) { this.validateCredentials = validateCredentials; }++ public List<String> getValidateForComputeEnvironments() { return validateForComputeEnvironments; }+ public void setValidateForComputeEnvironments(List<String> validateForComputeEnvironments) {+ this.validateForComputeEnvironments = validateForComputeEnvironments;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/OAuth2ClientApplication.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/OAuth2ClientApplication.javanew file mode 100644index 0000000000..c447b98738--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/OAuth2ClientApplication.java@@ -0,0 +1,25 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class OAuth2ClientApplication {+ @JsonProperty("UserManagedClientApplicationClientId")+ private String userManagedClientApplicationClientId;++ @JsonProperty("AWSManagedClientApplicationReference")+ private String awsManagedClientApplicationReference;++ public String getUserManagedClientApplicationClientId() { return userManagedClientApplicationClientId; }+ public void setUserManagedClientApplicationClientId(String userManagedClientApplicationClientId) {+ this.userManagedClientApplicationClientId = userManagedClientApplicationClientId;+ }++ public String getAwsManagedClientApplicationReference() { return awsManagedClientApplicationReference; }+ public void setAwsManagedClientApplicationReference(String awsManagedClientApplicationReference) {+ this.awsManagedClientApplicationReference = awsManagedClientApplicationReference;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/OAuth2Properties.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/OAuth2Properties.javanew file mode 100644index 0000000000..351f2d7e2c--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/OAuth2Properties.java@@ -0,0 +1,69 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import com.fasterxml.jackson.databind.JsonNode;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.Map;++/**+ * OAuth2 settings of a connection. The two credential-bearing members,+ * {@code AuthorizationCodeProperties} and {@code OAuth2Credentials}, exist only on the input+ * shape: AWS accepts them on Create and Update and never returns them, so {@link Connection}+ * is built through {@link #toOutput()}, which leaves them behind.+ */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class OAuth2Properties {+ @JsonProperty("OAuth2GrantType")+ private String oAuth2GrantType;++ @JsonProperty("OAuth2ClientApplication")+ private OAuth2ClientApplication oAuth2ClientApplication;++ @JsonProperty("TokenUrl")+ private String tokenUrl;++ @JsonProperty("TokenUrlParametersMap")+ private Map<String, String> tokenUrlParametersMap;++ @JsonProperty("AuthorizationCodeProperties")+ private JsonNode authorizationCodeProperties;++ @JsonProperty("OAuth2Credentials")+ private JsonNode oAuth2Credentials;++ public OAuth2Properties toOutput() {+ OAuth2Properties output = new OAuth2Properties();+ output.oAuth2GrantType = oAuth2GrantType;+ output.oAuth2ClientApplication = oAuth2ClientApplication;+ output.tokenUrl = tokenUrl;+ output.tokenUrlParametersMap = tokenUrlParametersMap;+ return output;+ }++ public String getOAuth2GrantType() { return oAuth2GrantType; }+ public void setOAuth2GrantType(String oAuth2GrantType) { this.oAuth2GrantType = oAuth2GrantType; }++ public OAuth2ClientApplication getOAuth2ClientApplication() { return oAuth2ClientApplication; }+ public void setOAuth2ClientApplication(OAuth2ClientApplication oAuth2ClientApplication) {+ this.oAuth2ClientApplication = oAuth2ClientApplication;+ }++ public String getTokenUrl() { return tokenUrl; }+ public void setTokenUrl(String tokenUrl) { this.tokenUrl = tokenUrl; }++ public Map<String, String> getTokenUrlParametersMap() { return tokenUrlParametersMap; }+ public void setTokenUrlParametersMap(Map<String, String> tokenUrlParametersMap) {+ this.tokenUrlParametersMap = tokenUrlParametersMap;+ }++ public JsonNode getAuthorizationCodeProperties() { return authorizationCodeProperties; }+ public void setAuthorizationCodeProperties(JsonNode authorizationCodeProperties) {+ this.authorizationCodeProperties = authorizationCodeProperties;+ }++ public JsonNode getOAuth2Credentials() { return oAuth2Credentials; }+ public void setOAuth2Credentials(JsonNode oAuth2Credentials) { this.oAuth2Credentials = oAuth2Credentials; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/PhysicalConnectionRequirements.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/PhysicalConnectionRequirements.javanew file mode 100644index 0000000000..971d4cc037--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/PhysicalConnectionRequirements.java@@ -0,0 +1,30 @@+package io.github.hectorvent.floci.services.glue.model;++import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.List;++/** The VPC placement a job or crawler needs to reach a connection's data store. */+@RegisterForReflection+@JsonInclude(JsonInclude.Include.NON_NULL)+public class PhysicalConnectionRequirements {+ @JsonProperty("SubnetId")+ private String subnetId;++ @JsonProperty("SecurityGroupIdList")+ private List<String> securityGroupIdList;++ @JsonProperty("AvailabilityZone")+ private String availabilityZone;++ public String getSubnetId() { return subnetId; }+ public void setSubnetId(String subnetId) { this.subnetId = subnetId; }++ public List<String> getSecurityGroupIdList() { return securityGroupIdList; }+ public void setSecurityGroupIdList(List<String> securityGroupIdList) { this.securityGroupIdList = securityGroupIdList; }++ public String getAvailabilityZone() { return availabilityZone; }+ public void setAvailabilityZone(String availabilityZone) { this.availabilityZone = availabilityZone; }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueConnectionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueConnectionIntegrationTest.javanew file mode 100644index 0000000000..0ae5f722c5--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueConnectionIntegrationTest.java@@ -0,0 +1,264 @@+package io.github.hectorvent.floci.services.glue;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasItem;+import static org.hamcrest.Matchers.hasKey;+import static org.hamcrest.Matchers.not;+import static org.hamcrest.Matchers.notNullValue;+import static org.hamcrest.Matchers.nullValue;++/**+ * The Data Catalog connection lifecycle over the JSON 1.1 protocol, in the order Terraform's+ * aws_glue_connection drives it: create with tags, read on every refresh, update by full+ * redefinition, delete. Each test uses its own names so the suite can run in any order.+ */+@QuarkusTest+class GlueConnectionIntegrationTest {++ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ private static String unique(String prefix) {+ return prefix + "-" + UUID.randomUUID().toString().substring(0, 8);+ }++ private static void createJdbcConnection(String name, String tagsJson) {+ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.CreateConnection")+ .body("""+ {+ "ConnectionInput": {+ "Name": "%s",+ "Description": "orders database",+ "ConnectionType": "JDBC",+ "MatchCriteria": ["orders"],+ "ConnectionProperties": {+ "JDBC_CONNECTION_URL": "jdbc:postgresql://db.internal:5432/orders",+ "USERNAME": "app",+ "PASSWORD": "s3cret"+ },+ "PhysicalConnectionRequirements": {+ "SubnetId": "subnet-0123456789abcdef0",+ "SecurityGroupIdList": ["sg-0123456789abcdef0"],+ "AvailabilityZone": "us-east-1a"+ }+ }%s+ }+ """.formatted(name, tagsJson))+ .when().post("/")+ .then()+ .statusCode(200)+ .body("CreateConnectionStatus", equalTo("READY"));+ }++ @Test+ void createConnection_thenGetConnection_returnsTheDefinitionWithServiceOwnedFields() {+ String name = unique("orders");+ createJdbcConnection(name, "");++ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.GetConnection")+ .body("{\"Name\": \"%s\"}".formatted(name))+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Connection.Name", equalTo(name))+ .body("Connection.ConnectionType", equalTo("JDBC"))+ .body("Connection.Description", equalTo("orders database"))+ .body("Connection.ConnectionProperties.PASSWORD", equalTo("s3cret"))+ .body("Connection.PhysicalConnectionRequirements.SubnetId", equalTo("subnet-0123456789abcdef0"))+ .body("Connection.Status", equalTo("READY"))+ .body("Connection.ConnectionSchemaVersion", equalTo(1))+ .body("Connection.CreationTime", notNullValue())+ .body("Connection.LastUpdatedTime", notNullValue());+ }++ @Test+ void getConnection_hidePassword_omitsThePasswordAndKeepsTheRest() {+ String name = unique("hidden");+ createJdbcConnection(name, "");++ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.GetConnection")+ .body("{\"Name\": \"%s\", \"HidePassword\": true}".formatted(name))+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Connection.ConnectionProperties", not(hasKey("PASSWORD")))+ .body("Connection.ConnectionProperties.USERNAME", equalTo("app"));+ }++ @Test+ void getConnections_filterByType_returnsOnlyMatchingConnections() {+ String jdbc = unique("jdbc");+ createJdbcConnection(jdbc, "");+ String kafka = unique("kafka");+ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.CreateConnection")+ .body("""+ {+ "ConnectionInput": {+ "Name": "%s",+ "ConnectionType": "KAFKA",+ "ConnectionProperties": {"KAFKA_BOOTSTRAP_SERVERS": "broker:9092"}+ }+ }+ """.formatted(kafka))+ .when().post("/")+ .then().statusCode(200);++ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.GetConnections")+ .body("{\"Filter\": {\"ConnectionType\": \"KAFKA\"}, \"MaxResults\": 100}")+ .when().post("/")+ .then()+ .statusCode(200)+ .body("ConnectionList.findAll { it.ConnectionType == 'KAFKA' }.Name", hasItem(kafka))+ .body("ConnectionList.findAll { it.ConnectionType != 'KAFKA' }.size()", equalTo(0))+ .body("NextToken", nullValue());+ }++ @Test+ void updateConnection_redefinesTheConnection_droppingMembersLeftOut() {+ String name = unique("redefined");+ createJdbcConnection(name, "");++ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.UpdateConnection")+ .body("""+ {+ "Name": "%s",+ "ConnectionInput": {+ "Name": "%s",+ "ConnectionType": "JDBC",+ "ConnectionProperties": {+ "JDBC_CONNECTION_URL": "jdbc:postgresql://db2.internal:5432/orders",+ "SECRET_ID": "prod/orders"+ }+ }+ }+ """.formatted(name, name))+ .when().post("/")+ .then().statusCode(200);++ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.GetConnection")+ .body("{\"Name\": \"%s\"}".formatted(name))+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Connection.ConnectionProperties.JDBC_CONNECTION_URL", equalTo("jdbc:postgresql://db2.internal:5432/orders"))+ .body("Connection.ConnectionProperties.SECRET_ID", equalTo("prod/orders"))+ .body("Connection.ConnectionProperties", not(hasKey("PASSWORD")))+ .body("Connection", not(hasKey("Description")))+ .body("Connection", not(hasKey("PhysicalConnectionRequirements")));+ }++ @Test+ void createConnectionWithTags_tagsReadableThroughGlueAndResourceGroupsTagging() {+ String name = unique("tagged");+ String arn = "arn:aws:glue:us-east-1:000000000000:connection/" + name;+ createJdbcConnection(name, ",\n \"Tags\": {\"env\": \"dev\"}");++ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.GetTags")+ .body("{\"ResourceArn\": \"%s\"}".formatted(arn))+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Tags.env", equalTo("dev"));++ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "ResourceGroupsTaggingAPI_20170126.GetResources")+ .body("{\"ResourceARNList\": [\"%s\"]}".formatted(arn))+ .when().post("/")+ .then()+ .statusCode(200)+ .body("ResourceTagMappingList.size()", equalTo(1))+ .body("ResourceTagMappingList[0].ResourceARN", equalTo(arn))+ .body("ResourceTagMappingList[0].Tags[0].Key", equalTo("env"))+ .body("ResourceTagMappingList[0].Tags[0].Value", equalTo("dev"));+ }++ @Test+ void deleteConnection_removesIt_andASecondDeleteIsEntityNotFound() {+ String name = unique("deleted");+ createJdbcConnection(name, "");++ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.DeleteConnection")+ .body("{\"ConnectionName\": \"%s\"}".formatted(name))+ .when().post("/")+ .then().statusCode(200);++ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.DeleteConnection")+ .body("{\"ConnectionName\": \"%s\"}".formatted(name))+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("EntityNotFoundException"));+ }++ @Test+ void batchDeleteConnection_reportsMissingNamesInErrors() {+ String name = unique("batch");+ createJdbcConnection(name, "");++ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.BatchDeleteConnection")+ .body("{\"ConnectionNameList\": [\"%s\", \"%s-absent\"]}".formatted(name, name))+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Succeeded[0]", equalTo(name))+ .body("Errors.'%s-absent'.ErrorCode".formatted(name), equalTo("EntityNotFoundException"));+ }++ @Test+ void createConnection_unknownPropertyKey_isInvalidInput() {+ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.CreateConnection")+ .body("""+ {+ "ConnectionInput": {+ "Name": "%s",+ "ConnectionType": "JDBC",+ "ConnectionProperties": {"HOSTNAME": "db"}+ }+ }+ """.formatted(unique("bad")))+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidInputException"));+ }++ @Test+ void testConnection_acceptsAnInlineDefinitionWithAnEmptyBody() {+ given().contentType(CONTENT_TYPE)+ .header("X-Amz-Target", "AWSGlue.TestConnection")+ .body("""+ {+ "TestConnectionInput": {+ "ConnectionType": "JDBC",+ "ConnectionProperties": {"JDBC_CONNECTION_URL": "jdbc:mysql://h:3306/d"}+ }+ }+ """)+ .when().post("/")+ .then().statusCode(200);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.javaindex 9c44ec9e7f..0843b32551 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java@@ -19,6 +19,7 @@ import java.util.List;import java.util.Map;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;@@ -624,4 +625,116 @@ class GlueJsonHandlerTest {return AccountAwareStorageBackend.inMemory("000000000000");}}++ private ObjectNode connectionInput(String name) {+ ObjectNode input = mapper.createObjectNode();+ input.put("Name", name);+ input.put("ConnectionType", "JDBC");+ input.put("Description", "orders db");+ input.putArray("MatchCriteria").add("orders");+ ObjectNode properties = input.putObject("ConnectionProperties");+ properties.put("JDBC_CONNECTION_URL", "jdbc:postgresql://db.internal:5432/orders");+ properties.put("USERNAME", "app");+ properties.put("PASSWORD", "s3cret");+ ObjectNode placement = input.putObject("PhysicalConnectionRequirements");+ placement.put("SubnetId", "subnet-0123456789abcdef0");+ placement.putArray("SecurityGroupIdList").add("sg-0123456789abcdef0");+ placement.put("AvailabilityZone", "us-east-1a");+ return input;+ }++ /**+ * The wire shape a Glue client reads back: CreateConnection answers with its status, and+ * GetConnection returns the definition under "Connection" with numeric timestamps and no+ * null-valued members, which is how the AWS SDK and the Terraform provider expect it.+ */+ @Test+ void createAndGetConnectionRoundTripTheDefinitionOnTheWire() throws Exception {+ ObjectNode create = mapper.createObjectNode();+ create.set("ConnectionInput", connectionInput("orders"));+ create.putObject("Tags").put("env", "dev");++ Response created = handler.handle("CreateConnection", create, REGION);+ assertEquals(200, created.getStatus());+ assertEquals("READY", mapper.valueToTree(created.getEntity()).get("CreateConnectionStatus").asText());++ Response got = handler.handle("GetConnection", mapper.createObjectNode().put("Name", "orders"), REGION);+ assertEquals(200, got.getStatus());+ JsonNode connection = mapper.valueToTree(got.getEntity()).get("Connection");+ assertEquals("orders", connection.get("Name").asText());+ assertEquals("JDBC", connection.get("ConnectionType").asText());+ assertEquals("s3cret", connection.get("ConnectionProperties").get("PASSWORD").asText());+ assertEquals("subnet-0123456789abcdef0", connection.get("PhysicalConnectionRequirements").get("SubnetId").asText());+ assertEquals("READY", connection.get("Status").asText());+ assertEquals(1, connection.get("ConnectionSchemaVersion").asInt());+ assertTrue(connection.get("CreationTime").isNumber());+ assertTrue(connection.get("LastUpdatedTime").isNumber());+ assertFalse(connection.has("LastUpdatedBy"));+ assertFalse(connection.has("AuthenticationConfiguration"));++ ObjectNode tags = mapper.createObjectNode();+ tags.put("ResourceArn", "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":connection/orders");+ JsonNode tagBody = mapper.valueToTree(handler.handle("GetTags", tags, REGION).getEntity());+ assertEquals("dev", tagBody.get("Tags").get("env").asText());+ }++ @Test+ void getConnectionsHonoursHidePasswordAndTheFilter() throws Exception {+ ObjectNode create = mapper.createObjectNode();+ create.set("ConnectionInput", connectionInput("orders"));+ handler.handle("CreateConnection", create, REGION);+ ObjectNode kafkaInput = connectionInput("events");+ kafkaInput.put("ConnectionType", "KAFKA");+ kafkaInput.putObject("ConnectionProperties").put("KAFKA_BOOTSTRAP_SERVERS", "broker:9092");+ ObjectNode createKafka = mapper.createObjectNode();+ createKafka.set("ConnectionInput", kafkaInput);+ handler.handle("CreateConnection", createKafka, REGION);++ ObjectNode list = mapper.createObjectNode();+ list.put("HidePassword", true);+ list.putObject("Filter").put("ConnectionType", "JDBC");+ JsonNode body = mapper.valueToTree(handler.handle("GetConnections", list, REGION).getEntity());+ assertEquals(1, body.get("ConnectionList").size());+ JsonNode orders = body.get("ConnectionList").get(0);+ assertEquals("orders", orders.get("Name").asText());+ assertFalse(orders.get("ConnectionProperties").has("PASSWORD"));+ assertEquals("app", orders.get("ConnectionProperties").get("USERNAME").asText());+ assertFalse(body.has("NextToken"));+ }++ @Test+ void updateDeleteAndBatchDeleteConnectionAnswerWithTheDocumentedBodies() throws Exception {+ ObjectNode create = mapper.createObjectNode();+ create.set("ConnectionInput", connectionInput("orders"));+ handler.handle("CreateConnection", create, REGION);++ ObjectNode update = mapper.createObjectNode();+ update.put("Name", "orders");+ ObjectNode redefinition = connectionInput("orders");+ redefinition.remove("Description");+ update.set("ConnectionInput", redefinition);+ Response updated = handler.handle("UpdateConnection", update, REGION);+ assertEquals(200, updated.getStatus());+ assertEquals(0, mapper.valueToTree(updated.getEntity()).size());+ JsonNode afterUpdate = mapper.valueToTree(+ handler.handle("GetConnection", mapper.createObjectNode().put("Name", "orders"), REGION).getEntity())+ .get("Connection");+ assertFalse(afterUpdate.has("Description"));++ ObjectNode batch = mapper.createObjectNode();+ batch.putArray("ConnectionNameList").add("orders").add("absent");+ JsonNode batchBody = mapper.valueToTree(handler.handle("BatchDeleteConnection", batch, REGION).getEntity());+ assertEquals("orders", batchBody.get("Succeeded").get(0).asText());+ assertEquals("EntityNotFoundException", batchBody.get("Errors").get("absent").get("ErrorCode").asText());++ AwsException gone = assertThrows(AwsException.class, () -> handler.handle(+ "DeleteConnection", mapper.createObjectNode().put("ConnectionName", "orders"), REGION));+ assertEquals("EntityNotFoundException", gone.getErrorCode());++ ObjectNode test = mapper.createObjectNode();+ ObjectNode inline = test.putObject("TestConnectionInput");+ inline.put("ConnectionType", "JDBC");+ inline.putObject("ConnectionProperties").put("JDBC_CONNECTION_URL", "jdbc:mysql://h:3306/d");+ assertEquals(200, handler.handle("TestConnection", test, REGION).getStatus());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.javaindex 0f67444e87..d4e0794c77 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java@@ -10,7 +10,11 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.glue.model.Classifier;import io.github.hectorvent.floci.services.glue.model.Column;import io.github.hectorvent.floci.services.glue.model.Database;+import io.github.hectorvent.floci.services.glue.model.Connection;+import io.github.hectorvent.floci.services.glue.model.ConnectionInput;import io.github.hectorvent.floci.services.glue.model.Crawler;+import io.github.hectorvent.floci.services.glue.model.AuthenticationConfiguration;+import io.github.hectorvent.floci.services.glue.model.PhysicalConnectionRequirements;import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;import io.github.hectorvent.floci.services.glue.model.Job;import io.github.hectorvent.floci.services.glue.model.JobCommand;@@ -35,6 +39,7 @@ import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.concurrent.TimeUnit;+import java.util.stream.IntStream;import java.util.concurrent.CountDownLatch;import static org.junit.jupiter.api.Assertions.assertEquals;@@ -88,6 +93,7 @@ class GlueServiceTest {new InMemoryStorage<String, Job>(),new InMemoryStorage<String, Crawler>(),new InMemoryStorage<String, Classifier>(),+ new InMemoryStorage<String, Connection>(),schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(null));glueService.createDatabase(new Database("db1"));}@@ -1385,7 +1391,7 @@ class GlueServiceTest {assertEquals("plain", errors.get(0).tableName());assertEquals(1, glueService.getTableVersions("db1", "plain").size(), "only the current version is left");- List<String> tooMany = java.util.stream.IntStream.rangeClosed(1, 101)+ List<String> tooMany = IntStream.rangeClosed(1, 101).mapToObj(String::valueOf).toList();AwsException overCap = assertThrows(AwsException.class,() -> glueService.batchDeleteTableVersions("db1", "plain", tooMany));@@ -1480,6 +1486,240 @@ class GlueServiceTest {assertNull(second.nextToken());}+ // ---- Connections -----------------------------------------------------------------------++ private static ConnectionInput jdbcConnection(String name) {+ ConnectionInput input = new ConnectionInput();+ input.setName(name);+ input.setConnectionType("JDBC");+ input.setDescription("orders db");+ input.setMatchCriteria(List.of("orders", "reporting"));+ Map<String, String> properties = new LinkedHashMap<>();+ properties.put("JDBC_CONNECTION_URL", "jdbc:postgresql://db.internal:5432/orders");+ properties.put("USERNAME", "app");+ properties.put("PASSWORD", "s3cret");+ input.setConnectionProperties(properties);+ PhysicalConnectionRequirements placement = new PhysicalConnectionRequirements();+ placement.setSubnetId("subnet-0123456789abcdef0");+ placement.setSecurityGroupIdList(List.of("sg-0123456789abcdef0"));+ placement.setAvailabilityZone("us-east-1a");+ input.setPhysicalConnectionRequirements(placement);+ return input;+ }++ @Test+ void connectionCanBeCreatedFetchedUpdatedAndDeleted() {+ assertEquals("READY", glueService.createConnection(jdbcConnection("orders"), null, REGION));++ Connection fetched = glueService.getConnection("orders", false);+ assertEquals("orders", fetched.getName());+ assertEquals("JDBC", fetched.getConnectionType());+ assertEquals("orders db", fetched.getDescription());+ assertEquals("s3cret", fetched.getConnectionProperties().get("PASSWORD"));+ assertEquals("subnet-0123456789abcdef0", fetched.getPhysicalConnectionRequirements().getSubnetId());+ assertEquals("READY", fetched.getStatus());+ assertEquals(1, fetched.getConnectionSchemaVersion());+ assertNotNull(fetched.getCreationTime());+ assertEquals(fetched.getCreationTime(), fetched.getLastUpdatedTime());++ // UpdateConnection redefines the connection: the description left out of the input is+ // gone afterwards, the name and creation time survive, and the update time moves.+ ConnectionInput redefinition = new ConnectionInput();+ redefinition.setName("orders");+ redefinition.setConnectionType("JDBC");+ redefinition.setConnectionProperties(Map.of("JDBC_CONNECTION_URL", "jdbc:postgresql://db2.internal:5432/orders"));+ glueService.updateConnection("orders", redefinition);++ Connection updated = glueService.getConnection("orders", false);+ assertEquals("jdbc:postgresql://db2.internal:5432/orders", updated.getConnectionProperties().get("JDBC_CONNECTION_URL"));+ assertNull(updated.getConnectionProperties().get("PASSWORD"));+ assertNull(updated.getDescription());+ assertNull(updated.getPhysicalConnectionRequirements());+ assertEquals(fetched.getCreationTime(), updated.getCreationTime());+ assertFalse(updated.getLastUpdatedTime().isBefore(fetched.getLastUpdatedTime()));++ glueService.deleteConnection("orders", REGION);++ AwsException ex = assertThrows(AwsException.class, () -> glueService.getConnection("orders", false));+ assertEquals("EntityNotFoundException", ex.getErrorCode());+ }++ @Test+ void connectionRejectsDuplicateCreationAndUpdateOfAMissingOne() {+ glueService.createConnection(jdbcConnection("dup"), null, REGION);++ AwsException duplicate = assertThrows(AwsException.class,+ () -> glueService.createConnection(jdbcConnection("dup"), null, REGION));+ assertEquals("AlreadyExistsException", duplicate.getErrorCode());++ AwsException missing = assertThrows(AwsException.class,+ () -> glueService.updateConnection("absent", jdbcConnection("absent")));+ assertEquals("EntityNotFoundException", missing.getErrorCode());++ AwsException missingDelete = assertThrows(AwsException.class,+ () -> glueService.deleteConnection("absent", REGION));+ assertEquals("EntityNotFoundException", missingDelete.getErrorCode());+ }++ @Test+ void connectionInputIsValidatedAgainstTheApiReference() {+ ConnectionInput noName = jdbcConnection("x");+ noName.setName(" ");+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.createConnection(noName, null, REGION)).getErrorCode());++ ConnectionInput longName = jdbcConnection("n".repeat(256));+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.createConnection(longName, null, REGION)).getErrorCode());++ ConnectionInput unknownType = jdbcConnection("t");+ unknownType.setConnectionType("FTP");+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.createConnection(unknownType, null, REGION)).getErrorCode());++ ConnectionInput noProperties = jdbcConnection("p");+ noProperties.setConnectionProperties(null);+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.createConnection(noProperties, null, REGION)).getErrorCode());++ ConnectionInput unknownKey = jdbcConnection("k");+ unknownKey.setConnectionProperties(Map.of("HOSTNAME", "db"));+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.createConnection(unknownKey, null, REGION)).getErrorCode());++ ConnectionInput tooManyCriteria = jdbcConnection("m");+ tooManyCriteria.setMatchCriteria(List.of("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"));+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.createConnection(tooManyCriteria, null, REGION)).getErrorCode());++ // A NETWORK connection carries no properties at all, only VPC placement.+ ConnectionInput network = jdbcConnection("vpc-only");+ network.setConnectionType("NETWORK");+ network.setConnectionProperties(Map.of());+ assertEquals("READY", glueService.createConnection(network, null, REGION));+ }++ @Test+ void hidePasswordDropsThePasswordPropertiesAndNothingElse() {+ ConnectionInput input = jdbcConnection("hidden");+ Map<String, String> properties = new LinkedHashMap<>(input.getConnectionProperties());+ properties.put("ENCRYPTED_PASSWORD", "AQICAHg...");+ properties.put("SECRET_ID", "prod/orders");+ input.setConnectionProperties(properties);+ glueService.createConnection(input, null, REGION);++ Connection hidden = glueService.getConnection("hidden", true);+ assertNull(hidden.getConnectionProperties().get("PASSWORD"));+ assertNull(hidden.getConnectionProperties().get("ENCRYPTED_PASSWORD"));+ assertEquals("app", hidden.getConnectionProperties().get("USERNAME"));+ assertEquals("prod/orders", hidden.getConnectionProperties().get("SECRET_ID"));++ // The stored connection is untouched: a later read without the flag has the password.+ assertEquals("s3cret", glueService.getConnection("hidden", false).getConnectionProperties().get("PASSWORD"));+ assertEquals("s3cret",+ glueService.getConnections(null, null, null, false, null, null).items().get(0)+ .getConnectionProperties().get("PASSWORD"));+ assertNull(glueService.getConnections(null, null, null, true, null, null).items().get(0)+ .getConnectionProperties().get("PASSWORD"));+ }++ @Test+ void getConnectionsFiltersByCriteriaTypeAndSchemaVersionAndPages() {+ glueService.createConnection(jdbcConnection("b-orders"), null, REGION);+ ConnectionInput kafka = jdbcConnection("a-events");+ kafka.setConnectionType("KAFKA");+ kafka.setMatchCriteria(List.of("events"));+ kafka.setConnectionProperties(Map.of("KAFKA_BOOTSTRAP_SERVERS", "broker:9092"));+ glueService.createConnection(kafka, null, REGION);+ ConnectionInput saas = jdbcConnection("c-crm");+ saas.setConnectionType("SALESFORCE");+ saas.setMatchCriteria(null);+ saas.setConnectionProperties(Map.of("ROLE_ARN", "arn:aws:iam::000000000000:role/crm"));+ AuthenticationConfiguration auth = new AuthenticationConfiguration();+ auth.setAuthenticationType("OAUTH2");+ auth.setSecretArn("arn:aws:secretsmanager:us-east-1:000000000000:secret:crm");+ auth.setBasicAuthenticationCredentials(Map.of("Username", "u", "Password", "p"));+ saas.setAuthenticationConfiguration(auth);+ glueService.createConnection(saas, null, REGION);++ List<Connection> all = glueService.getConnections(null, null, null, false, null, null).items();+ assertEquals(List.of("a-events", "b-orders", "c-crm"), all.stream().map(Connection::getName).toList());++ assertEquals(List.of("b-orders"),+ glueService.getConnections(List.of("orders", "reporting"), null, null, false, null, null)+ .items().stream().map(Connection::getName).toList());+ assertTrue(glueService.getConnections(List.of("orders", "missing"), null, null, false, null, null)+ .items().isEmpty());+ assertEquals(List.of("a-events"),+ glueService.getConnections(null, "KAFKA", null, false, null, null)+ .items().stream().map(Connection::getName).toList());+ assertEquals(List.of("c-crm"),+ glueService.getConnections(null, null, 2, false, null, null)+ .items().stream().map(Connection::getName).toList());++ // A credential given on create never comes back on a read.+ Connection crm = glueService.getConnection("c-crm", false);+ assertEquals("OAUTH2", crm.getAuthenticationConfiguration().getAuthenticationType());+ assertNull(crm.getAuthenticationConfiguration().getBasicAuthenticationCredentials());++ GlueService.Page<Connection> first = glueService.getConnections(null, null, null, false, 2, null);+ assertEquals(2, first.items().size());+ assertNotNull(first.nextToken());+ GlueService.Page<Connection> second = glueService.getConnections(null, null, null, false, 2, first.nextToken());+ assertEquals(List.of("c-crm"), second.items().stream().map(Connection::getName).toList());+ assertNull(second.nextToken());+ }++ @Test+ void batchDeleteConnectionReportsTheMissingNamesAndDeletesTheRest() {+ glueService.createConnection(jdbcConnection("keep-a"), null, REGION);+ glueService.createConnection(jdbcConnection("keep-b"), null, REGION);++ GlueService.BatchDeleteConnectionResult result =+ glueService.batchDeleteConnections(List.of("keep-a", "absent", "keep-b"), REGION);++ assertEquals(List.of("keep-a", "keep-b"), result.succeeded());+ assertEquals(1, result.errors().size());+ assertEquals("EntityNotFoundException", result.errors().get("absent").errorCode());+ assertTrue(glueService.getConnections(null, null, null, false, null, null).items().isEmpty());++ // The reference caps ConnectionNameList at 25 entries.+ List<String> twentySix = IntStream.rangeClosed(1, 26).mapToObj(i -> "c" + i).toList();+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.batchDeleteConnections(twentySix, REGION)).getErrorCode());+ }++ @Test+ void testConnectionAcceptsAnExistingNameOrAWellFormedInlineDefinition() {+ glueService.createConnection(jdbcConnection("probe"), null, REGION);+ glueService.testConnection("probe", null, null);+ glueService.testConnection(null, "JDBC", Map.of("JDBC_CONNECTION_URL", "jdbc:mysql://h:3306/d"));++ assertEquals("EntityNotFoundException",+ assertThrows(AwsException.class, () -> glueService.testConnection("absent", null, null)).getErrorCode());+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.testConnection(null, "JDBC", null)).getErrorCode());+ assertEquals("InvalidInputException",+ assertThrows(AwsException.class, () -> glueService.testConnection(null, null, Map.of())).getErrorCode());+ }++ @Test+ void connectionTagsAreKeptUnderTheConnectionArnAndRemovedOnDelete() {+ glueService.createConnection(jdbcConnection("tagged"), Map.of("env", "dev"), REGION);++ String arn = "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":connection/tagged";+ assertEquals("dev", glueService.getTags(arn, REGION).get("env"));++ glueService.tagResource(arn, Map.of("team", "data"), REGION);+ assertEquals("data", glueService.getTags(arn, REGION).get("team"));+ glueService.untagResource(arn, List.of("env"), REGION);+ assertNull(glueService.getTags(arn, REGION).get("env"));++ glueService.deleteConnection("tagged", REGION);+ assertEquals("EntityNotFoundException",+ assertThrows(AwsException.class, () -> glueService.getTags(arn, REGION)).getErrorCode());+ }+private static List<String> names(GlueService.Page<Table> page) {return page.items().stream().map(Table::getName).toList();}