Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(glue): Data Catalog connections (#3939)

Davida793776c1a99
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
index 6e5ff90a3d..ee5bebef7f 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
@@ -6,6 +6,20 @@ import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.services.glue.GlueClient;
import software.amazon.awssdk.services.glue.model.BatchCreatePartitionRequest;
+import software.amazon.awssdk.services.glue.model.BatchDeleteConnectionRequest;
+import software.amazon.awssdk.services.glue.model.BatchDeleteConnectionResponse;
+import software.amazon.awssdk.services.glue.model.Connection;
+import software.amazon.awssdk.services.glue.model.ConnectionInput;
+import software.amazon.awssdk.services.glue.model.ConnectionPropertyKey;
+import software.amazon.awssdk.services.glue.model.ConnectionType;
+import software.amazon.awssdk.services.glue.model.CreateConnectionRequest;
+import software.amazon.awssdk.services.glue.model.CreateConnectionResponse;
+import software.amazon.awssdk.services.glue.model.DeleteConnectionRequest;
+import software.amazon.awssdk.services.glue.model.GetConnectionRequest;
+import software.amazon.awssdk.services.glue.model.GetConnectionsFilter;
+import software.amazon.awssdk.services.glue.model.GetConnectionsRequest;
+import software.amazon.awssdk.services.glue.model.PhysicalConnectionRequirements;
+import software.amazon.awssdk.services.glue.model.UpdateConnectionRequest;
import software.amazon.awssdk.services.glue.model.BatchDeleteTableRequest;
import software.amazon.awssdk.services.glue.model.BatchGetPartitionRequest;
import software.amazon.awssdk.services.glue.model.BatchUpdatePartitionRequest;
@@ -54,6 +68,7 @@ import software.amazon.awssdk.services.glue.model.UserDefinedFunctionInput;
import software.amazon.awssdk.services.resourcegroupstaggingapi.ResourceGroupsTaggingApiClient;
import java.time.Instant;
+import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -69,6 +84,8 @@ class GlueCatalogTest {
private static final String SECOND_TABLE_NAME = "catalog_table_second";
private static final String FUNCTION_NAME = "catalog_function";
private static final String DATABASE_TAGGED_NAME = TestFixtures.uniqueName("catalog_tagged_db");
+ private static final String CONNECTION_NAME = TestFixtures.uniqueName("catalog_connection");
+ private static final String KAFKA_CONNECTION_NAME = TestFixtures.uniqueName("catalog_kafka_connection");
private static GlueClient glue;
private static ResourceGroupsTaggingApiClient tagging;
@@ -84,6 +101,12 @@ class GlueCatalogTest {
if (glue == null) {
return;
}
+ try {
+ glue.batchDeleteConnection(BatchDeleteConnectionRequest.builder()
+ .connectionNameList(CONNECTION_NAME, KAFKA_CONNECTION_NAME)
+ .build());
+ }
+ catch (Exception ignored) {}
try {
glue.deleteUserDefinedFunction(DeleteUserDefinedFunctionRequest.builder()
.databaseName(DATABASE_NAME)
@@ -531,6 +554,92 @@ class GlueCatalogTest {
});
}
+ @Test
+ @DisplayName("Connections are created, read with and without the password, redefined and deleted")
+ void connectionLifecycle() {
+ CreateConnectionResponse created = glue.createConnection(CreateConnectionRequest.builder()
+ .connectionInput(ConnectionInput.builder()
+ .name(CONNECTION_NAME)
+ .description("orders database")
+ .connectionType(ConnectionType.JDBC)
+ .matchCriteria("orders")
+ .connectionProperties(Map.of(
+ ConnectionPropertyKey.JDBC_CONNECTION_URL, "jdbc:postgresql://db.internal:5432/orders",
+ ConnectionPropertyKey.USERNAME, "app",
+ ConnectionPropertyKey.PASSWORD, "s3cret"))
+ .physicalConnectionRequirements(PhysicalConnectionRequirements.builder()
+ .subnetId("subnet-0123456789abcdef0")
+ .securityGroupIdList("sg-0123456789abcdef0")
+ .availabilityZone("us-east-1a")
+ .build())
+ .build())
+ .tags(Map.of("env", "dev"))
+ .build());
+ assertThat(created.createConnectionStatusAsString()).isEqualTo("READY");
+
+ Connection connection = glue.getConnection(GetConnectionRequest.builder()
+ .name(CONNECTION_NAME)
+ .build()).connection();
+ assertThat(connection.name()).isEqualTo(CONNECTION_NAME);
+ assertThat(connection.connectionType()).isEqualTo(ConnectionType.JDBC);
+ assertThat(connection.description()).isEqualTo("orders database");
+ assertThat(connection.connectionProperties()).containsEntry(ConnectionPropertyKey.PASSWORD, "s3cret");
+ assertThat(connection.physicalConnectionRequirements().subnetId()).isEqualTo("subnet-0123456789abcdef0");
+ assertThat(connection.statusAsString()).isEqualTo("READY");
+ assertThat(connection.connectionSchemaVersion()).isEqualTo(1);
+ assertThat(connection.creationTime()).isNotNull();
+ assertThat(connection.lastUpdatedTime()).isNotNull();
+
+ Connection hidden = glue.getConnection(GetConnectionRequest.builder()
+ .name(CONNECTION_NAME)
+ .hidePassword(true)
+ .build()).connection();
+ assertThat(hidden.connectionProperties()).doesNotContainKey(ConnectionPropertyKey.PASSWORD);
+ assertThat(hidden.connectionProperties()).containsEntry(ConnectionPropertyKey.USERNAME, "app");
+
+ glue.createConnection(CreateConnectionRequest.builder()
+ .connectionInput(ConnectionInput.builder()
+ .name(KAFKA_CONNECTION_NAME)
+ .connectionType(ConnectionType.KAFKA)
+ .connectionProperties(Map.of(ConnectionPropertyKey.KAFKA_BOOTSTRAP_SERVERS, "broker:9092"))
+ .build())
+ .build());
+ List<Connection> kafkaOnly = glue.getConnections(GetConnectionsRequest.builder()
+ .filter(GetConnectionsFilter.builder().connectionType(ConnectionType.KAFKA).build())
+ .build()).connectionList();
+ assertThat(kafkaOnly).extracting(Connection::name).contains(KAFKA_CONNECTION_NAME);
+ assertThat(kafkaOnly).extracting(Connection::connectionType).containsOnly(ConnectionType.KAFKA);
+
+ // UpdateConnection redefines the connection: the description is gone afterwards.
+ glue.updateConnection(UpdateConnectionRequest.builder()
+ .name(CONNECTION_NAME)
+ .connectionInput(ConnectionInput.builder()
+ .name(CONNECTION_NAME)
+ .connectionType(ConnectionType.JDBC)
+ .connectionProperties(Map.of(
+ ConnectionPropertyKey.JDBC_CONNECTION_URL, "jdbc:postgresql://db2.internal:5432/orders",
+ ConnectionPropertyKey.SECRET_ID, "prod/orders"))
+ .build())
+ .build());
+ Connection redefined = glue.getConnection(GetConnectionRequest.builder()
+ .name(CONNECTION_NAME)
+ .build()).connection();
+ assertThat(redefined.description()).isNull();
+ assertThat(redefined.connectionProperties()).doesNotContainKey(ConnectionPropertyKey.PASSWORD);
+ assertThat(redefined.connectionProperties()).containsEntry(ConnectionPropertyKey.SECRET_ID, "prod/orders");
+ assertThat(redefined.creationTime()).isEqualTo(connection.creationTime());
+
+ glue.deleteConnection(DeleteConnectionRequest.builder().connectionName(CONNECTION_NAME).build());
+ assertThatThrownBy(() -> glue.getConnection(GetConnectionRequest.builder().name(CONNECTION_NAME).build()))
+ .isInstanceOf(EntityNotFoundException.class);
+
+ BatchDeleteConnectionResponse batch = glue.batchDeleteConnection(BatchDeleteConnectionRequest.builder()
+ .connectionNameList(KAFKA_CONNECTION_NAME, CONNECTION_NAME)
+ .build());
+ assertThat(batch.succeeded()).containsExactly(KAFKA_CONNECTION_NAME);
+ assertThat(batch.errors()).containsOnlyKeys(CONNECTION_NAME);
+ }
+
private static TableInput tableInput(String description) {
return tableInput(TABLE_NAME, description);
}
diff --git a/docs/services/glue.md b/docs/services/glue.md
index f323578538..9e0f41c26b 100644
--- a/docs/services/glue.md
+++ b/docs/services/glue.md
@@ -3,7 +3,7 @@
**Protocol:** JSON 1.1
**Endpoint:** `http://localhost:4566/`
-Floci emulates the AWS Glue Data Catalog and Glue Schema Registry, allowing you to manage local data lake metadata and schema-version workflows.
+Floci emulates the AWS Glue Data Catalog (databases, tables, partitions, functions, connections) and Glue Schema Registry, allowing you to manage local data lake metadata and schema-version workflows.
## Supported Actions
@@ -64,6 +64,33 @@ only arises from a backfill failure.
| UpdateUserDefinedFunction | Updates a stored user-defined function. |
| DeleteUserDefinedFunction | Deletes a user-defined function from a database. |
+#### Connections
+
+| Action | Description |
+|--------|-------------|
+| CreateConnection | Creates a connection definition from a `ConnectionInput`, with optional `Tags`, and answers `CreateConnectionStatus: READY`. |
+| GetConnection | Returns a stored connection. `HidePassword` omits `PASSWORD` and `ENCRYPTED_PASSWORD` from `ConnectionProperties`. |
+| GetConnections | Lists connections, narrowed by `Filter.MatchCriteria`, `Filter.ConnectionType` and `Filter.ConnectionSchemaVersion`, paged by `MaxResults` and `NextToken`. |
+| UpdateConnection | Redefines a connection from a full `ConnectionInput`, as on AWS: members left out of the input are dropped; the name and `CreationTime` are kept. |
+| DeleteConnection | Deletes a connection and its tags. |
+| BatchDeleteConnection | Deletes up to 25 connections, reporting the ones not found in `Errors`. |
+| TestConnection | Accepts a connection name or an inline `TestConnectionInput` and answers with an empty body. |
+
+`ConnectionInput` is validated against the API reference: `Name` (1 to 255 characters), `ConnectionType`
+(the documented enumeration) and `ConnectionProperties` (the documented key list, at most 100 entries, and
+empty for a `NETWORK` connection) are required; `MatchCriteria` holds at most 10 entries. Credentials given
+under `AuthenticationConfiguration` are accepted and never returned by a read. A connection that uses
+`AuthenticationConfiguration` or the `SparkProperties`, `AthenaProperties` or `PythonProperties` maps reports
+`ConnectionSchemaVersion` 2; the classic JDBC, Kafka and network shape reports 1.
+
+`HidePassword` removes `PASSWORD` and `ENCRYPTED_PASSWORD`, the two members the API reference defines as the
+connection's password. The Kafka credentials (`KAFKA_CLIENT_KEYSTORE_PASSWORD`, `KAFKA_CLIENT_KEY_PASSWORD`,
+`KAFKA_SASL_PLAIN_PASSWORD`, `KAFKA_SASL_SCRAM_PASSWORD` and their `ENCRYPTED_` forms) are returned as stored: the
+reference does not say whether the flag covers them, so Floci does not guess.
+
+`TestConnection` is asynchronous on AWS and returns nothing; Floci checks the request's shape and accepts it
+without opening a socket to the data store, since no job or crawler runs against a connection yet.
+
#### Jobs
| Action | Description |
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
index e0216f8317..4179ddfa76 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
@@ -26,6 +26,7 @@ import java.util.Objects;
public class GlueJsonHandler {
private static final TypeReference<List<String>> STRING_LIST = new TypeReference<>() {};
+ private static final TypeReference<Map<String, String>> STRING_MAP = new TypeReference<>() {};
private static final TypeReference<List<Map<String, Object>>> MAP_LIST = new TypeReference<>() {};
private static final TypeReference<List<Partition>> PARTITION_LIST = new TypeReference<>() {};
@@ -285,6 +286,21 @@ public class GlueJsonHandler {
glueService.deleteCrawler(req.getName(), region);
yield Response.ok().build();
}
+ case "CreateConnection" -> handleCreateConnection(request, region);
+ case "GetConnection" -> handleGetConnection(request);
+ case "GetConnections" -> handleGetConnections(request);
+ case "UpdateConnection" -> handleUpdateConnection(request);
+ case "DeleteConnection" -> {
+ glueService.deleteConnection(request.path("ConnectionName").asText(null), region);
+ yield Response.ok(Map.of()).build();
+ }
+ case "BatchDeleteConnection" -> {
+ List<String> names = request.hasNonNull("ConnectionNameList")
+ ? mapper.convertValue(request.get("ConnectionNameList"), STRING_LIST)
+ : null;
+ yield Response.ok(glueService.batchDeleteConnections(names, region)).build();
+ }
+ case "TestConnection" -> handleTestConnection(request);
// Read-only Glue actions for resources the emulator does not model. The AWS SDK
// expects each to return a 200 with its result key present (empty), so we emit the
// documented empty shape rather than an InvalidAction 400 that callers can't read.
@@ -858,6 +874,59 @@ public class GlueJsonHandler {
return Response.ok().build();
}
+ private Response handleCreateConnection(JsonNode request, String region) throws Exception {
+ ConnectionInput input = request.hasNonNull("ConnectionInput")
+ ? mapper.treeToValue(request.get("ConnectionInput"), ConnectionInput.class)
+ : null;
+ Map<String, String> tags = request.hasNonNull("Tags")
+ ? mapper.convertValue(request.get("Tags"), STRING_MAP)
+ : null;
+ String status = glueService.createConnection(input, tags, region);
+ return Response.ok(Map.of("CreateConnectionStatus", status)).build();
+ }
+
+ private Response handleGetConnection(JsonNode request) {
+ Connection connection = glueService.getConnection(
+ request.path("Name").asText(null), request.path("HidePassword").asBoolean(false));
+ return Response.ok(Map.of("Connection", connection)).build();
+ }
+
+ private Response handleGetConnections(JsonNode request) {
+ JsonNode filter = request.path("Filter");
+ List<String> matchCriteria = filter.hasNonNull("MatchCriteria")
+ ? mapper.convertValue(filter.get("MatchCriteria"), STRING_LIST)
+ : null;
+ String connectionType = filter.path("ConnectionType").asText(null);
+ Integer schemaVersion = filter.hasNonNull("ConnectionSchemaVersion")
+ ? filter.get("ConnectionSchemaVersion").asInt()
+ : null;
+ GlueService.Page<Connection> page = glueService.getConnections(
+ matchCriteria, connectionType, schemaVersion,
+ request.path("HidePassword").asBoolean(false),
+ readMaxResults(request), readNextToken(request));
+ return Response.ok(pageResponse("ConnectionList", page.items(), page.nextToken())).build();
+ }
+
+ private Response handleUpdateConnection(JsonNode request) throws Exception {
+ ConnectionInput input = request.hasNonNull("ConnectionInput")
+ ? mapper.treeToValue(request.get("ConnectionInput"), ConnectionInput.class)
+ : null;
+ glueService.updateConnection(request.path("Name").asText(null), input);
+ return Response.ok(Map.of()).build();
+ }
+
+ private Response handleTestConnection(JsonNode request) {
+ JsonNode input = request.path("TestConnectionInput");
+ Map<String, String> properties = input.hasNonNull("ConnectionProperties")
+ ? mapper.convertValue(input.get("ConnectionProperties"), STRING_MAP)
+ : null;
+ glueService.testConnection(
+ request.path("ConnectionName").asText(null),
+ input.path("ConnectionType").asText(null),
+ properties);
+ return Response.ok(Map.of()).build();
+ }
+
private Response handleGetTags(JsonNode request, String region) {
String arn = request.path("ResourceArn").asText(null);
Map<String, String> tags = glueService.getTags(arn, region);
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index 619cb1d9d4..4c357ee34d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -8,6 +8,8 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.glue.model.Classifier;
import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.Connection;
+import io.github.hectorvent.floci.services.glue.model.ConnectionInput;
import io.github.hectorvent.floci.services.glue.model.Crawler;
import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;
import io.github.hectorvent.floci.services.glue.model.Database;
@@ -72,6 +74,45 @@ public class GlueService {
/** Measured against real Glue (us-west-2): a fourth index reports the limit. */
private static final int MAX_PARTITION_INDEXES_PER_TABLE = 3;
+ // Connection limits and enumerations from the Glue API reference (ConnectionInput and the
+ // Connection structure). AWS rejects a value outside them with InvalidInputException.
+ private static final int MAX_CONNECTION_NAME_LENGTH = 255;
+ private static final int MAX_CONNECTION_DESCRIPTION_LENGTH = 2048;
+ private static final int MAX_CONNECTION_MATCH_CRITERIA = 10;
+ private static final int MAX_CONNECTION_PROPERTIES = 100;
+ private static final int MAX_BATCH_DELETE_CONNECTIONS = 25;
+ private static final Set<String> CONNECTION_TYPES = Set.of(
+ "JDBC", "SFTP", "MONGODB", "KAFKA", "NETWORK", "MARKETPLACE", "CUSTOM", "SALESFORCE",
+ "VIEW_VALIDATION_REDSHIFT", "VIEW_VALIDATION_ATHENA", "GOOGLEADS", "GOOGLESHEETS",
+ "GOOGLEANALYTICS4", "SERVICENOW", "MARKETO", "SAPODATA", "ZENDESK", "JIRACLOUD", "NETSUITEERP",
+ "HUBSPOT", "FACEBOOKADS", "INSTAGRAMADS", "ZOHOCRM", "SALESFORCEPARDOT",
+ "SALESFORCEMARKETINGCLOUD", "ADOBEANALYTICS", "SLACK", "LINKEDIN", "MIXPANEL", "ASANA", "STRIPE",
+ "SMARTSHEET", "DATADOG", "WOOCOMMERCE", "INTERCOM", "SNAPCHATADS", "PAYPAL", "QUICKBOOKS",
+ "FACEBOOKPAGEINSIGHTS", "FRESHDESK", "TWILIO", "DOCUSIGNMONITOR", "FRESHSALES", "ZOOM",
+ "GOOGLESEARCHCONSOLE", "SALESFORCECOMMERCECLOUD", "SAPCONCUR", "DYNATRACE",
+ "MICROSOFTDYNAMIC365FINANCEANDOPS", "MICROSOFTTEAMS", "BLACKBAUDRAISEREDGENXT", "MAILCHIMP",
+ "GITLAB", "PENDO", "PRODUCTBOARD", "CIRCLECI", "PIPEDIVE", "SENDGRID", "AZURECOSMOS", "AZURESQL",
+ "BIGQUERY", "BLACKBAUD", "CLOUDERAHIVE", "CLOUDERAIMPALA", "CLOUDWATCH", "CLOUDWATCHMETRICS",
+ "CMDB", "DATALAKEGEN2", "DB2", "DB2AS400", "DOCUMENTDB", "DOMO", "DYNAMODB",
+ "GOOGLECLOUDSTORAGE", "HBASE", "KUSTOMER", "MICROSOFTDYNAMICS365CRM", "MONDAY", "MYSQL", "OKTA",
+ "OPENSEARCH", "ORACLE", "PIPEDRIVE", "POSTGRESQL", "SAPHANA", "SQLSERVER", "SYNAPSE", "TERADATA",
+ "TERADATANOS", "TIMESTREAM", "TPCDS", "VERTICA");
+ private static final Set<String> CONNECTION_PROPERTY_KEYS = Set.of(
+ "HOST", "PORT", "USERNAME", "PASSWORD", "ENCRYPTED_PASSWORD", "JDBC_DRIVER_JAR_URI",
+ "JDBC_DRIVER_CLASS_NAME", "JDBC_ENGINE", "JDBC_ENGINE_VERSION", "CONFIG_FILES", "INSTANCE_ID",
+ "JDBC_CONNECTION_URL", "JDBC_ENFORCE_SSL", "CUSTOM_JDBC_CERT", "SKIP_CUSTOM_JDBC_CERT_VALIDATION",
+ "CUSTOM_JDBC_CERT_STRING", "CONNECTION_URL", "KAFKA_BOOTSTRAP_SERVERS", "KAFKA_SSL_ENABLED",
+ "KAFKA_CUSTOM_CERT", "KAFKA_SKIP_CUSTOM_CERT_VALIDATION", "KAFKA_CLIENT_KEYSTORE",
+ "KAFKA_CLIENT_KEYSTORE_PASSWORD", "KAFKA_CLIENT_KEY_PASSWORD",
+ "ENCRYPTED_KAFKA_CLIENT_KEYSTORE_PASSWORD", "ENCRYPTED_KAFKA_CLIENT_KEY_PASSWORD",
+ "KAFKA_SASL_MECHANISM", "KAFKA_SASL_PLAIN_USERNAME", "KAFKA_SASL_PLAIN_PASSWORD",
+ "ENCRYPTED_KAFKA_SASL_PLAIN_PASSWORD", "KAFKA_SASL_SCRAM_USERNAME", "KAFKA_SASL_SCRAM_PASSWORD",
+ "KAFKA_SASL_SCRAM_SECRETS_ARN", "ENCRYPTED_KAFKA_SASL_SCRAM_PASSWORD", "KAFKA_SASL_GSSAPI_KEYTAB",
+ "KAFKA_SASL_GSSAPI_KRB5_CONF", "KAFKA_SASL_GSSAPI_SERVICE", "KAFKA_SASL_GSSAPI_PRINCIPAL",
+ "SECRET_ID", "CONNECTOR_URL", "CONNECTOR_TYPE", "CONNECTOR_CLASS_NAME", "ENDPOINT",
+ "ENDPOINT_TYPE", "ROLE_ARN", "REGION", "WORKGROUP_NAME", "CLUSTER_IDENTIFIER", "DATABASE");
+ private static final String CONNECTION_STATUS_READY = "READY";
+
// Glue's partition index states. FAILED also exists but is only reachable through a backfill
// failure, which is not emulated.
private static final String INDEX_STATUS_CREATING = "CREATING";
@@ -89,6 +130,7 @@ public class GlueService {
private final StorageBackend<String, Job> jobStore;
private final StorageBackend<String, Crawler> crawlerStore;
private final StorageBackend<String, Classifier> classifierStore;
+ private final StorageBackend<String, Connection> connectionStore;
private final GlueSchemaRegistryService schemaRegistryService;
private final RegionResolver regionResolver;
private final ResourceGroupsTaggingService resourceGroupsTaggingService;
@@ -111,6 +153,7 @@ public class GlueService {
this.jobStore = storageFactory.create("glue", "jobs.json", new TypeReference<>() {});
this.crawlerStore = storageFactory.create("glue", "crawlers.json", new TypeReference<>() {});
this.classifierStore = storageFactory.create("glue", "classifiers.json", new TypeReference<>() {});
+ this.connectionStore = storageFactory.create("glue", "connections.json", new TypeReference<>() {});
this.schemaRegistryService = schemaRegistryService;
this.regionResolver = regionResolver;
this.resourceGroupsTaggingService = resourceGroupsTaggingService;
@@ -127,6 +170,7 @@ public class GlueService {
StorageBackend<String, Job> jobStore,
StorageBackend<String, Crawler> crawlerStore,
StorageBackend<String, Classifier> classifierStore,
+ StorageBackend<String, Connection> connectionStore,
GlueSchemaRegistryService schemaRegistryService,
RegionResolver regionResolver,
ResourceGroupsTaggingService resourceGroupsTaggingService) {
@@ -141,6 +185,7 @@ public class GlueService {
this.jobStore = jobStore;
this.crawlerStore = crawlerStore;
this.classifierStore = classifierStore;
+ this.connectionStore = connectionStore;
this.schemaRegistryService = schemaRegistryService;
this.regionResolver = regionResolver;
this.resourceGroupsTaggingService = resourceGroupsTaggingService;
@@ -1337,6 +1382,10 @@ public class GlueService {
return regionResolver.buildArn("glue", region, "job/" + jobName);
}
+ private String connectionArn(String region, String connectionName) {
+ return regionResolver.buildArn("glue", region, "connection/" + connectionName);
+ }
+
private String crawlerArn(String region, String crawlerName) {
return regionResolver.buildArn("glue", region, "crawler/" + crawlerName);
}
@@ -1891,6 +1940,202 @@ public class GlueService {
LOG.infov("Deleted Glue Crawler: {0}", name);
}
+ // ---- Connections -----------------------------------------------------------------------
+
+ public String createConnection(ConnectionInput input, Map<String, String> tags, String region) {
+ validateConnectionInput(input);
+ // Connections keep their case, as jobs and crawlers do: AWS matches the name exactly.
+ String name = input.getName();
+ if (connectionStore.get(name).isPresent()) {
+ throw new AwsException("AlreadyExistsException", "Connection " + name + " already exists.", 400);
+ }
+ Instant now = Instant.now();
+ Connection connection = toConnection(input);
+ connection.setCreationTime(now);
+ connection.setLastUpdatedTime(now);
+ connectionStore.put(name, connection);
+ if (tags != null && !tags.isEmpty()) {
+ resourceGroupsTaggingService.tagResources(List.of(connectionArn(region, name)), tags, region);
+ }
+ LOG.infov("Created Glue Connection: {0}", name);
+ // Nothing is validated against the data store, so there is no IN_PROGRESS phase to report.
+ return CONNECTION_STATUS_READY;
+ }
+
+ public Connection getConnection(String name, boolean hidePassword) {
+ validateRequired(name, "Name");
+ Connection connection = connectionStore.get(name)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Connection " + name + " not found.", 400));
+ return hidePassword ? connection.withoutPassword() : connection;
+ }
+
+ /**
+ * Lists connections, narrowed by the GetConnections filter: every {@code MatchCriteria}
+ * entry must appear on the connection, and {@code ConnectionType} and
+ * {@code ConnectionSchemaVersion} must match exactly when given.
+ */
+ public Page<Connection> getConnections(List<String> matchCriteria, String connectionType,
+ Integer connectionSchemaVersion, boolean hidePassword,
+ Integer maxResults, String nextToken) {
+ List<Connection> matching = new ArrayList<>();
+ for (Connection connection : connectionStore.scan(k -> true)) {
+ if (matchCriteria != null && !matchCriteria.isEmpty()
+ && (connection.getMatchCriteria() == null
+ || !connection.getMatchCriteria().containsAll(matchCriteria))) {
+ continue;
+ }
+ if (connectionType != null && !connectionType.equals(connection.getConnectionType())) {
+ continue;
+ }
+ if (connectionSchemaVersion != null
+ && !connectionSchemaVersion.equals(connection.getConnectionSchemaVersion())) {
+ continue;
+ }
+ matching.add(hidePassword ? connection.withoutPassword() : connection);
+ }
+ matching.sort(Comparator.comparing(Connection::getName));
+ return paginate(matching, maxResults, nextToken);
+ }
+
+ /**
+ * UpdateConnection takes a ConnectionInput that "redefines the connection in question"
+ * (API reference), so the stored definition is replaced rather than merged: a member left
+ * out of the input is gone afterwards. The connection keeps its name and creation time.
+ */
+ public void updateConnection(String name, ConnectionInput input) {
+ validateRequired(name, "Name");
+ validateRequired(input, "ConnectionInput");
+ Connection existing = connectionStore.get(name)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Connection " + name + " not found.", 400));
+ validateConnectionInput(input);
+ Connection updated = toConnection(input);
+ updated.setName(existing.getName());
+ updated.setCreationTime(existing.getCreationTime());
+ updated.setLastUpdatedTime(Instant.now());
+ connectionStore.put(name, updated);
+ LOG.infov("Updated Glue Connection: {0}", name);
+ }
+
+ public void deleteConnection(String name, String region) {
+ validateRequired(name, "ConnectionName");
+ if (connectionStore.get(name).isEmpty()) {
+ throw new AwsException("EntityNotFoundException", "Connection " + name + " not found.", 400);
+ }
+ connectionStore.delete(name);
+ resourceGroupsTaggingService.deleteResources(List.of(connectionArn(region, name)), region);
+ LOG.infov("Deleted Glue Connection: {0}", name);
+ }
+
+ /** Deletes what it can; the names that could not be deleted come back keyed in the errors map. */
+ public BatchDeleteConnectionResult batchDeleteConnections(List<String> names, String region) {
+ validateRequired(names, "ConnectionNameList");
+ if (names.size() > MAX_BATCH_DELETE_CONNECTIONS) {
+ throw new AwsException("InvalidInputException",
+ "ConnectionNameList must have at most " + MAX_BATCH_DELETE_CONNECTIONS + " items.", 400);
+ }
+ List<String> succeeded = new ArrayList<>();
+ Map<String, ErrorDetail> errors = new LinkedHashMap<>();
+ for (String name : names) {
+ try {
+ deleteConnection(name, region);
+ succeeded.add(name);
+ } catch (AwsException e) {
+ errors.put(name, new ErrorDetail(e.getErrorCode(), e.getMessage()));
+ }
+ }
+ return new BatchDeleteConnectionResult(succeeded, errors);
+ }
+
+ /**
+ * TestConnection on AWS is asynchronous and answers with an empty body: the outcome is only
+ * visible later, on the connection's status. Floci never reaches the data store (no job can
+ * run against it yet), so the request is checked for shape and accepted. Either an existing
+ * connection is named, or an inline definition with a type and properties is given.
+ */
+ public void testConnection(String connectionName, String connectionType,
+ Map<String, String> connectionProperties) {
+ if (connectionName != null && !connectionName.isBlank()) {
+ getConnection(connectionName, false);
+ return;
+ }
+ validateRequired(connectionType, "TestConnectionInput.ConnectionType");
+ validateConnectionType(connectionType);
+ validateRequired(connectionProperties, "TestConnectionInput.ConnectionProperties");
+ validateConnectionProperties(connectionProperties);
+ }
+
+ private Connection toConnection(ConnectionInput input) {
+ Connection connection = new Connection();
+ connection.setName(input.getName());
+ connection.setDescription(input.getDescription());
+ connection.setConnectionType(input.getConnectionType());
+ connection.setMatchCriteria(input.getMatchCriteria());
+ connection.setConnectionProperties(new LinkedHashMap<>(input.getConnectionProperties()));
+ connection.setSparkProperties(input.getSparkProperties());
+ connection.setAthenaProperties(input.getAthenaProperties());
+ connection.setPythonProperties(input.getPythonProperties());
+ connection.setPhysicalConnectionRequirements(input.getPhysicalConnectionRequirements());
+ if (input.getAuthenticationConfiguration() != null) {
+ connection.setAuthenticationConfiguration(input.getAuthenticationConfiguration().toOutput());
+ }
+ connection.setStatus(CONNECTION_STATUS_READY);
+ // Schema version 2 "supports properties for specific compute environments" (Connection
+ // structure). A definition that uses any of those members is a version 2 connection;
+ // the classic JDBC/Kafka/Network shape is version 1.
+ boolean usesComputeEnvironmentMembers = input.getAuthenticationConfiguration() != null
+ || input.getSparkProperties() != null
+ || input.getAthenaProperties() != null
+ || input.getPythonProperties() != null;
+ connection.setConnectionSchemaVersion(usesComputeEnvironmentMembers ? 2 : 1);
+ return connection;
+ }
+
+ private void validateConnectionInput(ConnectionInput input) {
+ validateRequired(input, "ConnectionInput");
+ validateRequired(input.getName(), "ConnectionInput.Name");
+ if (input.getName().length() > MAX_CONNECTION_NAME_LENGTH) {
+ throw new AwsException("InvalidInputException",
+ "ConnectionInput.Name must be between 1 and " + MAX_CONNECTION_NAME_LENGTH + " characters.", 400);
+ }
+ validateRequired(input.getConnectionType(), "ConnectionInput.ConnectionType");
+ validateConnectionType(input.getConnectionType());
+ validateRequired(input.getConnectionProperties(), "ConnectionInput.ConnectionProperties");
+ validateConnectionProperties(input.getConnectionProperties());
+ if (input.getDescription() != null && input.getDescription().length() > MAX_CONNECTION_DESCRIPTION_LENGTH) {
+ throw new AwsException("InvalidInputException",
+ "ConnectionInput.Description must be at most " + MAX_CONNECTION_DESCRIPTION_LENGTH + " characters.", 400);
+ }
+ if (input.getMatchCriteria() != null && input.getMatchCriteria().size() > MAX_CONNECTION_MATCH_CRITERIA) {
+ throw new AwsException("InvalidInputException",
+ "ConnectionInput.MatchCriteria must have at most " + MAX_CONNECTION_MATCH_CRITERIA + " items.", 400);
+ }
+ }
+
+ private void validateConnectionType(String connectionType) {
+ if (!CONNECTION_TYPES.contains(connectionType)) {
+ throw new AwsException("InvalidInputException",
+ "Unsupported connection type: " + connectionType, 400);
+ }
+ }
+
+ private void validateConnectionProperties(Map<String, String> properties) {
+ if (properties.size() > MAX_CONNECTION_PROPERTIES) {
+ throw new AwsException("InvalidInputException",
+ "ConnectionProperties must have at most " + MAX_CONNECTION_PROPERTIES + " entries.", 400);
+ }
+ for (String key : properties.keySet()) {
+ if (!CONNECTION_PROPERTY_KEYS.contains(key)) {
+ throw new AwsException("InvalidInputException",
+ "Unsupported connection property key: " + key, 400);
+ }
+ }
+ }
+
+ @RegisterForReflection
+ public record BatchDeleteConnectionResult(
+ @JsonProperty("Succeeded") List<String> succeeded,
+ @JsonProperty("Errors") Map<String, ErrorDetail> errors) {}
+
public void tagResource(String arn, Map<String, String> tags, String region) {
validateArn(arn);
if (arn.contains(":registry/") || arn.contains(":schema/")) {
@@ -1937,6 +2182,9 @@ public class GlueService {
} else if (resource.startsWith("crawler/")) {
getCrawler(resource.substring(8));
return;
+ } else if (resource.startsWith("connection/")) {
+ getConnection(resource.substring(11), false);
+ return;
} else if (resource.startsWith("database/")) {
getDatabase(resource.substring(9));
return;
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/AuthenticationConfiguration.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/AuthenticationConfiguration.java
new file mode 100644
index 0000000000..16712f3b48
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/AuthenticationConfiguration.java
@@ -0,0 +1,65 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.Map;
+
+/**
+ * How a connection authenticates. {@code BasicAuthenticationCredentials} and
+ * {@code CustomAuthenticationCredentials} are input-only on AWS: a Get never echoes a
+ * credential, so {@link #toOutput()} drops them along with the OAuth2 secrets.
+ */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class AuthenticationConfiguration {
+ @JsonProperty("AuthenticationType")
+ private String authenticationType;
+
+ @JsonProperty("SecretArn")
+ private String secretArn;
+
+ @JsonProperty("KmsKeyArn")
+ private String kmsKeyArn;
+
+ @JsonProperty("OAuth2Properties")
+ private OAuth2Properties oAuth2Properties;
+
+ @JsonProperty("BasicAuthenticationCredentials")
+ private Map<String, String> basicAuthenticationCredentials;
+
+ @JsonProperty("CustomAuthenticationCredentials")
+ private Map<String, String> customAuthenticationCredentials;
+
+ public AuthenticationConfiguration toOutput() {
+ AuthenticationConfiguration output = new AuthenticationConfiguration();
+ output.authenticationType = authenticationType;
+ output.secretArn = secretArn;
+ output.kmsKeyArn = kmsKeyArn;
+ output.oAuth2Properties = oAuth2Properties == null ? null : oAuth2Properties.toOutput();
+ return output;
+ }
+
+ public String getAuthenticationType() { return authenticationType; }
+ public void setAuthenticationType(String authenticationType) { this.authenticationType = authenticationType; }
+
+ public String getSecretArn() { return secretArn; }
+ public void setSecretArn(String secretArn) { this.secretArn = secretArn; }
+
+ public String getKmsKeyArn() { return kmsKeyArn; }
+ public void setKmsKeyArn(String kmsKeyArn) { this.kmsKeyArn = kmsKeyArn; }
+
+ public OAuth2Properties getOAuth2Properties() { return oAuth2Properties; }
+ public void setOAuth2Properties(OAuth2Properties oAuth2Properties) { this.oAuth2Properties = oAuth2Properties; }
+
+ public Map<String, String> getBasicAuthenticationCredentials() { return basicAuthenticationCredentials; }
+ public void setBasicAuthenticationCredentials(Map<String, String> basicAuthenticationCredentials) {
+ this.basicAuthenticationCredentials = basicAuthenticationCredentials;
+ }
+
+ public Map<String, String> getCustomAuthenticationCredentials() { return customAuthenticationCredentials; }
+ public void setCustomAuthenticationCredentials(Map<String, String> customAuthenticationCredentials) {
+ this.customAuthenticationCredentials = customAuthenticationCredentials;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java
new file mode 100644
index 0000000000..95105c8fe3
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java
@@ -0,0 +1,183 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A Data Catalog connection as GetConnection returns it: the caller's {@link ConnectionInput}
+ * plus the members the service owns (timestamps, status, schema version). Credentials never
+ * reach this shape; see {@link AuthenticationConfiguration#toOutput()}.
+ */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Connection {
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("ConnectionType")
+ private String connectionType;
+
+ @JsonProperty("MatchCriteria")
+ private List<String> matchCriteria;
+
+ @JsonProperty("ConnectionProperties")
+ private Map<String, String> connectionProperties;
+
+ @JsonProperty("SparkProperties")
+ private Map<String, String> sparkProperties;
+
+ @JsonProperty("AthenaProperties")
+ private Map<String, String> athenaProperties;
+
+ @JsonProperty("PythonProperties")
+ private Map<String, String> pythonProperties;
+
+ @JsonProperty("PhysicalConnectionRequirements")
+ private PhysicalConnectionRequirements physicalConnectionRequirements;
+
+ @JsonProperty("CreationTime")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant creationTime;
+
+ @JsonProperty("LastUpdatedTime")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant lastUpdatedTime;
+
+ @JsonProperty("LastUpdatedBy")
+ private String lastUpdatedBy;
+
+ @JsonProperty("Status")
+ private String status;
+
+ @JsonProperty("StatusReason")
+ private String statusReason;
+
+ @JsonProperty("LastConnectionValidationTime")
+ @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+ private Instant lastConnectionValidationTime;
+
+ @JsonProperty("AuthenticationConfiguration")
+ private AuthenticationConfiguration authenticationConfiguration;
+
+ @JsonProperty("ConnectionSchemaVersion")
+ private Integer connectionSchemaVersion;
+
+ @JsonProperty("CompatibleComputeEnvironments")
+ private List<String> compatibleComputeEnvironments;
+
+ /**
+ * A copy with the password members of {@code ConnectionProperties} removed: what
+ * GetConnection and GetConnections return for {@code HidePassword=true}. The API reference
+ * names {@code PASSWORD} and {@code ENCRYPTED_PASSWORD} as the connection's password
+ * fields, the latter being the form stored under catalog password encryption.
+ */
+ public Connection withoutPassword() {
+ Connection copy = copy();
+ if (connectionProperties != null) {
+ Map<String, String> visible = new LinkedHashMap<>(connectionProperties);
+ visible.remove("PASSWORD");
+ visible.remove("ENCRYPTED_PASSWORD");
+ copy.connectionProperties = visible;
+ }
+ return copy;
+ }
+
+ private Connection copy() {
+ Connection copy = new Connection();
+ copy.name = name;
+ copy.description = description;
+ copy.connectionType = connectionType;
+ copy.matchCriteria = matchCriteria;
+ copy.connectionProperties = connectionProperties;
+ copy.sparkProperties = sparkProperties;
+ copy.athenaProperties = athenaProperties;
+ copy.pythonProperties = pythonProperties;
+ copy.physicalConnectionRequirements = physicalConnectionRequirements;
+ copy.creationTime = creationTime;
+ copy.lastUpdatedTime = lastUpdatedTime;
+ copy.lastUpdatedBy = lastUpdatedBy;
+ copy.status = status;
+ copy.statusReason = statusReason;
+ copy.lastConnectionValidationTime = lastConnectionValidationTime;
+ copy.authenticationConfiguration = authenticationConfiguration;
+ copy.connectionSchemaVersion = connectionSchemaVersion;
+ copy.compatibleComputeEnvironments = compatibleComputeEnvironments;
+ return copy;
+ }
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public String getDescription() { return description; }
+ public void setDescription(String description) { this.description = description; }
+
+ public String getConnectionType() { return connectionType; }
+ public void setConnectionType(String connectionType) { this.connectionType = connectionType; }
+
+ public List<String> getMatchCriteria() { return matchCriteria; }
+ public void setMatchCriteria(List<String> matchCriteria) { this.matchCriteria = matchCriteria; }
+
+ public Map<String, String> getConnectionProperties() { return connectionProperties; }
+ public void setConnectionProperties(Map<String, String> connectionProperties) {
+ this.connectionProperties = connectionProperties;
+ }
+
+ public Map<String, String> getSparkProperties() { return sparkProperties; }
+ public void setSparkProperties(Map<String, String> sparkProperties) { this.sparkProperties = sparkProperties; }
+
+ public Map<String, String> getAthenaProperties() { return athenaProperties; }
+ public void setAthenaProperties(Map<String, String> athenaProperties) { this.athenaProperties = athenaProperties; }
+
+ public Map<String, String> getPythonProperties() { return pythonProperties; }
+ public void setPythonProperties(Map<String, String> pythonProperties) { this.pythonProperties = pythonProperties; }
+
+ public PhysicalConnectionRequirements getPhysicalConnectionRequirements() { return physicalConnectionRequirements; }
+ public void setPhysicalConnectionRequirements(PhysicalConnectionRequirements physicalConnectionRequirements) {
+ this.physicalConnectionRequirements = physicalConnectionRequirements;
+ }
+
+ public Instant getCreationTime() { return creationTime; }
+ public void setCreationTime(Instant creationTime) { this.creationTime = creationTime; }
+
+ public Instant getLastUpdatedTime() { return lastUpdatedTime; }
+ public void setLastUpdatedTime(Instant lastUpdatedTime) { this.lastUpdatedTime = lastUpdatedTime; }
+
+ public String getLastUpdatedBy() { return lastUpdatedBy; }
+ public void setLastUpdatedBy(String lastUpdatedBy) { this.lastUpdatedBy = lastUpdatedBy; }
+
+ public String getStatus() { return status; }
+ public void setStatus(String status) { this.status = status; }
+
+ public String getStatusReason() { return statusReason; }
+ public void setStatusReason(String statusReason) { this.statusReason = statusReason; }
+
+ public Instant getLastConnectionValidationTime() { return lastConnectionValidationTime; }
+ public void setLastConnectionValidationTime(Instant lastConnectionValidationTime) {
+ this.lastConnectionValidationTime = lastConnectionValidationTime;
+ }
+
+ public AuthenticationConfiguration getAuthenticationConfiguration() { return authenticationConfiguration; }
+ public void setAuthenticationConfiguration(AuthenticationConfiguration authenticationConfiguration) {
+ this.authenticationConfiguration = authenticationConfiguration;
+ }
+
+ public Integer getConnectionSchemaVersion() { return connectionSchemaVersion; }
+ public void setConnectionSchemaVersion(Integer connectionSchemaVersion) {
+ this.connectionSchemaVersion = connectionSchemaVersion;
+ }
+
+ public List<String> getCompatibleComputeEnvironments() { return compatibleComputeEnvironments; }
+ public void setCompatibleComputeEnvironments(List<String> compatibleComputeEnvironments) {
+ this.compatibleComputeEnvironments = compatibleComputeEnvironments;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionInput.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionInput.java
new file mode 100644
index 0000000000..b61c572b30
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/ConnectionInput.java
@@ -0,0 +1,95 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The {@code ConnectionInput} structure of CreateConnection and UpdateConnection: the caller's
+ * full definition of a connection. On Update it redefines the connection rather than patching
+ * it, which is why it carries the required members again instead of an optional subset.
+ */
+@RegisterForReflection
+public class ConnectionInput {
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("ConnectionType")
+ private String connectionType;
+
+ @JsonProperty("MatchCriteria")
+ private List<String> matchCriteria;
+
+ @JsonProperty("ConnectionProperties")
+ private Map<String, String> connectionProperties;
+
+ @JsonProperty("SparkProperties")
+ private Map<String, String> sparkProperties;
+
+ @JsonProperty("AthenaProperties")
+ private Map<String, String> athenaProperties;
+
+ @JsonProperty("PythonProperties")
+ private Map<String, String> pythonProperties;
+
+ @JsonProperty("PhysicalConnectionRequirements")
+ private PhysicalConnectionRequirements physicalConnectionRequirements;
+
+ @JsonProperty("AuthenticationConfiguration")
+ private AuthenticationConfiguration authenticationConfiguration;
+
+ @JsonProperty("ValidateCredentials")
+ private Boolean validateCredentials;
+
+ @JsonProperty("ValidateForComputeEnvironments")
+ private List<String> validateForComputeEnvironments;
+
+ public String getName() { return name; }
+ public void setName(String name) { this.name = name; }
+
+ public String getDescription() { return description; }
+ public void setDescription(String description) { this.description = description; }
+
+ public String getConnectionType() { return connectionType; }
+ public void setConnectionType(String connectionType) { this.connectionType = connectionType; }
+
+ public List<String> getMatchCriteria() { return matchCriteria; }
+ public void setMatchCriteria(List<String> matchCriteria) { this.matchCriteria = matchCriteria; }
+
+ public Map<String, String> getConnectionProperties() { return connectionProperties; }
+ public void setConnectionProperties(Map<String, String> connectionProperties) {
+ this.connectionProperties = connectionProperties;
+ }
+
+ public Map<String, String> getSparkProperties() { return sparkProperties; }
+ public void setSparkProperties(Map<String, String> sparkProperties) { this.sparkProperties = sparkProperties; }
+
+ public Map<String, String> getAthenaProperties() { return athenaProperties; }
+ public void setAthenaProperties(Map<String, String> athenaProperties) { this.athenaProperties = athenaProperties; }
+
+ public Map<String, String> getPythonProperties() { return pythonProperties; }
+ public void setPythonProperties(Map<String, String> pythonProperties) { this.pythonProperties = pythonProperties; }
+
+ public PhysicalConnectionRequirements getPhysicalConnectionRequirements() { return physicalConnectionRequirements; }
+ public void setPhysicalConnectionRequirements(PhysicalConnectionRequirements physicalConnectionRequirements) {
+ this.physicalConnectionRequirements = physicalConnectionRequirements;
+ }
+
+ public AuthenticationConfiguration getAuthenticationConfiguration() { return authenticationConfiguration; }
+ public void setAuthenticationConfiguration(AuthenticationConfiguration authenticationConfiguration) {
+ this.authenticationConfiguration = authenticationConfiguration;
+ }
+
+ public Boolean getValidateCredentials() { return validateCredentials; }
+ public void setValidateCredentials(Boolean validateCredentials) { this.validateCredentials = validateCredentials; }
+
+ public List<String> getValidateForComputeEnvironments() { return validateForComputeEnvironments; }
+ public void setValidateForComputeEnvironments(List<String> validateForComputeEnvironments) {
+ this.validateForComputeEnvironments = validateForComputeEnvironments;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/OAuth2ClientApplication.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/OAuth2ClientApplication.java
new file mode 100644
index 0000000000..c447b98738
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/OAuth2ClientApplication.java
@@ -0,0 +1,25 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class OAuth2ClientApplication {
+ @JsonProperty("UserManagedClientApplicationClientId")
+ private String userManagedClientApplicationClientId;
+
+ @JsonProperty("AWSManagedClientApplicationReference")
+ private String awsManagedClientApplicationReference;
+
+ public String getUserManagedClientApplicationClientId() { return userManagedClientApplicationClientId; }
+ public void setUserManagedClientApplicationClientId(String userManagedClientApplicationClientId) {
+ this.userManagedClientApplicationClientId = userManagedClientApplicationClientId;
+ }
+
+ public String getAwsManagedClientApplicationReference() { return awsManagedClientApplicationReference; }
+ public void setAwsManagedClientApplicationReference(String awsManagedClientApplicationReference) {
+ this.awsManagedClientApplicationReference = awsManagedClientApplicationReference;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/OAuth2Properties.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/OAuth2Properties.java
new file mode 100644
index 0000000000..351f2d7e2c
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/OAuth2Properties.java
@@ -0,0 +1,69 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.Map;
+
+/**
+ * OAuth2 settings of a connection. The two credential-bearing members,
+ * {@code AuthorizationCodeProperties} and {@code OAuth2Credentials}, exist only on the input
+ * shape: AWS accepts them on Create and Update and never returns them, so {@link Connection}
+ * is built through {@link #toOutput()}, which leaves them behind.
+ */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class OAuth2Properties {
+ @JsonProperty("OAuth2GrantType")
+ private String oAuth2GrantType;
+
+ @JsonProperty("OAuth2ClientApplication")
+ private OAuth2ClientApplication oAuth2ClientApplication;
+
+ @JsonProperty("TokenUrl")
+ private String tokenUrl;
+
+ @JsonProperty("TokenUrlParametersMap")
+ private Map<String, String> tokenUrlParametersMap;
+
+ @JsonProperty("AuthorizationCodeProperties")
+ private JsonNode authorizationCodeProperties;
+
+ @JsonProperty("OAuth2Credentials")
+ private JsonNode oAuth2Credentials;
+
+ public OAuth2Properties toOutput() {
+ OAuth2Properties output = new OAuth2Properties();
+ output.oAuth2GrantType = oAuth2GrantType;
+ output.oAuth2ClientApplication = oAuth2ClientApplication;
+ output.tokenUrl = tokenUrl;
+ output.tokenUrlParametersMap = tokenUrlParametersMap;
+ return output;
+ }
+
+ public String getOAuth2GrantType() { return oAuth2GrantType; }
+ public void setOAuth2GrantType(String oAuth2GrantType) { this.oAuth2GrantType = oAuth2GrantType; }
+
+ public OAuth2ClientApplication getOAuth2ClientApplication() { return oAuth2ClientApplication; }
+ public void setOAuth2ClientApplication(OAuth2ClientApplication oAuth2ClientApplication) {
+ this.oAuth2ClientApplication = oAuth2ClientApplication;
+ }
+
+ public String getTokenUrl() { return tokenUrl; }
+ public void setTokenUrl(String tokenUrl) { this.tokenUrl = tokenUrl; }
+
+ public Map<String, String> getTokenUrlParametersMap() { return tokenUrlParametersMap; }
+ public void setTokenUrlParametersMap(Map<String, String> tokenUrlParametersMap) {
+ this.tokenUrlParametersMap = tokenUrlParametersMap;
+ }
+
+ public JsonNode getAuthorizationCodeProperties() { return authorizationCodeProperties; }
+ public void setAuthorizationCodeProperties(JsonNode authorizationCodeProperties) {
+ this.authorizationCodeProperties = authorizationCodeProperties;
+ }
+
+ public JsonNode getOAuth2Credentials() { return oAuth2Credentials; }
+ public void setOAuth2Credentials(JsonNode oAuth2Credentials) { this.oAuth2Credentials = oAuth2Credentials; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/PhysicalConnectionRequirements.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/PhysicalConnectionRequirements.java
new file mode 100644
index 0000000000..971d4cc037
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/PhysicalConnectionRequirements.java
@@ -0,0 +1,30 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.List;
+
+/** The VPC placement a job or crawler needs to reach a connection's data store. */
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class PhysicalConnectionRequirements {
+ @JsonProperty("SubnetId")
+ private String subnetId;
+
+ @JsonProperty("SecurityGroupIdList")
+ private List<String> securityGroupIdList;
+
+ @JsonProperty("AvailabilityZone")
+ private String availabilityZone;
+
+ public String getSubnetId() { return subnetId; }
+ public void setSubnetId(String subnetId) { this.subnetId = subnetId; }
+
+ public List<String> getSecurityGroupIdList() { return securityGroupIdList; }
+ public void setSecurityGroupIdList(List<String> securityGroupIdList) { this.securityGroupIdList = securityGroupIdList; }
+
+ public String getAvailabilityZone() { return availabilityZone; }
+ public void setAvailabilityZone(String availabilityZone) { this.availabilityZone = availabilityZone; }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueConnectionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueConnectionIntegrationTest.java
new file mode 100644
index 0000000000..0ae5f722c5
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueConnectionIntegrationTest.java
@@ -0,0 +1,264 @@
+package io.github.hectorvent.floci.services.glue;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ * The Data Catalog connection lifecycle over the JSON 1.1 protocol, in the order Terraform's
+ * aws_glue_connection drives it: create with tags, read on every refresh, update by full
+ * redefinition, delete. Each test uses its own names so the suite can run in any order.
+ */
+@QuarkusTest
+class GlueConnectionIntegrationTest {
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ private static String unique(String prefix) {
+ return prefix + "-" + UUID.randomUUID().toString().substring(0, 8);
+ }
+
+ private static void createJdbcConnection(String name, String tagsJson) {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateConnection")
+ .body("""
+ {
+ "ConnectionInput": {
+ "Name": "%s",
+ "Description": "orders database",
+ "ConnectionType": "JDBC",
+ "MatchCriteria": ["orders"],
+ "ConnectionProperties": {
+ "JDBC_CONNECTION_URL": "jdbc:postgresql://db.internal:5432/orders",
+ "USERNAME": "app",
+ "PASSWORD": "s3cret"
+ },
+ "PhysicalConnectionRequirements": {
+ "SubnetId": "subnet-0123456789abcdef0",
+ "SecurityGroupIdList": ["sg-0123456789abcdef0"],
+ "AvailabilityZone": "us-east-1a"
+ }
+ }%s
+ }
+ """.formatted(name, tagsJson))
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("CreateConnectionStatus", equalTo("READY"));
+ }
+
+ @Test
+ void createConnection_thenGetConnection_returnsTheDefinitionWithServiceOwnedFields() {
+ String name = unique("orders");
+ createJdbcConnection(name, "");
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetConnection")
+ .body("{\"Name\": \"%s\"}".formatted(name))
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("Connection.Name", equalTo(name))
+ .body("Connection.ConnectionType", equalTo("JDBC"))
+ .body("Connection.Description", equalTo("orders database"))
+ .body("Connection.ConnectionProperties.PASSWORD", equalTo("s3cret"))
+ .body("Connection.PhysicalConnectionRequirements.SubnetId", equalTo("subnet-0123456789abcdef0"))
+ .body("Connection.Status", equalTo("READY"))
+ .body("Connection.ConnectionSchemaVersion", equalTo(1))
+ .body("Connection.CreationTime", notNullValue())
+ .body("Connection.LastUpdatedTime", notNullValue());
+ }
+
+ @Test
+ void getConnection_hidePassword_omitsThePasswordAndKeepsTheRest() {
+ String name = unique("hidden");
+ createJdbcConnection(name, "");
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetConnection")
+ .body("{\"Name\": \"%s\", \"HidePassword\": true}".formatted(name))
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("Connection.ConnectionProperties", not(hasKey("PASSWORD")))
+ .body("Connection.ConnectionProperties.USERNAME", equalTo("app"));
+ }
+
+ @Test
+ void getConnections_filterByType_returnsOnlyMatchingConnections() {
+ String jdbc = unique("jdbc");
+ createJdbcConnection(jdbc, "");
+ String kafka = unique("kafka");
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateConnection")
+ .body("""
+ {
+ "ConnectionInput": {
+ "Name": "%s",
+ "ConnectionType": "KAFKA",
+ "ConnectionProperties": {"KAFKA_BOOTSTRAP_SERVERS": "broker:9092"}
+ }
+ }
+ """.formatted(kafka))
+ .when().post("/")
+ .then().statusCode(200);
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetConnections")
+ .body("{\"Filter\": {\"ConnectionType\": \"KAFKA\"}, \"MaxResults\": 100}")
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("ConnectionList.findAll { it.ConnectionType == 'KAFKA' }.Name", hasItem(kafka))
+ .body("ConnectionList.findAll { it.ConnectionType != 'KAFKA' }.size()", equalTo(0))
+ .body("NextToken", nullValue());
+ }
+
+ @Test
+ void updateConnection_redefinesTheConnection_droppingMembersLeftOut() {
+ String name = unique("redefined");
+ createJdbcConnection(name, "");
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.UpdateConnection")
+ .body("""
+ {
+ "Name": "%s",
+ "ConnectionInput": {
+ "Name": "%s",
+ "ConnectionType": "JDBC",
+ "ConnectionProperties": {
+ "JDBC_CONNECTION_URL": "jdbc:postgresql://db2.internal:5432/orders",
+ "SECRET_ID": "prod/orders"
+ }
+ }
+ }
+ """.formatted(name, name))
+ .when().post("/")
+ .then().statusCode(200);
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetConnection")
+ .body("{\"Name\": \"%s\"}".formatted(name))
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("Connection.ConnectionProperties.JDBC_CONNECTION_URL", equalTo("jdbc:postgresql://db2.internal:5432/orders"))
+ .body("Connection.ConnectionProperties.SECRET_ID", equalTo("prod/orders"))
+ .body("Connection.ConnectionProperties", not(hasKey("PASSWORD")))
+ .body("Connection", not(hasKey("Description")))
+ .body("Connection", not(hasKey("PhysicalConnectionRequirements")));
+ }
+
+ @Test
+ void createConnectionWithTags_tagsReadableThroughGlueAndResourceGroupsTagging() {
+ String name = unique("tagged");
+ String arn = "arn:aws:glue:us-east-1:000000000000:connection/" + name;
+ createJdbcConnection(name, ",\n \"Tags\": {\"env\": \"dev\"}");
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.GetTags")
+ .body("{\"ResourceArn\": \"%s\"}".formatted(arn))
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("Tags.env", equalTo("dev"));
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "ResourceGroupsTaggingAPI_20170126.GetResources")
+ .body("{\"ResourceARNList\": [\"%s\"]}".formatted(arn))
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("ResourceTagMappingList.size()", equalTo(1))
+ .body("ResourceTagMappingList[0].ResourceARN", equalTo(arn))
+ .body("ResourceTagMappingList[0].Tags[0].Key", equalTo("env"))
+ .body("ResourceTagMappingList[0].Tags[0].Value", equalTo("dev"));
+ }
+
+ @Test
+ void deleteConnection_removesIt_andASecondDeleteIsEntityNotFound() {
+ String name = unique("deleted");
+ createJdbcConnection(name, "");
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.DeleteConnection")
+ .body("{\"ConnectionName\": \"%s\"}".formatted(name))
+ .when().post("/")
+ .then().statusCode(200);
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.DeleteConnection")
+ .body("{\"ConnectionName\": \"%s\"}".formatted(name))
+ .when().post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("EntityNotFoundException"));
+ }
+
+ @Test
+ void batchDeleteConnection_reportsMissingNamesInErrors() {
+ String name = unique("batch");
+ createJdbcConnection(name, "");
+
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.BatchDeleteConnection")
+ .body("{\"ConnectionNameList\": [\"%s\", \"%s-absent\"]}".formatted(name, name))
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("Succeeded[0]", equalTo(name))
+ .body("Errors.'%s-absent'.ErrorCode".formatted(name), equalTo("EntityNotFoundException"));
+ }
+
+ @Test
+ void createConnection_unknownPropertyKey_isInvalidInput() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.CreateConnection")
+ .body("""
+ {
+ "ConnectionInput": {
+ "Name": "%s",
+ "ConnectionType": "JDBC",
+ "ConnectionProperties": {"HOSTNAME": "db"}
+ }
+ }
+ """.formatted(unique("bad")))
+ .when().post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("InvalidInputException"));
+ }
+
+ @Test
+ void testConnection_acceptsAnInlineDefinitionWithAnEmptyBody() {
+ given().contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", "AWSGlue.TestConnection")
+ .body("""
+ {
+ "TestConnectionInput": {
+ "ConnectionType": "JDBC",
+ "ConnectionProperties": {"JDBC_CONNECTION_URL": "jdbc:mysql://h:3306/d"}
+ }
+ }
+ """)
+ .when().post("/")
+ .then().statusCode(200);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
index 9c44ec9e7f..0843b32551 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueJsonHandlerTest.java
@@ -19,6 +19,7 @@ import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -624,4 +625,116 @@ class GlueJsonHandlerTest {
return AccountAwareStorageBackend.inMemory("000000000000");
}
}
+
+ private ObjectNode connectionInput(String name) {
+ ObjectNode input = mapper.createObjectNode();
+ input.put("Name", name);
+ input.put("ConnectionType", "JDBC");
+ input.put("Description", "orders db");
+ input.putArray("MatchCriteria").add("orders");
+ ObjectNode properties = input.putObject("ConnectionProperties");
+ properties.put("JDBC_CONNECTION_URL", "jdbc:postgresql://db.internal:5432/orders");
+ properties.put("USERNAME", "app");
+ properties.put("PASSWORD", "s3cret");
+ ObjectNode placement = input.putObject("PhysicalConnectionRequirements");
+ placement.put("SubnetId", "subnet-0123456789abcdef0");
+ placement.putArray("SecurityGroupIdList").add("sg-0123456789abcdef0");
+ placement.put("AvailabilityZone", "us-east-1a");
+ return input;
+ }
+
+ /**
+ * The wire shape a Glue client reads back: CreateConnection answers with its status, and
+ * GetConnection returns the definition under "Connection" with numeric timestamps and no
+ * null-valued members, which is how the AWS SDK and the Terraform provider expect it.
+ */
+ @Test
+ void createAndGetConnectionRoundTripTheDefinitionOnTheWire() throws Exception {
+ ObjectNode create = mapper.createObjectNode();
+ create.set("ConnectionInput", connectionInput("orders"));
+ create.putObject("Tags").put("env", "dev");
+
+ Response created = handler.handle("CreateConnection", create, REGION);
+ assertEquals(200, created.getStatus());
+ assertEquals("READY", mapper.valueToTree(created.getEntity()).get("CreateConnectionStatus").asText());
+
+ Response got = handler.handle("GetConnection", mapper.createObjectNode().put("Name", "orders"), REGION);
+ assertEquals(200, got.getStatus());
+ JsonNode connection = mapper.valueToTree(got.getEntity()).get("Connection");
+ assertEquals("orders", connection.get("Name").asText());
+ assertEquals("JDBC", connection.get("ConnectionType").asText());
+ assertEquals("s3cret", connection.get("ConnectionProperties").get("PASSWORD").asText());
+ assertEquals("subnet-0123456789abcdef0", connection.get("PhysicalConnectionRequirements").get("SubnetId").asText());
+ assertEquals("READY", connection.get("Status").asText());
+ assertEquals(1, connection.get("ConnectionSchemaVersion").asInt());
+ assertTrue(connection.get("CreationTime").isNumber());
+ assertTrue(connection.get("LastUpdatedTime").isNumber());
+ assertFalse(connection.has("LastUpdatedBy"));
+ assertFalse(connection.has("AuthenticationConfiguration"));
+
+ ObjectNode tags = mapper.createObjectNode();
+ tags.put("ResourceArn", "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":connection/orders");
+ JsonNode tagBody = mapper.valueToTree(handler.handle("GetTags", tags, REGION).getEntity());
+ assertEquals("dev", tagBody.get("Tags").get("env").asText());
+ }
+
+ @Test
+ void getConnectionsHonoursHidePasswordAndTheFilter() throws Exception {
+ ObjectNode create = mapper.createObjectNode();
+ create.set("ConnectionInput", connectionInput("orders"));
+ handler.handle("CreateConnection", create, REGION);
+ ObjectNode kafkaInput = connectionInput("events");
+ kafkaInput.put("ConnectionType", "KAFKA");
+ kafkaInput.putObject("ConnectionProperties").put("KAFKA_BOOTSTRAP_SERVERS", "broker:9092");
+ ObjectNode createKafka = mapper.createObjectNode();
+ createKafka.set("ConnectionInput", kafkaInput);
+ handler.handle("CreateConnection", createKafka, REGION);
+
+ ObjectNode list = mapper.createObjectNode();
+ list.put("HidePassword", true);
+ list.putObject("Filter").put("ConnectionType", "JDBC");
+ JsonNode body = mapper.valueToTree(handler.handle("GetConnections", list, REGION).getEntity());
+ assertEquals(1, body.get("ConnectionList").size());
+ JsonNode orders = body.get("ConnectionList").get(0);
+ assertEquals("orders", orders.get("Name").asText());
+ assertFalse(orders.get("ConnectionProperties").has("PASSWORD"));
+ assertEquals("app", orders.get("ConnectionProperties").get("USERNAME").asText());
+ assertFalse(body.has("NextToken"));
+ }
+
+ @Test
+ void updateDeleteAndBatchDeleteConnectionAnswerWithTheDocumentedBodies() throws Exception {
+ ObjectNode create = mapper.createObjectNode();
+ create.set("ConnectionInput", connectionInput("orders"));
+ handler.handle("CreateConnection", create, REGION);
+
+ ObjectNode update = mapper.createObjectNode();
+ update.put("Name", "orders");
+ ObjectNode redefinition = connectionInput("orders");
+ redefinition.remove("Description");
+ update.set("ConnectionInput", redefinition);
+ Response updated = handler.handle("UpdateConnection", update, REGION);
+ assertEquals(200, updated.getStatus());
+ assertEquals(0, mapper.valueToTree(updated.getEntity()).size());
+ JsonNode afterUpdate = mapper.valueToTree(
+ handler.handle("GetConnection", mapper.createObjectNode().put("Name", "orders"), REGION).getEntity())
+ .get("Connection");
+ assertFalse(afterUpdate.has("Description"));
+
+ ObjectNode batch = mapper.createObjectNode();
+ batch.putArray("ConnectionNameList").add("orders").add("absent");
+ JsonNode batchBody = mapper.valueToTree(handler.handle("BatchDeleteConnection", batch, REGION).getEntity());
+ assertEquals("orders", batchBody.get("Succeeded").get(0).asText());
+ assertEquals("EntityNotFoundException", batchBody.get("Errors").get("absent").get("ErrorCode").asText());
+
+ AwsException gone = assertThrows(AwsException.class, () -> handler.handle(
+ "DeleteConnection", mapper.createObjectNode().put("ConnectionName", "orders"), REGION));
+ assertEquals("EntityNotFoundException", gone.getErrorCode());
+
+ ObjectNode test = mapper.createObjectNode();
+ ObjectNode inline = test.putObject("TestConnectionInput");
+ inline.put("ConnectionType", "JDBC");
+ inline.putObject("ConnectionProperties").put("JDBC_CONNECTION_URL", "jdbc:mysql://h:3306/d");
+ assertEquals(200, handler.handle("TestConnection", test, REGION).getStatus());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
index 0f67444e87..d4e0794c77 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -10,7 +10,11 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.glue.model.Classifier;
import io.github.hectorvent.floci.services.glue.model.Column;
import io.github.hectorvent.floci.services.glue.model.Database;
+import io.github.hectorvent.floci.services.glue.model.Connection;
+import io.github.hectorvent.floci.services.glue.model.ConnectionInput;
import io.github.hectorvent.floci.services.glue.model.Crawler;
+import io.github.hectorvent.floci.services.glue.model.AuthenticationConfiguration;
+import io.github.hectorvent.floci.services.glue.model.PhysicalConnectionRequirements;
import io.github.hectorvent.floci.services.glue.model.CrawlerTargets;
import io.github.hectorvent.floci.services.glue.model.Job;
import io.github.hectorvent.floci.services.glue.model.JobCommand;
@@ -35,6 +39,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
import java.util.concurrent.CountDownLatch;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -88,6 +93,7 @@ class GlueServiceTest {
new InMemoryStorage<String, Job>(),
new InMemoryStorage<String, Crawler>(),
new InMemoryStorage<String, Classifier>(),
+ new InMemoryStorage<String, Connection>(),
schemaRegistryService, regionResolver, new ResourceGroupsTaggingService(null));
glueService.createDatabase(new Database("db1"));
}
@@ -1385,7 +1391,7 @@ class GlueServiceTest {
assertEquals("plain", errors.get(0).tableName());
assertEquals(1, glueService.getTableVersions("db1", "plain").size(), "only the current version is left");
- List<String> tooMany = java.util.stream.IntStream.rangeClosed(1, 101)
+ List<String> tooMany = IntStream.rangeClosed(1, 101)
.mapToObj(String::valueOf).toList();
AwsException overCap = assertThrows(AwsException.class,
() -> glueService.batchDeleteTableVersions("db1", "plain", tooMany));
@@ -1480,6 +1486,240 @@ class GlueServiceTest {
assertNull(second.nextToken());
}
+ // ---- Connections -----------------------------------------------------------------------
+
+ private static ConnectionInput jdbcConnection(String name) {
+ ConnectionInput input = new ConnectionInput();
+ input.setName(name);
+ input.setConnectionType("JDBC");
+ input.setDescription("orders db");
+ input.setMatchCriteria(List.of("orders", "reporting"));
+ Map<String, String> properties = new LinkedHashMap<>();
+ properties.put("JDBC_CONNECTION_URL", "jdbc:postgresql://db.internal:5432/orders");
+ properties.put("USERNAME", "app");
+ properties.put("PASSWORD", "s3cret");
+ input.setConnectionProperties(properties);
+ PhysicalConnectionRequirements placement = new PhysicalConnectionRequirements();
+ placement.setSubnetId("subnet-0123456789abcdef0");
+ placement.setSecurityGroupIdList(List.of("sg-0123456789abcdef0"));
+ placement.setAvailabilityZone("us-east-1a");
+ input.setPhysicalConnectionRequirements(placement);
+ return input;
+ }
+
+ @Test
+ void connectionCanBeCreatedFetchedUpdatedAndDeleted() {
+ assertEquals("READY", glueService.createConnection(jdbcConnection("orders"), null, REGION));
+
+ Connection fetched = glueService.getConnection("orders", false);
+ assertEquals("orders", fetched.getName());
+ assertEquals("JDBC", fetched.getConnectionType());
+ assertEquals("orders db", fetched.getDescription());
+ assertEquals("s3cret", fetched.getConnectionProperties().get("PASSWORD"));
+ assertEquals("subnet-0123456789abcdef0", fetched.getPhysicalConnectionRequirements().getSubnetId());
+ assertEquals("READY", fetched.getStatus());
+ assertEquals(1, fetched.getConnectionSchemaVersion());
+ assertNotNull(fetched.getCreationTime());
+ assertEquals(fetched.getCreationTime(), fetched.getLastUpdatedTime());
+
+ // UpdateConnection redefines the connection: the description left out of the input is
+ // gone afterwards, the name and creation time survive, and the update time moves.
+ ConnectionInput redefinition = new ConnectionInput();
+ redefinition.setName("orders");
+ redefinition.setConnectionType("JDBC");
+ redefinition.setConnectionProperties(Map.of("JDBC_CONNECTION_URL", "jdbc:postgresql://db2.internal:5432/orders"));
+ glueService.updateConnection("orders", redefinition);
+
+ Connection updated = glueService.getConnection("orders", false);
+ assertEquals("jdbc:postgresql://db2.internal:5432/orders", updated.getConnectionProperties().get("JDBC_CONNECTION_URL"));
+ assertNull(updated.getConnectionProperties().get("PASSWORD"));
+ assertNull(updated.getDescription());
+ assertNull(updated.getPhysicalConnectionRequirements());
+ assertEquals(fetched.getCreationTime(), updated.getCreationTime());
+ assertFalse(updated.getLastUpdatedTime().isBefore(fetched.getLastUpdatedTime()));
+
+ glueService.deleteConnection("orders", REGION);
+
+ AwsException ex = assertThrows(AwsException.class, () -> glueService.getConnection("orders", false));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void connectionRejectsDuplicateCreationAndUpdateOfAMissingOne() {
+ glueService.createConnection(jdbcConnection("dup"), null, REGION);
+
+ AwsException duplicate = assertThrows(AwsException.class,
+ () -> glueService.createConnection(jdbcConnection("dup"), null, REGION));
+ assertEquals("AlreadyExistsException", duplicate.getErrorCode());
+
+ AwsException missing = assertThrows(AwsException.class,
+ () -> glueService.updateConnection("absent", jdbcConnection("absent")));
+ assertEquals("EntityNotFoundException", missing.getErrorCode());
+
+ AwsException missingDelete = assertThrows(AwsException.class,
+ () -> glueService.deleteConnection("absent", REGION));
+ assertEquals("EntityNotFoundException", missingDelete.getErrorCode());
+ }
+
+ @Test
+ void connectionInputIsValidatedAgainstTheApiReference() {
+ ConnectionInput noName = jdbcConnection("x");
+ noName.setName(" ");
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.createConnection(noName, null, REGION)).getErrorCode());
+
+ ConnectionInput longName = jdbcConnection("n".repeat(256));
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.createConnection(longName, null, REGION)).getErrorCode());
+
+ ConnectionInput unknownType = jdbcConnection("t");
+ unknownType.setConnectionType("FTP");
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.createConnection(unknownType, null, REGION)).getErrorCode());
+
+ ConnectionInput noProperties = jdbcConnection("p");
+ noProperties.setConnectionProperties(null);
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.createConnection(noProperties, null, REGION)).getErrorCode());
+
+ ConnectionInput unknownKey = jdbcConnection("k");
+ unknownKey.setConnectionProperties(Map.of("HOSTNAME", "db"));
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.createConnection(unknownKey, null, REGION)).getErrorCode());
+
+ ConnectionInput tooManyCriteria = jdbcConnection("m");
+ tooManyCriteria.setMatchCriteria(List.of("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"));
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.createConnection(tooManyCriteria, null, REGION)).getErrorCode());
+
+ // A NETWORK connection carries no properties at all, only VPC placement.
+ ConnectionInput network = jdbcConnection("vpc-only");
+ network.setConnectionType("NETWORK");
+ network.setConnectionProperties(Map.of());
+ assertEquals("READY", glueService.createConnection(network, null, REGION));
+ }
+
+ @Test
+ void hidePasswordDropsThePasswordPropertiesAndNothingElse() {
+ ConnectionInput input = jdbcConnection("hidden");
+ Map<String, String> properties = new LinkedHashMap<>(input.getConnectionProperties());
+ properties.put("ENCRYPTED_PASSWORD", "AQICAHg...");
+ properties.put("SECRET_ID", "prod/orders");
+ input.setConnectionProperties(properties);
+ glueService.createConnection(input, null, REGION);
+
+ Connection hidden = glueService.getConnection("hidden", true);
+ assertNull(hidden.getConnectionProperties().get("PASSWORD"));
+ assertNull(hidden.getConnectionProperties().get("ENCRYPTED_PASSWORD"));
+ assertEquals("app", hidden.getConnectionProperties().get("USERNAME"));
+ assertEquals("prod/orders", hidden.getConnectionProperties().get("SECRET_ID"));
+
+ // The stored connection is untouched: a later read without the flag has the password.
+ assertEquals("s3cret", glueService.getConnection("hidden", false).getConnectionProperties().get("PASSWORD"));
+ assertEquals("s3cret",
+ glueService.getConnections(null, null, null, false, null, null).items().get(0)
+ .getConnectionProperties().get("PASSWORD"));
+ assertNull(glueService.getConnections(null, null, null, true, null, null).items().get(0)
+ .getConnectionProperties().get("PASSWORD"));
+ }
+
+ @Test
+ void getConnectionsFiltersByCriteriaTypeAndSchemaVersionAndPages() {
+ glueService.createConnection(jdbcConnection("b-orders"), null, REGION);
+ ConnectionInput kafka = jdbcConnection("a-events");
+ kafka.setConnectionType("KAFKA");
+ kafka.setMatchCriteria(List.of("events"));
+ kafka.setConnectionProperties(Map.of("KAFKA_BOOTSTRAP_SERVERS", "broker:9092"));
+ glueService.createConnection(kafka, null, REGION);
+ ConnectionInput saas = jdbcConnection("c-crm");
+ saas.setConnectionType("SALESFORCE");
+ saas.setMatchCriteria(null);
+ saas.setConnectionProperties(Map.of("ROLE_ARN", "arn:aws:iam::000000000000:role/crm"));
+ AuthenticationConfiguration auth = new AuthenticationConfiguration();
+ auth.setAuthenticationType("OAUTH2");
+ auth.setSecretArn("arn:aws:secretsmanager:us-east-1:000000000000:secret:crm");
+ auth.setBasicAuthenticationCredentials(Map.of("Username", "u", "Password", "p"));
+ saas.setAuthenticationConfiguration(auth);
+ glueService.createConnection(saas, null, REGION);
+
+ List<Connection> all = glueService.getConnections(null, null, null, false, null, null).items();
+ assertEquals(List.of("a-events", "b-orders", "c-crm"), all.stream().map(Connection::getName).toList());
+
+ assertEquals(List.of("b-orders"),
+ glueService.getConnections(List.of("orders", "reporting"), null, null, false, null, null)
+ .items().stream().map(Connection::getName).toList());
+ assertTrue(glueService.getConnections(List.of("orders", "missing"), null, null, false, null, null)
+ .items().isEmpty());
+ assertEquals(List.of("a-events"),
+ glueService.getConnections(null, "KAFKA", null, false, null, null)
+ .items().stream().map(Connection::getName).toList());
+ assertEquals(List.of("c-crm"),
+ glueService.getConnections(null, null, 2, false, null, null)
+ .items().stream().map(Connection::getName).toList());
+
+ // A credential given on create never comes back on a read.
+ Connection crm = glueService.getConnection("c-crm", false);
+ assertEquals("OAUTH2", crm.getAuthenticationConfiguration().getAuthenticationType());
+ assertNull(crm.getAuthenticationConfiguration().getBasicAuthenticationCredentials());
+
+ GlueService.Page<Connection> first = glueService.getConnections(null, null, null, false, 2, null);
+ assertEquals(2, first.items().size());
+ assertNotNull(first.nextToken());
+ GlueService.Page<Connection> second = glueService.getConnections(null, null, null, false, 2, first.nextToken());
+ assertEquals(List.of("c-crm"), second.items().stream().map(Connection::getName).toList());
+ assertNull(second.nextToken());
+ }
+
+ @Test
+ void batchDeleteConnectionReportsTheMissingNamesAndDeletesTheRest() {
+ glueService.createConnection(jdbcConnection("keep-a"), null, REGION);
+ glueService.createConnection(jdbcConnection("keep-b"), null, REGION);
+
+ GlueService.BatchDeleteConnectionResult result =
+ glueService.batchDeleteConnections(List.of("keep-a", "absent", "keep-b"), REGION);
+
+ assertEquals(List.of("keep-a", "keep-b"), result.succeeded());
+ assertEquals(1, result.errors().size());
+ assertEquals("EntityNotFoundException", result.errors().get("absent").errorCode());
+ assertTrue(glueService.getConnections(null, null, null, false, null, null).items().isEmpty());
+
+ // The reference caps ConnectionNameList at 25 entries.
+ List<String> twentySix = IntStream.rangeClosed(1, 26).mapToObj(i -> "c" + i).toList();
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.batchDeleteConnections(twentySix, REGION)).getErrorCode());
+ }
+
+ @Test
+ void testConnectionAcceptsAnExistingNameOrAWellFormedInlineDefinition() {
+ glueService.createConnection(jdbcConnection("probe"), null, REGION);
+ glueService.testConnection("probe", null, null);
+ glueService.testConnection(null, "JDBC", Map.of("JDBC_CONNECTION_URL", "jdbc:mysql://h:3306/d"));
+
+ assertEquals("EntityNotFoundException",
+ assertThrows(AwsException.class, () -> glueService.testConnection("absent", null, null)).getErrorCode());
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.testConnection(null, "JDBC", null)).getErrorCode());
+ assertEquals("InvalidInputException",
+ assertThrows(AwsException.class, () -> glueService.testConnection(null, null, Map.of())).getErrorCode());
+ }
+
+ @Test
+ void connectionTagsAreKeptUnderTheConnectionArnAndRemovedOnDelete() {
+ glueService.createConnection(jdbcConnection("tagged"), Map.of("env", "dev"), REGION);
+
+ String arn = "arn:aws:glue:" + REGION + ":" + ACCOUNT_ID + ":connection/tagged";
+ assertEquals("dev", glueService.getTags(arn, REGION).get("env"));
+
+ glueService.tagResource(arn, Map.of("team", "data"), REGION);
+ assertEquals("data", glueService.getTags(arn, REGION).get("team"));
+ glueService.untagResource(arn, List.of("env"), REGION);
+ assertNull(glueService.getTags(arn, REGION).get("env"));
+
+ glueService.deleteConnection("tagged", REGION);
+ assertEquals("EntityNotFoundException",
+ assertThrows(AwsException.class, () -> glueService.getTags(arn, REGION)).getErrorCode());
+ }
+
private static List<String> names(GlueService.Page<Table> page) {
return page.items().stream().map(Table::getName).toList();
}