Light, fluffy, and always free - The AWS Local Emulator alternative
feat(athena): create, update and delete data catalogs (#3381)
lex00cb6b4d8a8876
diff --git a/docs/services/athena.md b/docs/services/athena.mdindex 1f1b63305e..7fdd989d9d 100644--- a/docs/services/athena.md+++ b/docs/services/athena.md@@ -18,12 +18,17 @@ Floci emulates Amazon Athena with **real SQL execution** powered by a [floci-duc| `GetWorkGroup` | Returns information about a workgroup || `ListWorkGroups` | Lists all workgroups || `CreateWorkGroup` | Creates a new workgroup |-| `ListDataCatalogs` | - |-| `GetDataCatalog` | - |+| `ListDataCatalogs` | Lists the built-in `AwsDataCatalog` plus every registered data catalog |+| `GetDataCatalog` | Returns one data catalog, or `InvalidRequestException` when it does not exist |+| `CreateDataCatalog` | Registers a `LAMBDA`, `GLUE`, `HIVE` or `FEDERATED` data catalog |+| `UpdateDataCatalog` | Replaces the type, description and parameters of a data catalog |+| `DeleteDataCatalog` | Deletes a data catalog and returns the record it removed |+| `TagResource` | Adds tags to a workgroup or data catalog ARN |+| `UntagResource` | Removes tags from a workgroup or data catalog ARN || `ListDatabases` | - || `ListTableMetadata` | - || `GetTableMetadata` | - |-| `ListTagsForResource` | Returns the tags on a workgroup |+| `ListTagsForResource` | Returns the tags on a workgroup or data catalog || `DeleteWorkGroup` | Deletes a workgroup |<!-- floci:actions:end -->diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaJsonHandler.javaindex 7bb105e0dd..c7148f70fe 100644--- a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaJsonHandler.java@@ -8,10 +8,14 @@ import io.github.hectorvent.floci.services.athena.model.QueryExecution;import io.github.hectorvent.floci.services.athena.model.QueryExecutionContext;import io.github.hectorvent.floci.services.athena.model.ResultConfiguration;import io.github.hectorvent.floci.services.athena.model.ResultSet;+import io.github.hectorvent.floci.services.athena.model.WorkGroupTag;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import jakarta.ws.rs.core.Response;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;import java.util.Map;@ApplicationScoped@@ -74,10 +78,38 @@ public class AthenaJsonHandler {athenaService.createWorkGroup(createRequest, region);yield Response.ok(Map.of()).build();}- case "ListDataCatalogs" -> Response.ok(Map.of("DataCatalogsSummary", athenaService.listDataCatalogs())).build();+ case "ListDataCatalogs" ->+ Response.ok(Map.of("DataCatalogsSummary", athenaService.listDataCatalogs(region))).build();case "GetDataCatalog" -> {String name = request.has("Name") ? request.get("Name").asText() : AthenaService.DEFAULT_CATALOG;- yield Response.ok(Map.of("DataCatalog", athenaService.getDataCatalog(name))).build();+ yield Response.ok(Map.of("DataCatalog", athenaService.getDataCatalog(region, name))).build();+ }+ case "CreateDataCatalog" -> {+ Map<String, Object> catalog = athenaService.createDataCatalog(region,+ text(request, "Name"),+ text(request, "Type"),+ text(request, "Description"),+ parameters(request),+ tags(request));+ yield Response.ok(Map.of("DataCatalog", catalog)).build();+ }+ case "UpdateDataCatalog" -> {+ athenaService.updateDataCatalog(region,+ text(request, "Name"),+ text(request, "Type"),+ text(request, "Description"),+ parameters(request));+ yield Response.ok(Map.of()).build();+ }+ case "DeleteDataCatalog" -> Response.ok(+ Map.of("DataCatalog", athenaService.deleteDataCatalog(region, text(request, "Name")))).build();+ case "TagResource" -> {+ athenaService.tagResource(text(request, "ResourceARN"), tags(request));+ yield Response.ok(Map.of()).build();+ }+ case "UntagResource" -> {+ athenaService.untagResource(text(request, "ResourceARN"), tagKeys(request));+ yield Response.ok(Map.of()).build();}case "ListDatabases" -> {String catalog = request.has("CatalogName") ? request.get("CatalogName").asText() : AthenaService.DEFAULT_CATALOG;@@ -112,4 +144,43 @@ public class AthenaJsonHandler {default -> throw new AwsException("InvalidAction", "Action " + action + " is not supported", 400);};}++ private static String text(JsonNode request, String field) {+ JsonNode node = request.get(field);+ return node == null || node.isNull() ? null : node.asText();+ }++ private static Map<String, String> parameters(JsonNode request) {+ JsonNode node = request.get("Parameters");+ if (node == null || !node.isObject()) {+ return Map.of();+ }+ Map<String, String> parameters = new LinkedHashMap<>();+ node.fields().forEachRemaining(entry -> parameters.put(entry.getKey(), entry.getValue().asText()));+ return parameters;+ }++ private static List<WorkGroupTag> tags(JsonNode request) {+ JsonNode node = request.get("Tags");+ if (node == null || !node.isArray()) {+ return List.of();+ }+ List<WorkGroupTag> tags = new ArrayList<>();+ for (JsonNode tag : node) {+ tags.add(new WorkGroupTag(text(tag, "Key"), text(tag, "Value")));+ }+ return tags;+ }++ private static List<String> tagKeys(JsonNode request) {+ JsonNode node = request.get("TagKeys");+ if (node == null || !node.isArray()) {+ return List.of();+ }+ List<String> keys = new ArrayList<>();+ for (JsonNode key : node) {+ keys.add(key.asText());+ }+ return keys;+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.javaindex ac3bb2b200..6517470515 100644--- a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java@@ -37,6 +37,13 @@ public class AthenaService {private static final String DEFAULT_OUTPUT_BUCKET = "floci-athena-results";private static final String DEFAULT_WORKGROUP = "primary";private static final String DEFAULT_ENGINE_VERSION = "Athena engine version 3";+ private static final String WORKGROUP_RESOURCE = "workgroup/";+ private static final String DATA_CATALOG_RESOURCE = "datacatalog/";+ private static final Set<String> CATALOG_TYPES = Set.of("LAMBDA", "GLUE", "HIVE", "FEDERATED");+ private static final Set<String> CONNECTION_TYPES = Set.of(+ "DYNAMODB", "MYSQL", "POSTGRESQL", "REDSHIFT", "ORACLE", "SYNAPSE", "SQLSERVER", "DB2",+ "OPENSEARCH", "BIGQUERY", "GOOGLECLOUDSTORAGE", "HBASE", "DOCUMENTDB", "CMDB", "TPCDS",+ "TIMESTREAM", "SAPHANA", "SNOWFLAKE", "DATALAKEGEN2", "DB2AS400");private static final Pattern CREATE_DATABASE_PATTERN = Pattern.compile("^\\s*CREATE\\s+(?:DATABASE|SCHEMA)\\s+"+ "(IF\\s+NOT\\s+EXISTS\\s+)?"@@ -57,6 +64,7 @@ public class AthenaService {private final StorageBackend<String, QueryExecution> queryStore;private final StorageBackend<String, WorkGroup> workGroupStore;+ private final StorageBackend<String, DataCatalog> dataCatalogStore;private final FlociDuckClient duckClient;private final GlueService glueService;private final S3Service s3Service;@@ -76,6 +84,8 @@ public class AthenaService {new TypeReference<>() {});this.workGroupStore = storageFactory.create("athena", "workgroups.json",new TypeReference<>() {});+ this.dataCatalogStore = storageFactory.create("athena", "data-catalogs.json",+ new TypeReference<>() {});this.duckClient = duckClient;this.glueService = glueService;this.s3Service = s3Service;@@ -197,10 +207,7 @@ public class AthenaService {if (DEFAULT_WORKGROUP.equals(resolved)) {return primaryWorkGroupSummary();}- WorkGroup workGroup = workGroupStore.get(workGroupKey(region, resolved))- .orElseThrow(() -> new AwsException("InvalidRequestException",- "WorkGroup " + resolved + " is not found.", 400));- return toWorkGroupDetail(workGroup);+ return toWorkGroupDetail(requireWorkGroup(region, resolved));}public void deleteWorkGroup(String name, String region) {@@ -209,36 +216,62 @@ public class AthenaService {/*** github.com/floci-io/floci/issues/2791: terraform-provider-aws calls this on every- * aws_athena_workgroup refresh, tags or not. Only workgroup ARNs are supported - the- * primary workgroup can't be tagged and always answers with an empty list, matching a live- * account (CreateWorkGroup already rejects "primary", so it can never carry real tags).+ * aws_athena_workgroup refresh, tags or not. Athena tags a workgroup or a data catalog and+ * nothing else, so any other resource type is rejected. The built-in primary workgroup and+ * AwsDataCatalog are synthesized rather than stored here and always answer with an empty+ * list (CreateWorkGroup and CreateDataCatalog both reject those names, so neither can ever+ * carry real tags).*/public List<WorkGroupTag> listTagsForResource(String resourceArn) {- if (resourceArn == null || resourceArn.isBlank()) {- throw new AwsException("InvalidRequestException", "ResourceARN is required.", 400);+ AwsArnUtils.Arn arn = parseAthenaResourceArn(resourceArn);+ if (arn.resource().startsWith(WORKGROUP_RESOURCE)) {+ String name = arn.resource().substring(WORKGROUP_RESOURCE.length());+ if (DEFAULT_WORKGROUP.equals(name)) {+ return List.of();+ }+ return requireWorkGroup(arn.region(), name).getTags();}- AwsArnUtils.Arn arn;- try {- arn = AwsArnUtils.parse(resourceArn);- } catch (IllegalArgumentException e) {- throw new AwsException("InvalidRequestException", "Invalid ResourceARN: " + resourceArn, 400);+ String name = arn.resource().substring(DATA_CATALOG_RESOURCE.length());+ if (DEFAULT_CATALOG.equals(name)) {+ return List.of();}- // pgermosen (review, #3242): without this, an ARN naming a completely different- // service (e.g. a Neptune workgroup/-shaped resource that coincidentally shares this- // region and name) would resolve instead of being rejected the way a live account- // rejects a ResourceARN naming the wrong service.- if (!"athena".equals(arn.service()) || !arn.resource().startsWith("workgroup/")) {- throw new AwsException("InvalidRequestException",- "Unsupported resource type for ListTagsForResource: " + resourceArn, 400);+ return requireTaggedDataCatalog(arn.region(), name, resourceArn).getTags();+ }++ public synchronized void tagResource(String resourceArn, List<WorkGroupTag> tags) {+ if (tags == null || tags.isEmpty()) {+ throw new AwsException("InvalidRequestException", "Tags is required.", 400);}- String name = arn.resource().substring("workgroup/".length());- if (DEFAULT_WORKGROUP.equals(name)) {- return List.of();+ AwsArnUtils.Arn arn = parseAthenaResourceArn(resourceArn);+ if (arn.resource().startsWith(WORKGROUP_RESOURCE)) {+ String name = requireTaggableWorkGroupName(arn.resource());+ WorkGroup workGroup = requireWorkGroup(arn.region(), name);+ workGroup.setTags(mergeTags(workGroup.getTags(), tags));+ workGroupStore.put(workGroupKey(arn.region(), name), workGroup);+ return;}- WorkGroup workGroup = workGroupStore.get(workGroupKey(arn.region(), name))- .orElseThrow(() -> new AwsException("InvalidRequestException",- "WorkGroup " + name + " is not found.", 400));- return workGroup.getTags();+ String name = requireTaggableCatalogName(arn.resource());+ DataCatalog catalog = requireTaggedDataCatalog(arn.region(), name, resourceArn);+ catalog.setTags(mergeTags(catalog.getTags(), tags));+ dataCatalogStore.put(catalogKey(arn.region(), name), catalog);+ }++ public synchronized void untagResource(String resourceArn, List<String> tagKeys) {+ if (tagKeys == null || tagKeys.isEmpty()) {+ throw new AwsException("InvalidRequestException", "TagKeys is required.", 400);+ }+ AwsArnUtils.Arn arn = parseAthenaResourceArn(resourceArn);+ if (arn.resource().startsWith(WORKGROUP_RESOURCE)) {+ String name = requireTaggableWorkGroupName(arn.resource());+ WorkGroup workGroup = requireWorkGroup(arn.region(), name);+ workGroup.setTags(removeTags(workGroup.getTags(), tagKeys));+ workGroupStore.put(workGroupKey(arn.region(), name), workGroup);+ return;+ }+ String name = requireTaggableCatalogName(arn.resource());+ DataCatalog catalog = requireTaggedDataCatalog(arn.region(), name, resourceArn);+ catalog.setTags(removeTags(catalog.getTags(), tagKeys));+ dataCatalogStore.put(catalogKey(arn.region(), name), catalog);}public List<Map<String, Object>> listWorkGroups(String region) {@@ -251,12 +284,92 @@ public class AthenaService {return workGroups;}- public List<Map<String, Object>> listDataCatalogs() {- return List.of(Map.of("CatalogName", DEFAULT_CATALOG, "Type", "GLUE"));+ /**+ * The account's built-in Glue catalog first, then the catalogs registered through+ * {@code CreateDataCatalog}, sorted by name.+ */+ public List<Map<String, Object>> listDataCatalogs(String region) {+ List<Map<String, Object>> summaries = new ArrayList<>();+ summaries.add(Map.of("CatalogName", DEFAULT_CATALOG, "Type", "GLUE"));+ summaries.addAll(dataCatalogStore.scan(k -> k.startsWith(region + ":")).stream()+ .sorted(Comparator.comparing(DataCatalog::getName))+ .map(this::toCatalogSummary)+ .toList());+ return summaries;+ }++ public Map<String, Object> getDataCatalog(String region, String name) {+ String resolved = name == null || name.isBlank() ? DEFAULT_CATALOG : name;+ if (DEFAULT_CATALOG.equals(resolved)) {+ return Map.of("Name", DEFAULT_CATALOG, "Type", "GLUE");+ }+ return toCatalogDetail(requireDataCatalog(region, resolved));+ }++ public Map<String, Object> createDataCatalog(String region,+ String name,+ String type,+ String description,+ Map<String, String> parameters,+ List<WorkGroupTag> tags) {+ validateCatalogName(name);+ if (DEFAULT_CATALOG.equals(name)) {+ throw new AwsException("InvalidRequestException",+ DEFAULT_CATALOG + " is a reserved data catalog name.", 400);+ }+ validateCatalogType(type);+ if (dataCatalogStore.get(catalogKey(region, name)).isPresent()) {+ throw new AwsException("InvalidRequestException",+ "DataCatalog " + name + " already exists.", 400);+ }++ DataCatalog catalog = new DataCatalog();+ catalog.setName(name);+ catalog.setType(type);+ catalog.setDescription(description);+ catalog.setParameters(normalizeParameters(parameters));+ // AWS creates LAMBDA, GLUE and HIVE catalogs synchronously and FEDERATED ones+ // asynchronously. Registration is instantaneous here, so all four report the terminal+ // status rather than a CREATE_IN_PROGRESS that no worker would ever advance.+ catalog.setStatus("CREATE_COMPLETE");+ catalog.setConnectionType(resolveConnectionType(type, catalog.getParameters()));+ catalog.setTags(normalizeTags(tags));+ dataCatalogStore.put(catalogKey(region, name), catalog);+ return toCatalogDetail(catalog);+ }++ /**+ * UpdateDataCatalog replaces Type, Description and Parameters wholesale, the way AWS does.+ * Tags are untouched: they move only through TagResource and UntagResource.+ */+ public void updateDataCatalog(String region,+ String name,+ String type,+ String description,+ Map<String, String> parameters) {+ validateCatalogName(name);+ if (DEFAULT_CATALOG.equals(name)) {+ throw new AwsException("InvalidRequestException",+ DEFAULT_CATALOG + " cannot be modified.", 400);+ }+ validateCatalogType(type);+ DataCatalog catalog = requireDataCatalog(region, name);+ catalog.setType(type);+ catalog.setDescription(description);+ catalog.setParameters(normalizeParameters(parameters));+ catalog.setConnectionType(resolveConnectionType(type, catalog.getParameters()));+ dataCatalogStore.put(catalogKey(region, name), catalog);}- public Map<String, Object> getDataCatalog(String name) {- return Map.of("Name", name == null || name.isBlank() ? DEFAULT_CATALOG : name, "Type", "GLUE");+ public Map<String, Object> deleteDataCatalog(String region, String name) {+ validateCatalogName(name);+ if (DEFAULT_CATALOG.equals(name)) {+ throw new AwsException("InvalidRequestException",+ DEFAULT_CATALOG + " cannot be deleted.", 400);+ }+ DataCatalog catalog = requireDataCatalog(region, name);+ dataCatalogStore.delete(catalogKey(region, name));+ return toCatalogDetail(catalog);}public List<Map<String, Object>> listDatabases(String catalog) {@@ -524,6 +637,177 @@ public class AthenaService {.toList();}+ private List<WorkGroupTag> mergeTags(List<WorkGroupTag> current, List<WorkGroupTag> added) {+ Map<String, String> merged = currentTagsByKey(current);+ for (WorkGroupTag tag : added) {+ if (tag == null || tag.getKey() == null || tag.getKey().isBlank()) {+ throw new AwsException("InvalidRequestException", "Tag keys must not be empty.", 400);+ }+ merged.put(tag.getKey(), tag.getValue() == null ? "" : tag.getValue());+ }+ return toTagList(merged);+ }++ private List<WorkGroupTag> removeTags(List<WorkGroupTag> current, List<String> tagKeys) {+ Map<String, String> remaining = currentTagsByKey(current);+ tagKeys.forEach(remaining::remove);+ return toTagList(remaining);+ }++ private Map<String, String> currentTagsByKey(List<WorkGroupTag> current) {+ Map<String, String> byKey = new LinkedHashMap<>();+ for (WorkGroupTag tag : normalizeTags(current)) {+ if (tag.getKey() != null && !tag.getKey().isBlank()) {+ byKey.put(tag.getKey(), tag.getValue());+ }+ }+ return byKey;+ }++ private List<WorkGroupTag> toTagList(Map<String, String> values) {+ List<WorkGroupTag> tags = new ArrayList<>();+ values.forEach((key, value) -> tags.add(new WorkGroupTag(key, value)));+ return tags;+ }++ /**+ * Parses a ResourceARN for the tag operations and rejects anything Athena does not tag.+ *+ * <p>pgermosen (review, #3242): the service check matters. Without it, an ARN naming a+ * completely different service (a Neptune workgroup/-shaped resource that happens to share+ * this region and name, say) would resolve instead of being rejected the way a live account+ * rejects a ResourceARN naming the wrong service.+ */+ private AwsArnUtils.Arn parseAthenaResourceArn(String resourceArn) {+ if (resourceArn == null || resourceArn.isBlank()) {+ throw new AwsException("InvalidRequestException", "ResourceARN is required.", 400);+ }+ AwsArnUtils.Arn arn;+ try {+ arn = AwsArnUtils.parse(resourceArn);+ } catch (IllegalArgumentException e) {+ throw new AwsException("InvalidRequestException", "Invalid ResourceARN: " + resourceArn, 400);+ }+ boolean taggable = "athena".equals(arn.service())+ && (arn.resource().startsWith(WORKGROUP_RESOURCE)+ || arn.resource().startsWith(DATA_CATALOG_RESOURCE));+ if (!taggable) {+ throw new AwsException("InvalidRequestException",+ "ResourceARN " + resourceArn + " is not an Athena workgroup or data catalog.", 400);+ }+ return arn;+ }++ private WorkGroup requireWorkGroup(String region, String name) {+ return workGroupStore.get(workGroupKey(region, name))+ .orElseThrow(() -> new AwsException("InvalidRequestException",+ "WorkGroup " + name + " is not found.", 400));+ }++ private DataCatalog requireDataCatalog(String region, String name) {+ return dataCatalogStore.get(catalogKey(region, name))+ .orElseThrow(() -> new AwsException("InvalidRequestException",+ "DataCatalog " + name + " was not found.", 400));+ }++ /**+ * The tag operations answer ResourceNotFoundException, which is the error the Athena API+ * model declares for them. GetDataCatalog, UpdateDataCatalog and DeleteDataCatalog declare+ * only InvalidRequestException, so they keep {@link #requireDataCatalog}.+ */+ private DataCatalog requireTaggedDataCatalog(String region, String name, String resourceArn) {+ return dataCatalogStore.get(catalogKey(region, name))+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",+ "Resource " + resourceArn + " was not found.", 400));+ }++ private String requireTaggableWorkGroupName(String resource) {+ String name = resource.substring(WORKGROUP_RESOURCE.length());+ if (DEFAULT_WORKGROUP.equals(name)) {+ throw new AwsException("InvalidRequestException",+ "The " + DEFAULT_WORKGROUP + " workgroup cannot be tagged.", 400);+ }+ return name;+ }++ private String requireTaggableCatalogName(String resource) {+ String name = resource.substring(DATA_CATALOG_RESOURCE.length());+ if (DEFAULT_CATALOG.equals(name)) {+ throw new AwsException("InvalidRequestException",+ DEFAULT_CATALOG + " cannot be tagged.", 400);+ }+ return name;+ }++ private Map<String, Object> toCatalogSummary(DataCatalog catalog) {+ Map<String, Object> summary = new LinkedHashMap<>();+ summary.put("CatalogName", catalog.getName());+ summary.put("Type", catalog.getType());+ summary.put("Status", catalog.getStatus());+ if (catalog.getConnectionType() != null) {+ summary.put("ConnectionType", catalog.getConnectionType());+ }+ return summary;+ }++ private Map<String, Object> toCatalogDetail(DataCatalog catalog) {+ Map<String, Object> detail = new LinkedHashMap<>();+ detail.put("Name", catalog.getName());+ detail.put("Type", catalog.getType());+ detail.put("Status", catalog.getStatus());+ if (catalog.getDescription() != null) {+ detail.put("Description", catalog.getDescription());+ }+ detail.put("Parameters", catalog.getParameters() == null ? Map.of() : catalog.getParameters());+ if (catalog.getConnectionType() != null) {+ detail.put("ConnectionType", catalog.getConnectionType());+ }+ return detail;+ }++ private static Map<String, String> normalizeParameters(Map<String, String> parameters) {+ return parameters == null ? new LinkedHashMap<>() : new LinkedHashMap<>(parameters);+ }++ /**+ * A FEDERATED catalog carries its connection type in the {@code connection-type} parameter.+ * The other catalog types have none.+ */+ private static String resolveConnectionType(String type, Map<String, String> parameters) {+ if (!"FEDERATED".equals(type) || parameters == null) {+ return null;+ }+ String connectionType = parameters.get("connection-type");+ if (connectionType == null) {+ return null;+ }+ String normalized = connectionType.toUpperCase(Locale.ROOT);+ return CONNECTION_TYPES.contains(normalized) ? normalized : null;+ }++ private static void validateCatalogName(String name) {+ if (name == null || name.isBlank()) {+ throw new AwsException("InvalidRequestException", "Name is required.", 400);+ }+ if (name.length() > 256) {+ throw new AwsException("InvalidRequestException", "Name must be 1 to 256 characters.", 400);+ }+ }++ private static void validateCatalogType(String type) {+ if (type == null || type.isBlank()) {+ throw new AwsException("InvalidRequestException", "Type is required.", 400);+ }+ if (!CATALOG_TYPES.contains(type)) {+ throw new AwsException("InvalidRequestException",+ "Invalid data catalog type: " + type + ". Valid values are " + CATALOG_TYPES + ".", 400);+ }+ }++ private String catalogKey(String region, String name) {+ return region + ":" + name;+ }+private String workGroupKey(String region, String name) {return region + ":" + name;}diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/model/DataCatalog.java b/src/main/java/io/github/hectorvent/floci/services/athena/model/DataCatalog.javanew file mode 100644index 0000000000..b9684dff21--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/athena/model/DataCatalog.java@@ -0,0 +1,51 @@+package io.github.hectorvent.floci.services.athena.model;++import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++@RegisterForReflection+public class DataCatalog {++ @JsonProperty("Name")+ private String name;++ @JsonProperty("Description")+ private String description;++ @JsonProperty("Type")+ private String type;++ @JsonProperty("Status")+ private String status;++ @JsonProperty("ConnectionType")+ private String connectionType;++ @JsonProperty("Parameters")+ private Map<String, String> parameters = new LinkedHashMap<>();++ @JsonProperty("Tags")+ private List<WorkGroupTag> tags = new ArrayList<>();++ public DataCatalog() {}++ public String getName() { return name; }+ public void setName(String name) { this.name = name; }+ public String getDescription() { return description; }+ public void setDescription(String description) { this.description = description; }+ public String getType() { return type; }+ public void setType(String type) { this.type = type; }+ public String getStatus() { return status; }+ public void setStatus(String status) { this.status = status; }+ public String getConnectionType() { return connectionType; }+ public void setConnectionType(String connectionType) { this.connectionType = connectionType; }+ public Map<String, String> getParameters() { return parameters; }+ public void setParameters(Map<String, String> parameters) { this.parameters = parameters; }+ public List<WorkGroupTag> getTags() { return tags; }+ public void setTags(List<WorkGroupTag> tags) { this.tags = tags; }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/athena/AthenaDataCatalogIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/athena/AthenaDataCatalogIntegrationTest.javanew file mode 100644index 0000000000..0325e687b3--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/athena/AthenaDataCatalogIntegrationTest.java@@ -0,0 +1,489 @@+package io.github.hectorvent.floci.services.athena;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasItem;+import static org.hamcrest.Matchers.not;+import static org.hamcrest.Matchers.nullValue;++/**+ * Data catalog lifecycle behind terraform-provider-aws's aws_athena_data_catalog resource:+ * CreateDataCatalog, GetDataCatalog, ListDataCatalogs, UpdateDataCatalog, DeleteDataCatalog+ * and the tag operations on a datacatalog ARN.+ */+@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class AthenaDataCatalogIntegrationTest {++ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";+ private static final String CATALOG = "floci-lambda-catalog";+ private static final String FEDERATED_CATALOG = "floci-federated-catalog";+ private static final String CATALOG_ARN = "arn:aws:athena:us-east-1:000000000000:datacatalog/" + CATALOG;+ private static final String WORKGROUP = "floci-catalog-workgroup";+ private static final String WORKGROUP_ARN = "arn:aws:athena:us-east-1:000000000000:workgroup/" + WORKGROUP;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ @Order(1)+ void createDataCatalogEchoesTypeParametersAndTerminalStatus() {+ given()+ .header("X-Amz-Target", "AmazonAthena.CreateDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("""+ {+ "Name": "%s",+ "Type": "LAMBDA",+ "Description": "Lambda-backed federation",+ "Parameters": {+ "metadata-function": "arn:aws:lambda:us-east-1:000000000000:function:meta",+ "record-function": "arn:aws:lambda:us-east-1:000000000000:function:record"+ },+ "Tags": [ { "Key": "Project", "Value": "Floci" } ]+ }+ """.formatted(CATALOG))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DataCatalog.Name", equalTo(CATALOG))+ .body("DataCatalog.Type", equalTo("LAMBDA"))+ .body("DataCatalog.Status", equalTo("CREATE_COMPLETE"))+ .body("DataCatalog.Description", equalTo("Lambda-backed federation"))+ .body("DataCatalog.Parameters.'metadata-function'",+ equalTo("arn:aws:lambda:us-east-1:000000000000:function:meta"));+ }++ @Test+ @Order(2)+ void createDataCatalogRejectsADuplicateName() {+ given()+ .header("X-Amz-Target", "AmazonAthena.CreateDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"" + CATALOG + "\", \"Type\": \"LAMBDA\" }")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));+ }++ @Test+ @Order(3)+ void getDataCatalogSeesTheCreatedCatalog() {+ given()+ .header("X-Amz-Target", "AmazonAthena.GetDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"" + CATALOG + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DataCatalog.Name", equalTo(CATALOG))+ .body("DataCatalog.Type", equalTo("LAMBDA"))+ .body("DataCatalog.Parameters.'record-function'",+ equalTo("arn:aws:lambda:us-east-1:000000000000:function:record"));+ }++ @Test+ @Order(4)+ void listDataCatalogsKeepsTheDefaultFirstAndIncludesTheCreatedCatalog() {+ given()+ .header("X-Amz-Target", "AmazonAthena.ListDataCatalogs")+ .contentType(CONTENT_TYPE)+ .body("{}")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DataCatalogsSummary[0].CatalogName", equalTo("AwsDataCatalog"))+ .body("DataCatalogsSummary[0].Type", equalTo("GLUE"))+ .body("DataCatalogsSummary.CatalogName", hasItem(CATALOG))+ .body("DataCatalogsSummary.Type", hasItem("LAMBDA"));+ }++ @Test+ @Order(5)+ void getDataCatalogStillServesTheBuiltInGlueCatalog() {+ given()+ .header("X-Amz-Target", "AmazonAthena.GetDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"AwsDataCatalog\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DataCatalog.Name", equalTo("AwsDataCatalog"))+ .body("DataCatalog.Type", equalTo("GLUE"));+ }++ @Test+ @Order(6)+ void getDataCatalogAnswersInvalidRequestExceptionForAnAbsentCatalog() {+ given()+ .header("X-Amz-Target", "AmazonAthena.GetDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"floci-absent-catalog\" }")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));+ }++ @Test+ @Order(7)+ void createTimeTagsAreVisibleThroughListTagsForResource() {+ given()+ .header("X-Amz-Target", "AmazonAthena.ListTagsForResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"" + CATALOG_ARN + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Tags[0].Key", equalTo("Project"))+ .body("Tags[0].Value", equalTo("Floci"));+ }++ @Test+ @Order(8)+ void tagResourceAndUntagResourceRoundTripOnACatalogArn() {+ given()+ .header("X-Amz-Target", "AmazonAthena.TagResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"" + CATALOG_ARN + "\", "+ + "\"Tags\": [ { \"Key\": \"Environment\", \"Value\": \"test\" } ] }")+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .header("X-Amz-Target", "AmazonAthena.ListTagsForResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"" + CATALOG_ARN + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Tags.Key", hasItem("Environment"))+ .body("Tags.Key", hasItem("Project"));++ given()+ .header("X-Amz-Target", "AmazonAthena.UntagResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"" + CATALOG_ARN + "\", \"TagKeys\": [ \"Environment\" ] }")+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .header("X-Amz-Target", "AmazonAthena.ListTagsForResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"" + CATALOG_ARN + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Tags.Key", not(hasItem("Environment")))+ .body("Tags.Key", hasItem("Project"));+ }++ @Test+ @Order(9)+ void workGroupTagsAreReachableThroughTheSameTagOperations() {+ given()+ .header("X-Amz-Target", "AmazonAthena.CreateWorkGroup")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"" + WORKGROUP + "\", "+ + "\"Tags\": [ { \"Key\": \"Team\", \"Value\": \"Data\" } ] }")+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .header("X-Amz-Target", "AmazonAthena.TagResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"" + WORKGROUP_ARN + "\", "+ + "\"Tags\": [ { \"Key\": \"Owner\", \"Value\": \"Analytics\" } ] }")+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .header("X-Amz-Target", "AmazonAthena.ListTagsForResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"" + WORKGROUP_ARN + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Tags.Key", hasItem("Team"))+ .body("Tags.Key", hasItem("Owner"));++ given()+ .header("X-Amz-Target", "AmazonAthena.UntagResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"" + WORKGROUP_ARN + "\", \"TagKeys\": [ \"Owner\" ] }")+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .header("X-Amz-Target", "AmazonAthena.ListTagsForResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"" + WORKGROUP_ARN + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Tags.Key", hasItem("Team"))+ .body("Tags.Key", not(hasItem("Owner")));+ }++ @Test+ @Order(10)+ void updateDataCatalogReplacesTypeDescriptionAndParameters() {+ given()+ .header("X-Amz-Target", "AmazonAthena.UpdateDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("""+ {+ "Name": "%s",+ "Type": "HIVE",+ "Description": "Hive metastore",+ "Parameters": { "metadata-function": "arn:aws:lambda:us-east-1:000000000000:function:hive" }+ }+ """.formatted(CATALOG))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body(equalTo("{}"));++ given()+ .header("X-Amz-Target", "AmazonAthena.GetDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"" + CATALOG + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DataCatalog.Type", equalTo("HIVE"))+ .body("DataCatalog.Description", equalTo("Hive metastore"))+ .body("DataCatalog.Parameters.'metadata-function'",+ equalTo("arn:aws:lambda:us-east-1:000000000000:function:hive"))+ .body("DataCatalog.Parameters.'record-function'", nullValue());+ }++ @Test+ @Order(11)+ void updateDataCatalogLeavesTagsAlone() {+ given()+ .header("X-Amz-Target", "AmazonAthena.ListTagsForResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"" + CATALOG_ARN + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Tags.Key", hasItem("Project"));+ }++ @Test+ @Order(12)+ void federatedCatalogReportsItsConnectionType() {+ given()+ .header("X-Amz-Target", "AmazonAthena.CreateDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("""+ {+ "Name": "%s",+ "Type": "FEDERATED",+ "Parameters": {+ "connection-type": "MYSQL",+ "connection-arn": "arn:aws:glue:us-east-1:000000000000:connection/mysql"+ }+ }+ """.formatted(FEDERATED_CATALOG))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DataCatalog.Type", equalTo("FEDERATED"))+ .body("DataCatalog.ConnectionType", equalTo("MYSQL"))+ .body("DataCatalog.Status", equalTo("CREATE_COMPLETE"));++ given()+ .header("X-Amz-Target", "AmazonAthena.DeleteDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"" + FEDERATED_CATALOG + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(13)+ void theBuiltInCatalogIsReservedAgainstCreateUpdateDeleteAndTagging() {+ given()+ .header("X-Amz-Target", "AmazonAthena.CreateDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"AwsDataCatalog\", \"Type\": \"GLUE\" }")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));++ given()+ .header("X-Amz-Target", "AmazonAthena.UpdateDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"AwsDataCatalog\", \"Type\": \"GLUE\" }")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));++ given()+ .header("X-Amz-Target", "AmazonAthena.DeleteDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"AwsDataCatalog\" }")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));++ given()+ .header("X-Amz-Target", "AmazonAthena.TagResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"arn:aws:athena:us-east-1:000000000000:datacatalog/AwsDataCatalog\", "+ + "\"Tags\": [ { \"Key\": \"Environment\", \"Value\": \"test\" } ] }")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));+ }++ @Test+ @Order(14)+ void listTagsForResourceOnTheBuiltInCatalogReturnsAnEmptyList() {+ given()+ .header("X-Amz-Target", "AmazonAthena.ListTagsForResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"arn:aws:athena:us-east-1:000000000000:datacatalog/AwsDataCatalog\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Tags.size()", equalTo(0));+ }++ @Test+ @Order(15)+ void createDataCatalogRejectsAnUnmodelledType() {+ given()+ .header("X-Amz-Target", "AmazonAthena.CreateDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"floci-bad-type\", \"Type\": \"MONGODB\" }")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));+ }++ @Test+ @Order(16)+ void tagOperationsRejectACatalogArnWithNoBackingResource() {+ given()+ .header("X-Amz-Target", "AmazonAthena.ListTagsForResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"arn:aws:athena:us-east-1:000000000000:datacatalog/absent\" }")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ResourceNotFoundException"));++ given()+ .header("X-Amz-Target", "AmazonAthena.TagResource")+ .contentType(CONTENT_TYPE)+ .body("{ \"ResourceARN\": \"arn:aws:athena:us-east-1:000000000000:datacatalog/absent\", "+ + "\"Tags\": [ { \"Key\": \"Environment\", \"Value\": \"test\" } ] }")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ResourceNotFoundException"));+ }++ @Test+ @Order(17)+ void deleteDataCatalogReturnsTheCatalogAndRemovesIt() {+ given()+ .header("X-Amz-Target", "AmazonAthena.DeleteDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"" + CATALOG + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DataCatalog.Name", equalTo(CATALOG))+ .body("DataCatalog.Type", equalTo("HIVE"));++ given()+ .header("X-Amz-Target", "AmazonAthena.GetDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"" + CATALOG + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));++ given()+ .header("X-Amz-Target", "AmazonAthena.ListDataCatalogs")+ .contentType(CONTENT_TYPE)+ .body("{}")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DataCatalogsSummary.CatalogName", not(hasItem(CATALOG)));+ }++ @Test+ @Order(18)+ void deleteDataCatalogAnswersInvalidRequestExceptionForAnAbsentCatalog() {+ given()+ .header("X-Amz-Target", "AmazonAthena.DeleteDataCatalog")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"" + CATALOG + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/athena/AthenaListTagsForResourceIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/athena/AthenaListTagsForResourceIntegrationTest.javaindex ed5cdc85cf..9d979683ec 100644--- a/src/test/java/io/github/hectorvent/floci/services/athena/AthenaListTagsForResourceIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/athena/AthenaListTagsForResourceIntegrationTest.java@@ -126,12 +126,14 @@ class AthenaListTagsForResourceIntegrationTest {@Testvoid listTagsForResourceRejectsUnsupportedResourceType() {+ // Athena tags a workgroup or a data catalog and nothing else, so a capacity+ // reservation ARN is rejected even though it is a well-formed Athena ARN.given().header("X-Amz-Target", "AmazonAthena.ListTagsForResource").contentType(CONTENT_TYPE).body("""{- "ResourceARN": "arn:aws:athena:us-east-1:000000000000:datacatalog/AwsDataCatalog"+ "ResourceARN": "arn:aws:athena:us-east-1:000000000000:capacityreservation/reserved"}""").when()