Light, fluffy, and always free - The AWS Local Emulator alternative
fix: support Glue BatchDeleteTable (#1209)
David Phillips1ea22d0f60cb
diff --git a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats b/compatibility-tests/sdk-test-awscli/test/glue-catalog.batsindex ed823bf218..9cf076d28b 100644--- a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats+++ b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats@@ -4,6 +4,7 @@ setup() {load 'test_helper/common-setup'DB_NAME="$(unique_name glue-catalog-db)"TABLE_NAME="catalog_table"+ SECOND_TABLE_NAME="catalog_table_second"FUNCTION_NAME="catalog_function"}@@ -11,6 +12,9 @@ teardown() {aws_cmd glue delete-user-defined-function \--database-name "$DB_NAME" \--function-name "$FUNCTION_NAME" >/dev/null 2>&1 || true+ aws_cmd glue delete-table \+ --database-name "$DB_NAME" \+ --name "$SECOND_TABLE_NAME" >/dev/null 2>&1 || trueaws_cmd glue delete-table \--database-name "$DB_NAME" \--name "$TABLE_NAME" >/dev/null 2>&1 || true@@ -24,11 +28,12 @@ create_database() {}table_input() {- local description="$1"+ local name="${1:-$TABLE_NAME}"+ local description="$2"jq -n \- --arg name "$TABLE_NAME" \+ --arg name "$name" \--arg description "$description" \- --arg location "s3://floci-glue-catalog/$DB_NAME/$TABLE_NAME/" \+ --arg location "s3://floci-glue-catalog/$DB_NAME/$name/" \'{Name: $name,Description: $description,@@ -56,9 +61,11 @@ table_input() {}create_table() {+ local name="${1:-$TABLE_NAME}"+ local description="${2:-created}"aws_cmd glue create-table \--database-name "$DB_NAME" \- --table-input "$(table_input created)" >/dev/null+ --table-input "$(table_input "$name" "$description")" >/dev/null}function_input() {@@ -125,7 +132,7 @@ function_input() {run aws_cmd glue update-table \--database-name "$DB_NAME" \--version-id "$version_id" \- --table-input "$(table_input updated)"+ --table-input "$(table_input "$TABLE_NAME" updated)"assert_successrun aws_cmd glue get-table \@@ -161,6 +168,34 @@ function_input() {assert_failure}+@test "Glue catalog: batch delete table" {+ run create_database+ assert_success++ run create_table "$TABLE_NAME" "batch delete first table"+ assert_success++ run create_table "$SECOND_TABLE_NAME" "batch delete second table"+ assert_success++ run aws_cmd glue batch-delete-table \+ --database-name "$DB_NAME" \+ --tables-to-delete "$TABLE_NAME" "$SECOND_TABLE_NAME"+ assert_success+ error_count=$(echo "$output" | jq '.Errors | length')+ [ "$error_count" = "0" ]++ run aws_cmd glue get-table \+ --database-name "$DB_NAME" \+ --name "$TABLE_NAME"+ assert_failure++ run aws_cmd glue get-table \+ --database-name "$DB_NAME" \+ --name "$SECOND_TABLE_NAME"+ assert_failure+}+@test "Glue catalog: user-defined function lifecycle" {run create_databaseassert_successdiff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.javaindex ec138841f2..9dc52e7b7a 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java@@ -5,6 +5,7 @@ import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.DisplayName;import org.junit.jupiter.api.Test;import software.amazon.awssdk.services.glue.GlueClient;+import software.amazon.awssdk.services.glue.model.BatchDeleteTableRequest;import software.amazon.awssdk.services.glue.model.Column;import software.amazon.awssdk.services.glue.model.CreateDatabaseRequest;import software.amazon.awssdk.services.glue.model.CreatePartitionRequest;@@ -43,7 +44,9 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;class GlueCatalogTest {private static final String DATABASE_NAME = TestFixtures.uniqueName("catalog_db");+ private static final String BATCH_DELETE_DATABASE_NAME = TestFixtures.uniqueName("catalog_batch_delete_db");private static final String TABLE_NAME = "catalog_table";+ private static final String SECOND_TABLE_NAME = "catalog_table_second";private static final String FUNCTION_NAME = "catalog_function";private static GlueClient glue;@@ -65,6 +68,13 @@ class GlueCatalogTest {.build());}catch (Exception ignored) {}+ try {+ glue.deleteTable(DeleteTableRequest.builder()+ .databaseName(BATCH_DELETE_DATABASE_NAME)+ .name(SECOND_TABLE_NAME)+ .build());+ }+ catch (Exception ignored) {}try {glue.deleteTable(DeleteTableRequest.builder().databaseName(DATABASE_NAME)@@ -72,6 +82,12 @@ class GlueCatalogTest {.build());}catch (Exception ignored) {}+ try {+ glue.deleteDatabase(DeleteDatabaseRequest.builder()+ .name(BATCH_DELETE_DATABASE_NAME)+ .build());+ }+ catch (Exception ignored) {}try {glue.deleteDatabase(DeleteDatabaseRequest.builder().name(DATABASE_NAME)@@ -204,13 +220,61 @@ class GlueCatalogTest {.isInstanceOf(EntityNotFoundException.class);}+ @Test+ void batchDeleteTable() {+ glue.createDatabase(CreateDatabaseRequest.builder()+ .databaseInput(DatabaseInput.builder()+ .name(BATCH_DELETE_DATABASE_NAME)+ .description("catalog batch delete database")+ .build())+ .build());++ glue.createTable(CreateTableRequest.builder()+ .databaseName(BATCH_DELETE_DATABASE_NAME)+ .tableInput(tableInput(BATCH_DELETE_DATABASE_NAME, TABLE_NAME, "batch delete first table"))+ .build());+ glue.createTable(CreateTableRequest.builder()+ .databaseName(BATCH_DELETE_DATABASE_NAME)+ .tableInput(tableInput(BATCH_DELETE_DATABASE_NAME, SECOND_TABLE_NAME, "batch delete second table"))+ .build());++ var response = glue.batchDeleteTable(BatchDeleteTableRequest.builder()+ .databaseName(BATCH_DELETE_DATABASE_NAME)+ .tablesToDelete(TABLE_NAME, SECOND_TABLE_NAME)+ .build());+ assertThat(response.errors()).isEmpty();++ assertThatThrownBy(() -> glue.getTable(GetTableRequest.builder()+ .databaseName(BATCH_DELETE_DATABASE_NAME)+ .name(TABLE_NAME)+ .build()))+ .isInstanceOf(EntityNotFoundException.class);+ assertThatThrownBy(() -> glue.getTable(GetTableRequest.builder()+ .databaseName(BATCH_DELETE_DATABASE_NAME)+ .name(SECOND_TABLE_NAME)+ .build()))+ .isInstanceOf(EntityNotFoundException.class);++ glue.deleteDatabase(DeleteDatabaseRequest.builder()+ .name(BATCH_DELETE_DATABASE_NAME)+ .build());+ }+private static TableInput tableInput(String description) {+ return tableInput(TABLE_NAME, description);+ }++ private static TableInput tableInput(String tableName, String description) {+ return tableInput(DATABASE_NAME, tableName, description);+ }++ private static TableInput tableInput(String databaseName, String tableName, String description) {return TableInput.builder()- .name(TABLE_NAME)+ .name(tableName).description(description).parameters(Map.of("classification", "json")).storageDescriptor(StorageDescriptor.builder()- .location("s3://floci-glue-catalog/" + DATABASE_NAME + "/" + TABLE_NAME + "/")+ .location("s3://floci-glue-catalog/" + databaseName + "/" + tableName + "/").inputFormat("org.apache.hadoop.mapred.TextInputFormat").outputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat").serdeInfo(SerDeInfo.builder()diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.javaindex 42992a7f12..e027f78350 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.glue;+import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsException;@@ -27,6 +28,8 @@ import java.util.Objects;@ApplicationScopedpublic class GlueJsonHandler {+ private static final TypeReference<List<String>> STRING_LIST = new TypeReference<>() {};+private final GlueService glueService;private final GlueSchemaRegistryService schemaRegistryService;private final ObjectMapper mapper;@@ -91,6 +94,11 @@ public class GlueJsonHandler {glueService.deleteTable(dbName, tableName);yield Response.ok().build();}+ case "BatchDeleteTable" -> {+ String dbName = request.get("DatabaseName").asText();+ List<String> tableNames = mapper.convertValue(request.get("TablesToDelete"), STRING_LIST);+ yield Response.ok(Map.of("Errors", glueService.batchDeleteTables(dbName, tableNames))).build();+ }case "CreatePartition" -> {String dbName = request.get("DatabaseName").asText();String tableName = request.get("TableName").asText();diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.javaindex dff410a1cf..0ddee2b631 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.glue;+import com.fasterxml.jackson.annotation.JsonProperty;import com.fasterxml.jackson.core.type.TypeReference;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;@@ -27,6 +28,7 @@ import java.util.LinkedHashMap;import java.util.List;import java.util.Locale;import java.util.Map;+import java.util.Optional;import java.util.regex.Pattern;import java.util.regex.PatternSyntaxException;@@ -170,6 +172,23 @@ public class GlueService {LOG.infov("Deleted Glue Table: {0}.{1}", databaseName, tableName);}+ public List<BatchDeleteTableError> batchDeleteTables(String databaseName, List<String> tableNames) {+ getDatabase(databaseName);+ List<BatchDeleteTableError> errors = new ArrayList<>();+ for (String tableName : tableNames) {+ String key = tableKey(databaseName, tableName);+ Optional<Table> table = tableStore.get(key);+ if (table.isEmpty()) {+ errors.add(new BatchDeleteTableError(+ tableName,+ new ErrorDetail("EntityNotFoundException", "Table " + tableName + " not found")));+ continue;+ }+ deleteTable(databaseName, tableName);+ }+ return errors;+ }+public void createPartition(String databaseName, String tableName, Partition partition) {Table table = getTable(databaseName, tableName);String key = tableKey(databaseName, tableName) + ":" + String.join(",", partition.getValues());@@ -338,6 +357,14 @@ public class GlueService {public record UserDefinedFunctionPage(List<UserDefinedFunction> functions, String nextToken) {}+ public record BatchDeleteTableError(+ @JsonProperty("TableName") String tableName,+ @JsonProperty("ErrorDetail") ErrorDetail errorDetail) {}++ public record ErrorDetail(+ @JsonProperty("ErrorCode") String errorCode,+ @JsonProperty("ErrorMessage") String errorMessage) {}+private static Table copyTable(Table source) {Table copy = new Table();copy.setName(source.getName());diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.javaindex 9468408df5..be81ebf991 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java@@ -482,6 +482,25 @@ class GlueServiceTest {() -> glueService.getTable("mixedcasedatabase", "mixedcasetabledeletedwithdatabase"));}+ @Test+ void batchDeleteTablesDeletesExistingTablesAndReportsMissingTables() {+ Table table = new Table();+ table.setName("existing");+ glueService.createTable("db1", table);++ List<GlueService.BatchDeleteTableError> errors =+ glueService.batchDeleteTables("db1", List.of("existing", "missing"));++ assertEquals(1, errors.size());+ assertEquals("missing", errors.get(0).tableName());+ GlueService.ErrorDetail errorDetail = errors.get(0).errorDetail();+ assertEquals("EntityNotFoundException", errorDetail.errorCode());+ assertEquals("Table missing not found", errorDetail.errorMessage());+ AwsException ex = assertThrows(AwsException.class,+ () -> glueService.getTable("db1", "existing"));+ assertEquals("EntityNotFoundException", ex.getErrorCode());+ }+@Testvoid getTableVersionsReturnsEmptyListForAthenaCompatibility() {assertTrue(glueService.getTableVersions().isEmpty());