Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix: support Glue BatchDeleteTable (#1209)

David Phillips1ea22d0f60cb
diff --git a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
index ed823bf218..9cf076d28b 100644
--- a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
+++ b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
@@ -4,6 +4,7 @@ setup() {
load 'test_helper/common-setup'
DB_NAME="$(unique_name glue-catalog-db)"
TABLE_NAME="catalog_table"
+ SECOND_TABLE_NAME="catalog_table_second"
FUNCTION_NAME="catalog_function"
}
@@ -11,6 +12,9 @@ teardown() {
aws_cmd glue delete-user-defined-function \
--database-name "$DB_NAME" \
--function-name "$FUNCTION_NAME" >/dev/null 2>&1 || true
+ aws_cmd glue delete-table \
+ --database-name "$DB_NAME" \
+ --name "$SECOND_TABLE_NAME" >/dev/null 2>&1 || true
aws_cmd glue delete-table \
--database-name "$DB_NAME" \
--name "$TABLE_NAME" >/dev/null 2>&1 || true
@@ -24,11 +28,12 @@ create_database() {
}
table_input() {
- local description="$1"
+ local name="${1:-$TABLE_NAME}"
+ local description="$2"
jq -n \
- --arg name "$TABLE_NAME" \
+ --arg name "$name" \
--arg description "$description" \
- --arg location "s3://floci-glue-catalog/$DB_NAME/$TABLE_NAME/" \
+ --arg location "s3://floci-glue-catalog/$DB_NAME/$name/" \
'{
Name: $name,
Description: $description,
@@ -56,9 +61,11 @@ table_input() {
}
create_table() {
+ local name="${1:-$TABLE_NAME}"
+ local description="${2:-created}"
aws_cmd glue create-table \
--database-name "$DB_NAME" \
- --table-input "$(table_input created)" >/dev/null
+ --table-input "$(table_input "$name" "$description")" >/dev/null
}
function_input() {
@@ -125,7 +132,7 @@ function_input() {
run aws_cmd glue update-table \
--database-name "$DB_NAME" \
--version-id "$version_id" \
- --table-input "$(table_input updated)"
+ --table-input "$(table_input "$TABLE_NAME" updated)"
assert_success
run aws_cmd glue get-table \
@@ -161,6 +168,34 @@ function_input() {
assert_failure
}
+@test "Glue catalog: batch delete table" {
+ run create_database
+ assert_success
+
+ run create_table "$TABLE_NAME" "batch delete first table"
+ assert_success
+
+ run create_table "$SECOND_TABLE_NAME" "batch delete second table"
+ assert_success
+
+ run aws_cmd glue batch-delete-table \
+ --database-name "$DB_NAME" \
+ --tables-to-delete "$TABLE_NAME" "$SECOND_TABLE_NAME"
+ assert_success
+ error_count=$(echo "$output" | jq '.Errors | length')
+ [ "$error_count" = "0" ]
+
+ run aws_cmd glue get-table \
+ --database-name "$DB_NAME" \
+ --name "$TABLE_NAME"
+ assert_failure
+
+ run aws_cmd glue get-table \
+ --database-name "$DB_NAME" \
+ --name "$SECOND_TABLE_NAME"
+ assert_failure
+}
+
@test "Glue catalog: user-defined function lifecycle" {
run create_database
assert_success
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
index ec138841f2..9dc52e7b7a 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
@@ -5,6 +5,7 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.services.glue.GlueClient;
+import software.amazon.awssdk.services.glue.model.BatchDeleteTableRequest;
import software.amazon.awssdk.services.glue.model.Column;
import software.amazon.awssdk.services.glue.model.CreateDatabaseRequest;
import software.amazon.awssdk.services.glue.model.CreatePartitionRequest;
@@ -43,7 +44,9 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
class GlueCatalogTest {
private static final String DATABASE_NAME = TestFixtures.uniqueName("catalog_db");
+ private static final String BATCH_DELETE_DATABASE_NAME = TestFixtures.uniqueName("catalog_batch_delete_db");
private static final String TABLE_NAME = "catalog_table";
+ private static final String SECOND_TABLE_NAME = "catalog_table_second";
private static final String FUNCTION_NAME = "catalog_function";
private static GlueClient glue;
@@ -65,6 +68,13 @@ class GlueCatalogTest {
.build());
}
catch (Exception ignored) {}
+ try {
+ glue.deleteTable(DeleteTableRequest.builder()
+ .databaseName(BATCH_DELETE_DATABASE_NAME)
+ .name(SECOND_TABLE_NAME)
+ .build());
+ }
+ catch (Exception ignored) {}
try {
glue.deleteTable(DeleteTableRequest.builder()
.databaseName(DATABASE_NAME)
@@ -72,6 +82,12 @@ class GlueCatalogTest {
.build());
}
catch (Exception ignored) {}
+ try {
+ glue.deleteDatabase(DeleteDatabaseRequest.builder()
+ .name(BATCH_DELETE_DATABASE_NAME)
+ .build());
+ }
+ catch (Exception ignored) {}
try {
glue.deleteDatabase(DeleteDatabaseRequest.builder()
.name(DATABASE_NAME)
@@ -204,13 +220,61 @@ class GlueCatalogTest {
.isInstanceOf(EntityNotFoundException.class);
}
+ @Test
+ void batchDeleteTable() {
+ glue.createDatabase(CreateDatabaseRequest.builder()
+ .databaseInput(DatabaseInput.builder()
+ .name(BATCH_DELETE_DATABASE_NAME)
+ .description("catalog batch delete database")
+ .build())
+ .build());
+
+ glue.createTable(CreateTableRequest.builder()
+ .databaseName(BATCH_DELETE_DATABASE_NAME)
+ .tableInput(tableInput(BATCH_DELETE_DATABASE_NAME, TABLE_NAME, "batch delete first table"))
+ .build());
+ glue.createTable(CreateTableRequest.builder()
+ .databaseName(BATCH_DELETE_DATABASE_NAME)
+ .tableInput(tableInput(BATCH_DELETE_DATABASE_NAME, SECOND_TABLE_NAME, "batch delete second table"))
+ .build());
+
+ var response = glue.batchDeleteTable(BatchDeleteTableRequest.builder()
+ .databaseName(BATCH_DELETE_DATABASE_NAME)
+ .tablesToDelete(TABLE_NAME, SECOND_TABLE_NAME)
+ .build());
+ assertThat(response.errors()).isEmpty();
+
+ assertThatThrownBy(() -> glue.getTable(GetTableRequest.builder()
+ .databaseName(BATCH_DELETE_DATABASE_NAME)
+ .name(TABLE_NAME)
+ .build()))
+ .isInstanceOf(EntityNotFoundException.class);
+ assertThatThrownBy(() -> glue.getTable(GetTableRequest.builder()
+ .databaseName(BATCH_DELETE_DATABASE_NAME)
+ .name(SECOND_TABLE_NAME)
+ .build()))
+ .isInstanceOf(EntityNotFoundException.class);
+
+ glue.deleteDatabase(DeleteDatabaseRequest.builder()
+ .name(BATCH_DELETE_DATABASE_NAME)
+ .build());
+ }
+
private static TableInput tableInput(String description) {
+ return tableInput(TABLE_NAME, description);
+ }
+
+ private static TableInput tableInput(String tableName, String description) {
+ return tableInput(DATABASE_NAME, tableName, description);
+ }
+
+ private static TableInput tableInput(String databaseName, String tableName, String description) {
return TableInput.builder()
- .name(TABLE_NAME)
+ .name(tableName)
.description(description)
.parameters(Map.of("classification", "json"))
.storageDescriptor(StorageDescriptor.builder()
- .location("s3://floci-glue-catalog/" + DATABASE_NAME + "/" + TABLE_NAME + "/")
+ .location("s3://floci-glue-catalog/" + databaseName + "/" + tableName + "/")
.inputFormat("org.apache.hadoop.mapred.TextInputFormat")
.outputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat")
.serdeInfo(SerDeInfo.builder()
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
index 42992a7f12..e027f78350 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.glue;
+import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.core.common.AwsException;
@@ -27,6 +28,8 @@ import java.util.Objects;
@ApplicationScoped
public class GlueJsonHandler {
+ private static final TypeReference<List<String>> STRING_LIST = new TypeReference<>() {};
+
private final GlueService glueService;
private final GlueSchemaRegistryService schemaRegistryService;
private final ObjectMapper mapper;
@@ -91,6 +94,11 @@ public class GlueJsonHandler {
glueService.deleteTable(dbName, tableName);
yield Response.ok().build();
}
+ case "BatchDeleteTable" -> {
+ String dbName = request.get("DatabaseName").asText();
+ List<String> tableNames = mapper.convertValue(request.get("TablesToDelete"), STRING_LIST);
+ yield Response.ok(Map.of("Errors", glueService.batchDeleteTables(dbName, tableNames))).build();
+ }
case "CreatePartition" -> {
String dbName = request.get("DatabaseName").asText();
String tableName = request.get("TableName").asText();
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index dff410a1cf..0ddee2b631 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.glue;
+import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.type.TypeReference;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
@@ -27,6 +28,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
@@ -170,6 +172,23 @@ public class GlueService {
LOG.infov("Deleted Glue Table: {0}.{1}", databaseName, tableName);
}
+ public List<BatchDeleteTableError> batchDeleteTables(String databaseName, List<String> tableNames) {
+ getDatabase(databaseName);
+ List<BatchDeleteTableError> errors = new ArrayList<>();
+ for (String tableName : tableNames) {
+ String key = tableKey(databaseName, tableName);
+ Optional<Table> table = tableStore.get(key);
+ if (table.isEmpty()) {
+ errors.add(new BatchDeleteTableError(
+ tableName,
+ new ErrorDetail("EntityNotFoundException", "Table " + tableName + " not found")));
+ continue;
+ }
+ deleteTable(databaseName, tableName);
+ }
+ return errors;
+ }
+
public void createPartition(String databaseName, String tableName, Partition partition) {
Table table = getTable(databaseName, tableName);
String key = tableKey(databaseName, tableName) + ":" + String.join(",", partition.getValues());
@@ -338,6 +357,14 @@ public class GlueService {
public record UserDefinedFunctionPage(List<UserDefinedFunction> functions, String nextToken) {}
+ public record BatchDeleteTableError(
+ @JsonProperty("TableName") String tableName,
+ @JsonProperty("ErrorDetail") ErrorDetail errorDetail) {}
+
+ public record ErrorDetail(
+ @JsonProperty("ErrorCode") String errorCode,
+ @JsonProperty("ErrorMessage") String errorMessage) {}
+
private static Table copyTable(Table source) {
Table copy = new Table();
copy.setName(source.getName());
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
index 9468408df5..be81ebf991 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -482,6 +482,25 @@ class GlueServiceTest {
() -> glueService.getTable("mixedcasedatabase", "mixedcasetabledeletedwithdatabase"));
}
+ @Test
+ void batchDeleteTablesDeletesExistingTablesAndReportsMissingTables() {
+ Table table = new Table();
+ table.setName("existing");
+ glueService.createTable("db1", table);
+
+ List<GlueService.BatchDeleteTableError> errors =
+ glueService.batchDeleteTables("db1", List.of("existing", "missing"));
+
+ assertEquals(1, errors.size());
+ assertEquals("missing", errors.get(0).tableName());
+ GlueService.ErrorDetail errorDetail = errors.get(0).errorDetail();
+ assertEquals("EntityNotFoundException", errorDetail.errorCode());
+ assertEquals("Table missing not found", errorDetail.errorMessage());
+ AwsException ex = assertThrows(AwsException.class,
+ () -> glueService.getTable("db1", "existing"));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
@Test
void getTableVersionsReturnsEmptyListForAthenaCompatibility() {
assertTrue(glueService.getTableVersions().isEmpty());