Light, fluffy, and always free - The AWS Local Emulator alternative
fix: support Glue table statistics deletion (#1324)
David Phillips5a964e516d61
diff --git a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats b/compatibility-tests/sdk-test-awscli/test/glue-catalog.batsindex 4bee571d2d..76b5f8a559 100644--- a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats+++ b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats@@ -230,6 +230,28 @@ function_input() {[ "$missing_column_name" = "missing" ][ "$error_code" = "EntityNotFoundException" ]+ run aws_cmd glue delete-column-statistics-for-table \+ --database-name "$DB_NAME" \+ --table-name "$TABLE_NAME" \+ --column-name id+ assert_success++ run aws_cmd glue delete-column-statistics-for-table \+ --database-name "$DB_NAME" \+ --table-name "$TABLE_NAME" \+ --column-name id+ assert_success++ run aws_cmd glue get-column-statistics-for-table \+ --database-name "$DB_NAME" \+ --table-name "$TABLE_NAME" \+ --column-names id+ assert_success+ deleted_statistics_count=$(json_get "$output" '.ColumnStatisticsList | length')+ deleted_error_code=$(json_get "$output" '.Errors[0].Error.ErrorCode')+ [ "$deleted_statistics_count" = "0" ]+ [ "$deleted_error_code" = "EntityNotFoundException" ]+run aws_cmd glue create-partition \--database-name "$DB_NAME" \--table-name "$TABLE_NAME" \diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.javaindex 556b03b47e..21bf83915b 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java@@ -19,12 +19,14 @@ import software.amazon.awssdk.services.glue.model.CreateTableRequest;import software.amazon.awssdk.services.glue.model.CreateUserDefinedFunctionRequest;import software.amazon.awssdk.services.glue.model.DatabaseInput;import software.amazon.awssdk.services.glue.model.DeleteColumnStatisticsForPartitionRequest;+import software.amazon.awssdk.services.glue.model.DeleteColumnStatisticsForTableRequest;import software.amazon.awssdk.services.glue.model.DeleteDatabaseRequest;import software.amazon.awssdk.services.glue.model.DeletePartitionRequest;import software.amazon.awssdk.services.glue.model.DeleteTableRequest;import software.amazon.awssdk.services.glue.model.DeleteUserDefinedFunctionRequest;import software.amazon.awssdk.services.glue.model.EntityNotFoundException;import software.amazon.awssdk.services.glue.model.GetColumnStatisticsForPartitionRequest;+import software.amazon.awssdk.services.glue.model.GetColumnStatisticsForTableRequest;import software.amazon.awssdk.services.glue.model.GetDatabaseRequest;import software.amazon.awssdk.services.glue.model.GetDatabasesRequest;import software.amazon.awssdk.services.glue.model.GetPartitionRequest;@@ -43,6 +45,7 @@ import software.amazon.awssdk.services.glue.model.SerDeInfo;import software.amazon.awssdk.services.glue.model.StorageDescriptor;import software.amazon.awssdk.services.glue.model.TableInput;import software.amazon.awssdk.services.glue.model.UpdateColumnStatisticsForPartitionRequest;+import software.amazon.awssdk.services.glue.model.UpdateColumnStatisticsForTableRequest;import software.amazon.awssdk.services.glue.model.UpdatePartitionRequest;import software.amazon.awssdk.services.glue.model.UpdateTableRequest;import software.amazon.awssdk.services.glue.model.UpdateUserDefinedFunctionRequest;@@ -162,6 +165,44 @@ class GlueCatalogTest {.build()).tableList()).extracting(table -> table.name()).contains(TABLE_NAME);+ glue.updateColumnStatisticsForTable(UpdateColumnStatisticsForTableRequest.builder()+ .databaseName(DATABASE_NAME)+ .tableName(TABLE_NAME)+ .columnStatisticsList(ColumnStatistics.builder()+ .columnName("id")+ .columnType("int")+ .analyzedTime(Instant.EPOCH)+ .statisticsData(ColumnStatisticsData.builder()+ .type("LONG")+ .longColumnStatisticsData(LongColumnStatisticsData.builder()+ .minimumValue(1L)+ .maximumValue(10L)+ .numberOfNulls(0L)+ .numberOfDistinctValues(10L)+ .build())+ .build())+ .build())+ .build());+ glue.deleteColumnStatisticsForTable(DeleteColumnStatisticsForTableRequest.builder()+ .databaseName(DATABASE_NAME)+ .tableName(TABLE_NAME)+ .columnName("id")+ .build());+ glue.deleteColumnStatisticsForTable(DeleteColumnStatisticsForTableRequest.builder()+ .databaseName(DATABASE_NAME)+ .tableName(TABLE_NAME)+ .columnName("id")+ .build());+ var deletedStatistics = glue.getColumnStatisticsForTable(GetColumnStatisticsForTableRequest.builder()+ .databaseName(DATABASE_NAME)+ .tableName(TABLE_NAME)+ .columnNames("id")+ .build());+ assertThat(deletedStatistics.columnStatisticsList())+ .isEmpty();+ assertThat(deletedStatistics.errors())+ .singleElement()+ .satisfies(error -> assertThat(error.error().errorCode()).isEqualTo("EntityNotFoundException"));glue.updateTable(UpdateTableRequest.builder().databaseName(DATABASE_NAME)diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.javaindex 41fb8a2faa..3ebb17a068 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java@@ -116,6 +116,7 @@ public class GlueJsonHandler {}case "UpdateColumnStatisticsForTable" -> handleUpdateColumnStatisticsForTable(request);case "GetColumnStatisticsForTable" -> handleGetColumnStatisticsForTable(request);+ case "DeleteColumnStatisticsForTable" -> handleDeleteColumnStatisticsForTable(request);case "CreatePartition" -> {String dbName = request.get("DatabaseName").asText();String tableName = request.get("TableName").asText();@@ -191,6 +192,14 @@ public class GlueJsonHandler {.build();}+ private Response handleDeleteColumnStatisticsForTable(JsonNode request) {+ String dbName = request.get("DatabaseName").asText();+ String tableName = request.get("TableName").asText();+ String columnName = request.get("ColumnName").asText();+ glueService.deleteColumnStatisticsForTable(dbName, tableName, columnName);+ return Response.ok().build();+ }+private Response handleBatchCreatePartition(JsonNode request) {String dbName = request.get("DatabaseName").asText();String tableName = request.get("TableName").asText();diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.javaindex 657a6f1d68..7c6dbb0010 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java@@ -308,6 +308,11 @@ public class GlueService {return new ColumnStatisticsResult(columnStatistics, errors);}+ public void deleteColumnStatisticsForTable(String databaseName, String tableName, String columnName) {+ getTable(databaseName, tableName);+ columnStatisticsStore.delete(columnStatisticsKey(databaseName, tableName, columnName));+ }+public void createPartition(String databaseName, String tableName, Partition partition) {Table table = getTable(databaseName, tableName);String key = partitionKey(databaseName, tableName, partition.getValues());diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.javaindex b886d1fbfa..f0210c1c3c 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java@@ -804,6 +804,23 @@ class GlueServiceTest {assertEquals("LONG", ((Map<?, ?>) fetched.columnStatisticsList().get(0).get("StatisticsData")).get("Type"));}+ @Test+ void columnStatisticsCanBeDeleted() {+ Table table = new Table();+ table.setName("plain");+ glueService.createTable("db1", table);+ glueService.updateColumnStatisticsForTable("db1", "plain", List.of(columnStatistics("id")));++ glueService.deleteColumnStatisticsForTable("db1", "plain", "id");+ glueService.deleteColumnStatisticsForTable("db1", "plain", "id");++ GlueService.ColumnStatisticsResult fetched =+ glueService.getColumnStatisticsForTable("db1", "plain", List.of("id"));+ assertTrue(fetched.columnStatisticsList().isEmpty());+ assertEquals(1, fetched.errors().size());+ assertEquals("EntityNotFoundException", fetched.errors().getFirst().error().errorCode());+ }+@Testvoid getColumnStatisticsReportsMissingColumns() {Table table = new Table();