Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix: support Glue table statistics deletion (#1324)

David Phillips5a964e516d61
diff --git a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
index 4bee571d2d..76b5f8a559 100644
--- a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
+++ b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
@@ -230,6 +230,28 @@ function_input() {
[ "$missing_column_name" = "missing" ]
[ "$error_code" = "EntityNotFoundException" ]
+ run aws_cmd glue delete-column-statistics-for-table \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME" \
+ --column-name id
+ assert_success
+
+ run aws_cmd glue delete-column-statistics-for-table \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME" \
+ --column-name id
+ assert_success
+
+ run aws_cmd glue get-column-statistics-for-table \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME" \
+ --column-names id
+ assert_success
+ deleted_statistics_count=$(json_get "$output" '.ColumnStatisticsList | length')
+ deleted_error_code=$(json_get "$output" '.Errors[0].Error.ErrorCode')
+ [ "$deleted_statistics_count" = "0" ]
+ [ "$deleted_error_code" = "EntityNotFoundException" ]
+
run aws_cmd glue create-partition \
--database-name "$DB_NAME" \
--table-name "$TABLE_NAME" \
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
index 556b03b47e..21bf83915b 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
@@ -19,12 +19,14 @@ import software.amazon.awssdk.services.glue.model.CreateTableRequest;
import software.amazon.awssdk.services.glue.model.CreateUserDefinedFunctionRequest;
import software.amazon.awssdk.services.glue.model.DatabaseInput;
import software.amazon.awssdk.services.glue.model.DeleteColumnStatisticsForPartitionRequest;
+import software.amazon.awssdk.services.glue.model.DeleteColumnStatisticsForTableRequest;
import software.amazon.awssdk.services.glue.model.DeleteDatabaseRequest;
import software.amazon.awssdk.services.glue.model.DeletePartitionRequest;
import software.amazon.awssdk.services.glue.model.DeleteTableRequest;
import software.amazon.awssdk.services.glue.model.DeleteUserDefinedFunctionRequest;
import software.amazon.awssdk.services.glue.model.EntityNotFoundException;
import software.amazon.awssdk.services.glue.model.GetColumnStatisticsForPartitionRequest;
+import software.amazon.awssdk.services.glue.model.GetColumnStatisticsForTableRequest;
import software.amazon.awssdk.services.glue.model.GetDatabaseRequest;
import software.amazon.awssdk.services.glue.model.GetDatabasesRequest;
import software.amazon.awssdk.services.glue.model.GetPartitionRequest;
@@ -43,6 +45,7 @@ import software.amazon.awssdk.services.glue.model.SerDeInfo;
import software.amazon.awssdk.services.glue.model.StorageDescriptor;
import software.amazon.awssdk.services.glue.model.TableInput;
import software.amazon.awssdk.services.glue.model.UpdateColumnStatisticsForPartitionRequest;
+import software.amazon.awssdk.services.glue.model.UpdateColumnStatisticsForTableRequest;
import software.amazon.awssdk.services.glue.model.UpdatePartitionRequest;
import software.amazon.awssdk.services.glue.model.UpdateTableRequest;
import software.amazon.awssdk.services.glue.model.UpdateUserDefinedFunctionRequest;
@@ -162,6 +165,44 @@ class GlueCatalogTest {
.build()).tableList())
.extracting(table -> table.name())
.contains(TABLE_NAME);
+ glue.updateColumnStatisticsForTable(UpdateColumnStatisticsForTableRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .columnStatisticsList(ColumnStatistics.builder()
+ .columnName("id")
+ .columnType("int")
+ .analyzedTime(Instant.EPOCH)
+ .statisticsData(ColumnStatisticsData.builder()
+ .type("LONG")
+ .longColumnStatisticsData(LongColumnStatisticsData.builder()
+ .minimumValue(1L)
+ .maximumValue(10L)
+ .numberOfNulls(0L)
+ .numberOfDistinctValues(10L)
+ .build())
+ .build())
+ .build())
+ .build());
+ glue.deleteColumnStatisticsForTable(DeleteColumnStatisticsForTableRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .columnName("id")
+ .build());
+ glue.deleteColumnStatisticsForTable(DeleteColumnStatisticsForTableRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .columnName("id")
+ .build());
+ var deletedStatistics = glue.getColumnStatisticsForTable(GetColumnStatisticsForTableRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .columnNames("id")
+ .build());
+ assertThat(deletedStatistics.columnStatisticsList())
+ .isEmpty();
+ assertThat(deletedStatistics.errors())
+ .singleElement()
+ .satisfies(error -> assertThat(error.error().errorCode()).isEqualTo("EntityNotFoundException"));
glue.updateTable(UpdateTableRequest.builder()
.databaseName(DATABASE_NAME)
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
index 41fb8a2faa..3ebb17a068 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
@@ -116,6 +116,7 @@ public class GlueJsonHandler {
}
case "UpdateColumnStatisticsForTable" -> handleUpdateColumnStatisticsForTable(request);
case "GetColumnStatisticsForTable" -> handleGetColumnStatisticsForTable(request);
+ case "DeleteColumnStatisticsForTable" -> handleDeleteColumnStatisticsForTable(request);
case "CreatePartition" -> {
String dbName = request.get("DatabaseName").asText();
String tableName = request.get("TableName").asText();
@@ -191,6 +192,14 @@ public class GlueJsonHandler {
.build();
}
+ private Response handleDeleteColumnStatisticsForTable(JsonNode request) {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ String columnName = request.get("ColumnName").asText();
+ glueService.deleteColumnStatisticsForTable(dbName, tableName, columnName);
+ return Response.ok().build();
+ }
+
private Response handleBatchCreatePartition(JsonNode request) {
String dbName = request.get("DatabaseName").asText();
String tableName = request.get("TableName").asText();
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index 657a6f1d68..7c6dbb0010 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -308,6 +308,11 @@ public class GlueService {
return new ColumnStatisticsResult(columnStatistics, errors);
}
+ public void deleteColumnStatisticsForTable(String databaseName, String tableName, String columnName) {
+ getTable(databaseName, tableName);
+ columnStatisticsStore.delete(columnStatisticsKey(databaseName, tableName, columnName));
+ }
+
public void createPartition(String databaseName, String tableName, Partition partition) {
Table table = getTable(databaseName, tableName);
String key = partitionKey(databaseName, tableName, partition.getValues());
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
index b886d1fbfa..f0210c1c3c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -804,6 +804,23 @@ class GlueServiceTest {
assertEquals("LONG", ((Map<?, ?>) fetched.columnStatisticsList().get(0).get("StatisticsData")).get("Type"));
}
+ @Test
+ void columnStatisticsCanBeDeleted() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+ glueService.updateColumnStatisticsForTable("db1", "plain", List.of(columnStatistics("id")));
+
+ glueService.deleteColumnStatisticsForTable("db1", "plain", "id");
+ glueService.deleteColumnStatisticsForTable("db1", "plain", "id");
+
+ GlueService.ColumnStatisticsResult fetched =
+ glueService.getColumnStatisticsForTable("db1", "plain", List.of("id"));
+ assertTrue(fetched.columnStatisticsList().isEmpty());
+ assertEquals(1, fetched.errors().size());
+ assertEquals("EntityNotFoundException", fetched.errors().getFirst().error().errorCode());
+ }
+
@Test
void getColumnStatisticsReportsMissingColumns() {
Table table = new Table();