Light, fluffy, and always free - The AWS Local Emulator alternative
fix(glue): default missing required column statistics counts (#6366)
Aniket Shuklafb08a5ca55b6
diff --git a/docs/services/glue.md b/docs/services/glue.mdindex 5b877f7c18..f9392dec20 100644--- a/docs/services/glue.md+++ b/docs/services/glue.md@@ -5,6 +5,10 @@Floci emulates the AWS Glue Data Catalog (databases, tables, partitions, functions, connections) and Glue Schema Registry, allowing you to manage local data lake metadata and schema-version workflows.+Table and partition column-statistics reads fill omitted required numeric members with `0` for+string, binary, long, double, decimal, date, and boolean statistics. Supplied values and stored+records are preserved; optional minimum and maximum values remain absent when omitted.+## Supported Actions### Data Catalogdiff --git a/pom.xml b/pom.xmlindex ccca191e64..31c62f94aa 100644--- a/pom.xml+++ b/pom.xml@@ -336,6 +336,12 @@<version>${aws.sdk.version}</version><scope>test</scope></dependency>+ <dependency>+ <groupId>software.amazon.awssdk</groupId>+ <artifactId>glue</artifactId>+ <version>${aws.sdk.version}</version>+ <scope>test</scope>+ </dependency><dependency><groupId>software.amazon.awssdk</groupId><artifactId>bedrock</artifactId>diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.javaindex 01170d969a..ee91cdb5b6 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java@@ -88,7 +88,12 @@ public class GlueService {"StringColumnStatisticsData",List.of("MaximumLength", "AverageLength", "NumberOfNulls", "NumberOfDistinctValues"),"BinaryColumnStatisticsData",- List.of("MaximumLength", "AverageLength", "NumberOfNulls"));+ List.of("MaximumLength", "AverageLength", "NumberOfNulls"),+ "LongColumnStatisticsData", List.of("NumberOfNulls", "NumberOfDistinctValues"),+ "DoubleColumnStatisticsData", List.of("NumberOfNulls", "NumberOfDistinctValues"),+ "DecimalColumnStatisticsData", List.of("NumberOfNulls", "NumberOfDistinctValues"),+ "DateColumnStatisticsData", List.of("NumberOfNulls", "NumberOfDistinctValues"),+ "BooleanColumnStatisticsData", List.of("NumberOfTrues", "NumberOfFalses", "NumberOfNulls"));private static final Pattern COMPARISON_EXPRESSION = Pattern.compile("\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*(=|<>|<=|>=|<|>)\\s*('?[^']*'?|[^\\s]+)\\s*",Pattern.CASE_INSENSITIVE);@@ -1310,7 +1315,7 @@ public class GlueService {}/**- * Glue reads back the numeric string and binary statistics a writer left out as 0, and clients such as Trino+ * Glue reads back the required numeric statistics a writer left out as 0, and clients such as Trino* unbox them without a null check, so every statistic read back carries all of them, whatever the stored* record holds.*/diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueStatisticsCountDefaultsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueStatisticsCountDefaultsIntegrationTest.javanew file mode 100644index 0000000000..be8baf3c27--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueStatisticsCountDefaultsIntegrationTest.java@@ -0,0 +1,120 @@+package io.github.hectorvent.floci.services.glue;++import io.quarkus.test.junit.QuarkusTest;+import io.restassured.RestAssured;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;+import software.amazon.awssdk.regions.Region;+import software.amazon.awssdk.services.glue.GlueClient;+import software.amazon.awssdk.services.glue.model.BooleanColumnStatisticsData;+import software.amazon.awssdk.services.glue.model.Column;+import software.amazon.awssdk.services.glue.model.ColumnStatistics;+import software.amazon.awssdk.services.glue.model.ColumnStatisticsData;+import software.amazon.awssdk.services.glue.model.DatabaseInput;+import software.amazon.awssdk.services.glue.model.DateColumnStatisticsData;+import software.amazon.awssdk.services.glue.model.DecimalColumnStatisticsData;+import software.amazon.awssdk.services.glue.model.DoubleColumnStatisticsData;+import software.amazon.awssdk.services.glue.model.LongColumnStatisticsData;+import software.amazon.awssdk.services.glue.model.PartitionInput;+import software.amazon.awssdk.services.glue.model.StorageDescriptor;+import software.amazon.awssdk.services.glue.model.TableInput;++import java.net.URI;+import java.time.Instant;+import java.util.Arrays;+import java.util.List;+import java.util.UUID;++import static org.junit.jupiter.api.Assertions.assertEquals;++@QuarkusTest+class GlueStatisticsCountDefaultsIntegrationTest {++ @ParameterizedTest+ @CsvSource({"LONG,false,false", "LONG,true,false", "DOUBLE,false,false", "DOUBLE,true,false",+ "DECIMAL,false,false", "DECIMAL,true,false", "DATE,false,false", "DATE,true,false",+ "BOOLEAN,false,false", "BOOLEAN,true,false", "LONG,false,true", "LONG,true,true",+ "DOUBLE,false,true", "DOUBLE,true,true", "DECIMAL,false,true", "DECIMAL,true,true",+ "DATE,false,true", "DATE,true,true", "BOOLEAN,false,true", "BOOLEAN,true,true"})+ void sdkReadsMissingCountsAsZeroAndPreservesSuppliedCounts(String type, boolean partition, boolean supplied) {+ String database = "counts_" + UUID.randomUUID().toString().replace("-", "");+ String columnType = switch (type) {+ case "LONG" -> "bigint";+ case "DOUBLE" -> "double";+ case "DECIMAL" -> "decimal(10,2)";+ case "DATE" -> "date";+ default -> "boolean";+ };+ try (GlueClient client = GlueClient.builder()+ .endpointOverride(URI.create("http://localhost:" + RestAssured.port))+ .httpClientBuilder(UrlConnectionHttpClient.builder()).region(Region.US_EAST_1)+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("741579900001", "test")))+ .build()) {+ client.createDatabase(request -> request.databaseInput(DatabaseInput.builder().name(database).build()));+ try {+ StorageDescriptor descriptor = StorageDescriptor.builder()+ .columns(Column.builder().name("value").type(columnType).build()).build();+ client.createTable(request -> request.databaseName(database).tableInput(TableInput.builder().name("stats")+ .storageDescriptor(descriptor).partitionKeys(Column.builder().name("day").type("string").build())+ .build()));+ if (partition) {+ client.createPartition(request -> request.databaseName(database).tableName("stats")+ .partitionInput(PartitionInput.builder().values("one").storageDescriptor(descriptor).build()));+ }+ Long nulls = supplied ? 7L : null;+ ColumnStatisticsData.Builder data = ColumnStatisticsData.builder().type(type);+ switch (type) {+ case "LONG" -> data.longColumnStatisticsData(LongColumnStatisticsData.builder().numberOfNulls(nulls).build());+ case "DOUBLE" -> data.doubleColumnStatisticsData(DoubleColumnStatisticsData.builder().numberOfNulls(nulls).build());+ case "DECIMAL" -> data.decimalColumnStatisticsData(DecimalColumnStatisticsData.builder().numberOfNulls(nulls).build());+ case "DATE" -> data.dateColumnStatisticsData(DateColumnStatisticsData.builder().numberOfNulls(nulls).build());+ default -> data.booleanColumnStatisticsData(BooleanColumnStatisticsData.builder().numberOfNulls(nulls).build());+ }+ ColumnStatistics statistics = ColumnStatistics.builder().columnName("value").columnType(columnType)+ .analyzedTime(Instant.now()).statisticsData(data.build()).build();+ if (partition) {+ client.updateColumnStatisticsForPartition(request -> request.databaseName(database).tableName("stats")+ .partitionValues("one").columnStatisticsList(statistics));+ } else {+ client.updateColumnStatisticsForTable(request -> request.databaseName(database).tableName("stats")+ .columnStatisticsList(statistics));+ }+ List<ColumnStatistics> values = partition+ ? client.getColumnStatisticsForPartition(request -> request.databaseName(database).tableName("stats")+ .partitionValues("one").columnNames("value")).columnStatisticsList()+ : client.getColumnStatisticsForTable(request -> request.databaseName(database).tableName("stats")+ .columnNames("value")).columnStatisticsList();+ assertEquals(1, values.size());+ ColumnStatisticsData result = values.get(0).statisticsData();+ List<Long> counts = switch (type) {+ case "LONG" -> Arrays.asList(result.longColumnStatisticsData().numberOfNulls(),+ result.longColumnStatisticsData().numberOfDistinctValues());+ case "DOUBLE" -> Arrays.asList(result.doubleColumnStatisticsData().numberOfNulls(),+ result.doubleColumnStatisticsData().numberOfDistinctValues());+ case "DECIMAL" -> Arrays.asList(result.decimalColumnStatisticsData().numberOfNulls(),+ result.decimalColumnStatisticsData().numberOfDistinctValues());+ case "DATE" -> Arrays.asList(result.dateColumnStatisticsData().numberOfNulls(),+ result.dateColumnStatisticsData().numberOfDistinctValues());+ default -> Arrays.asList(result.booleanColumnStatisticsData().numberOfNulls(),+ result.booleanColumnStatisticsData().numberOfTrues(), result.booleanColumnStatisticsData().numberOfFalses());+ };+ long expectedNulls = supplied ? 7L : 0L;+ assertEquals(type.equals("BOOLEAN") ? List.of(expectedNulls, 0L, 0L) : List.of(expectedNulls, 0L), counts);+ if (!type.equals("BOOLEAN")) {+ List<?> bounds = switch (type) {+ case "LONG" -> Arrays.asList(result.longColumnStatisticsData().minimumValue(), result.longColumnStatisticsData().maximumValue());+ case "DOUBLE" -> Arrays.asList(result.doubleColumnStatisticsData().minimumValue(), result.doubleColumnStatisticsData().maximumValue());+ case "DECIMAL" -> Arrays.asList(result.decimalColumnStatisticsData().minimumValue(), result.decimalColumnStatisticsData().maximumValue());+ default -> Arrays.asList(result.dateColumnStatisticsData().minimumValue(), result.dateColumnStatisticsData().maximumValue());+ };+ assertEquals(Arrays.asList(null, null), bounds);+ }+ } finally {+ client.deleteDatabase(request -> request.name(database));+ }+ }+ }+}