Light, fluffy, and always free - The AWS Local Emulator alternative
fix(glue): read back omitted numeric string and binary column statistics as 0 (#5747)
galsuchetzky-actb2d98d6d08c8
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.javaindex 49d47ed6ee..9c326399a1 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java@@ -84,6 +84,11 @@ public class GlueService {static final String COLUMN_TYPE = "ColumnType";static final String ANALYZED_TIME = "AnalyzedTime";static final String STATISTICS_DATA = "StatisticsData";+ private static final Map<String, List<String>> NUMERIC_STATISTICS_FIELDS = Map.of(+ "StringColumnStatisticsData",+ List.of("MaximumLength", "AverageLength", "NumberOfNulls", "NumberOfDistinctValues"),+ "BinaryColumnStatisticsData",+ List.of("MaximumLength", "AverageLength", "NumberOfNulls"));private static final Pattern COMPARISON_EXPRESSION = Pattern.compile("\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*(=|<>|<=|>=|<|>)\\s*('?[^']*'?|[^\\s]+)\\s*",Pattern.CASE_INSENSITIVE);@@ -794,7 +799,7 @@ public class GlueService {Optional<Map<String, Object>> statistics =columnStatisticsStore.get(columnStatisticsKey(table.getDatabaseName(), table.getName(), columnName));if (statistics.isPresent()) {- columnStatistics.add(new LinkedHashMap<>(statistics.get()));+ columnStatistics.add(withNumericDefaults(statistics.get()));}else {errors.add(new ColumnError(@@ -1122,6 +1127,7 @@ public class GlueService {List<ColumnError> errors = new ArrayList<>();for (String columnName : columnNames) {partitionColumnStatisticsStore.get(partitionColumnStatisticsKey(databaseName, tableName, partitionValues, columnName))+ .map(GlueService::withNumericDefaults).ifPresentOrElse(columnStatistics::add, () -> errors.add(columnStatisticsNotFoundError(columnName)));}return new ColumnStatisticsResult(columnStatistics, errors);@@ -1303,6 +1309,34 @@ public class GlueService {return stringValue;}+ /**+ * Glue reads back the numeric string and binary statistics a writer left out as 0, and clients such as Trino+ * unbox them without a null check, so every statistic read back carries all of them, whatever the stored+ * record holds.+ */+ private static Map<String, Object> withNumericDefaults(Map<String, Object> statistics) {+ Map<String, Object> stored = new LinkedHashMap<>(statistics);+ if (!(stored.get(STATISTICS_DATA) instanceof Map<?, ?> data)) {+ return stored;+ }+ Map<String, Object> storedData = copyOf(data);+ NUMERIC_STATISTICS_FIELDS.forEach((typeKey, fields) -> {+ if (storedData.get(typeKey) instanceof Map<?, ?> typed) {+ Map<String, Object> filled = copyOf(typed);+ fields.forEach(field -> filled.putIfAbsent(field, 0));+ storedData.put(typeKey, filled);+ }+ });+ stored.put(STATISTICS_DATA, storedData);+ return stored;+ }++ private static Map<String, Object> copyOf(Map<?, ?> source) {+ Map<String, Object> copy = new LinkedHashMap<>();+ source.forEach((key, value) -> copy.put(String.valueOf(key), value));+ return copy;+ }+private static Object requireColumnStatisticsField(Map<String, Object> statistics, String field) {Object value = statistics.get(field);if (value == null) {diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.javaindex dd18099ac2..4a1596db72 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java@@ -609,6 +609,62 @@ class GlueServiceTest {assertEquals("Cannot find partition.", ex.getMessage());}+ @Test+ void stringStatisticsWithoutAverageLengthReadBackWithZero() {+ Table table = new Table();+ table.setName("nulls");+ glueService.createTable("db1", table);+ Map<String, Object> statistics = Map.of(+ "ColumnName", "a",+ "ColumnType", "string",+ "AnalyzedTime", Instant.EPOCH,+ "StatisticsData", Map.of(+ "Type", "STRING",+ "StringColumnStatisticsData", Map.of("NumberOfNulls", 1, "NumberOfDistinctValues", 0)));++ glueService.updateColumnStatisticsForTable("db1", "nulls", List.of(statistics));+ Map<String, Object> stored = glueService.getColumnStatisticsForTable("db1", "nulls", List.of("a"))+ .columnStatisticsList().get(0);++ @SuppressWarnings("unchecked")+ Map<String, Object> data = (Map<String, Object>) ((Map<String, Object>) stored.get("StatisticsData"))+ .get("StringColumnStatisticsData");+ assertEquals(0, data.get("AverageLength"));+ assertEquals(0, data.get("MaximumLength"));+ assertEquals(1, data.get("NumberOfNulls"));+ Map<String, Object> raw = columnStatisticsStore.scan(key -> true).get(0);+ assertFalse(((Map<?, ?>) ((Map<?, ?>) raw.get("StatisticsData")).get("StringColumnStatisticsData"))+ .containsKey("AverageLength"));+ }++ @Test+ void partitionBinaryStatisticsWithoutLengthsReadBackWithZero() {+ Table table = new Table();+ table.setName("blobs");+ glueService.createTable("db1", table);+ Partition partition = new Partition();+ partition.setValues(List.of("1"));+ glueService.createPartition("db1", "blobs", partition);+ Map<String, Object> statistics = Map.of(+ "ColumnName", "payload",+ "ColumnType", "binary",+ "AnalyzedTime", Instant.EPOCH,+ "StatisticsData", Map.of(+ "Type", "BINARY",+ "BinaryColumnStatisticsData", Map.of("NumberOfNulls", 2)));++ glueService.updateColumnStatisticsForPartition("db1", "blobs", List.of("1"), List.of(statistics));+ Map<String, Object> stored = glueService.getColumnStatisticsForPartition(+ "db1", "blobs", List.of("1"), List.of("payload")).columnStatisticsList().get(0);++ @SuppressWarnings("unchecked")+ Map<String, Object> data = (Map<String, Object>) ((Map<String, Object>) stored.get("StatisticsData"))+ .get("BinaryColumnStatisticsData");+ assertEquals(0, data.get("MaximumLength"));+ assertEquals(0, data.get("AverageLength"));+ assertEquals(2, data.get("NumberOfNulls"));+ }+@Testvoid partitionColumnStatisticsCanBeUpdatedFetchedAndDeleted() {Table table = new Table();