Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(glue): read back omitted numeric string and binary column statistics as 0 (#5747)

galsuchetzky-actb2d98d6d08c8
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index 49d47ed6ee..9c326399a1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -84,6 +84,11 @@ public class GlueService {
static final String COLUMN_TYPE = "ColumnType";
static final String ANALYZED_TIME = "AnalyzedTime";
static final String STATISTICS_DATA = "StatisticsData";
+ private static final Map<String, List<String>> NUMERIC_STATISTICS_FIELDS = Map.of(
+ "StringColumnStatisticsData",
+ List.of("MaximumLength", "AverageLength", "NumberOfNulls", "NumberOfDistinctValues"),
+ "BinaryColumnStatisticsData",
+ List.of("MaximumLength", "AverageLength", "NumberOfNulls"));
private static final Pattern COMPARISON_EXPRESSION = Pattern.compile(
"\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*(=|<>|<=|>=|<|>)\\s*('?[^']*'?|[^\\s]+)\\s*",
Pattern.CASE_INSENSITIVE);
@@ -794,7 +799,7 @@ public class GlueService {
Optional<Map<String, Object>> statistics =
columnStatisticsStore.get(columnStatisticsKey(table.getDatabaseName(), table.getName(), columnName));
if (statistics.isPresent()) {
- columnStatistics.add(new LinkedHashMap<>(statistics.get()));
+ columnStatistics.add(withNumericDefaults(statistics.get()));
}
else {
errors.add(new ColumnError(
@@ -1122,6 +1127,7 @@ public class GlueService {
List<ColumnError> errors = new ArrayList<>();
for (String columnName : columnNames) {
partitionColumnStatisticsStore.get(partitionColumnStatisticsKey(databaseName, tableName, partitionValues, columnName))
+ .map(GlueService::withNumericDefaults)
.ifPresentOrElse(columnStatistics::add, () -> errors.add(columnStatisticsNotFoundError(columnName)));
}
return new ColumnStatisticsResult(columnStatistics, errors);
@@ -1303,6 +1309,34 @@ public class GlueService {
return stringValue;
}
+ /**
+ * Glue reads back the numeric string and binary statistics a writer left out as 0, and clients such as Trino
+ * unbox them without a null check, so every statistic read back carries all of them, whatever the stored
+ * record holds.
+ */
+ private static Map<String, Object> withNumericDefaults(Map<String, Object> statistics) {
+ Map<String, Object> stored = new LinkedHashMap<>(statistics);
+ if (!(stored.get(STATISTICS_DATA) instanceof Map<?, ?> data)) {
+ return stored;
+ }
+ Map<String, Object> storedData = copyOf(data);
+ NUMERIC_STATISTICS_FIELDS.forEach((typeKey, fields) -> {
+ if (storedData.get(typeKey) instanceof Map<?, ?> typed) {
+ Map<String, Object> filled = copyOf(typed);
+ fields.forEach(field -> filled.putIfAbsent(field, 0));
+ storedData.put(typeKey, filled);
+ }
+ });
+ stored.put(STATISTICS_DATA, storedData);
+ return stored;
+ }
+
+ private static Map<String, Object> copyOf(Map<?, ?> source) {
+ Map<String, Object> copy = new LinkedHashMap<>();
+ source.forEach((key, value) -> copy.put(String.valueOf(key), value));
+ return copy;
+ }
+
private static Object requireColumnStatisticsField(Map<String, Object> statistics, String field) {
Object value = statistics.get(field);
if (value == null) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
index dd18099ac2..4a1596db72 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -609,6 +609,62 @@ class GlueServiceTest {
assertEquals("Cannot find partition.", ex.getMessage());
}
+ @Test
+ void stringStatisticsWithoutAverageLengthReadBackWithZero() {
+ Table table = new Table();
+ table.setName("nulls");
+ glueService.createTable("db1", table);
+ Map<String, Object> statistics = Map.of(
+ "ColumnName", "a",
+ "ColumnType", "string",
+ "AnalyzedTime", Instant.EPOCH,
+ "StatisticsData", Map.of(
+ "Type", "STRING",
+ "StringColumnStatisticsData", Map.of("NumberOfNulls", 1, "NumberOfDistinctValues", 0)));
+
+ glueService.updateColumnStatisticsForTable("db1", "nulls", List.of(statistics));
+ Map<String, Object> stored = glueService.getColumnStatisticsForTable("db1", "nulls", List.of("a"))
+ .columnStatisticsList().get(0);
+
+ @SuppressWarnings("unchecked")
+ Map<String, Object> data = (Map<String, Object>) ((Map<String, Object>) stored.get("StatisticsData"))
+ .get("StringColumnStatisticsData");
+ assertEquals(0, data.get("AverageLength"));
+ assertEquals(0, data.get("MaximumLength"));
+ assertEquals(1, data.get("NumberOfNulls"));
+ Map<String, Object> raw = columnStatisticsStore.scan(key -> true).get(0);
+ assertFalse(((Map<?, ?>) ((Map<?, ?>) raw.get("StatisticsData")).get("StringColumnStatisticsData"))
+ .containsKey("AverageLength"));
+ }
+
+ @Test
+ void partitionBinaryStatisticsWithoutLengthsReadBackWithZero() {
+ Table table = new Table();
+ table.setName("blobs");
+ glueService.createTable("db1", table);
+ Partition partition = new Partition();
+ partition.setValues(List.of("1"));
+ glueService.createPartition("db1", "blobs", partition);
+ Map<String, Object> statistics = Map.of(
+ "ColumnName", "payload",
+ "ColumnType", "binary",
+ "AnalyzedTime", Instant.EPOCH,
+ "StatisticsData", Map.of(
+ "Type", "BINARY",
+ "BinaryColumnStatisticsData", Map.of("NumberOfNulls", 2)));
+
+ glueService.updateColumnStatisticsForPartition("db1", "blobs", List.of("1"), List.of(statistics));
+ Map<String, Object> stored = glueService.getColumnStatisticsForPartition(
+ "db1", "blobs", List.of("1"), List.of("payload")).columnStatisticsList().get(0);
+
+ @SuppressWarnings("unchecked")
+ Map<String, Object> data = (Map<String, Object>) ((Map<String, Object>) stored.get("StatisticsData"))
+ .get("BinaryColumnStatisticsData");
+ assertEquals(0, data.get("MaximumLength"));
+ assertEquals(0, data.get("AverageLength"));
+ assertEquals(2, data.get("NumberOfNulls"));
+ }
+
@Test
void partitionColumnStatisticsCanBeUpdatedFetchedAndDeleted() {
Table table = new Table();