Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix: support Glue column statistics (#1309)

David Phillipse8e44e70bce3
diff --git a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
index f415a92228..462023382f 100644
--- a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
+++ b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
@@ -71,6 +71,25 @@ create_table() {
--table-input "$(table_input "$name" "$description")" >/dev/null
}
+column_statistics_input() {
+ jq -n '[
+ {
+ ColumnName: "id",
+ ColumnType: "int",
+ AnalyzedTime: "2026-06-08T00:00:00Z",
+ StatisticsData: {
+ Type: "LONG",
+ LongColumnStatisticsData: {
+ MinimumValue: 1,
+ MaximumValue: 10,
+ NumberOfNulls: 0,
+ NumberOfDistinctValues: 10
+ }
+ }
+ }
+ ]'
+}
+
function_input() {
local owner="$1"
jq -n \
@@ -183,6 +202,28 @@ function_input() {
[ "$archived_version" = "0" ]
[ "$archived_description" = "created" ]
+ run aws_cmd glue update-column-statistics-for-table \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME" \
+ --column-statistics-list "$(column_statistics_input)"
+ assert_success
+
+ run aws_cmd glue get-column-statistics-for-table \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME" \
+ --column-names id missing
+ assert_success
+ column_name=$(json_get "$output" '.ColumnStatisticsList[0].ColumnName')
+ statistics_type=$(json_get "$output" '.ColumnStatisticsList[0].StatisticsData.Type')
+ minimum_value=$(json_get "$output" '.ColumnStatisticsList[0].StatisticsData.LongColumnStatisticsData.MinimumValue')
+ missing_column_name=$(json_get "$output" '.Errors[0].ColumnName')
+ error_code=$(json_get "$output" '.Errors[0].Error.ErrorCode')
+ [ "$column_name" = "id" ]
+ [ "$statistics_type" = "LONG" ]
+ [ "$minimum_value" = "1" ]
+ [ "$missing_column_name" = "missing" ]
+ [ "$error_code" = "EntityNotFoundException" ]
+
run aws_cmd glue create-partition \
--database-name "$DB_NAME" \
--table-name "$TABLE_NAME" \
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
index 5839f554bb..5b6850db0f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
@@ -29,6 +29,7 @@ import java.util.Objects;
public class GlueJsonHandler {
private static final TypeReference<List<String>> STRING_LIST = new TypeReference<>() {};
+ private static final TypeReference<List<Map<String, Object>>> MAP_LIST = new TypeReference<>() {};
private final GlueService glueService;
private final GlueSchemaRegistryService schemaRegistryService;
@@ -112,6 +113,8 @@ public class GlueJsonHandler {
List<String> tableNames = mapper.convertValue(request.get("TablesToDelete"), STRING_LIST);
yield Response.ok(Map.of("Errors", glueService.batchDeleteTables(dbName, tableNames))).build();
}
+ case "UpdateColumnStatisticsForTable" -> handleUpdateColumnStatisticsForTable(request);
+ case "GetColumnStatisticsForTable" -> handleGetColumnStatisticsForTable(request);
case "CreatePartition" -> {
String dbName = request.get("DatabaseName").asText();
String tableName = request.get("TableName").asText();
@@ -156,6 +159,27 @@ public class GlueJsonHandler {
};
}
+ private Response handleUpdateColumnStatisticsForTable(JsonNode request) {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ List<Map<String, Object>> columnStatistics =
+ mapper.convertValue(request.get("ColumnStatisticsList"), MAP_LIST);
+ glueService.updateColumnStatisticsForTable(dbName, tableName, columnStatistics);
+ return Response.ok(Map.of("Errors", List.of())).build();
+ }
+
+ private Response handleGetColumnStatisticsForTable(JsonNode request) {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ List<String> columnNames = mapper.convertValue(request.get("ColumnNames"), STRING_LIST);
+ GlueService.ColumnStatisticsResult result =
+ glueService.getColumnStatisticsForTable(dbName, tableName, columnNames);
+ return Response.ok(Map.of(
+ "ColumnStatisticsList", result.columnStatisticsList(),
+ "Errors", result.errors()))
+ .build();
+ }
+
private Response handleCreateUserDefinedFunction(JsonNode request) throws Exception {
String dbName = request.get("DatabaseName").asText();
UserDefinedFunction function = mapper.treeToValue(request.get("FunctionInput"), UserDefinedFunction.class);
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index be927c90f3..bc6491e4fc 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -39,10 +39,15 @@ public class GlueService {
private static final Logger LOG = Logger.getLogger(GlueService.class);
private static final int MAX_FUNCTION_PATTERN_LENGTH = 255;
private static final int MAX_FUNCTION_RESULTS = 100;
+ static final String COLUMN_NAME = "ColumnName";
+ static final String COLUMN_TYPE = "ColumnType";
+ static final String ANALYZED_TIME = "AnalyzedTime";
+ static final String STATISTICS_DATA = "StatisticsData";
private final StorageBackend<String, Database> databaseStore;
private final StorageBackend<String, Table> tableStore;
private final StorageBackend<String, Table> tableVersionStore;
+ private final StorageBackend<String, Map<String, Object>> columnStatisticsStore;
private final StorageBackend<String, Partition> partitionStore;
private final StorageBackend<String, UserDefinedFunction> functionStore;
private final GlueSchemaRegistryService schemaRegistryService;
@@ -57,6 +62,7 @@ public class GlueService {
this.databaseStore = storageFactory.create("glue", "databases.json", new TypeReference<>() {});
this.tableStore = storageFactory.create("glue", "tables.json", new TypeReference<>() {});
this.tableVersionStore = storageFactory.create("glue", "table_versions.json", new TypeReference<>() {});
+ this.columnStatisticsStore = storageFactory.create("glue", "column_statistics.json", new TypeReference<>() {});
this.partitionStore = storageFactory.create("glue", "partitions.json", new TypeReference<>() {});
this.functionStore = storageFactory.create("glue", "functions.json", new TypeReference<>() {});
this.schemaRegistryService = schemaRegistryService;
@@ -67,6 +73,7 @@ public class GlueService {
GlueService(StorageBackend<String, Database> databaseStore,
StorageBackend<String, Table> tableStore,
StorageBackend<String, Table> tableVersionStore,
+ StorageBackend<String, Map<String, Object>> columnStatisticsStore,
StorageBackend<String, Partition> partitionStore,
StorageBackend<String, UserDefinedFunction> functionStore,
GlueSchemaRegistryService schemaRegistryService,
@@ -75,6 +82,7 @@ public class GlueService {
this.databaseStore = databaseStore;
this.tableStore = tableStore;
this.tableVersionStore = tableVersionStore;
+ this.columnStatisticsStore = columnStatisticsStore;
this.partitionStore = partitionStore;
this.functionStore = functionStore;
this.schemaRegistryService = schemaRegistryService;
@@ -218,6 +226,9 @@ public class GlueService {
tableVersionStore.keys().stream()
.filter(versionKey -> versionKey.startsWith(key + ":"))
.forEach(tableVersionStore::delete);
+ columnStatisticsStore.keys().stream()
+ .filter(statisticsKey -> statisticsKey.startsWith(key + ":"))
+ .forEach(columnStatisticsStore::delete);
partitionStore.scan(k -> k.startsWith(key + ":")).forEach(p -> {
partitionStore.delete(key + ":" + String.join(",", p.getValues()));
});
@@ -241,6 +252,44 @@ public class GlueService {
return errors;
}
+ public void updateColumnStatisticsForTable(
+ String databaseName,
+ String tableName,
+ List<Map<String, Object>> columnStatistics) {
+ Table table = getTable(databaseName, tableName);
+ for (Map<String, Object> statistics : columnStatistics) {
+ String columnNameString = requireColumnStatisticsString(statistics, COLUMN_NAME);
+ requireColumnStatisticsString(statistics, COLUMN_TYPE);
+ requireColumnStatisticsField(statistics, ANALYZED_TIME);
+ requireColumnStatisticsField(statistics, STATISTICS_DATA);
+ columnStatisticsStore.put(
+ columnStatisticsKey(table.getDatabaseName(), table.getName(), columnNameString),
+ new LinkedHashMap<>(statistics));
+ }
+ }
+
+ public ColumnStatisticsResult getColumnStatisticsForTable(
+ String databaseName,
+ String tableName,
+ List<String> columnNames) {
+ Table table = getTable(databaseName, tableName);
+ List<Map<String, Object>> columnStatistics = new ArrayList<>();
+ List<ColumnError> errors = new ArrayList<>();
+ for (String columnName : columnNames) {
+ Optional<Map<String, Object>> statistics =
+ columnStatisticsStore.get(columnStatisticsKey(table.getDatabaseName(), table.getName(), columnName));
+ if (statistics.isPresent()) {
+ columnStatistics.add(new LinkedHashMap<>(statistics.get()));
+ }
+ else {
+ errors.add(new ColumnError(
+ columnName,
+ new ErrorDetail("EntityNotFoundException", "Statistics do not exist for this column")));
+ }
+ }
+ return new ColumnStatisticsResult(columnStatistics, errors);
+ }
+
public void createPartition(String databaseName, String tableName, Partition partition) {
Table table = getTable(databaseName, tableName);
String key = tableKey(databaseName, tableName) + ":" + String.join(",", partition.getValues());
@@ -376,6 +425,26 @@ public class GlueService {
return tableKey(databaseName, tableName) + ":" + versionId;
}
+ private static String columnStatisticsKey(String databaseName, String tableName, String columnName) {
+ return tableKey(databaseName, tableName) + ":" + normalizeName(columnName);
+ }
+
+ private static String requireColumnStatisticsString(Map<String, Object> statistics, String field) {
+ Object value = requireColumnStatisticsField(statistics, field);
+ if (!(value instanceof String stringValue) || stringValue.isBlank()) {
+ throw new AwsException("InvalidInputException", field + " is required", 400);
+ }
+ return stringValue;
+ }
+
+ private static Object requireColumnStatisticsField(Map<String, Object> statistics, String field) {
+ Object value = statistics.get(field);
+ if (value == null) {
+ throw new AwsException("InvalidInputException", field + " is required", 400);
+ }
+ return value;
+ }
+
private static String normalizeName(String name) {
return name.toLowerCase(Locale.ROOT);
}
@@ -425,6 +494,14 @@ public class GlueService {
@JsonProperty("ErrorCode") String errorCode,
@JsonProperty("ErrorMessage") String errorMessage) {}
+ public record ColumnStatisticsResult(
+ @JsonProperty("ColumnStatisticsList") List<Map<String, Object>> columnStatisticsList,
+ @JsonProperty("Errors") List<ColumnError> errors) {}
+
+ public record ColumnError(
+ @JsonProperty("ColumnName") String columnName,
+ @JsonProperty("Error") ErrorDetail error) {}
+
private static Table copyTable(Table source) {
Table copy = new Table();
copy.setName(source.getName());
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
index da0e37f62e..7cdd698f3f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -21,6 +21,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.time.Instant;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -48,6 +49,7 @@ class GlueServiceTest {
private GlueSchemaRegistryService schemaRegistryService;
private StorageBackend<String, Table> tableStore;
private StorageBackend<String, Table> tableVersionStore;
+ private StorageBackend<String, Map<String, Object>> columnStatisticsStore;
private StorageBackend<String, Partition> partitionStore;
@BeforeEach
@@ -57,11 +59,13 @@ class GlueServiceTest {
schemaRegistryService = new GlueSchemaRegistryService(storageFactory, regionResolver);
tableStore = new InMemoryStorage<>();
tableVersionStore = new InMemoryStorage<>();
+ columnStatisticsStore = new InMemoryStorage<>();
partitionStore = new InMemoryStorage<>();
glueService = new GlueService(
new InMemoryStorage<String, Database>(),
tableStore,
tableVersionStore,
+ columnStatisticsStore,
partitionStore,
new InMemoryStorage<String, UserDefinedFunction>(),
schemaRegistryService, regionResolver, new ResourceGroupsTaggingService());
@@ -575,6 +579,73 @@ class GlueServiceTest {
assertEquals("1", tableVersions.get(0).get("VersionId"));
}
+ @Test
+ void columnStatisticsCanBeUpdatedAndRetrieved() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+ Map<String, Object> statistics = columnStatistics("id");
+
+ glueService.updateColumnStatisticsForTable("db1", "plain", List.of(statistics));
+
+ GlueService.ColumnStatisticsResult fetched =
+ glueService.getColumnStatisticsForTable("db1", "plain", List.of("id"));
+ assertEquals(1, fetched.columnStatisticsList().size());
+ assertTrue(fetched.errors().isEmpty());
+ assertEquals("id", fetched.columnStatisticsList().get(0).get(GlueService.COLUMN_NAME));
+ assertEquals("LONG", ((Map<?, ?>) fetched.columnStatisticsList().get(0).get("StatisticsData")).get("Type"));
+ }
+
+ @Test
+ void getColumnStatisticsReportsMissingColumns() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+
+ GlueService.ColumnStatisticsResult fetched =
+ glueService.getColumnStatisticsForTable("db1", "plain", List.of("missing"));
+
+ assertTrue(fetched.columnStatisticsList().isEmpty());
+ assertEquals(1, fetched.errors().size());
+ assertEquals("missing", fetched.errors().getFirst().columnName());
+ assertEquals("EntityNotFoundException", fetched.errors().getFirst().error().errorCode());
+ assertEquals("Statistics do not exist for this column", fetched.errors().getFirst().error().errorMessage());
+ }
+
+ @Test
+ void updateColumnStatisticsRejectsMissingRequiredFields() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+
+ for (String field : List.of(
+ GlueService.COLUMN_NAME,
+ GlueService.COLUMN_TYPE,
+ GlueService.ANALYZED_TIME,
+ GlueService.STATISTICS_DATA)) {
+ Map<String, Object> statistics = new LinkedHashMap<>(columnStatistics("id"));
+ statistics.remove(field);
+
+ AwsException exception = assertThrows(AwsException.class,
+ () -> glueService.updateColumnStatisticsForTable("db1", "plain", List.of(statistics)));
+
+ assertEquals("InvalidInputException", exception.getErrorCode());
+ assertEquals(field + " is required", exception.getMessage());
+ }
+ }
+
+ @Test
+ void deleteTableDeletesColumnStatistics() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+ glueService.updateColumnStatisticsForTable("db1", "plain", List.of(columnStatistics("id")));
+
+ glueService.deleteTable("db1", "plain");
+
+ assertTrue(columnStatisticsStore.scan(k -> true).isEmpty());
+ }
+
@Test
void userDefinedFunctionsCanBeCreatedListedUpdatedAndDeleted() {
UserDefinedFunction function = new UserDefinedFunction();
@@ -687,6 +758,21 @@ class GlueServiceTest {
return table;
}
+ private static Map<String, Object> columnStatistics(String columnName) {
+ Map<String, Object> statistics = new LinkedHashMap<>();
+ statistics.put(GlueService.COLUMN_NAME, columnName);
+ statistics.put(GlueService.COLUMN_TYPE, "int");
+ statistics.put(GlueService.ANALYZED_TIME, Instant.parse("2026-06-08T00:00:00Z"));
+ statistics.put(GlueService.STATISTICS_DATA, Map.of(
+ "Type", "LONG",
+ "LongColumnStatisticsData", Map.of(
+ "MinimumValue", 1,
+ "MaximumValue", 10,
+ "NumberOfNulls", 0,
+ "NumberOfDistinctValues", 10)));
+ return statistics;
+ }
+
private static final class InMemoryStorageFactory extends StorageFactory {
private InMemoryStorageFactory() {
super(null, null);