Light, fluffy, and always free - The AWS Local Emulator alternative
fix: support Glue column statistics (#1309)
David Phillipse8e44e70bce3
diff --git a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats b/compatibility-tests/sdk-test-awscli/test/glue-catalog.batsindex f415a92228..462023382f 100644--- a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats+++ b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats@@ -71,6 +71,25 @@ create_table() {--table-input "$(table_input "$name" "$description")" >/dev/null}+column_statistics_input() {+ jq -n '[+ {+ ColumnName: "id",+ ColumnType: "int",+ AnalyzedTime: "2026-06-08T00:00:00Z",+ StatisticsData: {+ Type: "LONG",+ LongColumnStatisticsData: {+ MinimumValue: 1,+ MaximumValue: 10,+ NumberOfNulls: 0,+ NumberOfDistinctValues: 10+ }+ }+ }+ ]'+}+function_input() {local owner="$1"jq -n \@@ -183,6 +202,28 @@ function_input() {[ "$archived_version" = "0" ][ "$archived_description" = "created" ]+ run aws_cmd glue update-column-statistics-for-table \+ --database-name "$DB_NAME" \+ --table-name "$TABLE_NAME" \+ --column-statistics-list "$(column_statistics_input)"+ assert_success++ run aws_cmd glue get-column-statistics-for-table \+ --database-name "$DB_NAME" \+ --table-name "$TABLE_NAME" \+ --column-names id missing+ assert_success+ column_name=$(json_get "$output" '.ColumnStatisticsList[0].ColumnName')+ statistics_type=$(json_get "$output" '.ColumnStatisticsList[0].StatisticsData.Type')+ minimum_value=$(json_get "$output" '.ColumnStatisticsList[0].StatisticsData.LongColumnStatisticsData.MinimumValue')+ missing_column_name=$(json_get "$output" '.Errors[0].ColumnName')+ error_code=$(json_get "$output" '.Errors[0].Error.ErrorCode')+ [ "$column_name" = "id" ]+ [ "$statistics_type" = "LONG" ]+ [ "$minimum_value" = "1" ]+ [ "$missing_column_name" = "missing" ]+ [ "$error_code" = "EntityNotFoundException" ]+run aws_cmd glue create-partition \--database-name "$DB_NAME" \--table-name "$TABLE_NAME" \diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.javaindex 5839f554bb..5b6850db0f 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java@@ -29,6 +29,7 @@ import java.util.Objects;public class GlueJsonHandler {private static final TypeReference<List<String>> STRING_LIST = new TypeReference<>() {};+ private static final TypeReference<List<Map<String, Object>>> MAP_LIST = new TypeReference<>() {};private final GlueService glueService;private final GlueSchemaRegistryService schemaRegistryService;@@ -112,6 +113,8 @@ public class GlueJsonHandler {List<String> tableNames = mapper.convertValue(request.get("TablesToDelete"), STRING_LIST);yield Response.ok(Map.of("Errors", glueService.batchDeleteTables(dbName, tableNames))).build();}+ case "UpdateColumnStatisticsForTable" -> handleUpdateColumnStatisticsForTable(request);+ case "GetColumnStatisticsForTable" -> handleGetColumnStatisticsForTable(request);case "CreatePartition" -> {String dbName = request.get("DatabaseName").asText();String tableName = request.get("TableName").asText();@@ -156,6 +159,27 @@ public class GlueJsonHandler {};}+ private Response handleUpdateColumnStatisticsForTable(JsonNode request) {+ String dbName = request.get("DatabaseName").asText();+ String tableName = request.get("TableName").asText();+ List<Map<String, Object>> columnStatistics =+ mapper.convertValue(request.get("ColumnStatisticsList"), MAP_LIST);+ glueService.updateColumnStatisticsForTable(dbName, tableName, columnStatistics);+ return Response.ok(Map.of("Errors", List.of())).build();+ }++ private Response handleGetColumnStatisticsForTable(JsonNode request) {+ String dbName = request.get("DatabaseName").asText();+ String tableName = request.get("TableName").asText();+ List<String> columnNames = mapper.convertValue(request.get("ColumnNames"), STRING_LIST);+ GlueService.ColumnStatisticsResult result =+ glueService.getColumnStatisticsForTable(dbName, tableName, columnNames);+ return Response.ok(Map.of(+ "ColumnStatisticsList", result.columnStatisticsList(),+ "Errors", result.errors()))+ .build();+ }+private Response handleCreateUserDefinedFunction(JsonNode request) throws Exception {String dbName = request.get("DatabaseName").asText();UserDefinedFunction function = mapper.treeToValue(request.get("FunctionInput"), UserDefinedFunction.class);diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.javaindex be927c90f3..bc6491e4fc 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java@@ -39,10 +39,15 @@ public class GlueService {private static final Logger LOG = Logger.getLogger(GlueService.class);private static final int MAX_FUNCTION_PATTERN_LENGTH = 255;private static final int MAX_FUNCTION_RESULTS = 100;+ static final String COLUMN_NAME = "ColumnName";+ static final String COLUMN_TYPE = "ColumnType";+ static final String ANALYZED_TIME = "AnalyzedTime";+ static final String STATISTICS_DATA = "StatisticsData";private final StorageBackend<String, Database> databaseStore;private final StorageBackend<String, Table> tableStore;private final StorageBackend<String, Table> tableVersionStore;+ private final StorageBackend<String, Map<String, Object>> columnStatisticsStore;private final StorageBackend<String, Partition> partitionStore;private final StorageBackend<String, UserDefinedFunction> functionStore;private final GlueSchemaRegistryService schemaRegistryService;@@ -57,6 +62,7 @@ public class GlueService {this.databaseStore = storageFactory.create("glue", "databases.json", new TypeReference<>() {});this.tableStore = storageFactory.create("glue", "tables.json", new TypeReference<>() {});this.tableVersionStore = storageFactory.create("glue", "table_versions.json", new TypeReference<>() {});+ this.columnStatisticsStore = storageFactory.create("glue", "column_statistics.json", new TypeReference<>() {});this.partitionStore = storageFactory.create("glue", "partitions.json", new TypeReference<>() {});this.functionStore = storageFactory.create("glue", "functions.json", new TypeReference<>() {});this.schemaRegistryService = schemaRegistryService;@@ -67,6 +73,7 @@ public class GlueService {GlueService(StorageBackend<String, Database> databaseStore,StorageBackend<String, Table> tableStore,StorageBackend<String, Table> tableVersionStore,+ StorageBackend<String, Map<String, Object>> columnStatisticsStore,StorageBackend<String, Partition> partitionStore,StorageBackend<String, UserDefinedFunction> functionStore,GlueSchemaRegistryService schemaRegistryService,@@ -75,6 +82,7 @@ public class GlueService {this.databaseStore = databaseStore;this.tableStore = tableStore;this.tableVersionStore = tableVersionStore;+ this.columnStatisticsStore = columnStatisticsStore;this.partitionStore = partitionStore;this.functionStore = functionStore;this.schemaRegistryService = schemaRegistryService;@@ -218,6 +226,9 @@ public class GlueService {tableVersionStore.keys().stream().filter(versionKey -> versionKey.startsWith(key + ":")).forEach(tableVersionStore::delete);+ columnStatisticsStore.keys().stream()+ .filter(statisticsKey -> statisticsKey.startsWith(key + ":"))+ .forEach(columnStatisticsStore::delete);partitionStore.scan(k -> k.startsWith(key + ":")).forEach(p -> {partitionStore.delete(key + ":" + String.join(",", p.getValues()));});@@ -241,6 +252,44 @@ public class GlueService {return errors;}+ public void updateColumnStatisticsForTable(+ String databaseName,+ String tableName,+ List<Map<String, Object>> columnStatistics) {+ Table table = getTable(databaseName, tableName);+ for (Map<String, Object> statistics : columnStatistics) {+ String columnNameString = requireColumnStatisticsString(statistics, COLUMN_NAME);+ requireColumnStatisticsString(statistics, COLUMN_TYPE);+ requireColumnStatisticsField(statistics, ANALYZED_TIME);+ requireColumnStatisticsField(statistics, STATISTICS_DATA);+ columnStatisticsStore.put(+ columnStatisticsKey(table.getDatabaseName(), table.getName(), columnNameString),+ new LinkedHashMap<>(statistics));+ }+ }++ public ColumnStatisticsResult getColumnStatisticsForTable(+ String databaseName,+ String tableName,+ List<String> columnNames) {+ Table table = getTable(databaseName, tableName);+ List<Map<String, Object>> columnStatistics = new ArrayList<>();+ List<ColumnError> errors = new ArrayList<>();+ for (String columnName : columnNames) {+ Optional<Map<String, Object>> statistics =+ columnStatisticsStore.get(columnStatisticsKey(table.getDatabaseName(), table.getName(), columnName));+ if (statistics.isPresent()) {+ columnStatistics.add(new LinkedHashMap<>(statistics.get()));+ }+ else {+ errors.add(new ColumnError(+ columnName,+ new ErrorDetail("EntityNotFoundException", "Statistics do not exist for this column")));+ }+ }+ return new ColumnStatisticsResult(columnStatistics, errors);+ }+public void createPartition(String databaseName, String tableName, Partition partition) {Table table = getTable(databaseName, tableName);String key = tableKey(databaseName, tableName) + ":" + String.join(",", partition.getValues());@@ -376,6 +425,26 @@ public class GlueService {return tableKey(databaseName, tableName) + ":" + versionId;}+ private static String columnStatisticsKey(String databaseName, String tableName, String columnName) {+ return tableKey(databaseName, tableName) + ":" + normalizeName(columnName);+ }++ private static String requireColumnStatisticsString(Map<String, Object> statistics, String field) {+ Object value = requireColumnStatisticsField(statistics, field);+ if (!(value instanceof String stringValue) || stringValue.isBlank()) {+ throw new AwsException("InvalidInputException", field + " is required", 400);+ }+ return stringValue;+ }++ private static Object requireColumnStatisticsField(Map<String, Object> statistics, String field) {+ Object value = statistics.get(field);+ if (value == null) {+ throw new AwsException("InvalidInputException", field + " is required", 400);+ }+ return value;+ }+private static String normalizeName(String name) {return name.toLowerCase(Locale.ROOT);}@@ -425,6 +494,14 @@ public class GlueService {@JsonProperty("ErrorCode") String errorCode,@JsonProperty("ErrorMessage") String errorMessage) {}+ public record ColumnStatisticsResult(+ @JsonProperty("ColumnStatisticsList") List<Map<String, Object>> columnStatisticsList,+ @JsonProperty("Errors") List<ColumnError> errors) {}++ public record ColumnError(+ @JsonProperty("ColumnName") String columnName,+ @JsonProperty("Error") ErrorDetail error) {}+private static Table copyTable(Table source) {Table copy = new Table();copy.setName(source.getName());diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.javaindex da0e37f62e..7cdd698f3f 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java@@ -21,6 +21,7 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import java.time.Instant;+import java.util.LinkedHashMap;import java.util.List;import java.util.Map;@@ -48,6 +49,7 @@ class GlueServiceTest {private GlueSchemaRegistryService schemaRegistryService;private StorageBackend<String, Table> tableStore;private StorageBackend<String, Table> tableVersionStore;+ private StorageBackend<String, Map<String, Object>> columnStatisticsStore;private StorageBackend<String, Partition> partitionStore;@BeforeEach@@ -57,11 +59,13 @@ class GlueServiceTest {schemaRegistryService = new GlueSchemaRegistryService(storageFactory, regionResolver);tableStore = new InMemoryStorage<>();tableVersionStore = new InMemoryStorage<>();+ columnStatisticsStore = new InMemoryStorage<>();partitionStore = new InMemoryStorage<>();glueService = new GlueService(new InMemoryStorage<String, Database>(),tableStore,tableVersionStore,+ columnStatisticsStore,partitionStore,new InMemoryStorage<String, UserDefinedFunction>(),schemaRegistryService, regionResolver, new ResourceGroupsTaggingService());@@ -575,6 +579,73 @@ class GlueServiceTest {assertEquals("1", tableVersions.get(0).get("VersionId"));}+ @Test+ void columnStatisticsCanBeUpdatedAndRetrieved() {+ Table table = new Table();+ table.setName("plain");+ glueService.createTable("db1", table);+ Map<String, Object> statistics = columnStatistics("id");++ glueService.updateColumnStatisticsForTable("db1", "plain", List.of(statistics));++ GlueService.ColumnStatisticsResult fetched =+ glueService.getColumnStatisticsForTable("db1", "plain", List.of("id"));+ assertEquals(1, fetched.columnStatisticsList().size());+ assertTrue(fetched.errors().isEmpty());+ assertEquals("id", fetched.columnStatisticsList().get(0).get(GlueService.COLUMN_NAME));+ assertEquals("LONG", ((Map<?, ?>) fetched.columnStatisticsList().get(0).get("StatisticsData")).get("Type"));+ }++ @Test+ void getColumnStatisticsReportsMissingColumns() {+ Table table = new Table();+ table.setName("plain");+ glueService.createTable("db1", table);++ GlueService.ColumnStatisticsResult fetched =+ glueService.getColumnStatisticsForTable("db1", "plain", List.of("missing"));++ assertTrue(fetched.columnStatisticsList().isEmpty());+ assertEquals(1, fetched.errors().size());+ assertEquals("missing", fetched.errors().getFirst().columnName());+ assertEquals("EntityNotFoundException", fetched.errors().getFirst().error().errorCode());+ assertEquals("Statistics do not exist for this column", fetched.errors().getFirst().error().errorMessage());+ }++ @Test+ void updateColumnStatisticsRejectsMissingRequiredFields() {+ Table table = new Table();+ table.setName("plain");+ glueService.createTable("db1", table);++ for (String field : List.of(+ GlueService.COLUMN_NAME,+ GlueService.COLUMN_TYPE,+ GlueService.ANALYZED_TIME,+ GlueService.STATISTICS_DATA)) {+ Map<String, Object> statistics = new LinkedHashMap<>(columnStatistics("id"));+ statistics.remove(field);++ AwsException exception = assertThrows(AwsException.class,+ () -> glueService.updateColumnStatisticsForTable("db1", "plain", List.of(statistics)));++ assertEquals("InvalidInputException", exception.getErrorCode());+ assertEquals(field + " is required", exception.getMessage());+ }+ }++ @Test+ void deleteTableDeletesColumnStatistics() {+ Table table = new Table();+ table.setName("plain");+ glueService.createTable("db1", table);+ glueService.updateColumnStatisticsForTable("db1", "plain", List.of(columnStatistics("id")));++ glueService.deleteTable("db1", "plain");++ assertTrue(columnStatisticsStore.scan(k -> true).isEmpty());+ }+@Testvoid userDefinedFunctionsCanBeCreatedListedUpdatedAndDeleted() {UserDefinedFunction function = new UserDefinedFunction();@@ -687,6 +758,21 @@ class GlueServiceTest {return table;}+ private static Map<String, Object> columnStatistics(String columnName) {+ Map<String, Object> statistics = new LinkedHashMap<>();+ statistics.put(GlueService.COLUMN_NAME, columnName);+ statistics.put(GlueService.COLUMN_TYPE, "int");+ statistics.put(GlueService.ANALYZED_TIME, Instant.parse("2026-06-08T00:00:00Z"));+ statistics.put(GlueService.STATISTICS_DATA, Map.of(+ "Type", "LONG",+ "LongColumnStatisticsData", Map.of(+ "MinimumValue", 1,+ "MaximumValue", 10,+ "NumberOfNulls", 0,+ "NumberOfDistinctValues", 10)));+ return statistics;+ }+private static final class InMemoryStorageFactory extends StorageFactory {private InMemoryStorageFactory() {super(null, null);