Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix: support Glue partition APIs (#1315)

David Phillips59a2659e41eb
diff --git a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
index 462023382f..4bee571d2d 100644
--- a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
+++ b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
@@ -59,7 +59,13 @@ table_input() {
}
}
]
- }
+ },
+ PartitionKeys: [
+ {
+ Name: "year",
+ Type: "int"
+ }
+ ]
}'
}
@@ -230,12 +236,77 @@ function_input() {
--partition-input '{"Values":["2026"]}'
assert_success
+ run aws_cmd glue batch-create-partition \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME" \
+ --partition-input-list '[{"Values":["2027"]},{"Values":["2028"]}]'
+ assert_success
+ batch_create_error_count=$(json_get "$output" '.Errors | length')
+ [ "$batch_create_error_count" = "0" ]
+
+ run aws_cmd glue batch-create-partition \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME" \
+ --partition-input-list '[{"Values":["2027"]}]'
+ assert_success
+ duplicate_partition_value=$(json_get "$output" '.Errors[0].PartitionValues[0]')
+ duplicate_error_code=$(json_get "$output" '.Errors[0].ErrorDetail.ErrorCode')
+ [ "$duplicate_partition_value" = "2027" ]
+ [ "$duplicate_error_code" = "AlreadyExistsException" ]
+
+ batch_update_entries='[
+ {"PartitionValueList":["2026"],"PartitionInput":{"Values":["2026"],"Parameters":{"after":"yes"}}},
+ {"PartitionValueList":["missing"],"PartitionInput":{"Values":["missing"]}}
+ ]'
+ run aws_cmd glue batch-update-partition \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME" \
+ --entries "$batch_update_entries"
+ assert_success
+ updated_missing_partition_value=$(json_get "$output" '.Errors[0].PartitionValueList[0]')
+ updated_missing_error_code=$(json_get "$output" '.Errors[0].ErrorDetail.ErrorCode')
+ [ "$updated_missing_partition_value" = "missing" ]
+ [ "$updated_missing_error_code" = "EntityNotFoundException" ]
+
+ run aws_cmd glue update-partition \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME" \
+ --partition-value-list 2027 \
+ --partition-input '{"Values":["2027"],"Parameters":{"updated":"yes"}}'
+ assert_success
+
+ run aws_cmd glue update-partition \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME" \
+ --partition-value-list missing \
+ --partition-input '{"Values":["missing"]}'
+ assert_failure
+ [[ "$output" == *"EntityNotFoundException"* ]]
+
run aws_cmd glue get-partitions \
--database-name "$DB_NAME" \
--table-name "$TABLE_NAME"
assert_success
value=$(json_get "$output" '.Partitions[0].Values[0]')
+ parameter=$(json_get "$output" '.Partitions[0].Parameters.after')
[ "$value" = "2026" ]
+ [ "$parameter" = "yes" ]
+
+ run aws_cmd glue get-partitions \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME" \
+ --expression "year >= 2026 AND year <= 2027"
+ assert_success
+ filtered_count=$(json_get "$output" '.Partitions | length')
+ [ "$filtered_count" = "2" ]
+
+ run aws_cmd glue get-partitions \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME" \
+ --expression "year in (2026, 2028)"
+ assert_success
+ filtered_count=$(json_get "$output" '.Partitions | length')
+ [ "$filtered_count" = "2" ]
run aws_cmd glue delete-table \
--database-name "$DB_NAME" \
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
index 824d48dbc7..556b03b47e 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java
@@ -5,37 +5,52 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.services.glue.GlueClient;
+import software.amazon.awssdk.services.glue.model.BatchCreatePartitionRequest;
import software.amazon.awssdk.services.glue.model.BatchDeleteTableRequest;
+import software.amazon.awssdk.services.glue.model.BatchGetPartitionRequest;
+import software.amazon.awssdk.services.glue.model.BatchUpdatePartitionRequest;
+import software.amazon.awssdk.services.glue.model.BatchUpdatePartitionRequestEntry;
import software.amazon.awssdk.services.glue.model.Column;
+import software.amazon.awssdk.services.glue.model.ColumnStatistics;
+import software.amazon.awssdk.services.glue.model.ColumnStatisticsData;
import software.amazon.awssdk.services.glue.model.CreateDatabaseRequest;
import software.amazon.awssdk.services.glue.model.CreatePartitionRequest;
import software.amazon.awssdk.services.glue.model.CreateTableRequest;
import software.amazon.awssdk.services.glue.model.CreateUserDefinedFunctionRequest;
import software.amazon.awssdk.services.glue.model.DatabaseInput;
+import software.amazon.awssdk.services.glue.model.DeleteColumnStatisticsForPartitionRequest;
import software.amazon.awssdk.services.glue.model.DeleteDatabaseRequest;
+import software.amazon.awssdk.services.glue.model.DeletePartitionRequest;
import software.amazon.awssdk.services.glue.model.DeleteTableRequest;
import software.amazon.awssdk.services.glue.model.DeleteUserDefinedFunctionRequest;
import software.amazon.awssdk.services.glue.model.EntityNotFoundException;
+import software.amazon.awssdk.services.glue.model.GetColumnStatisticsForPartitionRequest;
import software.amazon.awssdk.services.glue.model.GetDatabaseRequest;
import software.amazon.awssdk.services.glue.model.GetDatabasesRequest;
+import software.amazon.awssdk.services.glue.model.GetPartitionRequest;
import software.amazon.awssdk.services.glue.model.GetPartitionsRequest;
import software.amazon.awssdk.services.glue.model.GetTableRequest;
import software.amazon.awssdk.services.glue.model.GetTablesRequest;
import software.amazon.awssdk.services.glue.model.GetUserDefinedFunctionRequest;
import software.amazon.awssdk.services.glue.model.GetUserDefinedFunctionsRequest;
+import software.amazon.awssdk.services.glue.model.LongColumnStatisticsData;
import software.amazon.awssdk.services.glue.model.PartitionInput;
+import software.amazon.awssdk.services.glue.model.PartitionValueList;
import software.amazon.awssdk.services.glue.model.PrincipalType;
import software.amazon.awssdk.services.glue.model.ResourceType;
import software.amazon.awssdk.services.glue.model.ResourceUri;
import software.amazon.awssdk.services.glue.model.SerDeInfo;
import software.amazon.awssdk.services.glue.model.StorageDescriptor;
import software.amazon.awssdk.services.glue.model.TableInput;
+import software.amazon.awssdk.services.glue.model.UpdateColumnStatisticsForPartitionRequest;
+import software.amazon.awssdk.services.glue.model.UpdatePartitionRequest;
import software.amazon.awssdk.services.glue.model.UpdateTableRequest;
import software.amazon.awssdk.services.glue.model.UpdateUserDefinedFunctionRequest;
import software.amazon.awssdk.services.glue.model.UserDefinedFunction;
import software.amazon.awssdk.services.glue.model.UserDefinedFunctionInput;
import software.amazon.awssdk.services.resourcegroupstaggingapi.ResourceGroupsTaggingApiClient;
+import java.time.Instant;
import java.util.Map;
import java.util.stream.Collectors;
@@ -167,16 +182,189 @@ class GlueCatalogTest {
.values("2026")
.build())
.build());
+ glue.createPartition(CreatePartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionInput(PartitionInput.builder()
+ .values("2027")
+ .build())
+ .build());
+ var batchCreatePartition = glue.batchCreatePartition(BatchCreatePartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionInputList(PartitionInput.builder()
+ .values("2028")
+ .build(),
+ PartitionInput.builder()
+ .values("2029")
+ .build())
+ .build());
+ assertThat(batchCreatePartition.errors()).isEmpty();
+ assertThat(glue.batchCreatePartition(BatchCreatePartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionInputList(PartitionInput.builder()
+ .values("2028")
+ .build())
+ .build()).errors())
+ .singleElement()
+ .satisfies(error -> {
+ assertThat(error.partitionValues()).containsExactly("2028");
+ assertThat(error.errorDetail().errorCode()).isEqualTo("AlreadyExistsException");
+ });
+ assertThat(glue.batchUpdatePartition(BatchUpdatePartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .entries(
+ BatchUpdatePartitionRequestEntry.builder()
+ .partitionValueList("2026")
+ .partitionInput(PartitionInput.builder()
+ .values("2026")
+ .parameters(Map.of("after", "yes"))
+ .build())
+ .build(),
+ BatchUpdatePartitionRequestEntry.builder()
+ .partitionValueList("missing")
+ .partitionInput(PartitionInput.builder()
+ .values("missing")
+ .build())
+ .build())
+ .build()).errors())
+ .singleElement()
+ .satisfies(error -> {
+ assertThat(error.partitionValueList()).containsExactly("missing");
+ assertThat(error.errorDetail().errorCode()).isEqualTo("EntityNotFoundException");
+ });
+ var partition = glue.getPartition(GetPartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionValues("2026")
+ .build()).partition();
+ assertThat(partition.values()).containsExactly("2026");
+ assertThat(partition.parameters()).containsEntry("after", "yes");
+ glue.updatePartition(UpdatePartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionValueList("2027")
+ .partitionInput(PartitionInput.builder()
+ .values("2027")
+ .parameters(Map.of("updated", "yes"))
+ .build())
+ .build());
+ assertThat(glue.getPartition(GetPartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionValues("2027")
+ .build()).partition().parameters())
+ .containsEntry("updated", "yes");
+ assertThatThrownBy(() -> glue.updatePartition(UpdatePartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionValueList("missing")
+ .partitionInput(PartitionInput.builder()
+ .values("missing")
+ .build())
+ .build()))
+ .isInstanceOf(EntityNotFoundException.class);
+ assertThat(glue.batchGetPartition(BatchGetPartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionsToGet(
+ PartitionValueList.builder()
+ .values("2026")
+ .build(),
+ PartitionValueList.builder()
+ .values("missing")
+ .build())
+ .build()).partitions())
+ .singleElement()
+ .satisfies(batchPartition -> assertThat(batchPartition.values()).containsExactly("2026"));
+ assertThat(glue.getPartitions(GetPartitionsRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .expression("year = 2026")
+ .build()).partitions())
+ .singleElement()
+ .satisfies(filteredPartition -> {
+ assertThat(filteredPartition.databaseName()).isEqualTo(DATABASE_NAME);
+ assertThat(filteredPartition.tableName()).isEqualTo(TABLE_NAME);
+ assertThat(filteredPartition.values()).containsExactly("2026");
+ });
assertThat(glue.getPartitions(GetPartitionsRequest.builder()
.databaseName(DATABASE_NAME)
.tableName(TABLE_NAME)
+ .expression("year >= 2026 AND year <= 2027")
.build()).partitions())
+ .extracting(partitionInput -> partitionInput.values().get(0))
+ .containsExactlyInAnyOrder("2026", "2027");
+ assertThat(glue.getPartitions(GetPartitionsRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .expression("year in (2026, 2028)")
+ .build()).partitions())
+ .extracting(partitionInput -> partitionInput.values().get(0))
+ .containsExactlyInAnyOrder("2026", "2028");
+ glue.updateColumnStatisticsForPartition(UpdateColumnStatisticsForPartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionValues("2026")
+ .columnStatisticsList(ColumnStatistics.builder()
+ .columnName("id")
+ .columnType("int")
+ .analyzedTime(Instant.EPOCH)
+ .statisticsData(ColumnStatisticsData.builder()
+ .type("LONG")
+ .longColumnStatisticsData(LongColumnStatisticsData.builder()
+ .minimumValue(1L)
+ .maximumValue(10L)
+ .numberOfNulls(0L)
+ .numberOfDistinctValues(10L)
+ .build())
+ .build())
+ .build())
+ .build());
+ var statistics = glue.getColumnStatisticsForPartition(GetColumnStatisticsForPartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionValues("2026")
+ .columnNames("id", "missing")
+ .build());
+ assertThat(statistics.columnStatisticsList())
+ .singleElement()
+ .satisfies(columnStatistics -> {
+ assertThat(columnStatistics.columnName()).isEqualTo("id");
+ assertThat(columnStatistics.statisticsData().longColumnStatisticsData().maximumValue()).isEqualTo(10);
+ });
+ assertThat(statistics.errors())
.singleElement()
- .satisfies(partition -> {
- assertThat(partition.databaseName()).isEqualTo(DATABASE_NAME);
- assertThat(partition.tableName()).isEqualTo(TABLE_NAME);
- assertThat(partition.values()).containsExactly("2026");
+ .satisfies(error -> {
+ assertThat(error.columnName()).isEqualTo("missing");
+ assertThat(error.error().errorCode()).isEqualTo("EntityNotFoundException");
});
+ glue.deleteColumnStatisticsForPartition(DeleteColumnStatisticsForPartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionValues("2026")
+ .columnName("id")
+ .build());
+ assertThat(glue.getColumnStatisticsForPartition(GetColumnStatisticsForPartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionValues("2026")
+ .columnNames("id")
+ .build()).columnStatisticsList())
+ .isEmpty();
+ glue.deletePartition(DeletePartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionValues("2027")
+ .build());
+ assertThatThrownBy(() -> glue.getPartition(GetPartitionRequest.builder()
+ .databaseName(DATABASE_NAME)
+ .tableName(TABLE_NAME)
+ .partitionValues("2027")
+ .build()))
+ .isInstanceOf(EntityNotFoundException.class);
glue.createUserDefinedFunction(CreateUserDefinedFunctionRequest.builder()
.databaseName(DATABASE_NAME)
@@ -324,6 +512,10 @@ class GlueCatalogTest {
.parameters(Map.of("comment", "identifier"))
.build())
.build())
+ .partitionKeys(Column.builder()
+ .name("year")
+ .type("int")
+ .build())
.build();
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
index 5b6850db0f..41fb8a2faa 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
@@ -30,6 +30,7 @@ public class GlueJsonHandler {
private static final TypeReference<List<String>> STRING_LIST = new TypeReference<>() {};
private static final TypeReference<List<Map<String, Object>>> MAP_LIST = new TypeReference<>() {};
+ private static final TypeReference<List<Partition>> PARTITION_LIST = new TypeReference<>() {};
private final GlueService glueService;
private final GlueSchemaRegistryService schemaRegistryService;
@@ -122,11 +123,21 @@ public class GlueJsonHandler {
glueService.createPartition(dbName, tableName, partition);
yield Response.ok().build();
}
+ case "BatchCreatePartition" -> handleBatchCreatePartition(request);
+ case "BatchUpdatePartition" -> handleBatchUpdatePartition(request);
+ case "GetPartition" -> handleGetPartition(request);
+ case "BatchGetPartition" -> handleBatchGetPartition(request);
case "GetPartitions" -> {
String dbName = request.get("DatabaseName").asText();
String tableName = request.get("TableName").asText();
- yield Response.ok(Map.of("Partitions", glueService.getPartitions(dbName, tableName))).build();
+ String expression = request.path("Expression").asText(null);
+ yield Response.ok(Map.of("Partitions", glueService.getPartitions(dbName, tableName, expression))).build();
}
+ case "DeletePartition" -> handleDeletePartition(request);
+ case "UpdatePartition" -> handleUpdatePartition(request);
+ case "UpdateColumnStatisticsForPartition" -> handleUpdateColumnStatisticsForPartition(request);
+ case "GetColumnStatisticsForPartition" -> handleGetColumnStatisticsForPartition(request);
+ case "DeleteColumnStatisticsForPartition" -> handleDeleteColumnStatisticsForPartition(request);
case "CreateUserDefinedFunction" -> handleCreateUserDefinedFunction(request);
case "GetUserDefinedFunction" -> handleGetUserDefinedFunction(request);
case "GetUserDefinedFunctions" -> handleGetUserDefinedFunctions(request);
@@ -180,6 +191,92 @@ public class GlueJsonHandler {
.build();
}
+ private Response handleBatchCreatePartition(JsonNode request) {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ List<Partition> partitions = mapper.convertValue(request.get("PartitionInputList"), PARTITION_LIST);
+ return Response.ok(Map.of("Errors", glueService.batchCreatePartitions(dbName, tableName, partitions))).build();
+ }
+
+ private Response handleBatchUpdatePartition(JsonNode request) {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ List<GlueService.BatchUpdatePartitionEntry> entries =
+ mapper.convertValue(request.get("Entries"), new TypeReference<>() {});
+ return Response.ok(Map.of("Errors", glueService.batchUpdatePartitions(dbName, tableName, entries))).build();
+ }
+
+ private Response handleGetPartition(JsonNode request) {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ List<String> partitionValues = mapper.convertValue(request.get("PartitionValues"), STRING_LIST);
+ return Response.ok(Map.of(
+ "Partition",
+ glueService.getPartition(dbName, tableName, partitionValues)))
+ .build();
+ }
+
+ private Response handleBatchGetPartition(JsonNode request) {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ List<Map<String, Object>> partitionsToGet = mapper.convertValue(request.get("PartitionsToGet"), MAP_LIST);
+ List<List<String>> partitionValues = partitionsToGet.stream()
+ .map(partition -> mapper.convertValue(partition.get("Values"), STRING_LIST))
+ .toList();
+ return Response.ok(Map.of(
+ "Partitions", glueService.batchGetPartitions(dbName, tableName, partitionValues),
+ "UnprocessedKeys", List.of()))
+ .build();
+ }
+
+ private Response handleDeletePartition(JsonNode request) {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ List<String> partitionValues = mapper.convertValue(request.get("PartitionValues"), STRING_LIST);
+ glueService.deletePartition(dbName, tableName, partitionValues);
+ return Response.ok().build();
+ }
+
+ private Response handleUpdatePartition(JsonNode request) throws Exception {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ List<String> partitionValues = mapper.convertValue(request.get("PartitionValueList"), STRING_LIST);
+ Partition partition = mapper.treeToValue(request.get("PartitionInput"), Partition.class);
+ glueService.updatePartition(dbName, tableName, partitionValues, partition);
+ return Response.ok().build();
+ }
+
+ private Response handleUpdateColumnStatisticsForPartition(JsonNode request) {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ List<String> partitionValues = mapper.convertValue(request.get("PartitionValues"), STRING_LIST);
+ List<Map<String, Object>> columnStatistics = mapper.convertValue(request.get("ColumnStatisticsList"), MAP_LIST);
+ glueService.updateColumnStatisticsForPartition(dbName, tableName, partitionValues, columnStatistics);
+ return Response.ok(Map.of("Errors", List.of())).build();
+ }
+
+ private Response handleGetColumnStatisticsForPartition(JsonNode request) {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ List<String> partitionValues = mapper.convertValue(request.get("PartitionValues"), STRING_LIST);
+ List<String> columnNames = mapper.convertValue(request.get("ColumnNames"), STRING_LIST);
+ GlueService.ColumnStatisticsResult result = glueService.getColumnStatisticsForPartition(
+ dbName, tableName, partitionValues, columnNames);
+ return Response.ok(Map.of(
+ "ColumnStatisticsList", result.columnStatisticsList(),
+ "Errors", result.errors()))
+ .build();
+ }
+
+ private Response handleDeleteColumnStatisticsForPartition(JsonNode request) {
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ List<String> partitionValues = mapper.convertValue(request.get("PartitionValues"), STRING_LIST);
+ String columnName = request.get("ColumnName").asText();
+ glueService.deleteColumnStatisticsForPartition(dbName, tableName, partitionValues, columnName);
+ return Response.ok().build();
+ }
+
private Response handleCreateUserDefinedFunction(JsonNode request) throws Exception {
String dbName = request.get("DatabaseName").asText();
UserDefinedFunction function = mapper.treeToValue(request.get("FunctionInput"), UserDefinedFunction.class);
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index bc6491e4fc..657a6f1d68 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -22,14 +22,18 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
+import java.util.Base64;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
@@ -43,12 +47,19 @@ public class GlueService {
static final String COLUMN_TYPE = "ColumnType";
static final String ANALYZED_TIME = "AnalyzedTime";
static final String STATISTICS_DATA = "StatisticsData";
+ private static final Pattern COMPARISON_EXPRESSION = Pattern.compile(
+ "\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*(=|<>|<=|>=|<|>)\\s*('?[^']*'?|[^\\s]+)\\s*",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern IN_EXPRESSION = Pattern.compile(
+ "\\s*([A-Za-z_][A-Za-z0-9_]*)\\s+in\\s*\\((.*)\\)\\s*",
+ Pattern.CASE_INSENSITIVE);
private final StorageBackend<String, Database> databaseStore;
private final StorageBackend<String, Table> tableStore;
private final StorageBackend<String, Table> tableVersionStore;
private final StorageBackend<String, Map<String, Object>> columnStatisticsStore;
private final StorageBackend<String, Partition> partitionStore;
+ private final StorageBackend<String, Map<String, Object>> partitionColumnStatisticsStore;
private final StorageBackend<String, UserDefinedFunction> functionStore;
private final GlueSchemaRegistryService schemaRegistryService;
private final RegionResolver regionResolver;
@@ -64,6 +75,8 @@ public class GlueService {
this.tableVersionStore = storageFactory.create("glue", "table_versions.json", new TypeReference<>() {});
this.columnStatisticsStore = storageFactory.create("glue", "column_statistics.json", new TypeReference<>() {});
this.partitionStore = storageFactory.create("glue", "partitions.json", new TypeReference<>() {});
+ this.partitionColumnStatisticsStore = storageFactory.create(
+ "glue", "partition_column_statistics.json", new TypeReference<>() {});
this.functionStore = storageFactory.create("glue", "functions.json", new TypeReference<>() {});
this.schemaRegistryService = schemaRegistryService;
this.regionResolver = regionResolver;
@@ -75,6 +88,7 @@ public class GlueService {
StorageBackend<String, Table> tableVersionStore,
StorageBackend<String, Map<String, Object>> columnStatisticsStore,
StorageBackend<String, Partition> partitionStore,
+ StorageBackend<String, Map<String, Object>> partitionColumnStatisticsStore,
StorageBackend<String, UserDefinedFunction> functionStore,
GlueSchemaRegistryService schemaRegistryService,
RegionResolver regionResolver,
@@ -84,6 +98,7 @@ public class GlueService {
this.tableVersionStore = tableVersionStore;
this.columnStatisticsStore = columnStatisticsStore;
this.partitionStore = partitionStore;
+ this.partitionColumnStatisticsStore = partitionColumnStatisticsStore;
this.functionStore = functionStore;
this.schemaRegistryService = schemaRegistryService;
this.regionResolver = regionResolver;
@@ -229,9 +244,12 @@ public class GlueService {
columnStatisticsStore.keys().stream()
.filter(statisticsKey -> statisticsKey.startsWith(key + ":"))
.forEach(columnStatisticsStore::delete);
- partitionStore.scan(k -> k.startsWith(key + ":")).forEach(p -> {
- partitionStore.delete(key + ":" + String.join(",", p.getValues()));
- });
+ partitionStore.keys().stream()
+ .filter(partitionKey -> partitionKey.startsWith(key + ":"))
+ .forEach(partitionStore::delete);
+ partitionColumnStatisticsStore.keys().stream()
+ .filter(statisticsKey -> statisticsKey.startsWith(key + ":"))
+ .forEach(partitionColumnStatisticsStore::delete);
LOG.infov("Deleted Glue Table: {0}.{1}", databaseName, tableName);
}
@@ -292,15 +310,140 @@ public class GlueService {
public void createPartition(String databaseName, String tableName, Partition partition) {
Table table = getTable(databaseName, tableName);
- String key = tableKey(databaseName, tableName) + ":" + String.join(",", partition.getValues());
+ String key = partitionKey(databaseName, tableName, partition.getValues());
partition.setDatabaseName(table.getDatabaseName());
partition.setTableName(table.getName());
+ if (partition.getCreationTime() == null) {
+ partition.setCreationTime(Instant.now());
+ }
partitionStore.put(key, partition);
}
+ public List<BatchCreatePartitionError> batchCreatePartitions(
+ String databaseName,
+ String tableName,
+ List<Partition> partitions) {
+ getTable(databaseName, tableName);
+ List<BatchCreatePartitionError> errors = new ArrayList<>();
+ for (Partition partition : partitions) {
+ String key = partitionKey(databaseName, tableName, partition.getValues());
+ if (partitionStore.get(key).isPresent()) {
+ errors.add(new BatchCreatePartitionError(
+ partition.getValues(),
+ new ErrorDetail("AlreadyExistsException", "Partition already exists.")));
+ continue;
+ }
+ createPartition(databaseName, tableName, partition);
+ }
+ return errors;
+ }
+
public List<Partition> getPartitions(String databaseName, String tableName) {
+ return getPartitions(databaseName, tableName, null);
+ }
+
+ public Partition getPartition(String databaseName, String tableName, List<String> partitionValues) {
+ getTable(databaseName, tableName);
+ return partitionStore.get(partitionKey(databaseName, tableName, partitionValues))
+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Cannot find partition.", 400));
+ }
+
+ public List<Partition> batchGetPartitions(String databaseName, String tableName, List<List<String>> partitionValues) {
+ getTable(databaseName, tableName);
+ List<Partition> partitions = new ArrayList<>();
+ for (List<String> values : partitionValues) {
+ partitionStore.get(partitionKey(databaseName, tableName, values)).ifPresent(partitions::add);
+ }
+ return partitions;
+ }
+
+ public List<BatchUpdatePartitionError> batchUpdatePartitions(
+ String databaseName,
+ String tableName,
+ List<BatchUpdatePartitionEntry> entries) {
+ Table table = getTable(databaseName, tableName);
+ List<BatchUpdatePartitionError> errors = new ArrayList<>();
+ for (BatchUpdatePartitionEntry entry : entries) {
+ String key = partitionKey(databaseName, tableName, entry.partitionValueList());
+ Optional<Partition> existing = partitionStore.get(key);
+ if (existing.isEmpty()) {
+ errors.add(new BatchUpdatePartitionError(
+ entry.partitionValueList(),
+ new ErrorDetail(
+ "EntityNotFoundException",
+ "Partition [" + String.join(", ", entry.partitionValueList()) + "] not found")));
+ continue;
+ }
+
+ putUpdatedPartition(table, key, existing.get(), entry.partitionInput());
+ }
+ return errors;
+ }
+
+ public List<Partition> getPartitions(String databaseName, String tableName, String expression) {
+ Table table = getTable(databaseName, tableName);
String prefix = tableKey(databaseName, tableName) + ":";
- return partitionStore.scan(k -> k.startsWith(prefix));
+ return partitionStore.scan(k -> k.startsWith(prefix)).stream()
+ .filter(partition -> matchesPartitionExpression(table, partition, expression))
+ .toList();
+ }
+
+ public void deletePartition(String databaseName, String tableName, List<String> partitionValues) {
+ getPartition(databaseName, tableName, partitionValues);
+ String key = partitionKey(databaseName, tableName, partitionValues);
+ partitionStore.delete(key);
+ partitionColumnStatisticsStore.keys().stream()
+ .filter(statisticsKey -> statisticsKey.startsWith(key + ":"))
+ .forEach(partitionColumnStatisticsStore::delete);
+ }
+
+ public void updatePartition(String databaseName, String tableName, List<String> partitionValues, Partition partition) {
+ Table table = getTable(databaseName, tableName);
+ String key = partitionKey(databaseName, tableName, partitionValues);
+ Partition existing = partitionStore.get(key)
+ .orElseThrow(() -> new AwsException("EntityNotFoundException", "Partition not found.", 400));
+ putUpdatedPartition(table, key, existing, partition);
+ }
+
+ public void updateColumnStatisticsForPartition(
+ String databaseName,
+ String tableName,
+ List<String> partitionValues,
+ List<Map<String, Object>> columnStatistics) {
+ getPartition(databaseName, tableName, partitionValues);
+ for (Map<String, Object> statistics : columnStatistics) {
+ String columnNameString = requireColumnStatisticsString(statistics, COLUMN_NAME);
+ requireColumnStatisticsString(statistics, COLUMN_TYPE);
+ requireColumnStatisticsField(statistics, ANALYZED_TIME);
+ requireColumnStatisticsField(statistics, STATISTICS_DATA);
+ partitionColumnStatisticsStore.put(
+ partitionColumnStatisticsKey(databaseName, tableName, partitionValues, columnNameString),
+ new LinkedHashMap<>(statistics));
+ }
+ }
+
+ public ColumnStatisticsResult getColumnStatisticsForPartition(
+ String databaseName,
+ String tableName,
+ List<String> partitionValues,
+ List<String> columnNames) {
+ getPartition(databaseName, tableName, partitionValues);
+ List<Map<String, Object>> columnStatistics = new ArrayList<>();
+ List<ColumnError> errors = new ArrayList<>();
+ for (String columnName : columnNames) {
+ partitionColumnStatisticsStore.get(partitionColumnStatisticsKey(databaseName, tableName, partitionValues, columnName))
+ .ifPresentOrElse(columnStatistics::add, () -> errors.add(columnStatisticsNotFoundError(columnName)));
+ }
+ return new ColumnStatisticsResult(columnStatistics, errors);
+ }
+
+ public void deleteColumnStatisticsForPartition(
+ String databaseName,
+ String tableName,
+ List<String> partitionValues,
+ String columnName) {
+ getPartition(databaseName, tableName, partitionValues);
+ partitionColumnStatisticsStore.delete(partitionColumnStatisticsKey(databaseName, tableName, partitionValues, columnName));
}
public void createUserDefinedFunction(String databaseName, UserDefinedFunction function) {
@@ -429,6 +572,35 @@ public class GlueService {
return tableKey(databaseName, tableName) + ":" + normalizeName(columnName);
}
+ private static String partitionKey(String databaseName, String tableName, List<String> partitionValues) {
+ return tableKey(databaseName, tableName) + ":" + String.join(":", partitionValues.stream()
+ .map(GlueService::encodePartitionValue)
+ .toList());
+ }
+
+ private static String encodePartitionValue(String value) {
+ return Base64.getUrlEncoder().withoutPadding().encodeToString(value.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private static String partitionColumnStatisticsKey(
+ String databaseName,
+ String tableName,
+ List<String> partitionValues,
+ String columnName) {
+ return partitionKey(databaseName, tableName, partitionValues) + ":" + normalizeName(columnName);
+ }
+
+ private void putUpdatedPartition(Table table, String key, Partition existing, Partition partition) {
+ partition.setDatabaseName(table.getDatabaseName());
+ partition.setTableName(table.getName());
+ partition.setCreationTime(existing.getCreationTime());
+ String updatedKey = partitionKey(table.getDatabaseName(), table.getName(), partition.getValues());
+ if (!key.equals(updatedKey)) {
+ partitionStore.delete(key);
+ }
+ partitionStore.put(updatedKey, partition);
+ }
+
private static String requireColumnStatisticsString(Map<String, Object> statistics, String field) {
Object value = requireColumnStatisticsField(statistics, field);
if (!(value instanceof String stringValue) || stringValue.isBlank()) {
@@ -449,6 +621,198 @@ public class GlueService {
return name.toLowerCase(Locale.ROOT);
}
+ private static boolean matchesPartitionExpression(Table table, Partition partition, String expression) {
+ if (expression == null || expression.isBlank()) {
+ return true;
+ }
+ return matchesExpression(table, partition, expression);
+ }
+
+ private static boolean matchesExpression(Table table, Partition partition, String expression) {
+ String stripped = stripParentheses(expression.trim());
+ List<String> disjuncts = splitTopLevel(stripped, "OR");
+ if (disjuncts.size() > 1) {
+ return disjuncts.stream().anyMatch(disjunct -> matchesExpression(table, partition, disjunct));
+ }
+ List<String> conjuncts = splitTopLevel(stripped, "AND");
+ if (conjuncts.size() > 1) {
+ return conjuncts.stream().allMatch(conjunct -> matchesExpression(table, partition, conjunct));
+ }
+ return matchesPredicate(table, partition, stripped);
+ }
+
+ private static boolean matchesPredicate(Table table, Partition partition, String expression) {
+ Matcher inMatcher = IN_EXPRESSION.matcher(expression);
+ if (inMatcher.matches()) {
+ String partitionValue = partitionValue(table, partition, inMatcher.group(1));
+ if (partitionValue == null) {
+ return false;
+ }
+ return splitValues(inMatcher.group(2)).stream()
+ .map(GlueService::unquote)
+ .anyMatch(partitionValue::equals);
+ }
+ Matcher comparisonMatcher = COMPARISON_EXPRESSION.matcher(expression);
+ if (comparisonMatcher.matches()) {
+ String partitionValue = partitionValue(table, partition, comparisonMatcher.group(1));
+ if (partitionValue == null) {
+ return false;
+ }
+ return compare(partitionValue, comparisonMatcher.group(2), unquote(comparisonMatcher.group(3)));
+ }
+ throw new AwsException("InvalidInputException", "Unsupported partition expression: " + expression, 400);
+ }
+
+ private static String partitionValue(Table table, Partition partition, String partitionKeyName) {
+ int index = partitionKeyIndex(table, partitionKeyName);
+ if (index < 0 || partition.getValues() == null || partition.getValues().size() <= index) {
+ return null;
+ }
+ return partition.getValues().get(index);
+ }
+
+ private static boolean compare(String actual, String operator, String expected) {
+ int comparison = compareValues(actual, expected);
+ return switch (operator) {
+ case "=" -> comparison == 0;
+ case "<>" -> comparison != 0;
+ case "<" -> comparison < 0;
+ case "<=" -> comparison <= 0;
+ case ">" -> comparison > 0;
+ case ">=" -> comparison >= 0;
+ default -> true;
+ };
+ }
+
+ private static int compareValues(String left, String right) {
+ try {
+ return new BigDecimal(left).compareTo(new BigDecimal(right));
+ }
+ catch (NumberFormatException e) {
+ return left.compareTo(right);
+ }
+ }
+
+ private static String stripParentheses(String expression) {
+ String stripped = expression;
+ while (stripped.startsWith("(") && stripped.endsWith(")") && matchingOuterParentheses(stripped)) {
+ stripped = stripped.substring(1, stripped.length() - 1).trim();
+ }
+ return stripped;
+ }
+
+ private static boolean matchingOuterParentheses(String expression) {
+ int depth = 0;
+ boolean quoted = false;
+ for (int index = 0; index < expression.length(); index++) {
+ char current = expression.charAt(index);
+ if (current == '\'') {
+ quoted = !quoted;
+ }
+ if (quoted) {
+ continue;
+ }
+ if (current == '(') {
+ depth++;
+ }
+ else if (current == ')') {
+ depth--;
+ if (depth == 0 && index < expression.length() - 1) {
+ return false;
+ }
+ }
+ }
+ return depth == 0;
+ }
+
+ private static List<String> splitTopLevel(String expression, String operator) {
+ List<String> parts = new ArrayList<>();
+ int depth = 0;
+ boolean quoted = false;
+ int start = 0;
+ for (int index = 0; index < expression.length(); index++) {
+ char current = expression.charAt(index);
+ if (current == '\'') {
+ quoted = !quoted;
+ }
+ if (quoted) {
+ continue;
+ }
+ if (current == '(') {
+ depth++;
+ }
+ else if (current == ')') {
+ depth--;
+ }
+ else if (depth == 0 && matchesOperator(expression, index, operator)) {
+ parts.add(expression.substring(start, index).trim());
+ index += operator.length() - 1;
+ start = index + 1;
+ }
+ }
+ if (parts.isEmpty()) {
+ return List.of(expression);
+ }
+ parts.add(expression.substring(start).trim());
+ return parts;
+ }
+
+ private static boolean matchesOperator(String expression, int index, String operator) {
+ if (!expression.regionMatches(true, index, operator, 0, operator.length())) {
+ return false;
+ }
+ return isBoundary(expression, index - 1) && isBoundary(expression, index + operator.length());
+ }
+
+ private static boolean isBoundary(String expression, int index) {
+ return index < 0 || index >= expression.length() || Character.isWhitespace(expression.charAt(index));
+ }
+
+ private static List<String> splitValues(String values) {
+ List<String> result = new ArrayList<>();
+ boolean quoted = false;
+ int start = 0;
+ for (int index = 0; index < values.length(); index++) {
+ char current = values.charAt(index);
+ if (current == '\'') {
+ quoted = !quoted;
+ }
+ else if (current == ',' && !quoted) {
+ result.add(values.substring(start, index).trim());
+ start = index + 1;
+ }
+ }
+ result.add(values.substring(start).trim());
+ return result;
+ }
+
+ private static String unquote(String value) {
+ String stripped = value.trim();
+ if (stripped.length() >= 2 && stripped.startsWith("'") && stripped.endsWith("'")) {
+ return stripped.substring(1, stripped.length() - 1);
+ }
+ return stripped;
+ }
+
+ private static int partitionKeyIndex(Table table, String partitionKeyName) {
+ List<Column> partitionKeys = table.getPartitionKeys();
+ if (partitionKeys == null) {
+ return -1;
+ }
+ for (int index = 0; index < partitionKeys.size(); index++) {
+ if (partitionKeyName.equals(partitionKeys.get(index).getName())) {
+ return index;
+ }
+ }
+ return -1;
+ }
+
+ private static ColumnError columnStatisticsNotFoundError(String columnName) {
+ return new ColumnError(
+ columnName,
+ new ErrorDetail("EntityNotFoundException", "Statistics do not exist for this column"));
+ }
+
private String databaseArn(String region, String databaseName) {
return regionResolver.buildArn("glue", region, "database/" + databaseName);
}
@@ -490,6 +854,18 @@ public class GlueService {
@JsonProperty("TableName") String tableName,
@JsonProperty("ErrorDetail") ErrorDetail errorDetail) {}
+ public record BatchCreatePartitionError(
+ @JsonProperty("PartitionValues") List<String> partitionValues,
+ @JsonProperty("ErrorDetail") ErrorDetail errorDetail) {}
+
+ public record BatchUpdatePartitionEntry(
+ @JsonProperty("PartitionValueList") List<String> partitionValueList,
+ @JsonProperty("PartitionInput") Partition partitionInput) {}
+
+ public record BatchUpdatePartitionError(
+ @JsonProperty("PartitionValueList") List<String> partitionValueList,
+ @JsonProperty("ErrorDetail") ErrorDetail errorDetail) {}
+
public record ErrorDetail(
@JsonProperty("ErrorCode") String errorCode,
@JsonProperty("ErrorMessage") String errorMessage) {}
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
index 7cdd698f3f..b886d1fbfa 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -67,6 +67,7 @@ class GlueServiceTest {
tableVersionStore,
columnStatisticsStore,
partitionStore,
+ new InMemoryStorage<String, Map<String, Object>>(),
new InMemoryStorage<String, UserDefinedFunction>(),
schemaRegistryService, regionResolver, new ResourceGroupsTaggingService());
glueService.createDatabase(new Database("db1"));
@@ -346,6 +347,213 @@ class GlueServiceTest {
assertTrue(partitionStore.scan(k -> true).isEmpty());
}
+ @Test
+ void partitionsCanBeFetchedFilteredAndDeleted() {
+ Table table = new Table();
+ table.setName("plain");
+ table.setPartitionKeys(List.of(new Column("part", "int")));
+ glueService.createTable("db1", table);
+ Partition first = new Partition();
+ first.setValues(List.of("1"));
+ glueService.createPartition("db1", "plain", first);
+ Partition second = new Partition();
+ second.setValues(List.of("2"));
+ glueService.createPartition("db1", "plain", second);
+
+ assertEquals(List.of("1"), glueService.getPartition("db1", "plain", List.of("1")).getValues());
+ assertEquals(List.of(List.of("1")), glueService.batchGetPartitions(
+ "db1", "plain", List.of(List.of("1"), List.of("missing"))).stream()
+ .map(Partition::getValues)
+ .toList());
+ assertEquals(List.of(List.of("1")), glueService.getPartitions("db1", "plain", "part = 1").stream()
+ .map(Partition::getValues)
+ .toList());
+ assertEquals(List.of(List.of("1"), List.of("2")), glueService.getPartitions("db1", "plain", "part >= 1 AND part <= 2").stream()
+ .map(Partition::getValues)
+ .sorted((left, right) -> left.get(0).compareTo(right.get(0)))
+ .toList());
+ assertEquals(List.of(List.of("1")), glueService.getPartitions("db1", "plain", "part in (1, 3)").stream()
+ .map(Partition::getValues)
+ .toList());
+ AwsException invalidExpression = assertThrows(AwsException.class,
+ () -> glueService.getPartitions("db1", "plain", "part between 1 and 2"));
+ assertEquals("InvalidInputException", invalidExpression.getErrorCode());
+
+ glueService.deletePartition("db1", "plain", List.of("2"));
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> glueService.getPartition("db1", "plain", List.of("2")));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ }
+
+ @Test
+ void partitionKeysDoNotCollideForCommaSeparatedValues() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+ Partition first = new Partition();
+ first.setValues(List.of("a,b"));
+ Partition second = new Partition();
+ second.setValues(List.of("a", "b"));
+
+ glueService.createPartition("db1", "plain", first);
+ glueService.createPartition("db1", "plain", second);
+
+ assertEquals(List.of("a,b"), glueService.getPartition("db1", "plain", List.of("a,b")).getValues());
+ assertEquals(List.of("a", "b"), glueService.getPartition("db1", "plain", List.of("a", "b")).getValues());
+ }
+
+ @Test
+ void batchCreatePartitionCreatesNewPartitionsAndReportsDuplicates() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+ Partition first = new Partition();
+ first.setValues(List.of("2026"));
+ Partition duplicate = new Partition();
+ duplicate.setValues(List.of("2026"));
+ Partition second = new Partition();
+ second.setValues(List.of("2027"));
+
+ List<GlueService.BatchCreatePartitionError> firstResult =
+ glueService.batchCreatePartitions("db1", "plain", List.of(first, second));
+ List<GlueService.BatchCreatePartitionError> secondResult =
+ glueService.batchCreatePartitions("db1", "plain", List.of(duplicate));
+
+ assertTrue(firstResult.isEmpty());
+ assertEquals(1, secondResult.size());
+ assertEquals(List.of("2026"), secondResult.getFirst().partitionValues());
+ assertEquals("AlreadyExistsException", secondResult.getFirst().errorDetail().errorCode());
+ assertEquals(2, glueService.getPartitions("db1", "plain").size());
+ }
+
+ @Test
+ void batchUpdatePartitionUpdatesPartitionsAndReportsMissingPartitions() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+ Partition existing = new Partition();
+ existing.setValues(List.of("2026"));
+ existing.setParameters(Map.of("before", "yes"));
+ glueService.createPartition("db1", "plain", existing);
+ Partition updated = new Partition();
+ updated.setValues(List.of("2026"));
+ updated.setParameters(Map.of("after", "yes"));
+ Partition missing = new Partition();
+ missing.setValues(List.of("missing"));
+
+ List<GlueService.BatchUpdatePartitionError> result = glueService.batchUpdatePartitions("db1", "plain", List.of(
+ new GlueService.BatchUpdatePartitionEntry(List.of("2026"), updated),
+ new GlueService.BatchUpdatePartitionEntry(List.of("missing"), missing)));
+
+ assertEquals(Map.of("after", "yes"), glueService.getPartition("db1", "plain", List.of("2026")).getParameters());
+ assertEquals(1, result.size());
+ assertEquals(List.of("missing"), result.getFirst().partitionValueList());
+ assertEquals("EntityNotFoundException", result.getFirst().errorDetail().errorCode());
+ assertEquals("Partition [missing] not found", result.getFirst().errorDetail().errorMessage());
+ }
+
+ @Test
+ void updatePartitionUpdatesPartitionAndThrowsForMissingPartition() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+ Partition existing = new Partition();
+ existing.setValues(List.of("2026"));
+ existing.setParameters(Map.of("before", "yes"));
+ glueService.createPartition("db1", "plain", existing);
+ Partition updated = new Partition();
+ updated.setValues(List.of("2026"));
+ updated.setParameters(Map.of("after", "yes"));
+
+ glueService.updatePartition("db1", "plain", List.of("2026"), updated);
+
+ assertEquals(Map.of("after", "yes"), glueService.getPartition("db1", "plain", List.of("2026")).getParameters());
+ Partition missing = new Partition();
+ missing.setValues(List.of("missing"));
+ AwsException ex = assertThrows(AwsException.class,
+ () -> glueService.updatePartition("db1", "plain", List.of("missing"), missing));
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ assertEquals("Partition not found.", ex.getMessage());
+ }
+
+ @Test
+ void deletePartitionForMissingPartitionThrows() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> glueService.deletePartition("db1", "plain", List.of("missing")));
+
+ assertEquals("EntityNotFoundException", ex.getErrorCode());
+ assertEquals("Cannot find partition.", ex.getMessage());
+ }
+
+ @Test
+ void partitionColumnStatisticsCanBeUpdatedFetchedAndDeleted() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+ Partition partition = new Partition();
+ partition.setValues(List.of("1"));
+ glueService.createPartition("db1", "plain", partition);
+ Map<String, Object> statistics = Map.of(
+ "ColumnName", "id",
+ "ColumnType", "int",
+ "AnalyzedTime", Instant.EPOCH,
+ "StatisticsData", Map.of(
+ "Type", "LONG",
+ "LongColumnStatisticsData", Map.of(
+ "MinimumValue", 1,
+ "MaximumValue", 10,
+ "NumberOfNulls", 0,
+ "NumberOfDistinctValues", 10)));
+
+ glueService.updateColumnStatisticsForPartition("db1", "plain", List.of("1"), List.of(statistics));
+ GlueService.ColumnStatisticsResult result = glueService.getColumnStatisticsForPartition(
+ "db1", "plain", List.of("1"), List.of("id", "missing"));
+
+ assertEquals(List.of(statistics), result.columnStatisticsList());
+ assertEquals(1, result.errors().size());
+ assertEquals("missing", result.errors().get(0).columnName());
+ assertEquals(
+ new GlueService.ErrorDetail("EntityNotFoundException", "Statistics do not exist for this column"),
+ result.errors().get(0).error());
+
+ glueService.deleteColumnStatisticsForPartition("db1", "plain", List.of("1"), "id");
+ GlueService.ColumnStatisticsResult afterDelete = glueService.getColumnStatisticsForPartition(
+ "db1", "plain", List.of("1"), List.of("id"));
+
+ assertTrue(afterDelete.columnStatisticsList().isEmpty());
+ assertEquals(1, afterDelete.errors().size());
+ }
+
+ @Test
+ void updatePartitionColumnStatisticsRejectsMissingRequiredFields() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+ Partition partition = new Partition();
+ partition.setValues(List.of("1"));
+ glueService.createPartition("db1", "plain", partition);
+
+ for (String field : List.of(
+ GlueService.COLUMN_NAME,
+ GlueService.COLUMN_TYPE,
+ GlueService.ANALYZED_TIME,
+ GlueService.STATISTICS_DATA)) {
+ Map<String, Object> statistics = new LinkedHashMap<>(columnStatistics("id"));
+ statistics.remove(field);
+
+ AwsException exception = assertThrows(AwsException.class,
+ () -> glueService.updateColumnStatisticsForPartition("db1", "plain", List.of("1"), List.of(statistics)));
+
+ assertEquals("InvalidInputException", exception.getErrorCode());
+ assertEquals(field + " is required", exception.getMessage());
+ }
+ }
+
@Test
void deleteDatabaseDoesNotDeleteSimilarDatabaseNames() {
glueService.createDatabase(new Database("a"));